diff --git a/pom.xml b/pom.xml index eb0c8c4..0307752 100644 --- a/pom.xml +++ b/pom.xml @@ -145,5 +145,5 @@ pentaho-kettle-dataset - 3.4.0-SNAPSHOT + 3.4.1-SNAPSHOT diff --git a/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java b/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java index f2182dd..ce9aef9 100755 --- a/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java +++ b/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java @@ -55,6 +55,7 @@ import org.pentaho.di.dataset.spoon.dialog.DataSetGroupDialog; import org.pentaho.di.dataset.spoon.dialog.EditRowsDialog; import org.pentaho.di.dataset.spoon.dialog.TransUnitTestDialog; +import org.pentaho.di.dataset.spoon.xtpoint.TransMetaModifier; import org.pentaho.di.dataset.spoon.xtpoint.WriteToDataSetExtensionPoint; import org.pentaho.di.dataset.util.DataSetConst; import org.pentaho.di.dataset.util.FactoriesHierarchy; @@ -580,21 +581,27 @@ private boolean checkTestPresent( Spoon spoon, TransMeta transMeta ) { public void setGoldenDataSet() { Spoon spoon = ( (Spoon) SpoonFactory.getInstance() ); TransGraph transGraph = spoon.getActiveTransGraph(); - TransMeta transMeta = spoon.getActiveTransformation(); + TransMeta sourceTransMeta = spoon.getActiveTransformation(); StepMeta stepMeta = transGraph.getCurrentStep(); - if ( transGraph == null || transMeta == null || stepMeta == null ) { + if ( transGraph == null || sourceTransMeta == null || stepMeta == null ) { return; } IMetaStore metaStore = spoon.getMetaStore(); - if ( checkTestPresent( spoon, transMeta ) ) { + if ( checkTestPresent( spoon, sourceTransMeta ) ) { return; } - TransUnitTest unitTest = activeTests.get( transMeta ); + TransUnitTest unitTest = activeTests.get( sourceTransMeta ); try { FactoriesHierarchy hierarchy = getHierarchy(); + // Create a copy and modify the transformation + // This way we have + TransMetaModifier modifier = new TransMetaModifier( sourceTransMeta, unitTest ); + TransMeta transMeta = modifier.getTestTransformation( LogChannel.UI, sourceTransMeta, hierarchy ); + + MetaStoreFactory setFactory = hierarchy.getSetFactory(); List setNames = setFactory.getElementNames(); Collections.sort( setNames ); @@ -605,7 +612,15 @@ public void setGoldenDataSet() { // Now we need to map the fields from the step to golden data set fields... // - RowMetaInterface stepFields = transMeta.getPrevStepFields( stepMeta ); + RowMetaInterface stepFields; + try { + stepFields = transMeta.getPrevStepFields( stepMeta ); + } catch(KettleStepException e) { + // Ignore error: issues with not being able to get fields because of the unit test + // running in a different environment. + // + stepFields = new RowMeta(); + } RowMetaInterface setFields = dataSet.getSetRowMeta( false ); String[] stepFieldNames = stepFields.getFieldNames(); diff --git a/src/main/java/org/pentaho/di/dataset/spoon/xtpoint/ChangeTransMetaPriorToExecutionExtensionPoint.java b/src/main/java/org/pentaho/di/dataset/spoon/xtpoint/ChangeTransMetaPriorToExecutionExtensionPoint.java index 4efaa84..0f56fd3 100644 --- a/src/main/java/org/pentaho/di/dataset/spoon/xtpoint/ChangeTransMetaPriorToExecutionExtensionPoint.java +++ b/src/main/java/org/pentaho/di/dataset/spoon/xtpoint/ChangeTransMetaPriorToExecutionExtensionPoint.java @@ -107,110 +107,17 @@ public void callExtensionPoint( LogChannelInterface log, Object object ) throws } catch(MetaStoreException e) { throw new KettleException("Unable to load unit test '"+unitTestName+"'", e); } - - + if (unitTest==null) { throw new KettleException("Unit test '"+unitTestName+"' was not found or could not be loaded"); } - // OK, so now replace an input step with a data set attached with an Injector step... - // However, we don't want to have the user see this so we need to copy trans.transMeta first... - // - // Clone seems to has problems so we'll take the long (XML) way around... - InputStream stream; - try { - stream = new ByteArrayInputStream( transMeta.getXML().getBytes(Const.XML_ENCODING) ); - } catch ( UnsupportedEncodingException e ) { - throw new KettleException( "Encoding error", e ); - } - TransMeta copyTransMeta = new TransMeta(stream, transMeta.getRepository(), true, transMeta, null); - // Pass the metadata references... - // - copyTransMeta.setRepository( transMeta.getRepository() ); - copyTransMeta.setMetaStore( transMeta.getMetaStore() ); - copyTransMeta.setSharedObjects( transMeta.getSharedObjects() ); - - // Replace certain connections with another + // Get a modified copy of the transformation using the unit test information // - for (TransUnitTestDatabaseReplacement dbReplacement : unitTest.getDatabaseReplacements()) { - String sourceDatabaseName = transMeta.environmentSubstitute(dbReplacement.getOriginalDatabaseName()); - String replacementDatabaseName = transMeta.environmentSubstitute(dbReplacement.getReplacementDatabaseName()); - - DatabaseMeta sourceDatabaseMeta = copyTransMeta.findDatabase(sourceDatabaseName); - DatabaseMeta replacementDatabaseMeta = copyTransMeta.findDatabase(replacementDatabaseName); - if (sourceDatabaseMeta==null) { - throw new KettleException("Unable to find source database connection '"+sourceDatabaseName+"', can not be replaced"); - } - if (replacementDatabaseMeta==null) { - throw new KettleException("Unable to find replacement database connection '"+replacementDatabaseName+"', can not be used to replace"); - } - - if (log.isDetailed()) { - log.logDetailed("Replaced database connection '"+sourceDatabaseName+"' with connection '"+replacementDatabaseName+"'"); - } - sourceDatabaseMeta.replaceMeta(replacementDatabaseMeta); - } + TransMetaModifier modifier = new TransMetaModifier( transMeta, unitTest ); + TransMeta copyTransMeta = modifier.getTestTransformation( log, trans, factoriesHierarchy ); - // Set parameters and variables... - // - String[] parameters = copyTransMeta.listParameters(); - List variableValues = unitTest.getVariableValues(); - for (VariableValue variableValue : variableValues) { - String key = trans.environmentSubstitute( variableValue.getKey() ); - String value = trans.environmentSubstitute( variableValue.getValue() ); - if (StringUtils.isEmpty(key)) { - continue; - } - if (Const.indexOfString( key, parameters )<0) { - // set the variable in the transformation metadata... - // - copyTransMeta.setVariable( key, value ); - } else { - // Set the parameter value... - // - copyTransMeta.setParameterValue( key, value ); - } - } - - // Replace all steps with an Input Data Set marker with an Injector - // Replace all steps with a Golden Data Set marker with a Dummy - // Apply the tweaks to the steps: - // - Bypass : replace with Dummy - // - Remove : remove step and all connected hops. - // - // Loop over the original transformation to allow us to safely modify the copy - // - List steps = transMeta.getSteps(); - for (StepMeta step : steps) { - StepMeta stepMeta = copyTransMeta.findStep(step.getName()); - TransUnitTestSetLocation inputLocation = unitTest.findInputLocation( stepMeta.getName() ); - TransUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation( stepMeta.getName() ); - TransUnitTestTweak stepTweak = unitTest.findTweak( stepMeta.getName() ); - - // See if there's a unit test if the step isn't flagged... - // - if ( inputLocation!=null ) { - handleInputDataSet(log, inputLocation, unitTest, transMeta, stepMeta, factoriesHierarchy); - } - - // Capture golden data in a dummy step instead of the regular one? - // - if ( goldenLocation!=null) { - handleGoldenDataSet(log, goldenLocation, stepMeta); - } - - if ( stepTweak!=null && stepTweak.getTweak()!=null) { - switch(stepTweak.getTweak()) { - case NONE : break; - case REMOVE_STEP: handleTweakRemoveStep(log, copyTransMeta, stepMeta); break; - case BYPASS_STEP: handleTweakBypassStep(log, stepMeta); break; - default: break; - } - } - } - - // Now replace the metadata in the Trans object... // trans.setTransMeta( copyTransMeta ); @@ -227,95 +134,4 @@ public void callExtensionPoint( LogChannelInterface log, Object object ) throws } } } - - private void handleTweakBypassStep(LogChannelInterface log, StepMeta stepMeta) { - if (log.isDetailed()) { - log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Dummy for Bypass step tweak"); - } - - replaceStepWithDummy(log, stepMeta); - } - - private void handleTweakRemoveStep(LogChannelInterface log, TransMeta copyTransMeta, StepMeta stepMeta) { - if (log.isDetailed()) { - log.logDetailed("Removing step '"+stepMeta.getName()+"' for Remove step tweak"); - } - - // Remove all hops connecting to the step to be removed... - // - List prevSteps = copyTransMeta.findPreviousSteps(stepMeta); - for (StepMeta prevStep : prevSteps) { - TransHopMeta hop = copyTransMeta.findTransHop(prevStep, stepMeta); - if (hop!=null) { - int hopIndex = copyTransMeta.indexOfTransHop(hop); - copyTransMeta.removeTransHop(hopIndex); - } - } - List nextSteps = copyTransMeta.findNextSteps(stepMeta); - for (StepMeta nextStep : nextSteps) { - TransHopMeta hop = copyTransMeta.findTransHop(stepMeta, nextStep); - if (hop!=null) { - int hopIndex = copyTransMeta.indexOfTransHop(hop); - copyTransMeta.removeTransHop(hopIndex); - } - } - - int idx = copyTransMeta.indexOfStep(stepMeta); - if (idx>=0) { - copyTransMeta.removeStep(idx); - } - } - - private void handleGoldenDataSet( LogChannelInterface log, TransUnitTestSetLocation goldenSetName, StepMeta stepMeta) { - - if (log.isDetailed()) { - log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Dummy for golden dataset '"+goldenSetName+"'"); - } - - replaceStepWithDummy(log, stepMeta); - } - - private void replaceStepWithDummy(LogChannelInterface log, StepMeta stepMeta) { - DummyTransMeta dummyTransMeta = new DummyTransMeta(); - stepMeta.setStepMetaInterface( dummyTransMeta ); - stepMeta.setStepID( PluginRegistry.getInstance().getPluginId( StepPluginType.class, dummyTransMeta) ); - } - - private void handleInputDataSet( LogChannelInterface log, TransUnitTestSetLocation inputLocation, TransUnitTest unitTest, TransMeta transMeta, StepMeta stepMeta, FactoriesHierarchy factoriesHierarchy) throws KettleException { - - String inputSetName = inputLocation.getDataSetName(); - - if (log.isDetailed()) { - log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Injector for dataset '"+inputSetName+"'"); - } - - DataSet dataSet; - try { - dataSet = factoriesHierarchy.getSetFactory().loadElement(inputSetName); - } catch (MetaStoreException e) { - throw new KettleException("Unable to load data set '"+inputSetName+"'"); - } - - // OK, this step needs to be replaced by an Injector step... - // Which fields do we need to use? - // - final RowMetaInterface stepFields = DataSetConst.getStepOutputFields(log, dataSet, inputLocation); - - if (log.isDetailed()) { - log.logDetailed("Input Data Set '"+inputSetName+"' Injector fields : '"+stepFields.toString()); - } - - InjectorMeta injectorMeta = new InjectorMeta(); - injectorMeta.allocate( stepFields.size() ); - for (int x=0;x variableValues = unitTest.getVariableValues(); + for (VariableValue variableValue : variableValues) { + String key = space.environmentSubstitute( variableValue.getKey() ); + String value = space.environmentSubstitute( variableValue.getValue() ); + + if ( StringUtils.isEmpty(key)) { + continue; + } + if (Const.indexOfString( key, parameters )<0) { + // set the variable in the transformation metadata... + // + copyTransMeta.setVariable( key, value ); + } else { + // Set the parameter value... + // + copyTransMeta.setParameterValue( key, value ); + } + } + + // Replace all steps with an Input Data Set marker with an Injector + // Replace all steps with a Golden Data Set marker with a Dummy + // Apply the tweaks to the steps: + // - Bypass : replace with Dummy + // - Remove : remove step and all connected hops. + // + // Loop over the original transformation to allow us to safely modify the copy + // + List steps = transMeta.getSteps(); + for (StepMeta step : steps) { + StepMeta stepMeta = copyTransMeta.findStep(step.getName()); + TransUnitTestSetLocation inputLocation = unitTest.findInputLocation( stepMeta.getName() ); + TransUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation( stepMeta.getName() ); + TransUnitTestTweak stepTweak = unitTest.findTweak( stepMeta.getName() ); + + // See if there's a unit test if the step isn't flagged... + // + if ( inputLocation!=null ) { + handleInputDataSet(log, inputLocation, unitTest, transMeta, stepMeta, factoriesHierarchy); + } + + // Capture golden data in a dummy step instead of the regular one? + // + if ( goldenLocation!=null) { + handleGoldenDataSet(log, goldenLocation, stepMeta); + } + + if ( stepTweak!=null && stepTweak.getTweak()!=null) { + switch(stepTweak.getTweak()) { + case NONE : break; + case REMOVE_STEP: handleTweakRemoveStep(log, copyTransMeta, stepMeta); break; + case BYPASS_STEP: handleTweakBypassStep(log, stepMeta); break; + default: break; + } + } + } + + return copyTransMeta; + } + + private void handleInputDataSet( LogChannelInterface log, TransUnitTestSetLocation inputLocation, TransUnitTest unitTest, TransMeta transMeta, StepMeta stepMeta, FactoriesHierarchy factoriesHierarchy) throws KettleException { + + String inputSetName = inputLocation.getDataSetName(); + + if (log.isDetailed()) { + log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Injector for dataset '"+inputSetName+"'"); + } + + DataSet dataSet; + try { + dataSet = factoriesHierarchy.getSetFactory().loadElement(inputSetName); + } catch ( MetaStoreException e) { + throw new KettleException("Unable to load data set '"+inputSetName+"'"); + } + + // OK, this step needs to be replaced by an Injector step... + // Which fields do we need to use? + // + final RowMetaInterface stepFields = DataSetConst.getStepOutputFields(log, dataSet, inputLocation); + + if (log.isDetailed()) { + log.logDetailed("Input Data Set '"+inputSetName+"' Injector fields : '"+stepFields.toString()); + } + + InjectorMeta injectorMeta = new InjectorMeta(); + injectorMeta.allocate( stepFields.size() ); + for (int x=0;x prevSteps = copyTransMeta.findPreviousSteps(stepMeta); + for (StepMeta prevStep : prevSteps) { + TransHopMeta hop = copyTransMeta.findTransHop(prevStep, stepMeta); + if (hop!=null) { + int hopIndex = copyTransMeta.indexOfTransHop(hop); + copyTransMeta.removeTransHop(hopIndex); + } + } + List nextSteps = copyTransMeta.findNextSteps(stepMeta); + for (StepMeta nextStep : nextSteps) { + TransHopMeta hop = copyTransMeta.findTransHop(stepMeta, nextStep); + if (hop!=null) { + int hopIndex = copyTransMeta.indexOfTransHop(hop); + copyTransMeta.removeTransHop(hopIndex); + } + } + + int idx = copyTransMeta.indexOfStep(stepMeta); + if (idx>=0) { + copyTransMeta.removeStep(idx); + } + } + + + /** + * Gets transMeta + * + * @return value of transMeta + */ + public TransMeta getTransMeta() { + return transMeta; + } + + /** + * @param transMeta The transMeta to set + */ + public void setTransMeta( TransMeta transMeta ) { + this.transMeta = transMeta; + } + + /** + * Gets unitTest + * + * @return value of unitTest + */ + public TransUnitTest getUnitTest() { + return unitTest; + } + + /** + * @param unitTest The unitTest to set + */ + public void setUnitTest( TransUnitTest unitTest ) { + this.unitTest = unitTest; + } +}