diff --git a/pom.xml b/pom.xml
index eb0c8c4..0307752 100644
--- a/pom.xml
+++ b/pom.xml
@@ -145,5 +145,5 @@
pentaho-kettle-dataset
- 3.4.0-SNAPSHOT
+ 3.4.1-SNAPSHOT
diff --git a/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java b/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java
index f2182dd..ce9aef9 100755
--- a/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java
+++ b/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java
@@ -55,6 +55,7 @@
import org.pentaho.di.dataset.spoon.dialog.DataSetGroupDialog;
import org.pentaho.di.dataset.spoon.dialog.EditRowsDialog;
import org.pentaho.di.dataset.spoon.dialog.TransUnitTestDialog;
+import org.pentaho.di.dataset.spoon.xtpoint.TransMetaModifier;
import org.pentaho.di.dataset.spoon.xtpoint.WriteToDataSetExtensionPoint;
import org.pentaho.di.dataset.util.DataSetConst;
import org.pentaho.di.dataset.util.FactoriesHierarchy;
@@ -580,21 +581,27 @@ private boolean checkTestPresent( Spoon spoon, TransMeta transMeta ) {
public void setGoldenDataSet() {
Spoon spoon = ( (Spoon) SpoonFactory.getInstance() );
TransGraph transGraph = spoon.getActiveTransGraph();
- TransMeta transMeta = spoon.getActiveTransformation();
+ TransMeta sourceTransMeta = spoon.getActiveTransformation();
StepMeta stepMeta = transGraph.getCurrentStep();
- if ( transGraph == null || transMeta == null || stepMeta == null ) {
+ if ( transGraph == null || sourceTransMeta == null || stepMeta == null ) {
return;
}
IMetaStore metaStore = spoon.getMetaStore();
- if ( checkTestPresent( spoon, transMeta ) ) {
+ if ( checkTestPresent( spoon, sourceTransMeta ) ) {
return;
}
- TransUnitTest unitTest = activeTests.get( transMeta );
+ TransUnitTest unitTest = activeTests.get( sourceTransMeta );
try {
FactoriesHierarchy hierarchy = getHierarchy();
+ // Create a copy and modify the transformation
+ // This way we have
+ TransMetaModifier modifier = new TransMetaModifier( sourceTransMeta, unitTest );
+ TransMeta transMeta = modifier.getTestTransformation( LogChannel.UI, sourceTransMeta, hierarchy );
+
+
MetaStoreFactory setFactory = hierarchy.getSetFactory();
List setNames = setFactory.getElementNames();
Collections.sort( setNames );
@@ -605,7 +612,15 @@ public void setGoldenDataSet() {
// Now we need to map the fields from the step to golden data set fields...
//
- RowMetaInterface stepFields = transMeta.getPrevStepFields( stepMeta );
+ RowMetaInterface stepFields;
+ try {
+ stepFields = transMeta.getPrevStepFields( stepMeta );
+ } catch(KettleStepException e) {
+ // Ignore error: issues with not being able to get fields because of the unit test
+ // running in a different environment.
+ //
+ stepFields = new RowMeta();
+ }
RowMetaInterface setFields = dataSet.getSetRowMeta( false );
String[] stepFieldNames = stepFields.getFieldNames();
diff --git a/src/main/java/org/pentaho/di/dataset/spoon/xtpoint/ChangeTransMetaPriorToExecutionExtensionPoint.java b/src/main/java/org/pentaho/di/dataset/spoon/xtpoint/ChangeTransMetaPriorToExecutionExtensionPoint.java
index 4efaa84..0f56fd3 100644
--- a/src/main/java/org/pentaho/di/dataset/spoon/xtpoint/ChangeTransMetaPriorToExecutionExtensionPoint.java
+++ b/src/main/java/org/pentaho/di/dataset/spoon/xtpoint/ChangeTransMetaPriorToExecutionExtensionPoint.java
@@ -107,110 +107,17 @@ public void callExtensionPoint( LogChannelInterface log, Object object ) throws
} catch(MetaStoreException e) {
throw new KettleException("Unable to load unit test '"+unitTestName+"'", e);
}
-
-
+
if (unitTest==null) {
throw new KettleException("Unit test '"+unitTestName+"' was not found or could not be loaded");
}
- // OK, so now replace an input step with a data set attached with an Injector step...
- // However, we don't want to have the user see this so we need to copy trans.transMeta first...
- //
- // Clone seems to has problems so we'll take the long (XML) way around...
- InputStream stream;
- try {
- stream = new ByteArrayInputStream( transMeta.getXML().getBytes(Const.XML_ENCODING) );
- } catch ( UnsupportedEncodingException e ) {
- throw new KettleException( "Encoding error", e );
- }
- TransMeta copyTransMeta = new TransMeta(stream, transMeta.getRepository(), true, transMeta, null);
- // Pass the metadata references...
- //
- copyTransMeta.setRepository( transMeta.getRepository() );
- copyTransMeta.setMetaStore( transMeta.getMetaStore() );
- copyTransMeta.setSharedObjects( transMeta.getSharedObjects() );
-
- // Replace certain connections with another
+ // Get a modified copy of the transformation using the unit test information
//
- for (TransUnitTestDatabaseReplacement dbReplacement : unitTest.getDatabaseReplacements()) {
- String sourceDatabaseName = transMeta.environmentSubstitute(dbReplacement.getOriginalDatabaseName());
- String replacementDatabaseName = transMeta.environmentSubstitute(dbReplacement.getReplacementDatabaseName());
-
- DatabaseMeta sourceDatabaseMeta = copyTransMeta.findDatabase(sourceDatabaseName);
- DatabaseMeta replacementDatabaseMeta = copyTransMeta.findDatabase(replacementDatabaseName);
- if (sourceDatabaseMeta==null) {
- throw new KettleException("Unable to find source database connection '"+sourceDatabaseName+"', can not be replaced");
- }
- if (replacementDatabaseMeta==null) {
- throw new KettleException("Unable to find replacement database connection '"+replacementDatabaseName+"', can not be used to replace");
- }
-
- if (log.isDetailed()) {
- log.logDetailed("Replaced database connection '"+sourceDatabaseName+"' with connection '"+replacementDatabaseName+"'");
- }
- sourceDatabaseMeta.replaceMeta(replacementDatabaseMeta);
- }
+ TransMetaModifier modifier = new TransMetaModifier( transMeta, unitTest );
+ TransMeta copyTransMeta = modifier.getTestTransformation( log, trans, factoriesHierarchy );
- // Set parameters and variables...
- //
- String[] parameters = copyTransMeta.listParameters();
- List variableValues = unitTest.getVariableValues();
- for (VariableValue variableValue : variableValues) {
- String key = trans.environmentSubstitute( variableValue.getKey() );
- String value = trans.environmentSubstitute( variableValue.getValue() );
- if (StringUtils.isEmpty(key)) {
- continue;
- }
- if (Const.indexOfString( key, parameters )<0) {
- // set the variable in the transformation metadata...
- //
- copyTransMeta.setVariable( key, value );
- } else {
- // Set the parameter value...
- //
- copyTransMeta.setParameterValue( key, value );
- }
- }
-
- // Replace all steps with an Input Data Set marker with an Injector
- // Replace all steps with a Golden Data Set marker with a Dummy
- // Apply the tweaks to the steps:
- // - Bypass : replace with Dummy
- // - Remove : remove step and all connected hops.
- //
- // Loop over the original transformation to allow us to safely modify the copy
- //
- List steps = transMeta.getSteps();
- for (StepMeta step : steps) {
- StepMeta stepMeta = copyTransMeta.findStep(step.getName());
- TransUnitTestSetLocation inputLocation = unitTest.findInputLocation( stepMeta.getName() );
- TransUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation( stepMeta.getName() );
- TransUnitTestTweak stepTweak = unitTest.findTweak( stepMeta.getName() );
-
- // See if there's a unit test if the step isn't flagged...
- //
- if ( inputLocation!=null ) {
- handleInputDataSet(log, inputLocation, unitTest, transMeta, stepMeta, factoriesHierarchy);
- }
-
- // Capture golden data in a dummy step instead of the regular one?
- //
- if ( goldenLocation!=null) {
- handleGoldenDataSet(log, goldenLocation, stepMeta);
- }
-
- if ( stepTweak!=null && stepTweak.getTweak()!=null) {
- switch(stepTweak.getTweak()) {
- case NONE : break;
- case REMOVE_STEP: handleTweakRemoveStep(log, copyTransMeta, stepMeta); break;
- case BYPASS_STEP: handleTweakBypassStep(log, stepMeta); break;
- default: break;
- }
- }
- }
-
-
// Now replace the metadata in the Trans object...
//
trans.setTransMeta( copyTransMeta );
@@ -227,95 +134,4 @@ public void callExtensionPoint( LogChannelInterface log, Object object ) throws
}
}
}
-
- private void handleTweakBypassStep(LogChannelInterface log, StepMeta stepMeta) {
- if (log.isDetailed()) {
- log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Dummy for Bypass step tweak");
- }
-
- replaceStepWithDummy(log, stepMeta);
- }
-
- private void handleTweakRemoveStep(LogChannelInterface log, TransMeta copyTransMeta, StepMeta stepMeta) {
- if (log.isDetailed()) {
- log.logDetailed("Removing step '"+stepMeta.getName()+"' for Remove step tweak");
- }
-
- // Remove all hops connecting to the step to be removed...
- //
- List prevSteps = copyTransMeta.findPreviousSteps(stepMeta);
- for (StepMeta prevStep : prevSteps) {
- TransHopMeta hop = copyTransMeta.findTransHop(prevStep, stepMeta);
- if (hop!=null) {
- int hopIndex = copyTransMeta.indexOfTransHop(hop);
- copyTransMeta.removeTransHop(hopIndex);
- }
- }
- List nextSteps = copyTransMeta.findNextSteps(stepMeta);
- for (StepMeta nextStep : nextSteps) {
- TransHopMeta hop = copyTransMeta.findTransHop(stepMeta, nextStep);
- if (hop!=null) {
- int hopIndex = copyTransMeta.indexOfTransHop(hop);
- copyTransMeta.removeTransHop(hopIndex);
- }
- }
-
- int idx = copyTransMeta.indexOfStep(stepMeta);
- if (idx>=0) {
- copyTransMeta.removeStep(idx);
- }
- }
-
- private void handleGoldenDataSet( LogChannelInterface log, TransUnitTestSetLocation goldenSetName, StepMeta stepMeta) {
-
- if (log.isDetailed()) {
- log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Dummy for golden dataset '"+goldenSetName+"'");
- }
-
- replaceStepWithDummy(log, stepMeta);
- }
-
- private void replaceStepWithDummy(LogChannelInterface log, StepMeta stepMeta) {
- DummyTransMeta dummyTransMeta = new DummyTransMeta();
- stepMeta.setStepMetaInterface( dummyTransMeta );
- stepMeta.setStepID( PluginRegistry.getInstance().getPluginId( StepPluginType.class, dummyTransMeta) );
- }
-
- private void handleInputDataSet( LogChannelInterface log, TransUnitTestSetLocation inputLocation, TransUnitTest unitTest, TransMeta transMeta, StepMeta stepMeta, FactoriesHierarchy factoriesHierarchy) throws KettleException {
-
- String inputSetName = inputLocation.getDataSetName();
-
- if (log.isDetailed()) {
- log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Injector for dataset '"+inputSetName+"'");
- }
-
- DataSet dataSet;
- try {
- dataSet = factoriesHierarchy.getSetFactory().loadElement(inputSetName);
- } catch (MetaStoreException e) {
- throw new KettleException("Unable to load data set '"+inputSetName+"'");
- }
-
- // OK, this step needs to be replaced by an Injector step...
- // Which fields do we need to use?
- //
- final RowMetaInterface stepFields = DataSetConst.getStepOutputFields(log, dataSet, inputLocation);
-
- if (log.isDetailed()) {
- log.logDetailed("Input Data Set '"+inputSetName+"' Injector fields : '"+stepFields.toString());
- }
-
- InjectorMeta injectorMeta = new InjectorMeta();
- injectorMeta.allocate( stepFields.size() );
- for (int x=0;x variableValues = unitTest.getVariableValues();
+ for (VariableValue variableValue : variableValues) {
+ String key = space.environmentSubstitute( variableValue.getKey() );
+ String value = space.environmentSubstitute( variableValue.getValue() );
+
+ if ( StringUtils.isEmpty(key)) {
+ continue;
+ }
+ if (Const.indexOfString( key, parameters )<0) {
+ // set the variable in the transformation metadata...
+ //
+ copyTransMeta.setVariable( key, value );
+ } else {
+ // Set the parameter value...
+ //
+ copyTransMeta.setParameterValue( key, value );
+ }
+ }
+
+ // Replace all steps with an Input Data Set marker with an Injector
+ // Replace all steps with a Golden Data Set marker with a Dummy
+ // Apply the tweaks to the steps:
+ // - Bypass : replace with Dummy
+ // - Remove : remove step and all connected hops.
+ //
+ // Loop over the original transformation to allow us to safely modify the copy
+ //
+ List steps = transMeta.getSteps();
+ for (StepMeta step : steps) {
+ StepMeta stepMeta = copyTransMeta.findStep(step.getName());
+ TransUnitTestSetLocation inputLocation = unitTest.findInputLocation( stepMeta.getName() );
+ TransUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation( stepMeta.getName() );
+ TransUnitTestTweak stepTweak = unitTest.findTweak( stepMeta.getName() );
+
+ // See if there's a unit test if the step isn't flagged...
+ //
+ if ( inputLocation!=null ) {
+ handleInputDataSet(log, inputLocation, unitTest, transMeta, stepMeta, factoriesHierarchy);
+ }
+
+ // Capture golden data in a dummy step instead of the regular one?
+ //
+ if ( goldenLocation!=null) {
+ handleGoldenDataSet(log, goldenLocation, stepMeta);
+ }
+
+ if ( stepTweak!=null && stepTweak.getTweak()!=null) {
+ switch(stepTweak.getTweak()) {
+ case NONE : break;
+ case REMOVE_STEP: handleTweakRemoveStep(log, copyTransMeta, stepMeta); break;
+ case BYPASS_STEP: handleTweakBypassStep(log, stepMeta); break;
+ default: break;
+ }
+ }
+ }
+
+ return copyTransMeta;
+ }
+
+ private void handleInputDataSet( LogChannelInterface log, TransUnitTestSetLocation inputLocation, TransUnitTest unitTest, TransMeta transMeta, StepMeta stepMeta, FactoriesHierarchy factoriesHierarchy) throws KettleException {
+
+ String inputSetName = inputLocation.getDataSetName();
+
+ if (log.isDetailed()) {
+ log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Injector for dataset '"+inputSetName+"'");
+ }
+
+ DataSet dataSet;
+ try {
+ dataSet = factoriesHierarchy.getSetFactory().loadElement(inputSetName);
+ } catch ( MetaStoreException e) {
+ throw new KettleException("Unable to load data set '"+inputSetName+"'");
+ }
+
+ // OK, this step needs to be replaced by an Injector step...
+ // Which fields do we need to use?
+ //
+ final RowMetaInterface stepFields = DataSetConst.getStepOutputFields(log, dataSet, inputLocation);
+
+ if (log.isDetailed()) {
+ log.logDetailed("Input Data Set '"+inputSetName+"' Injector fields : '"+stepFields.toString());
+ }
+
+ InjectorMeta injectorMeta = new InjectorMeta();
+ injectorMeta.allocate( stepFields.size() );
+ for (int x=0;x prevSteps = copyTransMeta.findPreviousSteps(stepMeta);
+ for (StepMeta prevStep : prevSteps) {
+ TransHopMeta hop = copyTransMeta.findTransHop(prevStep, stepMeta);
+ if (hop!=null) {
+ int hopIndex = copyTransMeta.indexOfTransHop(hop);
+ copyTransMeta.removeTransHop(hopIndex);
+ }
+ }
+ List nextSteps = copyTransMeta.findNextSteps(stepMeta);
+ for (StepMeta nextStep : nextSteps) {
+ TransHopMeta hop = copyTransMeta.findTransHop(stepMeta, nextStep);
+ if (hop!=null) {
+ int hopIndex = copyTransMeta.indexOfTransHop(hop);
+ copyTransMeta.removeTransHop(hopIndex);
+ }
+ }
+
+ int idx = copyTransMeta.indexOfStep(stepMeta);
+ if (idx>=0) {
+ copyTransMeta.removeStep(idx);
+ }
+ }
+
+
+ /**
+ * Gets transMeta
+ *
+ * @return value of transMeta
+ */
+ public TransMeta getTransMeta() {
+ return transMeta;
+ }
+
+ /**
+ * @param transMeta The transMeta to set
+ */
+ public void setTransMeta( TransMeta transMeta ) {
+ this.transMeta = transMeta;
+ }
+
+ /**
+ * Gets unitTest
+ *
+ * @return value of unitTest
+ */
+ public TransUnitTest getUnitTest() {
+ return unitTest;
+ }
+
+ /**
+ * @param unitTest The unitTest to set
+ */
+ public void setUnitTest( TransUnitTest unitTest ) {
+ this.unitTest = unitTest;
+ }
+}