Skip to content
This repository has been archived by the owner on Dec 15, 2022. It is now read-only.

Commit

Permalink
Issue #37, golden data sets part
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattcasters
mattcasters committed Jan 21, 2019
1 parent 59616fe commit fb6ae17
Show file tree
Hide file tree
Showing 4 changed files with 294 additions and 194 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Expand Up @@ -145,5 +145,5 @@
</dependencyManagement>

<groupId>pentaho-kettle-dataset</groupId>
<version>3.4.0-SNAPSHOT</version>
<version>3.4.1-SNAPSHOT</version>
</project>
25 changes: 20 additions & 5 deletions src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java
View file
Expand Up @@ -55,6 +55,7 @@
import org.pentaho.di.dataset.spoon.dialog.DataSetGroupDialog;
import org.pentaho.di.dataset.spoon.dialog.EditRowsDialog;
import org.pentaho.di.dataset.spoon.dialog.TransUnitTestDialog;
import org.pentaho.di.dataset.spoon.xtpoint.TransMetaModifier;
import org.pentaho.di.dataset.spoon.xtpoint.WriteToDataSetExtensionPoint;
import org.pentaho.di.dataset.util.DataSetConst;
import org.pentaho.di.dataset.util.FactoriesHierarchy;
Expand Down Expand Up @@ -580,21 +581,27 @@ private boolean checkTestPresent( Spoon spoon, TransMeta transMeta ) {
public void setGoldenDataSet() {
Spoon spoon = ( (Spoon) SpoonFactory.getInstance() );
TransGraph transGraph = spoon.getActiveTransGraph();
TransMeta transMeta = spoon.getActiveTransformation();
TransMeta sourceTransMeta = spoon.getActiveTransformation();
StepMeta stepMeta = transGraph.getCurrentStep();
if ( transGraph == null || transMeta == null || stepMeta == null ) {
if ( transGraph == null || sourceTransMeta == null || stepMeta == null ) {
return;
}
IMetaStore metaStore = spoon.getMetaStore();

if ( checkTestPresent( spoon, transMeta ) ) {
if ( checkTestPresent( spoon, sourceTransMeta ) ) {
return;
}
TransUnitTest unitTest = activeTests.get( transMeta );
TransUnitTest unitTest = activeTests.get( sourceTransMeta );

try {
FactoriesHierarchy hierarchy = getHierarchy();

// Create a copy and modify the transformation
// This way we have
TransMetaModifier modifier = new TransMetaModifier( sourceTransMeta, unitTest );
TransMeta transMeta = modifier.getTestTransformation( LogChannel.UI, sourceTransMeta, hierarchy );


MetaStoreFactory<DataSet> setFactory = hierarchy.getSetFactory();
List<String> setNames = setFactory.getElementNames();
Collections.sort( setNames );
Expand All @@ -605,7 +612,15 @@ public void setGoldenDataSet() {

// Now we need to map the fields from the step to golden data set fields...
//
RowMetaInterface stepFields = transMeta.getPrevStepFields( stepMeta );
RowMetaInterface stepFields;
try {
stepFields = transMeta.getPrevStepFields( stepMeta );
} catch(KettleStepException e) {
// Ignore error: issues with not being able to get fields because of the unit test
// running in a different environment.
//
stepFields = new RowMeta();
}
RowMetaInterface setFields = dataSet.getSetRowMeta( false );

String[] stepFieldNames = stepFields.getFieldNames();
Expand Down
192 changes: 4 additions & 188 deletions ...a/org/pentaho/di/dataset/spoon/xtpoint/ChangeTransMetaPriorToExecutionExtensionPoint.java
View file
Expand Up @@ -107,110 +107,17 @@ public void callExtensionPoint( LogChannelInterface log, Object object ) throws
} catch(MetaStoreException e) {
throw new KettleException("Unable to load unit test '"+unitTestName+"'", e);
}



if (unitTest==null) {
throw new KettleException("Unit test '"+unitTestName+"' was not found or could not be loaded");
}

// OK, so now replace an input step with a data set attached with an Injector step...
// However, we don't want to have the user see this so we need to copy trans.transMeta first...
//
// Clone seems to has problems so we'll take the long (XML) way around...
InputStream stream;
try {
stream = new ByteArrayInputStream( transMeta.getXML().getBytes(Const.XML_ENCODING) );
} catch ( UnsupportedEncodingException e ) {
throw new KettleException( "Encoding error", e );
}
TransMeta copyTransMeta = new TransMeta(stream, transMeta.getRepository(), true, transMeta, null);
// Pass the metadata references...
//
copyTransMeta.setRepository( transMeta.getRepository() );
copyTransMeta.setMetaStore( transMeta.getMetaStore() );
copyTransMeta.setSharedObjects( transMeta.getSharedObjects() );

// Replace certain connections with another
// Get a modified copy of the transformation using the unit test information
//
for (TransUnitTestDatabaseReplacement dbReplacement : unitTest.getDatabaseReplacements()) {
String sourceDatabaseName = transMeta.environmentSubstitute(dbReplacement.getOriginalDatabaseName());
String replacementDatabaseName = transMeta.environmentSubstitute(dbReplacement.getReplacementDatabaseName());

DatabaseMeta sourceDatabaseMeta = copyTransMeta.findDatabase(sourceDatabaseName);
DatabaseMeta replacementDatabaseMeta = copyTransMeta.findDatabase(replacementDatabaseName);
if (sourceDatabaseMeta==null) {
throw new KettleException("Unable to find source database connection '"+sourceDatabaseName+"', can not be replaced");
}
if (replacementDatabaseMeta==null) {
throw new KettleException("Unable to find replacement database connection '"+replacementDatabaseName+"', can not be used to replace");
}

if (log.isDetailed()) {
log.logDetailed("Replaced database connection '"+sourceDatabaseName+"' with connection '"+replacementDatabaseName+"'");
}
sourceDatabaseMeta.replaceMeta(replacementDatabaseMeta);
}
TransMetaModifier modifier = new TransMetaModifier( transMeta, unitTest );
TransMeta copyTransMeta = modifier.getTestTransformation( log, trans, factoriesHierarchy );

// Set parameters and variables...
//
String[] parameters = copyTransMeta.listParameters();
List<VariableValue> variableValues = unitTest.getVariableValues();
for (VariableValue variableValue : variableValues) {
String key = trans.environmentSubstitute( variableValue.getKey() );
String value = trans.environmentSubstitute( variableValue.getValue() );

if (StringUtils.isEmpty(key)) {
continue;
}
if (Const.indexOfString( key, parameters )<0) {
// set the variable in the transformation metadata...
//
copyTransMeta.setVariable( key, value );
} else {
// Set the parameter value...
//
copyTransMeta.setParameterValue( key, value );
}
}

// Replace all steps with an Input Data Set marker with an Injector
// Replace all steps with a Golden Data Set marker with a Dummy
// Apply the tweaks to the steps:
// - Bypass : replace with Dummy
// - Remove : remove step and all connected hops.
//
// Loop over the original transformation to allow us to safely modify the copy
//
List<StepMeta> steps = transMeta.getSteps();
for (StepMeta step : steps) {
StepMeta stepMeta = copyTransMeta.findStep(step.getName());
TransUnitTestSetLocation inputLocation = unitTest.findInputLocation( stepMeta.getName() );
TransUnitTestSetLocation goldenLocation = unitTest.findGoldenLocation( stepMeta.getName() );
TransUnitTestTweak stepTweak = unitTest.findTweak( stepMeta.getName() );

// See if there's a unit test if the step isn't flagged...
//
if ( inputLocation!=null ) {
handleInputDataSet(log, inputLocation, unitTest, transMeta, stepMeta, factoriesHierarchy);
}

// Capture golden data in a dummy step instead of the regular one?
//
if ( goldenLocation!=null) {
handleGoldenDataSet(log, goldenLocation, stepMeta);
}

if ( stepTweak!=null && stepTweak.getTweak()!=null) {
switch(stepTweak.getTweak()) {
case NONE : break;
case REMOVE_STEP: handleTweakRemoveStep(log, copyTransMeta, stepMeta); break;
case BYPASS_STEP: handleTweakBypassStep(log, stepMeta); break;
default: break;
}
}
}


// Now replace the metadata in the Trans object...
//
trans.setTransMeta( copyTransMeta );
Expand All @@ -227,95 +134,4 @@ public void callExtensionPoint( LogChannelInterface log, Object object ) throws
}
}
}

private void handleTweakBypassStep(LogChannelInterface log, StepMeta stepMeta) {
if (log.isDetailed()) {
log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Dummy for Bypass step tweak");
}

replaceStepWithDummy(log, stepMeta);
}

private void handleTweakRemoveStep(LogChannelInterface log, TransMeta copyTransMeta, StepMeta stepMeta) {
if (log.isDetailed()) {
log.logDetailed("Removing step '"+stepMeta.getName()+"' for Remove step tweak");
}

// Remove all hops connecting to the step to be removed...
//
List<StepMeta> prevSteps = copyTransMeta.findPreviousSteps(stepMeta);
for (StepMeta prevStep : prevSteps) {
TransHopMeta hop = copyTransMeta.findTransHop(prevStep, stepMeta);
if (hop!=null) {
int hopIndex = copyTransMeta.indexOfTransHop(hop);
copyTransMeta.removeTransHop(hopIndex);
}
}
List<StepMeta> nextSteps = copyTransMeta.findNextSteps(stepMeta);
for (StepMeta nextStep : nextSteps) {
TransHopMeta hop = copyTransMeta.findTransHop(stepMeta, nextStep);
if (hop!=null) {
int hopIndex = copyTransMeta.indexOfTransHop(hop);
copyTransMeta.removeTransHop(hopIndex);
}
}

int idx = copyTransMeta.indexOfStep(stepMeta);
if (idx>=0) {
copyTransMeta.removeStep(idx);
}
}

private void handleGoldenDataSet( LogChannelInterface log, TransUnitTestSetLocation goldenSetName, StepMeta stepMeta) {

if (log.isDetailed()) {
log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Dummy for golden dataset '"+goldenSetName+"'");
}

replaceStepWithDummy(log, stepMeta);
}

private void replaceStepWithDummy(LogChannelInterface log, StepMeta stepMeta) {
DummyTransMeta dummyTransMeta = new DummyTransMeta();
stepMeta.setStepMetaInterface( dummyTransMeta );
stepMeta.setStepID( PluginRegistry.getInstance().getPluginId( StepPluginType.class, dummyTransMeta) );
}

private void handleInputDataSet( LogChannelInterface log, TransUnitTestSetLocation inputLocation, TransUnitTest unitTest, TransMeta transMeta, StepMeta stepMeta, FactoriesHierarchy factoriesHierarchy) throws KettleException {

String inputSetName = inputLocation.getDataSetName();

if (log.isDetailed()) {
log.logDetailed("Replacing step '"+stepMeta.getName()+"' with an Injector for dataset '"+inputSetName+"'");
}

DataSet dataSet;
try {
dataSet = factoriesHierarchy.getSetFactory().loadElement(inputSetName);
} catch (MetaStoreException e) {
throw new KettleException("Unable to load data set '"+inputSetName+"'");
}

// OK, this step needs to be replaced by an Injector step...
// Which fields do we need to use?
//
final RowMetaInterface stepFields = DataSetConst.getStepOutputFields(log, dataSet, inputLocation);

if (log.isDetailed()) {
log.logDetailed("Input Data Set '"+inputSetName+"' Injector fields : '"+stepFields.toString());
}

InjectorMeta injectorMeta = new InjectorMeta();
injectorMeta.allocate( stepFields.size() );
for (int x=0;x<stepFields.size();x++) {
injectorMeta.getFieldname()[x] = stepFields.getValueMeta( x ).getName();
injectorMeta.getType()[x] = stepFields.getValueMeta( x ).getType();
injectorMeta.getLength()[x] = stepFields.getValueMeta( x ).getLength();
injectorMeta.getPrecision()[x] = stepFields.getValueMeta( x ).getPrecision();

// Only the step metadata, type...
stepMeta.setStepMetaInterface( injectorMeta );
stepMeta.setStepID( PluginRegistry.getInstance().getPluginId( StepPluginType.class, injectorMeta) );
}
}
}

0 comments on commit fb6ae17

Please sign in to comment.