diff --git a/build-res/subfloor.xml b/build-res/subfloor.xml index df12d66..d7898f4 100755 --- a/build-res/subfloor.xml +++ b/build-res/subfloor.xml @@ -156,8 +156,8 @@ TYPICAL TARGET SEQUENCE - - + + diff --git a/src/org/pentaho/di/dataset/TransUnitTest.java b/src/org/pentaho/di/dataset/TransUnitTest.java index 79bf310..5fa3e74 100644 --- a/src/org/pentaho/di/dataset/TransUnitTest.java +++ b/src/org/pentaho/di/dataset/TransUnitTest.java @@ -51,12 +51,16 @@ public class TransUnitTest { @MetaStoreAttribute( key = "persist_filename") protected String filename; + + @MetaStoreAttribute( key = "database_replacements" ) + protected List databaseReplacements; public TransUnitTest() { inputDataSets = new ArrayList(); goldenDataSets = new ArrayList(); tweaks = new ArrayList(); type = TestType.NONE; + databaseReplacements = new ArrayList(); } public TransUnitTest( String name, String description, @@ -65,7 +69,8 @@ public TransUnitTest( String name, String description, List goldenDataSets, List tweaks, TestType type, - String filename) { + String filename, + List databaseReplacements) { this(); this.name = name; this.description = description; @@ -77,6 +82,7 @@ public TransUnitTest( String name, String description, this.tweaks = tweaks; this.type = type; this.filename = filename; + this.databaseReplacements = databaseReplacements; } @Override @@ -252,6 +258,15 @@ public void setFilename(String filename) { this.filename = filename; } + public List getDatabaseReplacements() { + return databaseReplacements; + } + + public void setDatabaseReplacements(List databaseReplacements) { + this.databaseReplacements = databaseReplacements; + } + + /** * Remove all input and golden data sets on the step with the provided name * @param stepname the name of the step for which we need to clear out all input and golden data sets @@ -271,7 +286,6 @@ public void removeInputAndGoldenDataSets(String stepname) { iterator.remove(); } } - } } diff --git a/src/org/pentaho/di/dataset/TransUnitTestDatabaseReplacement.java b/src/org/pentaho/di/dataset/TransUnitTestDatabaseReplacement.java new file mode 100644 index 0000000..37e8e8f --- /dev/null +++ b/src/org/pentaho/di/dataset/TransUnitTestDatabaseReplacement.java @@ -0,0 +1,43 @@ +package org.pentaho.di.dataset; + +import org.pentaho.metastore.persist.MetaStoreAttribute; + +/** + * This class simply replaces all occurrences of a certain database connection with another one. It allows developers to point to a test database for lookup data and database related steps like database lookup, dimension lookup and so on. + * + * @author matt + * + */ +public class TransUnitTestDatabaseReplacement { + + @MetaStoreAttribute(key = "original_connection") + private String originalDatabaseName; + + @MetaStoreAttribute(key = "replacement_connection") + private String replacementDatabaseName; + + public TransUnitTestDatabaseReplacement(String originalDatabaseName, String replacementDatabaseName) { + this(); + this.originalDatabaseName = originalDatabaseName; + this.replacementDatabaseName = replacementDatabaseName; + } + + public TransUnitTestDatabaseReplacement() { + } + + public String getOriginalDatabaseName() { + return originalDatabaseName; + } + + public void setOriginalDatabaseName(String originalDatabaseName) { + this.originalDatabaseName = originalDatabaseName; + } + + public String getReplacementDatabaseName() { + return replacementDatabaseName; + } + + public void setReplacementDatabaseName(String replacementDatabaseName) { + this.replacementDatabaseName = replacementDatabaseName; + } +} diff --git a/src/org/pentaho/di/dataset/spoon/DataSetHelper.java b/src/org/pentaho/di/dataset/spoon/DataSetHelper.java index 051a9c7..27cf6f7 100755 --- a/src/org/pentaho/di/dataset/spoon/DataSetHelper.java +++ b/src/org/pentaho/di/dataset/spoon/DataSetHelper.java @@ -767,8 +767,6 @@ public void detachUnitTest() { public void selectUnitTest() { - System.out.println("XXXXXX selectUnitTest()"); - Spoon spoon = ( (Spoon) SpoonFactory.getInstance() ); try { TransGraph transGraph = spoon.getActiveTransGraph(); diff --git a/src/org/pentaho/di/dataset/spoon/dialog/TransUnitTestDialog.java b/src/org/pentaho/di/dataset/spoon/dialog/TransUnitTestDialog.java index 28be0b5..8972b81 100644 --- a/src/org/pentaho/di/dataset/spoon/dialog/TransUnitTestDialog.java +++ b/src/org/pentaho/di/dataset/spoon/dialog/TransUnitTestDialog.java @@ -2,7 +2,6 @@ import java.util.Arrays; -import org.apache.commons.lang.StringUtils; import org.eclipse.swt.SWT; import org.eclipse.swt.custom.CCombo; import org.eclipse.swt.events.SelectionAdapter; @@ -25,10 +24,9 @@ import org.pentaho.di.core.Const; import org.pentaho.di.core.exception.KettleException; import org.pentaho.di.core.variables.Variables; -import org.pentaho.di.dataset.TestType; -import org.pentaho.di.dataset.TransTweak; import org.pentaho.di.dataset.TransUnitTest; -import org.pentaho.di.dataset.TransUnitTestTweak; +import org.pentaho.di.dataset.TransUnitTestDatabaseReplacement; +import org.pentaho.di.dataset.util.DataSetConst; import org.pentaho.di.i18n.BaseMessages; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.ui.core.PropsUI; @@ -46,19 +44,6 @@ public class TransUnitTestDialog extends Dialog { private static Class PKG = TransUnitTestDialog.class; // for i18n purposes, needed by Translator2!! - private static final String[] tweakDesc = new String[] { - BaseMessages.getString(PKG, "TransUnitTestDialog.Tweak.NONE.Desc"), - BaseMessages.getString(PKG, "TransUnitTestDialog.Tweak.BYPASS_STEP.Desc"), - BaseMessages.getString(PKG, "TransUnitTestDialog.Tweak.REMOVE_STEP.Desc"), - }; - - private static final String[] testTypeDesc = new String[] { - BaseMessages.getString(PKG, "TransUnitTestDialog.TestType.NONE.Desc"), - BaseMessages.getString(PKG, "TransUnitTestDialog.TestType.CONCEPTUAL.Desc"), - BaseMessages.getString(PKG, "TransUnitTestDialog.Tweak.DEVELOPMENT.Desc"), - BaseMessages.getString(PKG, "TransUnitTestDialog.Tweak.UNIT_TEST.Desc"), - }; - private TransUnitTest transUnitTest; protected TransMeta transMeta; @@ -69,7 +54,7 @@ public class TransUnitTestDialog extends Dialog { private CCombo wTestType; private TextVar wFilename; - private TableView wTweaks; + private TableView wDbReplacements; private Button wOK; private Button wCancel; @@ -166,7 +151,7 @@ public boolean open() { fdTestType.left = new FormAttachment(middle, 0); fdTestType.right = new FormAttachment(100, 0); wTestType.setLayoutData(fdTestType); - wTestType.setItems(testTypeDesc); + wTestType.setItems(DataSetConst.getTestTypeDescriptions()); lastControl = wTestType; // The optional filename of the test result... @@ -189,10 +174,10 @@ public boolean open() { lastControl = wFilename; - // The list of tweaks to the transformation + // The list of database replacements in the unit test transformation // Label wlFieldMapping = new Label(shell, SWT.NONE); - wlFieldMapping.setText(BaseMessages.getString(PKG, "TransUnitTestDialog.Tweaks.Label")); + wlFieldMapping.setText(BaseMessages.getString(PKG, "TransUnitTestDialog.DbReplacements.Label")); props.setLook(wlFieldMapping); FormData fdlUpIns = new FormData(); fdlUpIns.left = new FormAttachment(0, 0); @@ -211,26 +196,28 @@ public boolean open() { Button[] buttons = new Button[] { wOK, wCancel }; BaseStepDialog.positionBottomButtons(shell, buttons, margin, null); - // the transformation tweaks + // the database replacements // - String[] stepNames = transMeta.getStepNames(); - Arrays.sort(stepNames); + String[] dbNames = transMeta.getDatabaseNames(); + Arrays.sort(dbNames); ColumnInfo[] columns = new ColumnInfo[] { - new ColumnInfo(BaseMessages.getString(PKG, "TransUnitTestDialog.Tweak.ColumnInfo.Tweak"), - ColumnInfo.COLUMN_TYPE_CCOMBO, tweakDesc, false), - new ColumnInfo(BaseMessages.getString(PKG, "TransUnitTestDialog.Tweak.ColumnInfo.Step"), - ColumnInfo.COLUMN_TYPE_CCOMBO, stepNames, false), }; - - wTweaks = new TableView(new Variables(), shell, + new ColumnInfo(BaseMessages.getString(PKG, "TransUnitTestDialog.DbReplacement.ColumnInfo.OriginalDb"), + ColumnInfo.COLUMN_TYPE_CCOMBO, dbNames, false), + new ColumnInfo(BaseMessages.getString(PKG, "TransUnitTestDialog.DbReplacement.ColumnInfo.ReplacementDb"), + ColumnInfo.COLUMN_TYPE_CCOMBO, dbNames, false), }; + columns[0].setUsingVariables(true); + columns[1].setUsingVariables(true); + + wDbReplacements = new TableView(new Variables(), shell, SWT.BORDER | SWT.FULL_SELECTION | SWT.MULTI | SWT.V_SCROLL | SWT.H_SCROLL, columns, transUnitTest.getTweaks().size(), null, props); - FormData fdFieldMapping = new FormData(); - fdFieldMapping.left = new FormAttachment(0, 0); - fdFieldMapping.top = new FormAttachment(lastControl, margin); - fdFieldMapping.right = new FormAttachment(100, 0); - fdFieldMapping.bottom = new FormAttachment(wOK, -2 * margin); - wTweaks.setLayoutData(fdFieldMapping); + FormData fdDbReplacements = new FormData(); + fdDbReplacements.left = new FormAttachment(0, 0); + fdDbReplacements.top = new FormAttachment(lastControl, margin); + fdDbReplacements.right = new FormAttachment(100, 0); + fdDbReplacements.bottom = new FormAttachment(wOK, -2 * margin); + wDbReplacements.setLayoutData(fdDbReplacements); // Add listeners wOK.addListener(SWT.Selection, new Listener() { @@ -284,15 +271,17 @@ public void getData() { wName.setText( Const.NVL( transUnitTest.getName(), "" ) ); wDescription.setText( Const.NVL( transUnitTest.getDescription(), "" ) ); - wTestType.setText( Const.NVL( getTestTypeDescription(transUnitTest.getType()), "") ); + wTestType.setText( Const.NVL( DataSetConst.getTestTypeDescription(transUnitTest.getType()), "") ); wFilename.setText( Const.NVL( transUnitTest.getFilename(), "")); - for ( int i = 0; i < transUnitTest.getTweaks().size(); i++ ) { - TransUnitTestTweak tweak = transUnitTest.getTweaks().get( i ); + for ( int i = 0; i < transUnitTest.getDatabaseReplacements().size(); i++ ) { + TransUnitTestDatabaseReplacement dbReplacement = transUnitTest.getDatabaseReplacements().get( i ); int colnr = 1; - wTweaks.setText( Const.NVL( getTweakDescription(tweak.getTweak()), "" ), colnr++, i ); - wTweaks.setText( Const.NVL( tweak.getStepName(), "" ), colnr++, i ); + wDbReplacements.setText( Const.NVL( dbReplacement.getOriginalDatabaseName(), ""), colnr++, i ); + wDbReplacements.setText( Const.NVL( dbReplacement.getReplacementDatabaseName(), "" ), colnr++, i ); } + wDbReplacements.removeEmptyRows(); + wDbReplacements.setRowNums(); wName.setFocus(); } @@ -310,17 +299,17 @@ public void getInfo(TransUnitTest test) { test.setName(wName.getText()); test.setDescription(wDescription.getText()); - test.setType(getTestTypeForDescription(wTestType.getText())); + test.setType(DataSetConst.getTestTypeForDescription(wTestType.getText())); test.setFilename(wFilename.getText()); - test.getTweaks().clear(); - int nrFields = wTweaks.nrNonEmpty(); + test.getDatabaseReplacements().clear(); + int nrFields = wDbReplacements.nrNonEmpty(); for (int i=0;i databases = DataSetConst.getAvailableDatabases( transMeta.getRepository(), transMeta.getSharedObjects() ); - - try { - factoriesHierarchy = new FactoriesHierarchy( transMeta.getMetaStore(), databases ); - unitTest = factoriesHierarchy.getTestFactory().loadElement( unitTestName ); - } catch(MetaStoreException e) { - throw new KettleException("Unable to load unit test '"+unitTestName+"'", e); - } + List databases = DataSetConst.getAvailableDatabases( transMeta.getRepository(), transMeta.getSharedObjects() ); + try { + factoriesHierarchy = new FactoriesHierarchy( transMeta.getMetaStore(), databases ); + unitTest = factoriesHierarchy.getTestFactory().loadElement( unitTestName ); + } catch(MetaStoreException e) { + throw new KettleException("Unable to load unit test '"+unitTestName+"'", e); } + if (unitTest==null) { throw new KettleException("Unit test '"+unitTestName+"' was not found or could not be loaded"); @@ -94,6 +105,27 @@ public void callExtensionPoint( LogChannelInterface log, Object object ) throws copyTransMeta.setMetaStore( transMeta.getMetaStore() ); copyTransMeta.setSharedObjects( transMeta.getSharedObjects() ); + // Replace certain connections with another + // + for (TransUnitTestDatabaseReplacement dbReplacement : unitTest.getDatabaseReplacements()) { + String sourceDatabaseName = transMeta.environmentSubstitute(dbReplacement.getOriginalDatabaseName()); + String replacementDatabaseName = transMeta.environmentSubstitute(dbReplacement.getReplacementDatabaseName()); + + DatabaseMeta sourceDatabaseMeta = copyTransMeta.findDatabase(sourceDatabaseName); + DatabaseMeta replacementDatabaseMeta = copyTransMeta.findDatabase(replacementDatabaseName); + if (sourceDatabaseMeta==null) { + throw new KettleException("Unable to find source database connection '"+sourceDatabaseName+"', can not be replaced"); + } + if (replacementDatabaseMeta==null) { + throw new KettleException("Unable to find replacement database connection '"+replacementDatabaseName+"', can not be used to replace"); + } + + if (log.isDetailed()) { + log.logDetailed("Replaced database connection '"+sourceDatabaseName+"' with connection '"+replacementDatabaseName+"'"); + } + sourceDatabaseMeta.replaceMeta(replacementDatabaseMeta); + } + // Replace all steps with an Input Data Set marker with an Injector // Replace all steps with a Golden Data Set marker with a Dummy // Apply the tweaks to the steps: diff --git a/src/org/pentaho/di/dataset/spoon/xtpoint/ShowUnitTestMenuExtensionPoint.java b/src/org/pentaho/di/dataset/spoon/xtpoint/ShowUnitTestMenuExtensionPoint.java index d0bfedc..736b3b9 100644 --- a/src/org/pentaho/di/dataset/spoon/xtpoint/ShowUnitTestMenuExtensionPoint.java +++ b/src/org/pentaho/di/dataset/spoon/xtpoint/ShowUnitTestMenuExtensionPoint.java @@ -84,8 +84,7 @@ public void widgetSelected(SelectionEvent event) { public void widgetSelected(SelectionEvent event) { disableUnitTest(unitTestName); } }; - - + Menu menu = new Menu(spoon.getShell(), SWT.POP_UP); MenuItem newItem = new MenuItem(menu, SWT.PUSH); newItem.setText(BaseMessages.getString(PKG, "ShowUnitTestMenuExtensionPoint.UnitMenu.New.Label")); diff --git a/src/org/pentaho/di/dataset/spoon/xul/spoon_overlays.xul b/src/org/pentaho/di/dataset/spoon/xul/spoon_overlays.xul index c074836..5b1e573 100755 --- a/src/org/pentaho/di/dataset/spoon/xul/spoon_overlays.xul +++ b/src/org/pentaho/di/dataset/spoon/xul/spoon_overlays.xul @@ -33,8 +33,7 @@ - - + diff --git a/src/org/pentaho/di/dataset/steps/exectests/ExecuteTestsDialog.java b/src/org/pentaho/di/dataset/steps/exectests/ExecuteTestsDialog.java index b69777d..93630a5 100644 --- a/src/org/pentaho/di/dataset/steps/exectests/ExecuteTestsDialog.java +++ b/src/org/pentaho/di/dataset/steps/exectests/ExecuteTestsDialog.java @@ -18,7 +18,7 @@ import org.eclipse.swt.widgets.Shell; import org.eclipse.swt.widgets.Text; import org.pentaho.di.core.Const; -import org.pentaho.di.dataset.spoon.dialog.TransUnitTestDialog; +import org.pentaho.di.dataset.util.DataSetConst; import org.pentaho.di.i18n.BaseMessages; import org.pentaho.di.trans.TransMeta; import org.pentaho.di.trans.step.BaseStepMeta; @@ -95,7 +95,7 @@ public String open() { fdlTypeToExecute.top = new FormAttachment( lastControl, margin ); wlTypeToExecute.setLayoutData( fdlTypeToExecute ); wTypeToExecute = new CCombo( shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER ); - wTypeToExecute.setItems(TransUnitTestDialog.getTestTypeDescriptions()); + wTypeToExecute.setItems(DataSetConst.getTestTypeDescriptions()); props.setLook( wTypeToExecute ); FormData fdTypeToExecute = new FormData(); fdTypeToExecute.left = new FormAttachment( middle, 0 ); @@ -279,7 +279,7 @@ public void shellClosed( ShellEvent e ) { private void getData() { - wTypeToExecute.setText( TransUnitTestDialog.getTestTypeDescription(input.getTypeToExecute())); + wTypeToExecute.setText( DataSetConst.getTestTypeDescription(input.getTypeToExecute())); wTransformationNameField.setText(Const.NVL(input.getTransformationNameField(), "")); wUnitTestNameField.setText(Const.NVL(input.getUnitTestNameField(), "")); wDataSetNameField.setText(Const.NVL(input.getDataSetNameField(), "")); @@ -306,7 +306,7 @@ private void ok() { input.setChanged(); - input.setTypeToExecute( TransUnitTestDialog.getTestTypeForDescription(wTypeToExecute.getText()) ); + input.setTypeToExecute( DataSetConst.getTestTypeForDescription(wTypeToExecute.getText()) ); input.setTransformationNameField( wTransformationNameField.getText() ); input.setUnitTestNameField( wUnitTestNameField.getText() ); input.setDataSetNameField( wDataSetNameField.getText() ); diff --git a/src/org/pentaho/di/dataset/steps/exectests/messages/messages_en_US.properties b/src/org/pentaho/di/dataset/steps/exectests/messages/messages_en_US.properties index 4839a7f..775ec8f 100644 --- a/src/org/pentaho/di/dataset/steps/exectests/messages/messages_en_US.properties +++ b/src/org/pentaho/di/dataset/steps/exectests/messages/messages_en_US.properties @@ -8,6 +8,3 @@ ExecuteTestsDialog.StepNameField.Label = Step field name ExecuteTestsDialog.ErrorField.Label = Error field name ExecuteTestsDialog.CommentField.Label = Comment field name - - - diff --git a/src/org/pentaho/di/dataset/util/DataSetConst.java b/src/org/pentaho/di/dataset/util/DataSetConst.java index 9776ffb..c0e9c80 100755 --- a/src/org/pentaho/di/dataset/util/DataSetConst.java +++ b/src/org/pentaho/di/dataset/util/DataSetConst.java @@ -5,6 +5,7 @@ import java.util.List; import java.util.Map; +import org.apache.commons.lang.StringUtils; import org.pentaho.di.core.Const; import org.pentaho.di.core.RowMetaAndData; import org.pentaho.di.core.database.Database; @@ -20,11 +21,15 @@ import org.pentaho.di.dataset.DataSet; import org.pentaho.di.dataset.DataSetField; import org.pentaho.di.dataset.DataSetGroup; +import org.pentaho.di.dataset.TestType; +import org.pentaho.di.dataset.TransTweak; import org.pentaho.di.dataset.TransUnitTest; import org.pentaho.di.dataset.TransUnitTestFieldMapping; import org.pentaho.di.dataset.TransUnitTestSetLocation; +import org.pentaho.di.dataset.TransUnitTestTweak; import org.pentaho.di.dataset.UnitTestResult; import org.pentaho.di.dataset.spoon.xtpoint.RowCollection; +import org.pentaho.di.i18n.BaseMessages; import org.pentaho.di.repository.ObjectId; import org.pentaho.di.repository.Repository; import org.pentaho.di.shared.SharedObjectInterface; @@ -34,6 +39,8 @@ import org.pentaho.di.trans.step.StepMeta; public class DataSetConst { + private static Class PKG = DataSetConst.class; // for i18n purposes, needed by Translator2!! + public static String DATA_SET_GROUP_TYPE_NAME = "Data Set Group"; public static String DATA_SET_GROUP_TYPE_DESCRIPTION = "A collection of data sets and unit tests"; @@ -66,6 +73,19 @@ public class DataSetConst { public static final String ROW_COLLECTION_MAP = "RowCollectionMap"; public static final String UNIT_TEST_RESULTS = "UnitTestResults"; + private static final String[] tweakDesc = new String[] { + BaseMessages.getString(PKG, "DataSetConst.Tweak.NONE.Desc"), + BaseMessages.getString(PKG, "DataSetConst.Tweak.BYPASS_STEP.Desc"), + BaseMessages.getString(PKG, "DataSetConst.Tweak.REMOVE_STEP.Desc"), + }; + + private static final String[] testTypeDesc = new String[] { + BaseMessages.getString(PKG, "DataSetConst.TestType.NONE.Desc"), + BaseMessages.getString(PKG, "DataSetConst.TestType.CONCEPTUAL.Desc"), + BaseMessages.getString(PKG, "DataSetConst.TestType.DEVELOPMENT.Desc"), + BaseMessages.getString(PKG, "DataSetConst.TestType.UNIT_TEST.Desc"), + }; + public static final DataSet findDataSet( List list, String dataSetName ) { if ( Const.isEmpty( dataSetName ) ) { return null; @@ -162,6 +182,7 @@ public static final void clearStepDataSetIndicators(TransMeta transMeta) { if (attributes!=null) { attributes.remove( DataSetConst.ATTR_STEP_DATASET_INPUT ); attributes.remove( DataSetConst.ATTR_STEP_DATASET_GOLDEN); + attributes.remove( DataSetConst.ATTR_STEP_TWEAK); } } } @@ -186,6 +207,16 @@ public static final void loadStepDataSetIndicators(TransMeta transMeta, TransUni stepMeta.setAttribute( DataSetConst.ATTR_GROUP_DATASET, DataSetConst.ATTR_STEP_DATASET_GOLDEN, location.getDataSetName()); } } + + // Load the tweak indicators? + // + List tweaks = test.getTweaks(); + for (TransUnitTestTweak tweak : tweaks) { + StepMeta stepMeta = transMeta.findStep(tweak.getStepName()); + if (stepMeta!=null && tweak.getTweak()!=null) { + stepMeta.setAttribute(DataSetConst.ATTR_GROUP_DATASET, DataSetConst.ATTR_STEP_TWEAK, tweak.getTweak().name()); + } + } } /** @@ -378,4 +409,59 @@ public static final String getNameFromPath(String path) { return path; } } + + /** + * Get the TransTweak for a tweak description (from the dialog) + * @param tweakDescription The description to look for + * @return the tweak or NONE if nothing matched + */ + public TransTweak getTweakForDescription(String tweakDescription) { + if (StringUtils.isEmpty(tweakDescription)) { + return TransTweak.NONE; + } + int index = Const.indexOfString(tweakDescription, tweakDesc); + if (index<0) { + return TransTweak.NONE; + } + return TransTweak.values()[index]; + } + + public static final String getTestTypeDescription(TestType testType) { + int index = 0; // NONE + if (testType!=null) { + TestType[] testTypes = TestType.values(); + for (int i=0;i