Skip to content
This repository has been archived by the owner on Dec 15, 2022. It is now read-only.

Commit

Permalink
Issue #36 for version 3.3.0
Browse files Browse the repository at this point in the history
  • Loading branch information
@mattcasters
mattcasters committed Jan 13, 2019
1 parent 6806066 commit 8081341
Show file tree
Hide file tree
Showing 9 changed files with 243 additions and 123 deletions.
2 changes: 1 addition & 1 deletion pom.xml
View file
Expand Up @@ -145,5 +145,5 @@
</dependencyManagement>

<groupId>pentaho-kettle-dataset</groupId>
<version>3.2.0-SNAPSHOT</version>
<version>3.3.0-SNAPSHOT</version>
</project>
143 changes: 82 additions & 61 deletions src/main/java/org/pentaho/di/dataset/TransUnitTest.java
View file
Expand Up @@ -22,28 +22,24 @@

package org.pentaho.di.dataset;

import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

import org.apache.commons.lang.StringUtils;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.logging.LogChannelInterface;
import org.pentaho.di.core.row.RowMeta;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.dataset.spoon.xtpoint.RowCollection;
import org.pentaho.di.dataset.util.DataSetConst;
import org.pentaho.di.dataset.util.FactoriesHierarchy;
import org.pentaho.metastore.persist.MetaStoreAttribute;
import org.pentaho.metastore.persist.MetaStoreElementType;

import java.io.File;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;

/**
* This class describes a test-case where a transformation output is verified against golden data.
*
* @author matt
*
* @author matt
*/
@MetaStoreElementType(
name = "Kettle Transformation Unit Test",
Expand All @@ -63,7 +59,7 @@ public class TransUnitTest {

@MetaStoreAttribute( key = "transformation_filename" )
protected String transFilename; // file (3rd priority)

@MetaStoreAttribute( key = "input_data_sets" )
protected List<TransUnitTestSetLocation> inputDataSets;

Expand All @@ -73,10 +69,10 @@ public class TransUnitTest {
@MetaStoreAttribute( key = "trans_test_tweaks" )
protected List<TransUnitTestTweak> tweaks;

@MetaStoreAttribute( key = "test_type")
@MetaStoreAttribute( key = "test_type" )
protected TestType type;
@MetaStoreAttribute( key = "persist_filename")

@MetaStoreAttribute( key = "persist_filename" )
protected String filename;

@MetaStoreAttribute
Expand All @@ -88,6 +84,10 @@ public class TransUnitTest {
@MetaStoreAttribute
protected List<VariableValue> variableValues;

@MetaStoreAttribute
protected boolean autoOpening;


public TransUnitTest() {
inputDataSets = new ArrayList<TransUnitTestSetLocation>();
goldenDataSets = new ArrayList<TransUnitTestSetLocation>();
Expand All @@ -96,16 +96,18 @@ public TransUnitTest() {
databaseReplacements = new ArrayList<TransUnitTestDatabaseReplacement>();
variableValues = new ArrayList<>();
basePath = null;
}

public TransUnitTest( String name, String description,
String transObjectId, String transRepositoryPath, String transFilename,
List<TransUnitTestSetLocation> inputDataSets,
List<TransUnitTestSetLocation> goldenDataSets,
List<TransUnitTestTweak> tweaks,
TestType type,
String filename,
List<TransUnitTestDatabaseReplacement> databaseReplacements) {
autoOpening = false;
}

public TransUnitTest( String name, String description,
String transObjectId, String transRepositoryPath, String transFilename,
List<TransUnitTestSetLocation> inputDataSets,
List<TransUnitTestSetLocation> goldenDataSets,
List<TransUnitTestTweak> tweaks,
TestType type,
String filename,
List<TransUnitTestDatabaseReplacement> databaseReplacements,
boolean autoOpening ) {
this();
this.name = name;
this.description = description;
Expand All @@ -118,37 +120,38 @@ public TransUnitTest( String name, String description,
this.type = type;
this.filename = filename;
this.databaseReplacements = databaseReplacements;
this.autoOpening = autoOpening;
}

@Override
public boolean equals( Object obj ) {
if (obj==this) {
if ( obj == this ) {
return true;
}
if (!(obj instanceof TransUnitTest)) {
if ( !( obj instanceof TransUnitTest ) ) {
return false;
}
return ((TransUnitTest)obj).name.equalsIgnoreCase( name );
return ( (TransUnitTest) obj ).name.equalsIgnoreCase( name );
}

@Override
public int hashCode() {
return name.hashCode();
}


public TransUnitTestSetLocation findGoldenLocation(String stepName) {
for (TransUnitTestSetLocation location : goldenDataSets) {
if (stepName.equalsIgnoreCase( location.getStepname() )) {
public TransUnitTestSetLocation findGoldenLocation( String stepName ) {
for ( TransUnitTestSetLocation location : goldenDataSets ) {
if ( stepName.equalsIgnoreCase( location.getStepname() ) ) {
return location;
}
}
return null;
}

public TransUnitTestSetLocation findInputLocation(String stepName) {
for (TransUnitTestSetLocation location : inputDataSets) {
if (stepName.equalsIgnoreCase( location.getStepname() )) {
public TransUnitTestSetLocation findInputLocation( String stepName ) {
for ( TransUnitTestSetLocation location : inputDataSets ) {
if ( stepName.equalsIgnoreCase( location.getStepname() ) ) {
return location;
}
}
Expand All @@ -158,44 +161,45 @@ public TransUnitTestSetLocation findInputLocation(String stepName) {
/**
* Retrieve the golden data set for the specified location
*
* @param log the logging channel to log to
* @param log the logging channel to log to
* @param hierarchy The factories to load sets with
* @param location the location where we want to check against golden rows
* @param location the location where we want to check against golden rows
* @return The golden data set
*
* @throws KettleException
*/
public DataSet getGoldenDataSet(LogChannelInterface log, FactoriesHierarchy hierarchy, TransUnitTestSetLocation location) throws KettleException {
public DataSet getGoldenDataSet( LogChannelInterface log, FactoriesHierarchy hierarchy, TransUnitTestSetLocation location ) throws KettleException {

String stepName = location.getStepname();
String goldenDataSetName = location.getDataSetName();

try {
// Look in the golden data sets list for the mentioned step name
//
if (goldenDataSetName==null) {
throw new KettleException("Unable to find golden data set for step '"+stepName+"'");
if ( goldenDataSetName == null ) {
throw new KettleException( "Unable to find golden data set for step '" + stepName + "'" );
}

DataSet goldenDataSet = hierarchy.getSetFactory().loadElement( goldenDataSetName );
if (goldenDataSet==null) {
throw new KettleException("Unable to find golden data set '"+goldenDataSetName+"' for step '"+stepName+"'");
if ( goldenDataSet == null ) {
throw new KettleException( "Unable to find golden data set '" + goldenDataSetName + "' for step '" + stepName + "'" );
}

return goldenDataSet;

} catch ( Exception e ) {
throw new KettleException( "Unable to retrieve sorted golden row data set '"+stepName+"'", e );
throw new KettleException( "Unable to retrieve sorted golden row data set '" + stepName + "'", e );
}
}

/** Find the first tweak for a certain step
/**
* Find the first tweak for a certain step
*
* @param stepname the name of the step on which a tweak is put
* @return the first tweak for a certain step or null if nothing was found
*/
public TransUnitTestTweak findTweak(String stepname) {
for (TransUnitTestTweak tweak : tweaks) {
if (tweak.getStepName()!=null && tweak.getStepName().equalsIgnoreCase(stepname)) {
public TransUnitTestTweak findTweak( String stepname ) {
for ( TransUnitTestTweak tweak : tweaks ) {
if ( tweak.getStepName() != null && tweak.getStepName().equalsIgnoreCase( stepname ) ) {
return tweak;
}
}
Expand All @@ -204,42 +208,43 @@ public TransUnitTestTweak findTweak(String stepname) {

/**
* Remove all input and golden data sets on the step with the provided name
*
* @param stepname the name of the step for which we need to clear out all input and golden data sets
*/
public void removeInputAndGoldenDataSets(String stepname) {
public void removeInputAndGoldenDataSets( String stepname ) {

for (Iterator<TransUnitTestSetLocation> iterator = inputDataSets.iterator() ; iterator.hasNext() ; ) {
for ( Iterator<TransUnitTestSetLocation> iterator = inputDataSets.iterator(); iterator.hasNext(); ) {
TransUnitTestSetLocation inputLocation = iterator.next();
if (inputLocation.getStepname().equalsIgnoreCase(stepname)) {
if ( inputLocation.getStepname().equalsIgnoreCase( stepname ) ) {
iterator.remove();
}
}

for (Iterator<TransUnitTestSetLocation> iterator = goldenDataSets.iterator() ; iterator.hasNext() ; ) {
for ( Iterator<TransUnitTestSetLocation> iterator = goldenDataSets.iterator(); iterator.hasNext(); ) {
TransUnitTestSetLocation goldenLocation = iterator.next();
if (goldenLocation.getStepname().equalsIgnoreCase(stepname)) {
if ( goldenLocation.getStepname().equalsIgnoreCase( stepname ) ) {
iterator.remove();
}
}
}

public String calculateCompleteFilename( VariableSpace space) {
public String calculateCompleteFilename( VariableSpace space ) {

String baseFilePath = space.environmentSubstitute( basePath );
if ( StringUtils.isEmpty(baseFilePath) ) {
if ( StringUtils.isEmpty( baseFilePath ) ) {
// See if the base path environment variable is set
//
baseFilePath = space.getVariable( DataSetConst.VARIABLE_UNIT_TESTS_BASE_PATH );
}
if (StringUtils.isEmpty( baseFilePath )) {
baseFilePath="";
if ( StringUtils.isEmpty( baseFilePath ) ) {
baseFilePath = "";
}
if (StringUtils.isNotEmpty( baseFilePath )) {
if (!baseFilePath.endsWith( File.separator )) {
baseFilePath+=File.separator;
if ( StringUtils.isNotEmpty( baseFilePath ) ) {
if ( !baseFilePath.endsWith( File.separator ) ) {
baseFilePath += File.separator;
}
}
return baseFilePath+transFilename;
return baseFilePath + transFilename;
}

/**
Expand Down Expand Up @@ -449,4 +454,20 @@ public List<VariableValue> getVariableValues() {
public void setVariableValues( List<VariableValue> variableValues ) {
this.variableValues = variableValues;
}

/**
* Gets autoOpening
*
* @return value of autoOpening
*/
public boolean isAutoOpening() {
return autoOpening;
}

/**
* @param autoOpening The autoOpening to set
*/
public void setAutoOpening( boolean autoOpening ) {
this.autoOpening = autoOpening;
}
}
59 changes: 57 additions & 2 deletions src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java
View file
Expand Up @@ -118,7 +118,7 @@ public void updateMenu( Document doc ) {
public void manage() {
}

public List<DatabaseMeta> getAvailableDatabases( Repository repository ) throws KettleException {
public static List<DatabaseMeta> getAvailableDatabases( Repository repository ) throws KettleException {
List<DatabaseMeta> list = new ArrayList<DatabaseMeta>();

// Load database connections from the central repository if we're connected to one
Expand Down Expand Up @@ -725,7 +725,7 @@ public void clearGoldenDataSet() {
transGraph.redraw();
}

private FactoriesHierarchy getHierarchy() throws KettleException {
public static FactoriesHierarchy getHierarchy() throws KettleException {

try {
Spoon spoon = Spoon.getInstance();
Expand Down Expand Up @@ -1239,6 +1239,61 @@ public void switchUnitTest( TransUnitTest targetTest, TransMeta transMeta ) {
Spoon.getInstance().refreshGraph();
}

public static List<TransUnitTest> findTransformationUnitTest( TransMeta transMeta, IMetaStore metaStore ) {
MetaStoreFactory<TransUnitTest> factory = new MetaStoreFactory<TransUnitTest>( TransUnitTest.class, metaStore, PentahoDefaults.NAMESPACE );
List<TransUnitTest> tests = new ArrayList<TransUnitTest>();

try {

List<TransUnitTest> allTests = factory.getElements();
for ( TransUnitTest test : allTests ) {
// Match the filename
//
if ( StringUtils.isNotEmpty( transMeta.getFilename() ) ) {

// What's the transformation absolute URI
//
FileObject transFile = KettleVFS.getFileObject( transMeta.getFilename() );
String transUri = transFile.getName().getURI();

// What's the filename referenced in the test?
//
FileObject testTransFile = KettleVFS.getFileObject( test.calculateCompleteFilename(transMeta) );
if (testTransFile.exists()) {
String testTransUri = testTransFile.getName().getURI();

if ( transUri.equals( testTransUri ) ) {
tests.add( test );
}
}
} else {
if ( transMeta.getRepository() != null ) {
// No filename, check the object_id ...
//
if ( transMeta.getObjectId() != null && transMeta.getObjectId().getId().equals( test.getTransObjectId() ) ) {
tests.add( test );
} else {
// Try the repository path..
//
// What is the repsository path?
String repositoryPath = transMeta.getRepositoryDirectory().getPath() + "/" + transMeta.getName();
if ( repositoryPath.equals( test.getTransRepositoryPath() ) ) {
tests.add( test );
}
}
}
}
}

} catch ( Exception exception ) {
new ErrorDialog( Spoon.getInstance().getShell(),
BaseMessages.getString( PKG, "ShowUnitTestMenuExtensionPoint.ErrorFindingUnitTestsForTransformation.Title" ),
BaseMessages.getString( PKG, "ShowUnitTestMenuExtensionPoint.ErrorFindingUnitTestsForTransformation.Message" ),
exception );
}
return tests;
}

/**
* Gets activeTests
*
Expand Down

0 comments on commit 8081341

Please sign in to comment.