Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[PDI-19718] Dimension lookup: lookup field data type is not stored correctly #8789

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

[PDI-19718] Dimension lookup: lookup field data type is not stored correctly #8789

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
63e8c45
[PDI-19718] - Regression of test
petrprochy Jan 26, 2023
7c48d00
[PDI-19718] - Fixed lookup field data type
petrprochy Jan 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
39 changes: 25 additions & 14 deletions engine/src/main/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMeta.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
* Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com
*
*******************************************************************************
*
Expand All @@ -22,17 +22,9 @@

package org.pentaho.di.trans.steps.dimensionlookup;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

import org.pentaho.di.core.CheckResult;
import org.pentaho.di.core.CheckResultInterface;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.injection.AfterInjection;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.ProvidesModelerMeta;
import org.pentaho.di.core.SQLStatement;
import org.pentaho.di.core.database.Database;
Expand All @@ -42,6 +34,7 @@
import org.pentaho.di.core.exception.KettleStepException;
import org.pentaho.di.core.exception.KettleValueException;
import org.pentaho.di.core.exception.KettleXMLException;
import org.pentaho.di.core.injection.AfterInjection;
import org.pentaho.di.core.injection.Injection;
import org.pentaho.di.core.injection.InjectionSupported;
import org.pentaho.di.core.injection.InjectionTypeConverter;
Expand All @@ -52,6 +45,7 @@
import org.pentaho.di.core.row.value.ValueMetaDate;
import org.pentaho.di.core.row.value.ValueMetaFactory;
import org.pentaho.di.core.row.value.ValueMetaInteger;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.variables.VariableSpace;
import org.pentaho.di.core.xml.XMLHandler;
import org.pentaho.di.i18n.BaseMessages;
Expand All @@ -65,6 +59,12 @@
import org.pentaho.metastore.api.IMetaStore;
import org.w3c.dom.Node;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Date;
import java.util.List;

/**
* @author Matt
* @since on 14-may-2003
Expand Down Expand Up @@ -430,6 +430,20 @@ public void setFieldUpdate( int[] fieldUpdate ) {
this.fieldUpdate = fieldUpdate;
}

/**
* @param position
* Index to array of property fieldUpdate.
* @param desc
* Update type code or value meta type name.
*
* @see #getUpdateType(boolean, String)
*/
public void setFieldUpdateValue(int position, String desc) {
fieldUpdate[position] = getUpdateType( update, desc );
if ( !update ) {
returnType[position] = fieldUpdate[position];
}
}
/**
* @return Returns the returnType.
*/
Expand Down Expand Up @@ -937,7 +951,7 @@ private void readData( Node stepnode, List<? extends SharedObjectInterface> data
fieldStream[i] = XMLHandler.getTagValue( fnode, "name" );
fieldLookup[i] = XMLHandler.getTagValue( fnode, "lookup" );
upd = XMLHandler.getTagValue( fnode, "update" );
fieldUpdate[i] = getUpdateType( update, upd );
setFieldUpdateValue( i, upd );
}

if ( update ) {
Expand Down Expand Up @@ -998,10 +1012,7 @@ public void readRep( Repository rep, IMetaStore metaStore, ObjectId id_step, Lis
for ( int i = 0; i < nrfields; i++ ) {
fieldStream[i] = rep.getStepAttributeString( id_step, i, "field_name" );
fieldLookup[i] = rep.getStepAttributeString( id_step, i, "field_lookup" );
fieldUpdate[i] = getUpdateType( update, rep.getStepAttributeString( id_step, i, "field_update" ) );
if ( !update ) {
returnType[i] = fieldUpdate[i];
}
setFieldUpdateValue( i, rep.getStepAttributeString( id_step, i, "field_update" ) );
}

keyField = rep.getStepAttributeString( id_step, "return_name" );
Expand Down
101 changes: 55 additions & 46 deletions engine/src/test/java/org/pentaho/di/trans/steps/dimensionlookup/DimensionLookupMetaTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
* Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com
*
*******************************************************************************
*
Expand All @@ -22,25 +22,7 @@

package org.pentaho.di.trans.steps.dimensionlookup;

import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Random;
import java.util.UUID;

import org.apache.commons.lang.StringUtils;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
Expand All @@ -51,7 +33,6 @@
import org.pentaho.di.core.SQLStatement;
import org.pentaho.di.core.database.Database;
import org.pentaho.di.core.database.DatabaseMeta;
import org.pentaho.di.core.exception.KettleDatabaseException;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.logging.KettleLogStore;
import org.pentaho.di.core.logging.LogChannelInterface;
Expand All @@ -78,10 +59,28 @@
import org.pentaho.di.trans.steps.loadsave.validator.StringLoadSaveValidator;
import org.pentaho.metastore.api.IMetaStore;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;

import static org.junit.Assert.assertEquals;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;

public class DimensionLookupMetaTest implements InitializerInterface<StepMetaInterface> {
LoadSaveTester loadSaveTester;
UpdateLoadSaveValidator updateValidator;

Class<DimensionLookupMeta> testMetaClass = DimensionLookupMeta.class;
private ThreadLocal<DimensionLookupMeta> holdTestingMeta = new ThreadLocal<DimensionLookupMeta>();
private final ThreadLocal<DimensionLookupMeta> holdTestingMeta = new ThreadLocal<>();
@ClassRule public static RestorePDIEngineEnvironment env = new RestorePDIEngineEnvironment();

@BeforeClass
Expand All @@ -95,19 +94,19 @@ public void setUpLoadSave() throws Exception {
Arrays.asList( "schemaName", "tableName", "update", "dateField", "dateFrom", "dateTo", "keyField", "keyRename",
"autoIncrement", "versionField", "commitSize", "useBatchUpdate", "minYear", "maxYear", "techKeyCreation",
"cacheSize", "usingStartDateAlternative", "startDateAlternative", "startDateFieldName", "preloadingCache", "keyStream",
"keyLookup", "fieldStream", "fieldLookup", "fieldUpdate", "databaseMeta", "sequenceName" );
"keyLookup", "fieldStream", "fieldLookup", "fieldUpdate", "returnType", "databaseMeta", "sequenceName" );

Map<String, String> getterMap = new HashMap<String, String>() {
{
put( "useBatchUpdate", "useBatchUpdate" );
}
};
Map<String, String> setterMap = new HashMap<String, String>();
Map<String, String> setterMap = new HashMap<>();

FieldLoadSaveValidator<String[]> stringArrayLoadSaveValidator =
new ArrayLoadSaveValidator<String>( new StringLoadSaveValidator(), 5 );
new ArrayLoadSaveValidator<>( new StringLoadSaveValidator(), 5 );

Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();
Map<String, FieldLoadSaveValidator<?>> attrValidatorMap = new HashMap<>();
attrValidatorMap.put( "keyStream", stringArrayLoadSaveValidator );
attrValidatorMap.put( "keyLookup", stringArrayLoadSaveValidator );
attrValidatorMap.put( "fieldStream", stringArrayLoadSaveValidator );
Expand All @@ -117,13 +116,15 @@ public void setUpLoadSave() throws Exception {
// ValueMetaInterface.TYPE_STRING. This happens about once out of every 3 or so runs of
// the test which made it a bit difficult to track down.
// MB - 5/2016
attrValidatorMap.put( "fieldUpdate", new FieldUpdateIntArrayLoadSaveValidator( new NonZeroIntLoadSaveValidator(
attrValidatorMap.put( "update", updateValidator = new UpdateLoadSaveValidator() );
attrValidatorMap.put( "fieldUpdate", new PrimitiveIntArrayLoadSaveValidator( new NonZeroIntLoadSaveValidator(
DimensionLookupMeta.typeDesc.length ), 5 ) );
attrValidatorMap.put( "returnType", new ReturnTypeLoadSaveValidator() );
attrValidatorMap.put( "databaseMeta", new DatabaseMetaLoadSaveValidator() );
attrValidatorMap.put( "startDateAlternative", new IntLoadSaveValidator( DimensionLookupMeta.getStartDateAlternativeCodes().length ) );
attrValidatorMap.put( "sequenceName", new SequenceNameLoadSaveValidator() );

Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<String, FieldLoadSaveValidator<?>>();
Map<String, FieldLoadSaveValidator<?>> typeValidatorMap = new HashMap<>();

loadSaveTester =
new LoadSaveTester( testMetaClass, attributes, new ArrayList<String>(), new ArrayList<String>(),
Expand All @@ -149,6 +150,9 @@ public void modify( StepMetaInterface someMeta ) {

@Test
public void testSerialization() throws KettleException {
updateValidator.value = true;
loadSaveTester.testSerialization();
updateValidator.value = false;
loadSaveTester.testSerialization();
}

Expand Down Expand Up @@ -182,7 +186,7 @@ public void testGetFields() throws Exception {
meta.setFieldLookup( new String[] { "field1" } );
meta.setFieldStream( new String[] { "" } );
meta.setDatabaseMeta( dbMeta );
doReturn( extraFields ).when( meta ).getDatabaseTableFields( (Database) anyObject(), anyString(), anyString() );
doReturn( extraFields ).when( meta ).getDatabaseTableFields( anyObject(), anyString(), anyString() );
doReturn( mock( LogChannelInterface.class ) ).when( meta ).getLog();

RowMeta row = new RowMeta();
Expand Down Expand Up @@ -237,16 +241,15 @@ public String getFieldDefinition( ValueMetaInterface v, String tk, String pk, bo
}

@Test
public void testProvidesModelerMeta() throws Exception {
public void testProvidesModelerMeta() {

final RowMeta rowMeta = Mockito.mock( RowMeta.class );
final DimensionLookupMeta dimensionLookupMeta = new DimensionLookupMeta() {
@Override Database createDatabaseObject() {
return mock( Database.class );
}

@Override protected RowMetaInterface getDatabaseTableFields( Database db, String schemaName, String tableName )
throws KettleDatabaseException {
@Override protected RowMetaInterface getDatabaseTableFields( Database db, String schemaName, String tableName ) {
assertEquals( "aSchema", schemaName );
assertEquals( "aDimTable", tableName );
return rowMeta;
Expand Down Expand Up @@ -284,7 +287,6 @@ public void testProvidesModelerMeta() throws Exception {
// this special load/save handler for sequenceName.
// MB - 5/2016
public class SequenceNameLoadSaveValidator implements FieldLoadSaveValidator<String> {
final Random rand = new Random();
@Override
public String getTestObject() {
DimensionLookupMeta dlm = holdTestingMeta.get(); // get the currently-being tested meta
Expand All @@ -307,30 +309,37 @@ public boolean validateTestObject( String testObject, Object actual ) {
}
}

public class FieldUpdateIntArrayLoadSaveValidator extends PrimitiveIntArrayLoadSaveValidator {
/**
* This validator simulates behavior in {@link DimensionLookupMeta#actualizeWithInjectedValues()}.
*/
public class ReturnTypeLoadSaveValidator implements FieldLoadSaveValidator<int[]> {
@Override
public int[] getTestObject() {
final DimensionLookupMeta meta = holdTestingMeta.get();
return meta.isUpdate() ? new int[meta.getFieldUpdate().length] : meta.getFieldUpdate();
}

public FieldUpdateIntArrayLoadSaveValidator( FieldLoadSaveValidator<Integer> fieldValidator ) {
this( fieldValidator, null );
@Override
public boolean validateTestObject( int[] testObject, Object actual ) {
return Arrays.equals( testObject, (int[]) actual );
}
}

public FieldUpdateIntArrayLoadSaveValidator( FieldLoadSaveValidator<Integer> fieldValidator, Integer elements ) {
super( fieldValidator, elements );
public static class UpdateLoadSaveValidator implements FieldLoadSaveValidator<Boolean> {
boolean value;
@Override
public Boolean getTestObject() {
return value;
}

@Override
public int[] getTestObject() {
DimensionLookupMeta dlm = holdTestingMeta.get();
int[] testObject = super.getTestObject();
if ( !dlm.isUpdate() ) {
dlm.setReturnType( testObject );
}
return testObject;
public boolean validateTestObject( Boolean testObject, Object actual ) {
return testObject.equals( actual );
}
}


@Test
public void testPDI16559() throws Exception {
public void testPDI16559() {
DimensionLookupMeta dimensionLookup = new DimensionLookupMeta();
dimensionLookup.setKeyStream( new String[] { "test_field" } );
dimensionLookup.setKeyLookup( new String[] {} );
Expand Down
18 changes: 9 additions & 9 deletions ui/src/main/java/org/pentaho/di/ui/trans/steps/dimensionlookup/DimensionLookupDialog.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
*
* Pentaho Data Integration
*
* Copyright (C) 2002-2018 by Hitachi Vantara : http://www.pentaho.com
* Copyright (C) 2002-2023 by Hitachi Vantara : http://www.pentaho.com
*
*******************************************************************************
*
Expand All @@ -22,12 +22,6 @@

package org.pentaho.di.ui.trans.steps.dimensionlookup;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.eclipse.swt.SWT;
import org.eclipse.swt.custom.CCombo;
import org.eclipse.swt.custom.CTabFolder;
Expand Down Expand Up @@ -63,7 +57,6 @@
import org.eclipse.swt.widgets.TableItem;
import org.eclipse.swt.widgets.Text;
import org.pentaho.di.core.Const;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.core.Props;
import org.pentaho.di.core.SQLStatement;
import org.pentaho.di.core.database.Database;
Expand All @@ -72,6 +65,7 @@
import org.pentaho.di.core.plugins.PluginInterface;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.core.row.ValueMetaInterface;
import org.pentaho.di.core.util.Utils;
import org.pentaho.di.i18n.BaseMessages;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStepMeta;
Expand All @@ -89,6 +83,12 @@
import org.pentaho.di.ui.trans.step.TableItemInsertListener;
import org.pentaho.di.ui.util.HelpUtils;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;

/**
* Dialog for the Dimension Lookup/Update step.
*/
Expand Down Expand Up @@ -1454,7 +1454,7 @@ private void getInfo( DimensionLookupMeta in ) {
TableItem item = wUpIns.getNonEmpty( i );
in.getFieldLookup()[i] = item.getText( 1 );
in.getFieldStream()[i] = item.getText( 2 );
in.getFieldUpdate()[i] = DimensionLookupMeta.getUpdateType( in.isUpdate(), item.getText( 3 ) );
in.setFieldUpdateValue( i, item.getText( 3 ) );
}

in.setSchemaName( wSchema.getText() );
Expand Down