diff --git a/pom.xml b/pom.xml index 3cce0ce..fdd24fe 100644 --- a/pom.xml +++ b/pom.xml @@ -19,7 +19,7 @@ - 8.1-SNAPSHOT + 8.2.0.0-342 1.9.5 4.4 diff --git a/src/main/java/org/pentaho/di/dataset/DataSet.java b/src/main/java/org/pentaho/di/dataset/DataSet.java index c51eda4..54cfeaa 100755 --- a/src/main/java/org/pentaho/di/dataset/DataSet.java +++ b/src/main/java/org/pentaho/di/dataset/DataSet.java @@ -103,6 +103,7 @@ public RowMetaInterface getSetRowMeta( boolean columnName ) throws KettlePluginE field.getLength(), field.getPrecision() ); valueMeta.setComments( field.getComment() ); + valueMeta.setConversionMask( field.getFormat() ); rowMeta.addValueMeta( valueMeta ); } return rowMeta; diff --git a/src/main/java/org/pentaho/di/dataset/DataSetCsvGroup.java b/src/main/java/org/pentaho/di/dataset/DataSetCsvGroup.java index 55e5c60..74249d3 100644 --- a/src/main/java/org/pentaho/di/dataset/DataSetCsvGroup.java +++ b/src/main/java/org/pentaho/di/dataset/DataSetCsvGroup.java @@ -71,18 +71,20 @@ private static String getDataSetFolder( DataSetGroup group ) { private static void setValueFormats( RowMetaInterface rowMeta ) { for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) { - switch ( valueMeta.getType() ) { - case ValueMetaInterface.TYPE_INTEGER: - valueMeta.setConversionMask( "0" ); - break; - case ValueMetaInterface.TYPE_NUMBER: - valueMeta.setConversionMask( "0.#" ); - break; - case ValueMetaInterface.TYPE_DATE: - valueMeta.setConversionMask( "yyyyMMdd-HHmmss.SSS" ); - break; - default: - break; + if ( StringUtils.isEmpty( valueMeta.getConversionMask() ) ) { + switch ( valueMeta.getType() ) { + case ValueMetaInterface.TYPE_INTEGER: + valueMeta.setConversionMask( "0" ); + break; + case ValueMetaInterface.TYPE_NUMBER: + valueMeta.setConversionMask( "0.#" ); + break; + case ValueMetaInterface.TYPE_DATE: + valueMeta.setConversionMask( "yyyyMMdd-HHmmss.SSS" ); + break; + default: + break; + } } } } @@ -117,15 +119,15 @@ public static final List getAllRows( LogChannelInterface log, DataSetG if ( csvRecord.getRecordNumber() > 1 ) { Object[] row = RowDataUtil.allocateRowData( setRowMeta.size() ); for ( int i = 0; i < setRowMeta.size(); i++ ) { - ValueMetaInterface valueMeta = setRowMeta.getValueMeta( i ); + ValueMetaInterface valueMeta = setRowMeta.getValueMeta( i ).clone(); + constantValueMeta.setConversionMetadata( valueMeta ); String value = csvRecord.get( i ); - row[ i ] = valueMeta.convertDataFromString( value, constantValueMeta, null, null, ValueMetaInterface.TRIM_TYPE_NONE ); + row[ i ] = valueMeta.convertData( constantValueMeta, value ); } rows.add( row ); } } } - return rows; } catch ( Exception e ) { throw new KettleException( "Unable to get all rows for CSV data set '" + dataSet.getName() + "'", e ); @@ -204,12 +206,12 @@ public static final List getAllRows( LogChannelInterface log, DataSetG sortIndexes[ i ] = outputRowMeta.indexOfValue( sortFields.get( i ) ); } - if (outputRowMeta.isEmpty()) { - log.logError( "WARNING: No field mappings selected for data set '"+dataSet.getName()+"', returning empty set of rows" ); + if ( outputRowMeta.isEmpty() ) { + log.logError( "WARNING: No field mappings selected for data set '" + dataSet.getName() + "', returning empty set of rows" ); return new ArrayList<>(); } - if ( !sortFields.isEmpty()) { + if ( !sortFields.isEmpty() ) { // Sort the rows... // diff --git a/src/main/java/org/pentaho/di/dataset/DataSetField.java b/src/main/java/org/pentaho/di/dataset/DataSetField.java index 0c11eee..9099e4f 100644 --- a/src/main/java/org/pentaho/di/dataset/DataSetField.java +++ b/src/main/java/org/pentaho/di/dataset/DataSetField.java @@ -24,6 +24,8 @@ import org.pentaho.metastore.persist.MetaStoreAttribute; +import java.util.Objects; + public class DataSetField { @MetaStoreAttribute( key = "field_name" ) private String fieldName; @@ -43,11 +45,14 @@ public class DataSetField { @MetaStoreAttribute( key = "field_comment" ) private String comment; + @MetaStoreAttribute( key = "field_format" ) + private String format; + public DataSetField() { // Empty constructor for MetaStoreFactory. } - public DataSetField( String fieldName, String columnName, int type, int length, int precision, String comment ) { + public DataSetField( String fieldName, String columnName, int type, int length, int precision, String comment, String format ) { super(); this.fieldName = fieldName; this.columnName = columnName; @@ -55,76 +60,133 @@ public DataSetField( String fieldName, String columnName, int type, int length, this.length = length; this.precision = precision; this.comment = comment; + this.format = format; } - @Override - public boolean equals( Object obj ) { - if ( this == obj ) { + @Override public boolean equals( Object o ) { + if ( this == o ) { return true; } - if ( !( obj instanceof DataSetField ) ) { + if ( o == null || getClass() != o.getClass() ) { return false; } - DataSetField cmp = (DataSetField) obj; - return fieldName.equals( cmp.fieldName ) - && columnName.equals( cmp.columnName ) - && type == cmp.type - && length == cmp.length - && precision == cmp.precision - && ( comment == null && cmp.comment == null || comment != null && comment.equals( cmp.comment ) ); + DataSetField that = (DataSetField) o; + return Objects.equals( fieldName, that.fieldName ); } - @Override - public int hashCode() { - return fieldName.hashCode(); + @Override public int hashCode() { + return Objects.hash( fieldName ); } + /** + * Gets fieldName + * + * @return value of fieldName + */ public String getFieldName() { return fieldName; } + /** + * @param fieldName The fieldName to set + */ public void setFieldName( String fieldName ) { this.fieldName = fieldName; } + /** + * Gets columnName + * + * @return value of columnName + */ public String getColumnName() { return columnName; } + /** + * @param columnName The columnName to set + */ public void setColumnName( String columnName ) { this.columnName = columnName; } + /** + * Gets type + * + * @return value of type + */ public int getType() { return type; } + /** + * @param type The type to set + */ public void setType( int type ) { this.type = type; } + /** + * Gets length + * + * @return value of length + */ public int getLength() { return length; } + /** + * @param length The length to set + */ public void setLength( int length ) { this.length = length; } + /** + * Gets precision + * + * @return value of precision + */ public int getPrecision() { return precision; } + /** + * @param precision The precision to set + */ public void setPrecision( int precision ) { this.precision = precision; } + /** + * Gets comment + * + * @return value of comment + */ public String getComment() { return comment; } + /** + * @param comment The comment to set + */ public void setComment( String comment ) { this.comment = comment; } + /** + * Gets format + * + * @return value of format + */ + public String getFormat() { + return format; + } + + /** + * @param format The format to set + */ + public void setFormat( String format ) { + this.format = format; + } } diff --git a/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java b/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java index 637c3cb..5635f82 100755 --- a/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java +++ b/src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java @@ -797,7 +797,8 @@ public void createDataSetFromStep() { ValueMetaInterface valueMeta = rowMeta.getValueMeta( i ); String setFieldname = valueMeta.getName(); String columnName = "field" + i; - DataSetField field = new DataSetField( setFieldname, columnName, valueMeta.getType(), valueMeta.getLength(), valueMeta.getPrecision(), valueMeta.getComments() ); + DataSetField field = new DataSetField( setFieldname, columnName, valueMeta.getType(), valueMeta.getLength(), + valueMeta.getPrecision(), valueMeta.getComments(), valueMeta.getFormatMask() ); dataSet.getFields().add( field ); } diff --git a/src/main/java/org/pentaho/di/dataset/spoon/dialog/DataSetDialog.java b/src/main/java/org/pentaho/di/dataset/spoon/dialog/DataSetDialog.java index 2dce0bf..28b1e21 100644 --- a/src/main/java/org/pentaho/di/dataset/spoon/dialog/DataSetDialog.java +++ b/src/main/java/org/pentaho/di/dataset/spoon/dialog/DataSetDialog.java @@ -288,6 +288,8 @@ public void widgetSelected( SelectionEvent arg0 ) { ColumnInfo.COLUMN_TYPE_CCOMBO, new String[] { "" }, false ), new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldType" ), ColumnInfo.COLUMN_TYPE_CCOMBO, ValueMetaFactory.getAllValueMetaNames(), false ), + new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldFormat" ), + ColumnInfo.COLUMN_TYPE_TEXT, true, false ), new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldLength" ), ColumnInfo.COLUMN_TYPE_TEXT, true, false ), new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldPrecision" ), @@ -595,6 +597,7 @@ public void getData() { wFieldMapping.setText( Const.NVL( field.getFieldName(), "" ), colnr++, i ); wFieldMapping.setText( Const.NVL( field.getColumnName(), "" ), colnr++, i ); wFieldMapping.setText( ValueMetaFactory.getValueMetaName( field.getType() ), colnr++, i ); + wFieldMapping.setText( Const.NVL( field.getFormat(), ""), colnr++, i ); wFieldMapping.setText( field.getLength() >= 0 ? Integer.toString( field.getLength() ) : "", colnr++, i ); wFieldMapping.setText( field.getPrecision() >= 0 ? Integer.toString( field.getPrecision() ) : "", colnr++, i ); wFieldMapping.setText( Const.NVL( field.getComment(), "" ), colnr++, i ); @@ -624,11 +627,12 @@ public void getInfo( DataSet set ) { String fieldName = item.getText( colnr++ ); String columnName = item.getText( colnr++ ); int type = ValueMetaFactory.getIdForValueMeta( item.getText( colnr++ ) ); + String format = item.getText( colnr++ ); int length = Const.toInt( item.getText( colnr++ ), -1 ); int precision = Const.toInt( item.getText( colnr++ ), -1 ); String comment = item.getText( colnr++ ); - DataSetField field = new DataSetField( fieldName, columnName, type, length, precision, comment ); + DataSetField field = new DataSetField( fieldName, columnName, type, length, precision, comment, format ); set.getFields().add( field ); } diff --git a/src/main/java/org/pentaho/di/dataset/spoon/dialog/messages/messages_en_US.properties b/src/main/java/org/pentaho/di/dataset/spoon/dialog/messages/messages_en_US.properties index 32ca953..81fa038 100644 --- a/src/main/java/org/pentaho/di/dataset/spoon/dialog/messages/messages_en_US.properties +++ b/src/main/java/org/pentaho/di/dataset/spoon/dialog/messages/messages_en_US.properties @@ -12,6 +12,7 @@ DataSetDialog.FieldMapping.Label=The data set fields and their column names in t DataSetDialog.ColumnInfo.FieldName=Field name DataSetDialog.ColumnInfo.ColumnName=Column name DataSetDialog.ColumnInfo.FieldType=Type +DataSetDialog.ColumnInfo.FieldFormat = Format DataSetDialog.ColumnInfo.FieldLength=Length DataSetDialog.ColumnInfo.FieldPrecision=Precision DataSetDialog.ColumnInfo.Comment=Comment @@ -65,4 +66,3 @@ TransUnitTestSetLocationDialog.GetSortFields.Button=Get sort fields TransUnitTestSetLocationDialog.ColumnInfo.StepField=Step field TransUnitTestSetLocationDialog.ColumnInfo.DatasetField=Dataset field TransUnitTestSetLocationDialog.FieldOrder.Label=Field order: - diff --git a/src/test/java/org/pentaho/di/dataset/DataSetTest.java b/src/test/java/org/pentaho/di/dataset/DataSetTest.java index 2cbd4e8..a545a68 100755 --- a/src/test/java/org/pentaho/di/dataset/DataSetTest.java +++ b/src/test/java/org/pentaho/di/dataset/DataSetTest.java @@ -44,7 +44,7 @@ protected void setUp() throws Exception { List fields = new ArrayList<>(); for ( int i = 0; i < NR_FIELDS; i++ ) { - fields.add( new DataSetField( "field" + i, "column" + i, ValueMetaInterface.TYPE_STRING, 50, 0, "comment" + i ) ); + fields.add( new DataSetField( "field" + i, "column" + i, ValueMetaInterface.TYPE_STRING, 50, 0, "comment" + i, null ) ); } dataSet = new DataSet( NAME, DESC, dataSetGroup, TABLE, fields ); } diff --git a/src/test/java/org/pentaho/di/dataset/TransUnitTestExecutionTest.java b/src/test/java/org/pentaho/di/dataset/TransUnitTestExecutionTest.java index 0372260..e853525 100644 --- a/src/test/java/org/pentaho/di/dataset/TransUnitTestExecutionTest.java +++ b/src/test/java/org/pentaho/di/dataset/TransUnitTestExecutionTest.java @@ -137,9 +137,9 @@ protected void tearDown() throws Exception { private void createInputDataSet() throws KettleException { List fields = new ArrayList<>(); - fields.add( new DataSetField( "a", "column_a", ValueMetaInterface.TYPE_STRING, 20, 0, null ) ); - fields.add( new DataSetField( "b", "column_b", ValueMetaInterface.TYPE_STRING, 20, 0, null ) ); - fields.add( new DataSetField( "c", "column_c", ValueMetaInterface.TYPE_STRING, 20, 0, null ) ); + fields.add( new DataSetField( "a", "column_a", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) ); + fields.add( new DataSetField( "b", "column_b", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) ); + fields.add( new DataSetField( "c", "column_c", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) ); List rows = new ArrayList(); rows.add( new Object[] { "a1", "b1", "c1", } ); @@ -154,10 +154,10 @@ private void createGoldenDataSet() throws KettleException { // Add the fields in a different order to see if we can correctly compare data against it! // - fields.add( new DataSetField( "d", "column_d", ValueMetaInterface.TYPE_INTEGER, 6, 0, null ) ); - fields.add( new DataSetField( "c", "column_c", ValueMetaInterface.TYPE_STRING, 20, 0, null ) ); - fields.add( new DataSetField( "b", "column_b", ValueMetaInterface.TYPE_STRING, 20, 0, null ) ); - fields.add( new DataSetField( "a", "column_a", ValueMetaInterface.TYPE_STRING, 20, 0, null ) ); + fields.add( new DataSetField( "d", "column_d", ValueMetaInterface.TYPE_INTEGER, 6, 0, null, null ) ); + fields.add( new DataSetField( "c", "column_c", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) ); + fields.add( new DataSetField( "b", "column_b", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) ); + fields.add( new DataSetField( "a", "column_a", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) ); List rows = new ArrayList(); rows.add( new Object[] { 123456L, "c1", "b1", "a1", } );