Issue #49 : allow format masks

pom.xml

@@ -19,7 +19,7 @@
     </license>
   </licenses>
   <properties>
-    <pentaho-kettle.version>8.1-SNAPSHOT</pentaho-kettle.version>
+    <pentaho-kettle.version>8.2.0.0-342</pentaho-kettle.version>
     <mockito-all.version>1.9.5</mockito-all.version>
     <junit.version>4.4</junit.version>
   </properties>

src/main/java/org/pentaho/di/dataset/DataSet.java

@@ -103,6 +103,7 @@ public RowMetaInterface getSetRowMeta( boolean columnName ) throws KettlePluginE
         field.getLength(),
         field.getPrecision() );
       valueMeta.setComments( field.getComment() );
+      valueMeta.setConversionMask( field.getFormat() );
       rowMeta.addValueMeta( valueMeta );
     }
     return rowMeta;

src/main/java/org/pentaho/di/dataset/DataSetCsvGroup.java

@@ -71,18 +71,20 @@ private static String getDataSetFolder( DataSetGroup group ) {
 
   private static void setValueFormats( RowMetaInterface rowMeta ) {
     for ( ValueMetaInterface valueMeta : rowMeta.getValueMetaList() ) {
-      switch ( valueMeta.getType() ) {
-        case ValueMetaInterface.TYPE_INTEGER:
-          valueMeta.setConversionMask( "0" );
-          break;
-        case ValueMetaInterface.TYPE_NUMBER:
-          valueMeta.setConversionMask( "0.#" );
-          break;
-        case ValueMetaInterface.TYPE_DATE:
-          valueMeta.setConversionMask( "yyyyMMdd-HHmmss.SSS" );
-          break;
-        default:
-          break;
+      if ( StringUtils.isEmpty( valueMeta.getConversionMask() ) ) {
+        switch ( valueMeta.getType() ) {
+          case ValueMetaInterface.TYPE_INTEGER:
+            valueMeta.setConversionMask( "0" );
+            break;
+          case ValueMetaInterface.TYPE_NUMBER:
+            valueMeta.setConversionMask( "0.#" );
+            break;
+          case ValueMetaInterface.TYPE_DATE:
+            valueMeta.setConversionMask( "yyyyMMdd-HHmmss.SSS" );
+            break;
+          default:
+            break;
+        }
       }
     }
   }
@@ -117,15 +119,15 @@ public static final List<Object[]> getAllRows( LogChannelInterface log, DataSetG
           if ( csvRecord.getRecordNumber() > 1 ) {
             Object[] row = RowDataUtil.allocateRowData( setRowMeta.size() );
             for ( int i = 0; i < setRowMeta.size(); i++ ) {
-              ValueMetaInterface valueMeta = setRowMeta.getValueMeta( i );
+              ValueMetaInterface valueMeta = setRowMeta.getValueMeta( i ).clone();
+              constantValueMeta.setConversionMetadata( valueMeta );
               String value = csvRecord.get( i );
-              row[ i ] = valueMeta.convertDataFromString( value, constantValueMeta, null, null, ValueMetaInterface.TRIM_TYPE_NONE );
+              row[ i ] = valueMeta.convertData( constantValueMeta, value );
             }
             rows.add( row );
           }
         }
       }
-
       return rows;
     } catch ( Exception e ) {
       throw new KettleException( "Unable to get all rows for CSV data set '" + dataSet.getName() + "'", e );
@@ -204,12 +206,12 @@ public static final List<Object[]> getAllRows( LogChannelInterface log, DataSetG
         sortIndexes[ i ] = outputRowMeta.indexOfValue( sortFields.get( i ) );
       }
 
-      if (outputRowMeta.isEmpty()) {
-        log.logError( "WARNING: No field mappings selected for data set '"+dataSet.getName()+"', returning empty set of rows" );
+      if ( outputRowMeta.isEmpty() ) {
+        log.logError( "WARNING: No field mappings selected for data set '" + dataSet.getName() + "', returning empty set of rows" );
         return new ArrayList<>();
       }
 
-      if ( !sortFields.isEmpty()) {
+      if ( !sortFields.isEmpty() ) {
 
         // Sort the rows...
         //

src/main/java/org/pentaho/di/dataset/DataSetField.java

@@ -24,6 +24,8 @@
 
 import org.pentaho.metastore.persist.MetaStoreAttribute;
 
+import java.util.Objects;
+
 public class DataSetField {
   @MetaStoreAttribute( key = "field_name" )
   private String fieldName;
@@ -43,88 +45,148 @@ public class DataSetField {
   @MetaStoreAttribute( key = "field_comment" )
   private String comment;
 
+  @MetaStoreAttribute( key = "field_format" )
+  private String format;
+
   public DataSetField() {
     // Empty constructor for MetaStoreFactory.
   }
 
-  public DataSetField( String fieldName, String columnName, int type, int length, int precision, String comment ) {
+  public DataSetField( String fieldName, String columnName, int type, int length, int precision, String comment, String format ) {
     super();
     this.fieldName = fieldName;
     this.columnName = columnName;
     this.type = type;
     this.length = length;
     this.precision = precision;
     this.comment = comment;
+    this.format = format;
   }
 
-  @Override
-  public boolean equals( Object obj ) {
-    if ( this == obj ) {
+  @Override public boolean equals( Object o ) {
+    if ( this == o ) {
       return true;
     }
-    if ( !( obj instanceof DataSetField ) ) {
+    if ( o == null || getClass() != o.getClass() ) {
       return false;
     }
-    DataSetField cmp = (DataSetField) obj;
-    return fieldName.equals( cmp.fieldName )
-      && columnName.equals( cmp.columnName )
-      && type == cmp.type
-      && length == cmp.length
-      && precision == cmp.precision
-      && ( comment == null && cmp.comment == null || comment != null && comment.equals( cmp.comment ) );
+    DataSetField that = (DataSetField) o;
+    return Objects.equals( fieldName, that.fieldName );
   }
 
-  @Override
-  public int hashCode() {
-    return fieldName.hashCode();
+  @Override public int hashCode() {
+    return Objects.hash( fieldName );
   }
 
+  /**
+   * Gets fieldName
+   *
+   * @return value of fieldName
+   */
   public String getFieldName() {
     return fieldName;
   }
 
+  /**
+   * @param fieldName The fieldName to set
+   */
   public void setFieldName( String fieldName ) {
     this.fieldName = fieldName;
   }
 
+  /**
+   * Gets columnName
+   *
+   * @return value of columnName
+   */
   public String getColumnName() {
     return columnName;
   }
 
+  /**
+   * @param columnName The columnName to set
+   */
   public void setColumnName( String columnName ) {
     this.columnName = columnName;
   }
 
+  /**
+   * Gets type
+   *
+   * @return value of type
+   */
   public int getType() {
     return type;
   }
 
+  /**
+   * @param type The type to set
+   */
   public void setType( int type ) {
     this.type = type;
   }
 
+  /**
+   * Gets length
+   *
+   * @return value of length
+   */
   public int getLength() {
     return length;
   }
 
+  /**
+   * @param length The length to set
+   */
   public void setLength( int length ) {
     this.length = length;
   }
 
+  /**
+   * Gets precision
+   *
+   * @return value of precision
+   */
   public int getPrecision() {
     return precision;
   }
 
+  /**
+   * @param precision The precision to set
+   */
   public void setPrecision( int precision ) {
     this.precision = precision;
   }
 
+  /**
+   * Gets comment
+   *
+   * @return value of comment
+   */
   public String getComment() {
     return comment;
   }
 
+  /**
+   * @param comment The comment to set
+   */
   public void setComment( String comment ) {
     this.comment = comment;
   }
 
+  /**
+   * Gets format
+   *
+   * @return value of format
+   */
+  public String getFormat() {
+    return format;
+  }
+
+  /**
+   * @param format The format to set
+   */
+  public void setFormat( String format ) {
+    this.format = format;
+  }
 }

src/main/java/org/pentaho/di/dataset/spoon/DataSetHelper.java

@@ -797,7 +797,8 @@ public void createDataSetFromStep() {
         ValueMetaInterface valueMeta = rowMeta.getValueMeta( i );
         String setFieldname = valueMeta.getName();
         String columnName = "field" + i;
-        DataSetField field = new DataSetField( setFieldname, columnName, valueMeta.getType(), valueMeta.getLength(), valueMeta.getPrecision(), valueMeta.getComments() );
+        DataSetField field = new DataSetField( setFieldname, columnName, valueMeta.getType(), valueMeta.getLength(),
+          valueMeta.getPrecision(), valueMeta.getComments(), valueMeta.getFormatMask() );
         dataSet.getFields().add( field );
       }
 

src/main/java/org/pentaho/di/dataset/spoon/dialog/DataSetDialog.java

@@ -288,6 +288,8 @@ public void widgetSelected( SelectionEvent arg0 ) {
         ColumnInfo.COLUMN_TYPE_CCOMBO, new String[] { "" }, false ),
       new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldType" ),
         ColumnInfo.COLUMN_TYPE_CCOMBO, ValueMetaFactory.getAllValueMetaNames(), false ),
+      new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldFormat" ),
+        ColumnInfo.COLUMN_TYPE_TEXT, true, false ),
       new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldLength" ),
         ColumnInfo.COLUMN_TYPE_TEXT, true, false ),
       new ColumnInfo( BaseMessages.getString( PKG, "DataSetDialog.ColumnInfo.FieldPrecision" ),
@@ -595,6 +597,7 @@ public void getData() {
       wFieldMapping.setText( Const.NVL( field.getFieldName(), "" ), colnr++, i );
       wFieldMapping.setText( Const.NVL( field.getColumnName(), "" ), colnr++, i );
       wFieldMapping.setText( ValueMetaFactory.getValueMetaName( field.getType() ), colnr++, i );
+      wFieldMapping.setText( Const.NVL( field.getFormat(), ""), colnr++, i );
       wFieldMapping.setText( field.getLength() >= 0 ? Integer.toString( field.getLength() ) : "", colnr++, i );
       wFieldMapping.setText( field.getPrecision() >= 0 ? Integer.toString( field.getPrecision() ) : "", colnr++, i );
       wFieldMapping.setText( Const.NVL( field.getComment(), "" ), colnr++, i );
@@ -624,11 +627,12 @@ public void getInfo( DataSet set ) {
       String fieldName = item.getText( colnr++ );
       String columnName = item.getText( colnr++ );
       int type = ValueMetaFactory.getIdForValueMeta( item.getText( colnr++ ) );
+      String format = item.getText( colnr++ );
       int length = Const.toInt( item.getText( colnr++ ), -1 );
       int precision = Const.toInt( item.getText( colnr++ ), -1 );
       String comment = item.getText( colnr++ );
 
-      DataSetField field = new DataSetField( fieldName, columnName, type, length, precision, comment );
+      DataSetField field = new DataSetField( fieldName, columnName, type, length, precision, comment, format );
       set.getFields().add( field );
     }
 

src/main/java/org/pentaho/di/dataset/spoon/dialog/messages/messages_en_US.properties

@@ -12,6 +12,7 @@ DataSetDialog.FieldMapping.Label=The data set fields and their column names in t
 DataSetDialog.ColumnInfo.FieldName=Field name
 DataSetDialog.ColumnInfo.ColumnName=Column name
 DataSetDialog.ColumnInfo.FieldType=Type
+DataSetDialog.ColumnInfo.FieldFormat = Format
 DataSetDialog.ColumnInfo.FieldLength=Length
 DataSetDialog.ColumnInfo.FieldPrecision=Precision
 DataSetDialog.ColumnInfo.Comment=Comment
@@ -65,4 +66,3 @@ TransUnitTestSetLocationDialog.GetSortFields.Button=Get sort fields
 TransUnitTestSetLocationDialog.ColumnInfo.StepField=Step field
 TransUnitTestSetLocationDialog.ColumnInfo.DatasetField=Dataset field
 TransUnitTestSetLocationDialog.FieldOrder.Label=Field order:
-

src/test/java/org/pentaho/di/dataset/DataSetTest.java

@@ -44,7 +44,7 @@ protected void setUp() throws Exception {
 
     List<DataSetField> fields = new ArrayList<>();
     for ( int i = 0; i < NR_FIELDS; i++ ) {
-      fields.add( new DataSetField( "field" + i, "column" + i, ValueMetaInterface.TYPE_STRING, 50, 0, "comment" + i ) );
+      fields.add( new DataSetField( "field" + i, "column" + i, ValueMetaInterface.TYPE_STRING, 50, 0, "comment" + i, null ) );
     }
     dataSet = new DataSet( NAME, DESC, dataSetGroup, TABLE, fields );
   }

src/test/java/org/pentaho/di/dataset/TransUnitTestExecutionTest.java

@@ -137,9 +137,9 @@ protected void tearDown() throws Exception {
 
   private void createInputDataSet() throws KettleException {
     List<DataSetField> fields = new ArrayList<>();
-    fields.add( new DataSetField( "a", "column_a", ValueMetaInterface.TYPE_STRING, 20, 0, null ) );
-    fields.add( new DataSetField( "b", "column_b", ValueMetaInterface.TYPE_STRING, 20, 0, null ) );
-    fields.add( new DataSetField( "c", "column_c", ValueMetaInterface.TYPE_STRING, 20, 0, null ) );
+    fields.add( new DataSetField( "a", "column_a", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) );
+    fields.add( new DataSetField( "b", "column_b", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) );
+    fields.add( new DataSetField( "c", "column_c", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) );
 
     List<Object[]> rows = new ArrayList<Object[]>();
     rows.add( new Object[] { "a1", "b1", "c1", } );
@@ -154,10 +154,10 @@ private void createGoldenDataSet() throws KettleException {
 
     // Add the fields in a different order to see if we can correctly compare data against it!
     //
-    fields.add( new DataSetField( "d", "column_d", ValueMetaInterface.TYPE_INTEGER, 6, 0, null ) );
-    fields.add( new DataSetField( "c", "column_c", ValueMetaInterface.TYPE_STRING, 20, 0, null ) );
-    fields.add( new DataSetField( "b", "column_b", ValueMetaInterface.TYPE_STRING, 20, 0, null ) );
-    fields.add( new DataSetField( "a", "column_a", ValueMetaInterface.TYPE_STRING, 20, 0, null ) );
+    fields.add( new DataSetField( "d", "column_d", ValueMetaInterface.TYPE_INTEGER, 6, 0, null, null ) );
+    fields.add( new DataSetField( "c", "column_c", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) );
+    fields.add( new DataSetField( "b", "column_b", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) );
+    fields.add( new DataSetField( "a", "column_a", ValueMetaInterface.TYPE_STRING, 20, 0, null, null ) );
 
     List<Object[]> rows = new ArrayList<Object[]>();
     rows.add( new Object[] { 123456L, "c1", "b1", "a1", } );