Handle and test csv column length errors

csv.go

@@ -54,12 +54,6 @@ func copyCSVRows(i *Import, reader *csv.Reader, ignoreErrors bool, delimiter str
 		cols := make([]interface{}, len(columns))
 		record, err := reader.Read()
 
-		//Loop ensures we don't insert too many values and that
-		//values are properly converted into empty interfaces
-		for i, col := range record {
-			cols[i] = col
-		}
-
 		if err == io.EOF {
 			break
 		}
@@ -77,6 +71,12 @@ func copyCSVRows(i *Import, reader *csv.Reader, ignoreErrors bool, delimiter str
 			}
 		}
 
+		//Loop ensures we don't insert too many values and that
+		//values are properly converted into empty interfaces
+		for i, col := range record {
+			cols[i] = col
+		}
+
 		err = i.AddRow(cols...)
 
 		if err != nil {

download_samples.sh

@@ -13,6 +13,7 @@ function download_json_samples() {
 function download_csv_samples() {
     mkdir -p $SAMPLES_DIR
     cd $SAMPLES_DIR
+    wget -nc https://s3-eu-west-1.amazonaws.com/repostruct/all_repos_analyzed.csv.tar.gz && tar -xvzf all_repos_analyzed.csv.tar.gz
     wget -nc -O parking_garage_availability.csv https://data.montgomerycountymd.gov/api/views/qahs-fevu/rows.csv
     wget -nc -O local_severe_wheather_warning_systems.csv https://data.mo.gov/api/views/n59h-ggai/rows.csv
     wget -nc -O montgomery_crime.csv https://data.montgomerycountymd.gov/api/views/icn6-v9z3/rows.csv

test.sh

@@ -71,8 +71,9 @@ function import_and_test_csv() {
     local table=$1
     local filename=$2
     local delimiter=${3:-,}
+    local general_args=${4:-}
 
-    pgfutter --schema $DB_SCHEMA --db $DB_NAME --user $DB_USER csv "$filename" --delimiter $delimiter
+    pgfutter $general_args --schema $DB_SCHEMA --db $DB_NAME --user $DB_USER csv "$filename" --delimiter "$delimiter"
     if [ $? -ne 0 ]; then
         echo "pgfutter could not import $filename"
         exit 300
@@ -84,9 +85,11 @@ function import_and_test_csv() {
 
 recreate_db
 
+import_and_test_csv "all_repos_analyzed" "$SAMPLES_DIR/all_repos_analyzed.csv" " " "--ignore-errors"
 csv_with_wrong_delimiter_should_fail
 import_csv_and_skip_header_row_with_custom_fields
 import_csv_with_special_delimiter_and_trailing
+
 import_and_test_json "_2015_01_01_15" "$SAMPLES_DIR/2015-01-01-15.json"
 import_and_test_csv "parking_garage_availability" "$SAMPLES_DIR/parking_garage_availability.csv"
 import_and_test_csv "local_severe_wheather_warning_systems" "$SAMPLES_DIR/local_severe_wheather_warning_systems.csv"
@@ -100,3 +103,5 @@ import_and_test_csv "customer_complaints" "$SAMPLES_DIR/customer_complaints.csv"
 import_and_test_csv "whitehouse_visits_2014" "$SAMPLES_DIR/whitehouse_visits_2014.csv"
 import_and_test_csv "traffic_violations" "$SAMPLES_DIR/traffic_violations.csv"
 import_and_test_json "filepaths_1" "$SAMPLES_DIR/filepaths-1.json"
+
+recreate_db