This repository has been archived by the owner on Jan 21, 2021. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 14
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Updated browse tag export to properly quote tsv/csv.
- Loading branch information
Showing
8 changed files
with
147 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
34 changes: 34 additions & 0 deletions
34
src/main/groovy/org/transmartproject/batch/batchartifacts/CsvLineAggregator.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
package org.transmartproject.batch.batchartifacts | ||
|
||
import com.opencsv.CSVWriter | ||
import groovy.transform.CompileStatic | ||
import org.springframework.batch.item.file.transform.ExtractorLineAggregator | ||
|
||
/** | ||
* Writes lines in CSV format using the opencsv library. | ||
*/ | ||
@CompileStatic | ||
class CsvLineAggregator<T> extends ExtractorLineAggregator<T> { | ||
|
||
char separator = CSVWriter.DEFAULT_SEPARATOR | ||
char quotechar = CSVWriter.DEFAULT_QUOTE_CHARACTER | ||
char escapechar = CSVWriter.DEFAULT_ESCAPE_CHARACTER | ||
String lineEnd = CSVWriter.DEFAULT_LINE_END | ||
|
||
@Override | ||
protected String doAggregate(Object[] fields) { | ||
OutputStream out = new ByteArrayOutputStream() | ||
out.withWriter { writer -> | ||
CSVWriter csvWriter = new CSVWriter(writer, separator, quotechar, escapechar, lineEnd) | ||
List<String> outData = [] | ||
fields.each { outData << it.toString() } | ||
csvWriter.writeNext(outData.toArray(new String[0]), false) | ||
csvWriter.flush() | ||
csvWriter.close() | ||
} | ||
String result = out.toString() | ||
out.close() | ||
result | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
88 changes: 88 additions & 0 deletions
88
src/test/groovy/org/transmartproject/batch/batchartifacts/CsvLineAggregatorTests.groovy
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,88 @@ | ||
package org.transmartproject.batch.batchartifacts | ||
|
||
import org.junit.Test | ||
import org.springframework.batch.item.ExecutionContext | ||
import org.springframework.batch.item.ItemStreamReader | ||
import org.springframework.batch.item.file.FlatFileItemReader | ||
import org.springframework.batch.item.file.mapping.DefaultLineMapper | ||
import org.springframework.batch.item.file.mapping.FieldSetMapper | ||
import org.springframework.batch.item.file.separator.DefaultRecordSeparatorPolicy | ||
import org.springframework.batch.item.file.transform.DelimitedLineTokenizer | ||
import org.springframework.batch.item.file.transform.FieldSet | ||
import org.springframework.batch.item.file.transform.LineAggregator | ||
import org.springframework.batch.item.file.transform.PassThroughFieldExtractor | ||
import org.springframework.core.io.ByteArrayResource | ||
import org.springframework.core.io.Resource | ||
|
||
import static org.hamcrest.MatcherAssert.assertThat | ||
import static org.hamcrest.Matchers.* | ||
|
||
/** | ||
* Tests the {@link CsvLineAggregator}, to see if it writes lines such that | ||
* they can be read correctly by the {@link DelimitedLineTokenizer}. | ||
*/ | ||
class CsvLineAggregatorTests { | ||
|
||
static <T> void write(List<? extends T> items, LineAggregator<T> lineAggregator, Writer writer) { | ||
StringBuilder lines = new StringBuilder() | ||
for (T item : items) { | ||
lines.append(lineAggregator.aggregate(item)) | ||
} | ||
writer.write(lines.toString()) | ||
} | ||
|
||
static <T> List<T> read(ItemStreamReader<T> reader) { | ||
List<T> result = [] | ||
reader.open(new ExecutionContext()) | ||
List<String> item | ||
while ((item = reader.read()) != null) { | ||
result << item | ||
} | ||
reader.close() | ||
result | ||
} | ||
|
||
private static final List<List<String>> TEST_DATA = [ | ||
[ 'Column 1', 'Column 2', 'Column 3', 'Column 4' ], | ||
[ 'A', 'Test', 'Test with tab (\t).', 'Test should pass.' ], | ||
[ 'B', 'Test', 'Test with comma (,) in the text.', 'Test should pass.' ], | ||
[ 'C', 'Test', 'Test with newline (\n) in it.', 'Test should pass.' ], | ||
] | ||
|
||
@Test | ||
void testWriteCsvData() { | ||
// Writing test data to byte array | ||
ByteArrayOutputStream out = new ByteArrayOutputStream() | ||
out.withWriter { writer -> | ||
def aggregator = new CsvLineAggregator<List<String>>( | ||
separator: '\t', | ||
fieldExtractor: new PassThroughFieldExtractor<List<String>>(), | ||
) | ||
write(TEST_DATA, aggregator, writer) | ||
writer.flush() | ||
} | ||
|
||
// Reading data from byte array | ||
def tokenizer = new DelimitedLineTokenizer( | ||
delimiter: '\t', | ||
) | ||
def lineMapper = new DefaultLineMapper( | ||
lineTokenizer: tokenizer, | ||
fieldSetMapper: { FieldSet fs -> | ||
fs.values as List<String> | ||
} as FieldSetMapper<List<String>>, | ||
) | ||
|
||
Resource resource = new ByteArrayResource(out.toByteArray()) | ||
def reader = new FlatFileItemReader<List<String>>( | ||
lineMapper: lineMapper, | ||
recordSeparatorPolicy: new DefaultRecordSeparatorPolicy(), | ||
resource: resource, | ||
) | ||
reader.afterPropertiesSet() | ||
List<List<String>> result = read(reader) | ||
|
||
assertThat result, equalTo(TEST_DATA) | ||
} | ||
|
||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,5 +1,7 @@ | ||
concept_key tag_title tag_description index | ||
\ TITLE Human Chronic Obstructive Pulmonary Disorder Biomarker 2 | ||
\ ORGANISM Homo sapiens 3 | ||
\ FREE TEXT "Text with tab characters ( ), and | ||
new lines." 6 | ||
\Endpoints\FEV1 NAME FEV1/FVC ratio 4 | ||
\Endpoints\FEV1 SYNONYMS Tiffeneau-Pinelli 5 |