Skip to content

Commit 11fb86a

Browse files
tineveztinevez
authored
Merge pull request #13 from mastodon-sc/rework-csv-importer
Rework the CSV importer: use opencsv as deps.
2 parents 84dbd40 + e59b15c commit 11fb86a

File tree

5 files changed

+212
-193
lines changed

5 files changed

+212
-193
lines changed

‎pom.xml

+4-5
Original file line numberDiff line numberDiff line change
@@ -44,10 +44,9 @@
4444

4545
<!-- Other deps -->
4646
<dependency>
47-
<groupId>org.apache.commons</groupId>
48-
<artifactId>commons-csv</artifactId>
49-
</dependency>
50-
47+
<groupId>com.opencsv</groupId>
48+
<artifactId>opencsv</artifactId>
49+
</dependency>
5150
<dependency>
5251
<groupId>sc.fiji</groupId>
5352
<artifactId>bigdataviewer-core</artifactId>
@@ -101,7 +100,7 @@
101100
<license.organizationName>Mastodon authors</license.organizationName>
102101
<license.copyrightOwners>Tobias Pietzsch, Jean-Yves Tinevez</license.copyrightOwners>
103102

104-
<mastodon.version>1.0.0-beta-27</mastodon.version>
103+
<mastodon.version>1.0.0-beta-29</mastodon.version>
105104
<mastodon-tracking.version>1.0.0-beta-14</mastodon-tracking.version>
106105

107106
<!-- NB: Deploy releases to the SciJava Maven repository. -->

‎src/main/java/org/mastodon/mamut/io/csv/CSVImporter.java

+69-54
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,11 @@
3232
import java.io.FileReader;
3333
import java.io.IOException;
3434
import java.io.ObjectInputStream;
35-
import java.io.Reader;
3635
import java.util.HashMap;
36+
import java.util.Iterator;
3737
import java.util.Map;
3838
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
3939

40-
import org.apache.commons.csv.CSVFormat;
41-
import org.apache.commons.csv.CSVParser;
42-
import org.apache.commons.csv.CSVRecord;
4340
import org.mastodon.RefPool;
4441
import org.mastodon.collection.RefCollection;
4542
import org.mastodon.feature.Dimension;
@@ -60,6 +57,11 @@
6057
import org.scijava.plugin.Plugin;
6158
import org.scijava.util.VersionUtils;
6259

60+
import com.opencsv.CSVParser;
61+
import com.opencsv.CSVParserBuilder;
62+
import com.opencsv.CSVReader;
63+
import com.opencsv.CSVReaderBuilder;
64+
6365
import net.imglib2.algorithm.Algorithm;
6466

6567
public class CSVImporter implements Algorithm
@@ -137,32 +139,36 @@ public static Builder create()
137139
@Override
138140
public boolean checkInput()
139141
{
140-
final CSVFormat csvFormat = CSVFormat.EXCEL
141-
.builder()
142-
.setHeader()
143-
.setCommentMarker( '#' )
144-
.build();
145-
try (Reader in = new FileReader( filePath );
146-
CSVParser records = csvFormat.parse( in );)
142+
if ( separator == '\0' )
147143
{
148-
final Map< String, Integer > headerMap = records.getHeaderMap();
149-
if ( null == headerMap )
144+
try
150145
{
151-
errorMessage = "File " + filePath + " does not have a header.\n";
152-
return false;
146+
separator = AutoDetectCSVSeparator.autoDetect( filePath );
147+
}
148+
catch ( final IOException e1 )
149+
{
150+
separator = ',';
153151
}
154-
155152
}
156-
catch ( final FileNotFoundException e )
153+
154+
final CSVParser parser =
155+
new CSVParserBuilder()
156+
.withSeparator( separator )
157+
.withIgnoreQuotations( true )
158+
.build();
159+
try
157160
{
158-
errorMessage = "Could not find file " + filePath + "\n" + e.getMessage();
159-
return false;
161+
new CSVReaderBuilder( new FileReader( filePath ) )
162+
.withCSVParser( parser )
163+
.build();
160164
}
161-
catch ( final IOException e )
165+
catch ( final FileNotFoundException e )
162166
{
163-
errorMessage = "Error reading file " + filePath + "\n" + e.getMessage();
167+
errorMessage = "Could not find file: " + filePath;
168+
e.printStackTrace();
164169
return false;
165170
}
171+
166172
return true;
167173
}
168174

@@ -185,24 +191,34 @@ public boolean process()
185191
}
186192
}
187193

188-
final CSVFormat csvFormat = CSVFormat.EXCEL
189-
.builder()
190-
.setDelimiter( separator )
191-
.setHeader()
192-
.setCommentMarker( '#' )
193-
.build();
194-
195-
try (Reader in = new FileReader( filePath );
196-
CSVParser records = csvFormat.parse( in );)
194+
final CSVParser parser =
195+
new CSVParserBuilder()
196+
.withSeparator( separator )
197+
.withIgnoreQuotations( true )
198+
.build();
199+
try
197200
{
201+
final CSVReader reader = new CSVReaderBuilder( new FileReader( filePath ) )
202+
.withCSVParser( parser )
203+
.build();
204+
final Iterator< String[] > it = reader.iterator();
205+
206+
/*
207+
* Parse first line and reads it as the header of the file.
208+
*/
209+
210+
if ( !it.hasNext() )
211+
{
212+
errorMessage = "CSV file is empty.";
213+
return false;
214+
}
198215

199-
final Map< String, Integer > uncleanHeaderMap = records.getHeaderMap();
200-
final Map< String, Integer > headerMap = new HashMap<>( uncleanHeaderMap.size() );
201-
for ( final String uncleanKey : uncleanHeaderMap.keySet() )
216+
final String[] firstLine = it.next();
217+
final Map< String, Integer > headerMap = new HashMap<>( firstLine.length );
218+
for ( int i = 0; i < firstLine.length; i++ )
202219
{
203-
// Remove control and invisible chars.
204-
final String cleanKey = uncleanKey.trim().replaceAll( "\\p{C}", "" );
205-
headerMap.put( cleanKey, uncleanHeaderMap.get( uncleanKey ) );
220+
final String cleanKey = firstLine[ i ].trim().replaceAll( "\\p{C}", "" );
221+
headerMap.put( cleanKey, Integer.valueOf( i ) );
206222
}
207223

208224
/*
@@ -264,50 +280,55 @@ public boolean process()
264280
labelcol = headerMap.get( labelColumnName );
265281

266282
/*
267-
* Iterate over records.
283+
* Iterate over the rest of lines.
268284
*/
269285

270286
final WriteLock lock = graph.getLock().writeLock();
271287
lock.lock();
272288
final Spot vref = graph.vertexRef();
273289
final double[] pos = new double[ 3 ];
290+
274291
try
275292
{
276-
for ( final CSVRecord record : records )
293+
int lineNumber = 1;
294+
while ( it.hasNext() )
277295
{
296+
final String[] record = it.next();
297+
lineNumber++;
298+
278299
try
279300
{
280-
pos[ 0 ] = Double.parseDouble( record.get( xcol ) ) + xOrigin;
281-
pos[ 1 ] = Double.parseDouble( record.get( ycol ) ) + yOrigin;
282-
pos[ 2 ] = Double.parseDouble( record.get( zcol ) ) + zOrigin;
283-
final int t = Integer.parseInt( record.get( framecol ) );
301+
pos[ 0 ] = Double.parseDouble( record[ xcol ] ) + xOrigin;
302+
pos[ 1 ] = Double.parseDouble( record[ ycol ] ) + yOrigin;
303+
pos[ 2 ] = Double.parseDouble( record[ zcol ] ) + zOrigin;
304+
final int t = Integer.parseInt( record[ framecol ] );
284305

285306
final Spot spot = graph.addVertex( vref ).init( t, pos, radius );
286307
if ( null != idcol )
287308
{
288-
final int id = Integer.parseInt( record.get( idcol ) );
309+
final int id = Integer.parseInt( record[ idcol ] );
289310
originalIdFeature.set( spot, id );
290311
if ( null == labelcol )
291312
spot.setLabel( "" + id );
292313
}
314+
293315
if ( null != labelcol )
294316
{
295-
spot.setLabel( record.get( labelcol ) );
317+
spot.setLabel( record[ labelcol ] );
296318
}
297319
double q = 1.;
298320
if ( null != qualitycol )
299321
{
300-
q = Double.parseDouble( record.get( qualitycol ) );
322+
q = Double.parseDouble( record[ qualitycol ] );
301323
qualityFeature.set( spot, q );
302324
}
303325
}
304326
catch ( final NumberFormatException nfe )
305327
{
306328
nfe.printStackTrace();
307-
System.out.println( "Could not parse line " + record.getRecordNumber() + ". Malformed number, skipping.\n" + nfe.getMessage() );
329+
System.out.println( "Could not parse line " + lineNumber + ". Malformed number, skipping.\n" + nfe.getMessage() );
308330
continue;
309331
}
310-
311332
}
312333
}
313334
finally
@@ -318,14 +339,8 @@ public boolean process()
318339
}
319340
catch ( final FileNotFoundException e )
320341
{
342+
errorMessage = "Cannot find file " + filePath;
321343
e.printStackTrace();
322-
errorMessage = e.getMessage();
323-
return false;
324-
}
325-
catch ( final IOException e )
326-
{
327-
e.printStackTrace();
328-
errorMessage = e.getMessage();
329344
return false;
330345
}
331346

0 commit comments

Comments
 (0)