Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<parent>
<groupId>org.scijava</groupId>
<artifactId>pom-scijava</artifactId>
<version>37.0.0</version>
<version>43.0.0</version>
<relativePath />
</parent>

Expand Down Expand Up @@ -96,17 +96,26 @@
<license.organizationName>Saalfeld Lab</license.organizationName>
<license.copyrightOwners>Igor Pisarev, Stephan Saalfeld</license.copyrightOwners>

<scijava.jvm.version>17</scijava.jvm.version>
<scijava.jvm.version>11</scijava.jvm.version>
<maven-compiler-plugin.version>3.13.0</maven-compiler-plugin.version>
<extra-enforcer-rules.version>1.9.0</extra-enforcer-rules.version>


<!-- NB: Deploy releases to the SciJava Maven repository. -->
<releaseProfiles>sign,deploy-to-scijava</releaseProfiles>
<args4j.version>2.33</args4j.version>
<n5.version>3.2.0</n5.version>
<n5-imglib2.version>7.0.0</n5-imglib2.version>
<n5-ij.version>4.1.1</n5-ij.version>

<n5.version>4.0.0-alpha-12</n5.version>
<n5-aws-s3.version>4.4.0-alpha-9</n5-aws-s3.version>
<n5-blosc.version>2.0.0-alpha-4</n5-blosc.version>
<n5-google-cloud.version>5.2.0-alpha-7</n5-google-cloud.version>
<n5-hdf5.version>2.3.0-alpha-7</n5-hdf5.version>
<n5-ij.version>4.5.0-alpha-8-SNAPSHOT</n5-ij.version>
<n5-imglib2.version>7.1.0-alpha-8</n5-imglib2.version>
<n5-universe.version>2.4.0-alpha-9</n5-universe.version>
<n5-zarr.version>2.0.0-alpha-8</n5-zarr.version>
<n5-zstandard.version>2.0.0-alpha-4</n5-zstandard.version>

<bigdataviewer-core.version>10.4.14</bigdataviewer-core.version>

<kryo.version>4.0.2</kryo.version>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import org.janelia.saalfeldlab.n5.DatasetAttributes;
import org.janelia.saalfeldlab.n5.N5FSReader;
import org.janelia.saalfeldlab.n5.N5Reader;
import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
import org.janelia.saalfeldlab.n5.spark.supplier.N5ReaderSupplier;
import org.janelia.saalfeldlab.n5.spark.util.CmdUtils;
import org.janelia.saalfeldlab.n5.spark.util.N5SparkUtils;
Expand Down Expand Up @@ -219,7 +220,7 @@ public static <T extends NativeType<T> & RealType<T>> void createMaxIntensityPro
new BasicNameValuePair("dataset", datasetPath),
new BasicNameValuePair("call", "create-max-intensity-projection")
).toString();
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> N5SparkUtils.openWithLoaderCache(n5, datasetPath, new SoftRefLoaderCache<>()));
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> (CachedCellImg<T, ?>)N5Utils.open(n5, datasetPath));
final long[] cellGridDimensions = cellImg.getCellGrid().getGridDimensions();
numCells = Intervals.numElements(cellGridDimensions);
type = Util.getTypeFromInterval(cellImg).createVariable();
Expand Down Expand Up @@ -249,7 +250,7 @@ public static <T extends NativeType<T> & RealType<T>> void createMaxIntensityPro
new BasicNameValuePair("dataset", datasetPath),
new BasicNameValuePair("call", "create-max-intensity-projection")
).toString();
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> N5SparkUtils.openWithLoaderCache(n5, datasetPath, new SoftRefLoaderCache<>()));
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> (CachedCellImg<T, ?>)N5Utils.open(n5, datasetPath));
final RandomAccess<T> cellImgRandomAccess = cellImg.randomAccess();

final long[] cellMin = new long[dim], cellMax = new long[dim];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@
import org.janelia.saalfeldlab.n5.DatasetAttributes;
import org.janelia.saalfeldlab.n5.N5FSReader;
import org.janelia.saalfeldlab.n5.N5Reader;
import org.janelia.saalfeldlab.n5.imglib2.N5Utils;
import org.janelia.saalfeldlab.n5.spark.supplier.N5ReaderSupplier;
import org.janelia.saalfeldlab.n5.spark.util.N5SparkUtils;
import org.janelia.saalfeldlab.n5.spark.util.SliceDimension;
Expand Down Expand Up @@ -261,7 +262,7 @@ public static < T extends NativeType< T > & RealType< T >, U extends NativeType<
new BasicNameValuePair("dataset", datasetPath),
new BasicNameValuePair("call", "create-max-intensity-projection")
).toString();
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> N5SparkUtils.openWithLoaderCache(n5, datasetPath, new SoftRefLoaderCache<>()));
final CachedCellImg<T, ?> cellImg = Singleton.get(readerCacheKey, () -> (CachedCellImg<T, ?>)N5Utils.open(n5, datasetPath));

final CellGrid cellGrid = cellImg.getCellGrid();
final long[] slicePos = new long[ cellImg.numDimensions() ], cellPos = new long[ cellImg.numDimensions() ];
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ public static < T extends NativeType< T > > void convert(
new BasicNameValuePair("call", "slice-tiff-to-n5-spark")
).toString();

final RandomAccessibleInterval< T > tmpImg = Singleton.get(tmpReaderCacheKey, () -> N5Utils.open( n5Local, tmpDataset ));
final RandomAccessibleInterval< T > tmpImg = Singleton.get(tmpReaderCacheKey, () -> (RandomAccessibleInterval< T > )N5Utils.open( n5Local, tmpDataset ));

final Interval interval = N5SparkUtils.toInterval( minMaxTuple );
final RandomAccessibleInterval< T > tmpImgCrop = Views.offsetInterval( tmpImg, interval );
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ public static <T extends NativeType<T> & RealType<T>> void downsample(
new BasicNameValuePair("call", "n5-downsample-spark")
).toString();

final RandomAccessibleInterval< T > source = Singleton.get(imgCacheKey, () -> N5Utils.open( n5Writer, inputDatasetPath ));
final RandomAccessibleInterval< T > source = Singleton.get(imgCacheKey, () -> (RandomAccessibleInterval< T > )N5Utils.open( n5Writer, inputDatasetPath ));
final RandomAccessibleInterval<T> sourceBlock = Views.offsetInterval(source, sourceInterval);

/* test if empty */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -235,7 +235,7 @@ public static <T extends NativeType<T> & IntegerType<T>> void downsampleLabel(
new BasicNameValuePair("call", "n5-downsample-spark")
).toString();

final RandomAccessibleInterval< T > source = Singleton.get(imgCacheKey, () -> N5Utils.open( n5Writer, inputDatasetPath ));
final RandomAccessibleInterval< T > source = Singleton.get(imgCacheKey, () -> (RandomAccessibleInterval< T >)N5Utils.open( n5Writer, inputDatasetPath ));
final RandomAccessibleInterval<T> sourceBlock = Views.offsetInterval(source, sourceInterval);

/* test if empty */
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -202,7 +202,7 @@ public static < T extends NativeType< T > & RealType< T > > void downsampleWithO
new BasicNameValuePair("call", "n5-downsample-spark")
).toString();

final RandomAccessibleInterval< T > source = Singleton.get(imgCacheKey, () -> N5Utils.open( n5Writer, inputDatasetPath ));
final RandomAccessibleInterval< T > source = Singleton.get(imgCacheKey, () -> (RandomAccessibleInterval< T >)N5Utils.open( n5Writer, inputDatasetPath ));

// apply offset to source to align it with respect to the target block
final RandomAccessibleInterval< T > translatedSource = Views.translate( source, offset );
Expand Down
123 changes: 0 additions & 123 deletions src/main/java/org/janelia/saalfeldlab/n5/spark/util/N5SparkUtils.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,41 +28,12 @@
*/
package org.janelia.saalfeldlab.n5.spark.util;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;

import org.janelia.saalfeldlab.n5.DatasetAttributes;
import org.janelia.saalfeldlab.n5.N5Reader;
import org.janelia.saalfeldlab.n5.imglib2.N5CellLoader;

import net.imglib2.FinalInterval;
import net.imglib2.Interval;
import net.imglib2.cache.Cache;
import net.imglib2.cache.CacheLoader;
import net.imglib2.cache.LoaderCache;
import net.imglib2.cache.img.CachedCellImg;
import net.imglib2.cache.img.LoadedCellCacheLoader;
import net.imglib2.cache.ref.BoundedSoftRefLoaderCache;
import net.imglib2.img.basictypeaccess.AccessFlags;
import net.imglib2.img.basictypeaccess.ArrayDataAccessFactory;
import net.imglib2.img.cell.Cell;
import net.imglib2.img.cell.CellGrid;
import net.imglib2.img.cell.LazyCellImg;
import net.imglib2.type.NativeType;
import net.imglib2.type.PrimitiveType;
import net.imglib2.type.numeric.integer.ByteType;
import net.imglib2.type.numeric.integer.IntType;
import net.imglib2.type.numeric.integer.LongType;
import net.imglib2.type.numeric.integer.ShortType;
import net.imglib2.type.numeric.integer.UnsignedByteType;
import net.imglib2.type.numeric.integer.UnsignedIntType;
import net.imglib2.type.numeric.integer.UnsignedLongType;
import net.imglib2.type.numeric.integer.UnsignedShortType;
import net.imglib2.type.numeric.real.DoubleType;
import net.imglib2.type.numeric.real.FloatType;
import net.imglib2.util.Intervals;
import scala.Tuple2;

Expand Down Expand Up @@ -93,98 +64,4 @@ public static Interval toInterval( final Tuple2< long[], long[] > minMaxTuple )
);
}

/**
* Open an N5 dataset as a memory cached {@link LazyCellImg} with bounded cache size.
*
* @param n5
* @param dataset
* @param cacheSize
* @return
* @throws IOException
*/
@SuppressWarnings( { "unchecked", "rawtypes" } )
public static final < T extends NativeType< T > > CachedCellImg< T, ? > openWithBoundedCache(
final N5Reader n5,
final String dataset,
final int cacheSize ) throws IOException
{
return openWithLoaderCache( n5, dataset, new BoundedSoftRefLoaderCache<>(cacheSize));
}

public static final < T extends NativeType< T > > CachedCellImg< T, ? > openWithLoaderCache(
final N5Reader n5,
final String dataset,
final LoaderCache<Long, Cell<?>> loaderCache) throws IOException
{
final DatasetAttributes attributes = n5.getDatasetAttributes( dataset );
final long[] dimensions = attributes.getDimensions();
final int[] blockSize = attributes.getBlockSize();

final N5CellLoader< T > loader = new N5CellLoader<>( n5, dataset, blockSize );

final CellGrid grid = new CellGrid( dimensions, blockSize );

final CachedCellImg< T, ? > img;
final T type;
final Cache< Long, Cell< ? > > cache;
final Set< AccessFlags > accessFlags = AccessFlags.setOf();

switch ( attributes.getDataType() )
{
case INT8:
type = ( T )new ByteType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.BYTE, accessFlags ) );
break;
case UINT8:
type = ( T )new UnsignedByteType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.BYTE, accessFlags ) );
break;
case INT16:
type = ( T )new ShortType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.SHORT, accessFlags ) );
break;
case UINT16:
type = ( T )new UnsignedShortType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.SHORT, accessFlags ) );
break;
case INT32:
type = ( T )new IntType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.INT, accessFlags ) );
break;
case UINT32:
type = ( T )new UnsignedIntType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.INT, accessFlags ) );
break;
case INT64:
type = ( T )new LongType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.LONG, accessFlags ) );
break;
case UINT64:
type = ( T )new UnsignedLongType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.LONG, accessFlags ) );
break;
case FLOAT32:
type = ( T )new FloatType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.FLOAT, accessFlags ) );
break;
case FLOAT64:
type = ( T )new DoubleType();
cache = loaderCache.withLoader( ( CacheLoader )LoadedCellCacheLoader.get( grid, loader, type, accessFlags ) );
img = new CachedCellImg( grid, type, cache, ArrayDataAccessFactory.get( PrimitiveType.DOUBLE, accessFlags ) );
break;
default:
img = null;
}

return img;
}
}
Loading