diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Axis.java b/src/main/java/com/glencoesoftware/bioformats2raw/Axis.java new file mode 100644 index 00000000..30941956 --- /dev/null +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Axis.java @@ -0,0 +1,53 @@ +/** + * Copyright (c) 2025 Glencoe Software, Inc. All rights reserved. + * + * This software is distributed under the terms described by the LICENSE.txt + * file you can find at the root of the distribution bundle. If the file is + * missing please request a copy by contacting info@glencoesoftware.com + */ +package com.glencoesoftware.bioformats2raw; + +/** + * Describe an axis, including type and dimensions. + */ +public class Axis { + + private char type; + private int length; + private int chunkSize; + + /** + * Create a new Axis. + * + * @param t axis type (e.g. 'X') + * @param len axis length + * @param chunk chunk length (expected to be in range [1, len]) + */ + public Axis(char t, int len, int chunk) { + type = t; + length = len; + chunkSize = chunk; + } + + /** + * @return axis type (e.g. 'X') + */ + public char getType() { + return type; + } + + /** + * @return axis length + */ + public int getLength() { + return length; + } + + /** + * @return chunk length + */ + public int getChunkSize() { + return chunkSize; + } + +} diff --git a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java index c6424aa2..029043e5 100644 --- a/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java +++ b/src/main/java/com/glencoesoftware/bioformats2raw/Converter.java @@ -19,6 +19,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.HashMap; import java.util.List; @@ -36,6 +37,7 @@ import java.util.stream.IntStream; import loci.common.Constants; +import loci.common.Region; import loci.common.image.IImageScaler; import loci.common.image.SimpleImageScaler; import loci.common.services.DependencyException; @@ -168,6 +170,7 @@ public class Converter implements Callable { private volatile boolean reuseExistingResolutions = false; private volatile int minSize; private volatile boolean writeImageData = true; + private volatile boolean compactDims = false; /** Scaling implementation that will be used during downsampling. */ private volatile IImageScaler scaler = new SimpleImageScaler(); @@ -869,6 +872,22 @@ public void setMinImageSize(int min) { } } + /** + * Set whether a compact dimensional representation should be used, + * i.e. whether singleton dimensions should be omitted. + * Defaults to false. + * + * @param compact true for a compact dimensional representation + */ + @Option( + names = "--compact", + description = "Only write dimensions greater than 1", + defaultValue = "false" + ) + public void setCompactDimensions(boolean compact) { + compactDims = compact; + } + /** * Set the output dimension order. Defaults to XYZCT for compliance with * the OME-NGFF specification. @@ -1144,6 +1163,13 @@ public int getMinImageSize() { return minSize; } + /** + * @return true if dimensions have been compacted + */ + public boolean getCompactDimensions() { + return compactDims; + } + /** * @return current dimension roder */ @@ -1844,8 +1870,8 @@ private static void writeBytes( } private byte[] getTileDownsampled( - int series, int resolution, int plane, int xx, int yy, - int width, int height) + int series, int resolution, int plane, + Region boundingBox, List axes) throws FormatException, IOException, InterruptedException, EnumerationException, InvalidRangeException { @@ -1855,31 +1881,43 @@ private byte[] getTileDownsampled( final ZarrArray zarr = ZarrArray.open(getRootPath().resolve(pathName)); int[] dimensions = zarr.getShape(); int[] blockSizes = zarr.getChunks(); - int activeTileWidth = blockSizes[blockSizes.length - 1]; - int activeTileHeight = blockSizes[blockSizes.length - 2]; + int xDim = 1; + int yDim = 1; + int activeTileWidth = 1; + int activeTileHeight = 1; + for (int i=0; i axes) throws FormatException, IOException, InterruptedException, EnumerationException, InvalidRangeException { @@ -1903,7 +1941,9 @@ private byte[] getTile( try { if (reader.getResolutionCount() > 1 && reuseExistingResolutions) { reader.setResolution(resolution); - return reader.openBytes(plane, xx, yy, width, height); + return reader.openBytes(plane, + boundingBox.x, boundingBox.y, + boundingBox.width, boundingBox.height); } } finally { @@ -1912,7 +1952,9 @@ private byte[] getTile( if (resolution == 0) { reader = readers.take(); try { - return reader.openBytes(plane, xx, yy, width, height); + return reader.openBytes(plane, + boundingBox.x, boundingBox.y, + boundingBox.width, boundingBox.height); } finally { readers.put(reader); @@ -1921,8 +1963,7 @@ private byte[] getTile( else { Slf4JStopWatch t0 = new Slf4JStopWatch("getTileDownsampled"); try { - return getTileDownsampled( - series, resolution, plane, xx, yy, width, height); + return getTileDownsampled(series, resolution, plane, boundingBox, axes); } finally { t0.stop(); @@ -1937,24 +1978,109 @@ private byte[] getTile( * @param scaledWidth size of the X dimension at the current resolution * @param scaledHeight size of the Y dimension at the current resolution * @param scaledDepth size of the Z dimension at the current resolution + * @param scaledTileWidth chunk size in X + * @param scaledTileHeight chunk size in Y + * @param scaledChunkDepth chunk size in Z * @return dimension array ready for use with Zarr * @throws EnumerationException */ - private int[] getDimensions( - IFormatReader reader, int scaledWidth, int scaledHeight, int scaledDepth) + private List getDimensions( + IFormatReader reader, int scaledWidth, int scaledHeight, int scaledDepth, + int scaledTileWidth, int scaledTileHeight, int scaledChunkDepth) throws EnumerationException { + ArrayList axes = new ArrayList(); int sizeZ = reader.getSizeZ(); int sizeC = reader.getSizeC(); int sizeT = reader.getSizeT(); String o = new StringBuilder( dimensionOrder != null? dimensionOrder.toString() : reader.getDimensionOrder()).reverse().toString(); - int[] dimensions = new int[] {0, 0, scaledDepth, scaledHeight, scaledWidth}; - dimensions[o.indexOf("Z")] = sizeZ; - dimensions[o.indexOf("C")] = sizeC; - dimensions[o.indexOf("T")] = sizeT; - return dimensions; + + int spatialDims = 0; + for (char c : o.toCharArray()) { + switch (c) { + case 'X': + axes.add(new Axis(c, scaledWidth, scaledTileWidth)); + spatialDims++; + break; + case 'Y': + axes.add(new Axis(c, scaledHeight, scaledTileHeight)); + spatialDims++; + break; + case 'Z': + axes.add(new Axis(c, scaledDepth, scaledChunkDepth)); + spatialDims++; + break; + case 'C': + axes.add(new Axis(c, sizeC, 1)); + break; + case 'T': + axes.add(new Axis(c, sizeT, 1)); + break; + default: + LOGGER.trace("ignoring axis type {}", c); + } + } + + if (getCompactDimensions()) { + // if requested, omit any dimension that has length 1 + // this includes X and Y + for (int a=0; a axes) { + int[] shape = new int[axes.size()]; + for (int i=0; i axes) { + int[] chunk = new int[axes.size()]; + for (int i=0; i axes, int width, int height, int depth) { + int[] shape = new int[axes.size()]; + Arrays.fill(shape, 1); + for (int i=0; i axes) + throws EnumerationException { - String o = new StringBuilder( - dimensionOrder != null? dimensionOrder.toString() - : reader.getDimensionOrder()).reverse().toString(); int[] zct = reader.getZCTCoords(plane); - int[] offset = new int[] {0, 0, 0, y, x}; - offset[o.indexOf("Z")] = zct[0]; - offset[o.indexOf("C")] = zct[1]; - offset[o.indexOf("T")] = zct[2]; + int[] offset = new int[axes.size()]; + Arrays.fill(offset, 0); + for (int i=0; i axes) throws EnumerationException, FormatException, IOException, InterruptedException, InvalidRangeException { @@ -1995,23 +2140,40 @@ private void processChunk(int series, int resolution, int plane, int bpp = FormatTools.getBytesPerPixel(reader.getPixelType()); int[] zct; ByteArrayOutputStream chunkAsBytes = new ByteArrayOutputStream(); - int zOffset; - int zShape; + int xOffset = 0; + int xShape = 1; + int yOffset = 0; + int yShape = 1; + int zOffset = 0; + int zShape = 1; try { LOGGER.info("requesting tile to write at {} to {}", offset, pathName); //Get coords of current series zct = reader.getZCTCoords(plane); - String o = new StringBuilder( - dimensionOrder != null? dimensionOrder.toString() - : reader.getDimensionOrder()).reverse().toString(); - zOffset = offset[o.indexOf("Z")]; - zShape = shape[o.indexOf("Z")]; + for (int i=0; i activeAxes = + getDimensions(workingReader, scaledWidth, scaledHeight, scaledDepth, + activeTileWidth, activeTileHeight, activeChunkDepth); + DataType dataType = getZarrType(pixelType); String resolutionString = String.format( scaleFormatString, getScaleFormatStringArgs(series, resolution)); ArrayParams arrayParams = new ArrayParams() - .shape(getDimensions( - workingReader, scaledWidth, scaledHeight, scaledDepth)) - .chunks(new int[] {1, 1, activeChunkDepth, activeTileHeight, - activeTileWidth}) + .shape(getShapeArray(activeAxes)) + .chunks(getChunkSizeArray(activeAxes)) .dataType(dataType) .dimensionSeparator(getDimensionSeparator()) .compressor(CompressorFactory.create( @@ -2188,20 +2353,18 @@ public void saveResolutions(int series) futures.add(future); executor.execute(() -> { try { - int[] shape = {1, 1, 1, height, width}; + int[] shape = + createShape(activeAxes, width, height, depth); int[] offset; IFormatReader reader = readers.take(); try { - String o = new StringBuilder( - dimensionOrder != null? dimensionOrder.toString() - : reader.getDimensionOrder()).reverse().toString(); - shape[o.indexOf("Z")] = depth; - offset = getOffset(reader, xx, yy, plane); + offset = getOffset(reader, xx, yy, plane, activeAxes); } finally { readers.put(reader); } - processChunk(series, resolution, plane, offset, shape); + processChunk( + series, resolution, plane, offset, shape, activeAxes); LOGGER.info( "Successfully processed chunk; resolution={} plane={}" + " xx={} yy={} zz={} width={} height={} depth={}", @@ -2407,7 +2570,7 @@ private void saveHCSMetadata(IMetadata meta) throws IOException { * @throws InterruptedException */ private void setSeriesLevelMetadata(int series, int resolutions) - throws IOException, InterruptedException + throws IOException, InterruptedException, EnumerationException { LOGGER.debug("setSeriesLevelMetadata({}, {})", series, resolutions); String resolutionString = String.format( @@ -2445,17 +2608,14 @@ private void setSeriesLevelMetadata(int series, int resolutions) IFormatReader v = null; IMetadata meta = null; - String axisOrder = null; + List activeAxes = null; + try { v = readers.take(); meta = (IMetadata) v.getMetadataStore(); - if (dimensionOrder != null) { - axisOrder = dimensionOrder.toString(); - } - else { - axisOrder = v.getDimensionOrder(); - } + activeAxes = getDimensions(v, v.getSizeX(), v.getSizeY(), + v.getSizeZ(), 1, 1, 1); } finally { readers.put(v); @@ -2483,9 +2643,10 @@ private void setSeriesLevelMetadata(int series, int resolutions) scale.put("type", "scale"); List axisValues = new ArrayList(); double resolutionScale = Math.pow(PYRAMID_SCALE, r); - for (int i=axisOrder.length()-1; i>=0; i--) { - Quantity axisScale = getScale(meta, series, axisOrder, i); - String axisChar = axisOrder.substring(i, i + 1).toLowerCase(); + for (int i=0; i> axes = new ArrayList>(); - for (int i=axisOrder.length()-1; i>=0; i--) { - String axis = axisOrder.substring(i, i + 1).toLowerCase(); + for (int i=0; i> multiscales = getMultiscales("0"); + assertEquals(1, multiscales.size()); + Map multiscale = multiscales.get(0); + checkMultiscale(multiscale, "image"); + List> datasets = + (List>) multiscale.get("datasets"); + assertTrue(datasets.size() > 0); + assertEquals("0", datasets.get(0).get("path")); + + List> axes = + (List>) multiscale.get("axes"); + checkAxes(axes, "ZYX", null); + } + + /** + * Test compact representation of single XY plane. + */ + @Test + public void testCompactXY() throws Exception { + input = fake(); + assertTool("--compact"); + ZarrGroup group = ZarrGroup.open(output.toString()); + + // Check dimensions and block size + ZarrArray series0 = group.openArray("0/0"); + assertArrayEquals(new int[] {512, 512}, series0.getShape()); + assertArrayEquals(new int[] {512, 512}, series0.getChunks()); + + // Check special pixels for each plane + int[] shape = new int[] {512, 512}; + byte[] tile = new byte[512 * 512]; + int[] offset = new int[] {0, 0}; + + series0.read(tile, shape, offset); + int[] seriesPlaneNumberZCT = FakeReader.readSpecialPixels(tile); + assertArrayEquals(new int[] {0, 0, 0, 0, 0}, seriesPlaneNumberZCT); + + // check that 2 axes were written instead of 5 + List> multiscales = getMultiscales("0"); + assertEquals(1, multiscales.size()); + Map multiscale = multiscales.get(0); + checkMultiscale(multiscale, "image"); + List> datasets = + (List>) multiscale.get("datasets"); + assertTrue(datasets.size() > 0); + assertEquals("0", datasets.get(0).get("path")); + + List> axes = + (List>) multiscale.get("axes"); + checkAxes(axes, "YX", null); + } + + /** + * Test compact representation of dataset with Y=1. + */ + @Test + public void testCompactSingleY() throws Exception { + input = fake("sizeY", "1", "sizeZ", "300"); + assertTool("--compact"); + ZarrGroup group = ZarrGroup.open(output.toString()); + + // Check dimensions and block size + ZarrArray series0 = group.openArray("0/0"); + assertArrayEquals(new int[] {300, 512}, series0.getShape()); + assertArrayEquals(new int[] {1, 512}, series0.getChunks()); + + // Check special pixels for each plane + int[] shape = new int[] {1, 512}; + byte[] tile = new byte[512]; + int[] offset = new int[] {0, 0}; + + for (int z=0; z<300; z++) { + offset[0] = z; + series0.read(tile, shape, offset); + int[] seriesPlaneNumberZCT = FakeReader.readSpecialPixels(tile); + + // special pixels will be adjusted because it is uint8 data + int zSpecialPixel = z % 256; + assertArrayEquals(new int[] {0, zSpecialPixel, zSpecialPixel, 0, 0}, + seriesPlaneNumberZCT); + } + + // check that 2 axes were written instead of 5 + List> multiscales = getMultiscales("0"); + assertEquals(1, multiscales.size()); + Map multiscale = multiscales.get(0); + checkMultiscale(multiscale, "image"); + List> datasets = + (List>) multiscale.get("datasets"); + assertTrue(datasets.size() > 0); + assertEquals("0", datasets.get(0).get("path")); + + List> axes = + (List>) multiscale.get("axes"); + checkAxes(axes, "ZX", null); + } + + /** + * Test compact representation of XT data. + * This should throw an exception as there is only one + * spatial dimension greater than 1. + */ + @Test + public void testCompactInvalid2D() throws Exception { + input = fake("sizeY", "1", "sizeT", "4"); + assertThrows(ExecutionException.class, () -> { + assertTool("--compact"); + }); + } + + /** + * Test compact representation of single pixel. + * This should throw an exception. + */ + @Test + public void testCompactSinglePixel() throws Exception { + input = fake("sizeX", "1", "sizeY", "1"); + assertThrows(ExecutionException.class, () -> { + assertTool("--compact"); + }); + } + + /** + * Test compact representation of multiple channels and + * timepoints with a single Z section. + * The result should be a 4D array, not a 5D array. + */ + @Test + public void testCompact4D() throws Exception { + input = fake("sizeC", "4", "sizeT", "2"); + assertTool("--compact"); + ZarrGroup group = ZarrGroup.open(output.toString()); + + // Check dimensions and block size + ZarrArray series0 = group.openArray("0/0"); + assertArrayEquals(new int[] {2, 4, 512, 512}, series0.getShape()); + assertArrayEquals(new int[] {1, 1, 512, 512}, series0.getChunks()); + + // Check special pixels for each plane + int[] shape = new int[] {1, 1, 512, 512}; + byte[] tile = new byte[512 * 512]; + int[] offset = new int[] {0, 0, 0, 0}; + + int plane = 0; + for (int t=0; t<2; t++) { + for (int c=0; c<4; c++) { + offset[0] = t; + offset[1] = c; + series0.read(tile, shape, offset); + int[] seriesPlaneNumberZCT = FakeReader.readSpecialPixels(tile); + assertArrayEquals(new int[] {0, plane, 0, c, t}, seriesPlaneNumberZCT); + plane++; + } + } + + // check that 3 axes were written instead of 5 + List> multiscales = getMultiscales("0"); + assertEquals(1, multiscales.size()); + Map multiscale = multiscales.get(0); + checkMultiscale(multiscale, "image"); + List> datasets = + (List>) multiscale.get("datasets"); + assertTrue(datasets.size() > 0); + assertEquals("0", datasets.get(0).get("path")); + + List> axes = + (List>) multiscale.get("axes"); + checkAxes(axes, "TCYX", null); + } + /** * Test the progress listener API. */