Merge pull request #4181 from melissalinkert/ndpi-precompressed

Add support for reading compressed NDPI tiles

Author: dgault

components/bio-formats-tools/src/loci/formats/tools/ImageConverter.java

@@ -1263,6 +1263,14 @@ private boolean doTileConversion(IFormatWriter writer, String outputFile)
     if (writer instanceof DicomWriter ||
       (writer instanceof ImageWriter && ((ImageWriter) writer).getWriter(outputFile) instanceof DicomWriter))
     {
+      // if we asked to try a precompressed conversion,
+      // then the writer's tile sizes will have been set automatically
+      // according to the input data
+      // the conversion must then be performed tile-wise to match the tile sizes,
+      // even if precompression doesn't end up being possible
+      if (precompressed) {
+        return true;
+      }
       MetadataStore r = reader.getMetadataStore();
       return !(r instanceof IPyramidStore) || ((IPyramidStore) r).getResolutionCount(reader.getSeries()) > 1;
     }

components/formats-bsd/src/loci/formats/in/MinimalTiffReader.java

@@ -705,4 +705,73 @@ protected void initTiffParser() {
     tiffParser.setUse64BitOffsets(use64Bit);
   }
 
+  /**
+   * Get the index of the tile corresponding to given IFD (plane)
+   * and tile XY indexes.
+   *
+   * @param ifd IFD for the requested tile's plane
+   * @param x tile X index
+   * @param y tile Y index
+   * @return corresponding tile index
+   */
+  protected int getTileIndex(IFD ifd, int x, int y) throws FormatException {
+    int rows = (int) ifd.getTilesPerColumn();
+    int cols = (int) ifd.getTilesPerRow();
+
+    if (x < 0 || x >= cols) {
+      throw new IllegalArgumentException("X index " + x + " not in range [0, " + cols + ")");
+    }
+    if (y < 0 || y >= rows) {
+      throw new IllegalArgumentException("Y index " + y + " not in range [0, " + rows + ")");
+    }
+
+    return (cols * y) + x;
+  }
+
+  protected long getCompressedByteCount(IFD ifd, int x, int y) throws FormatException, IOException {
+    long[] byteCounts = ifd.getStripByteCounts();
+    int tileIndex = getTileIndex(ifd, x, y);
+    byte[] jpegTable = (byte[]) ifd.getIFDValue(IFD.JPEG_TABLES);
+    int jpegTableBytes = jpegTable == null ? 0 : jpegTable.length - 2;
+    long expectedBytes = byteCounts[tileIndex];
+    if (expectedBytes > 0) {
+      expectedBytes += jpegTableBytes;
+    }
+    if (expectedBytes < 0 || expectedBytes > Integer.MAX_VALUE) {
+      throw new IOException("Invalid compressed tile size: " + expectedBytes);
+    }
+    return expectedBytes;
+  }
+
+  protected byte[] copyTile(IFD ifd, byte[] buf, int x, int y) throws FormatException, IOException {
+    long[] offsets = ifd.getStripOffsets();
+    long[] byteCounts = ifd.getStripByteCounts();
+
+    int tileIndex = getTileIndex(ifd, x, y);
+
+    byte[] jpegTable = (byte[]) ifd.getIFDValue(IFD.JPEG_TABLES);
+    int jpegTableBytes = jpegTable == null ? 0 : jpegTable.length - 2;
+    long expectedBytes = getCompressedByteCount(ifd, x, y);
+
+    if (buf.length < expectedBytes) {
+      throw new IllegalArgumentException("Tile buffer too small: expected >=" +
+        expectedBytes + ", got " + buf.length);
+    }
+    else if (expectedBytes < 0 || expectedBytes > Integer.MAX_VALUE) {
+      throw new IOException("Invalid compressed tile size: " + expectedBytes);
+    }
+
+    if (jpegTable != null && expectedBytes > 0) {
+      System.arraycopy(jpegTable, 0, buf, 0, jpegTable.length - 2);
+      // skip over the duplicate SOI marker
+      tiffParser.getStream().seek(offsets[tileIndex] + 2);
+      tiffParser.getStream().readFully(buf, jpegTable.length - 2, (int) byteCounts[tileIndex]);
+    }
+    else if (byteCounts[tileIndex] > 0) {
+      tiffParser.getStream().seek(offsets[tileIndex]);
+      tiffParser.getStream().readFully(buf, 0, (int) byteCounts[tileIndex]);
+    }
+    return buf;
+  }
+
 }

components/formats-bsd/src/loci/formats/out/DicomWriter.java

@@ -113,6 +113,9 @@ public class DicomWriter extends FormatWriter implements IExtraMetadataWriter {
   private int baseTileHeight = 256;
   private int[] tileWidth;
   private int[] tileHeight;
+  private long[] tileWidthPointer;
+  private long[] tileHeightPointer;
+  private long[] tileCountPointer;
   private PlaneOffset[][] planeOffsets;
   private Integer currentPlane = null;
   private UIDCreator uids;
@@ -232,14 +235,7 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h)
     LOGGER.debug("savePrecompressedBytes(series={}, resolution={}, no={}, x={}, y={})",
       series, resolution, no, x, y);
 
-    // TODO: may want better handling of non-tiled "extra" images (e.g. label, macro)
     MetadataRetrieve r = getMetadataRetrieve();
-    if ((!(r instanceof IPyramidStore) ||
-      ((IPyramidStore) r).getResolutionCount(series) == 1) &&
-      !isFullPlane(x, y, w, h))
-    {
-      throw new FormatException("DicomWriter does not allow tiles for non-pyramid images");
-    }
 
     int bytesPerPixel = FormatTools.getBytesPerPixel(
       FormatTools.pixelTypeFromString(
@@ -279,6 +275,13 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h)
     boolean first = x == 0 && y == 0;
     boolean last = x + w == getSizeX() && y + h == getSizeY();
 
+    int width = getSizeX();
+    int height = getSizeY();
+    int sizeZ = r.getPixelsSizeZ(series).getValue().intValue();
+
+    int tileCountX = (int) Math.ceil((double) width / tileWidth[resolutionIndex]);
+    int tileCountY = (int) Math.ceil((double) height / tileHeight[resolutionIndex]);
+
     // the compression type isn't supplied to the writer until
     // after setId is called, so metadata that indicates or
     // depends on the compression type needs to be set in
@@ -296,6 +299,15 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h)
       if (getTIFFCompression() == TiffCompression.JPEG) {
         ifds[resolutionIndex][no].put(IFD.PHOTOMETRIC_INTERPRETATION, PhotoInterp.Y_CB_CR.getCode());
       }
+
+      out.seek(tileWidthPointer[resolutionIndex]);
+      out.writeShort((short) getTileSizeX());
+      out.seek(tileHeightPointer[resolutionIndex]);
+      out.writeShort((short) getTileSizeY());
+      out.seek(tileCountPointer[resolutionIndex]);
+
+      out.writeBytes(padString(String.valueOf(
+          tileCountX * tileCountY * sizeZ * r.getChannelCount(series))));
     }
 
     out.seek(out.length());
@@ -334,6 +346,17 @@ public void saveCompressedBytes(int no, byte[] buf, int x, int y, int w, int h)
     if (ifds[resolutionIndex][no] != null) {
       tileByteCounts = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_BYTE_COUNTS);
       tileOffsets = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_OFFSETS);
+
+      if (tileByteCounts.length < tileCountX * tileCountY) {
+        long[] newTileByteCounts = new long[tileCountX * tileCountY];
+        long[] newTileOffsets = new long[tileCountX * tileCountY];
+        System.arraycopy(tileByteCounts, 0, newTileByteCounts, 0, tileByteCounts.length);
+        System.arraycopy(tileOffsets, 0, newTileOffsets, 0, tileOffsets.length);
+        tileByteCounts = newTileByteCounts;
+        tileOffsets = newTileOffsets;
+        ifds[resolutionIndex][no].put(IFD.TILE_BYTE_COUNTS, tileByteCounts);
+        ifds[resolutionIndex][no].put(IFD.TILE_OFFSETS, tileOffsets);
+      }
     }
 
     if (tileByteCounts != null) {
@@ -367,13 +390,7 @@ public void saveBytes(int no, byte[] buf, int x, int y, int w, int h)
     int thisTileHeight = tileHeight[resolutionIndex];
 
     MetadataRetrieve r = getMetadataRetrieve();
-    if ((!(r instanceof IPyramidStore) ||
-      ((IPyramidStore) r).getResolutionCount(series) == 1) &&
-      !isFullPlane(x, y, w, h))
-    {
-      throw new FormatException("DicomWriter does not allow tiles for non-pyramid images");
-    }
-    else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
+    if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
       (w != thisTileWidth && x + w != getSizeX()) ||
       (h != thisTileHeight && y + h != getSizeY()))
     {
@@ -385,6 +402,10 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
     boolean first = x == 0 && y == 0;
     boolean last = x + w == getSizeX() && y + h == getSizeY();
 
+    int xTiles = (int) Math.ceil((double) getSizeX() / thisTileWidth);
+    int yTiles = (int) Math.ceil((double) getSizeY() / thisTileHeight);
+    int sizeZ = r.getPixelsSizeZ(series).getValue().intValue();
+
     // the compression type isn't supplied to the writer until
     // after setId is called, so metadata that indicates or
     // depends on the compression type needs to be set in
@@ -406,6 +427,15 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
           ifds[resolutionIndex][no].put(IFD.PHOTOMETRIC_INTERPRETATION, PhotoInterp.Y_CB_CR.getCode());
         }
       }
+
+      out.seek(tileWidthPointer[resolutionIndex]);
+      out.writeShort((short) getTileSizeX());
+      out.seek(tileHeightPointer[resolutionIndex]);
+      out.writeShort((short) getTileSizeY());
+      out.seek(tileCountPointer[resolutionIndex]);
+
+      out.writeBytes(padString(String.valueOf(
+          xTiles * yTiles * sizeZ * r.getChannelCount(series))));
     }
 
     // TILED_SPARSE, so the tile coordinates must be written
@@ -498,7 +528,6 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
     // in the IFD
     // this tries to calculate the index without assuming sequential tile
     // writing, but maybe there is a better way to calculate this?
-    int xTiles = (int) Math.ceil((double) getSizeX() / tileWidth[resolutionIndex]);
     int xTile = x / tileWidth[resolutionIndex];
     int yTile = y / tileHeight[resolutionIndex];
     int tileIndex = (yTile * xTiles) + xTile;
@@ -508,6 +537,17 @@ else if (x % thisTileWidth != 0 || y % thisTileHeight != 0 ||
     if (ifds[resolutionIndex][no] != null) {
       tileByteCounts = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_BYTE_COUNTS);
       tileOffsets = (long[]) ifds[resolutionIndex][no].getIFDValue(IFD.TILE_OFFSETS);
+
+      if (tileByteCounts.length < xTiles * yTiles) {
+        long[] newTileByteCounts = new long[xTiles * yTiles];
+        long[] newTileOffsets = new long[xTiles * yTiles];
+        System.arraycopy(tileByteCounts, 0, newTileByteCounts, 0, tileByteCounts.length);
+        System.arraycopy(tileOffsets, 0, newTileOffsets, 0, tileOffsets.length);
+        tileByteCounts = newTileByteCounts;
+        tileOffsets = newTileOffsets;
+        ifds[resolutionIndex][no].put(IFD.TILE_BYTE_COUNTS, tileByteCounts);
+        ifds[resolutionIndex][no].put(IFD.TILE_OFFSETS, tileOffsets);
+      }
     }
 
     if (compression == null || compression.equals(CompressionType.UNCOMPRESSED.getCompression())) {
@@ -640,6 +680,9 @@ public void setId(String id) throws FormatException, IOException {
     planeOffsets = new PlaneOffset[totalFiles][];
     tileWidth = new int[totalFiles];
     tileHeight = new int[totalFiles];
+    tileWidthPointer = new long[totalFiles];
+    tileHeightPointer = new long[totalFiles];
+    tileCountPointer = new long[totalFiles];
 
     // create UIDs that must be consistent across all files in the dataset
     String specimenUIDValue = uids.getUID();
@@ -739,8 +782,9 @@ public void setId(String id) throws FormatException, IOException {
         int tileCountX = (int) Math.ceil((double) width / tileWidth[resolutionIndex]);
         int tileCountY = (int) Math.ceil((double) height / tileHeight[resolutionIndex]);
         DicomTag numberOfFrames = new DicomTag(NUMBER_OF_FRAMES, IS);
+        // save space for up to 10 digits
         numberOfFrames.value = padString(String.valueOf(
-          tileCountX * tileCountY * sizeZ * r.getChannelCount(pyramid)));
+          tileCountX * tileCountY * sizeZ * r.getChannelCount(pyramid)), " ", 10);
         tags.add(numberOfFrames);
 
         DicomTag matrixFrames = new DicomTag(TOTAL_PIXEL_MATRIX_FOCAL_PLANES, UL);
@@ -1374,6 +1418,9 @@ public void close() throws IOException {
     ifds = null;
     tiffSaver = null;
     validPixelCount = null;
+    tileWidthPointer = null;
+    tileHeightPointer = null;
+    tileCountPointer = null;
 
     tagProviders.clear();
 
@@ -1382,33 +1429,46 @@ public void close() throws IOException {
 
   @Override
   public int setTileSizeX(int tileSize) throws FormatException {
-    // TODO: this currently enforces the same tile size across all resolutions
-    // since the tile size is written during setId
-    // the tile size should probably be configurable per resolution,
-    // for better pre-compressed tile support
     if (currentId == null) {
       baseTileWidth = tileSize;
+      return baseTileWidth;
     }
-    return baseTileWidth;
+
+    int resolutionIndex = getIndex(series, resolution);
+    tileWidth[resolutionIndex] = tileSize;
+    return tileWidth[resolutionIndex];
   }
 
   @Override
   public int getTileSizeX() {
-    return baseTileWidth;
+    if (currentId == null) {
+      return baseTileWidth;
+    }
+
+    int resolutionIndex = getIndex(series, resolution);
+    return tileWidth[resolutionIndex];
   }
 
   @Override
   public int setTileSizeY(int tileSize) throws FormatException {
-    // TODO: see note in setTileSizeX above
     if (currentId == null) {
       baseTileHeight = tileSize;
+      return baseTileHeight;
     }
-    return baseTileHeight;
+
+    int resolutionIndex = getIndex(series, resolution);
+    tileHeight[resolutionIndex] = tileSize;
+    return tileHeight[resolutionIndex];
   }
 
   @Override
   public int getTileSizeY() {
-    return baseTileHeight;
+    if (currentId == null) {
+      return baseTileHeight;
+    }
+
+    int resolutionIndex = getIndex(series, resolution);
+    return tileHeight[resolutionIndex];
   }
 
   // -- DicomWriter-specific methods --
@@ -1468,15 +1528,25 @@ private void writeTag(DicomTag tag) throws IOException {
         out.writeShort((short) getStoredLength(tag));
       }
 
+      int resolutionIndex = getIndex(series, resolution);
       if (tag.attribute == TRANSFER_SYNTAX_UID) {
-        transferSyntaxPointer[getIndex(series, resolution)] = out.getFilePointer();
+        transferSyntaxPointer[resolutionIndex] = out.getFilePointer();
       }
       else if (tag.attribute == LOSSY_IMAGE_COMPRESSION_METHOD) {
-        compressionMethodPointer[getIndex(series, resolution)] = out.getFilePointer();
+        compressionMethodPointer[resolutionIndex] = out.getFilePointer();
       }
       else if (tag.attribute == FILE_META_INFO_GROUP_LENGTH) {
         fileMetaLengthPointer = out.getFilePointer();
       }
+      else if (tag.attribute == ROWS) {
+        tileHeightPointer[resolutionIndex] = out.getFilePointer();
+      }
+      else if (tag.attribute == COLUMNS) {
+        tileWidthPointer[resolutionIndex] = out.getFilePointer();
+      }
+      else if (tag.attribute == NUMBER_OF_FRAMES) {
+        tileCountPointer[resolutionIndex] = out.getFilePointer();
+      }
 
       // sequences with no items still need to write a SequenceDelimitationItem below
       if (tag.children.size() == 0 && tag.value == null && tag.vr != SQ) {
@@ -1665,6 +1735,17 @@ private String padString(String value, String append) {
     return value + append;
   }
 
+  private String padString(String value, String append, int length) {
+    String rtn = "";
+    if (value != null) {
+      rtn += value;
+    }
+    while (rtn.length() < length) {
+      rtn += append;
+    }
+    return rtn;
+  }
+
   /**
    * @return transfer syntax UID corresponding to the current compression type
    */
@@ -1919,6 +2000,9 @@ private void writeIFDs(int resIndex) throws IOException {
     out.seek(ifdStart);
 
     for (int no=0; no<ifds[resIndex].length; no++) {
+      ifds[resIndex][no].put(IFD.TILE_WIDTH, tileWidth[resIndex]);
+      ifds[resIndex][no].put(IFD.TILE_LENGTH, tileHeight[resIndex]);
+
       try {
         tiffSaver.writeIFD(ifds[resIndex][no], 0, no < ifds[resIndex].length - 1);
       }