TIKA-4838: Expose the MP4 audio and video track codecs as FourCCs (audio:fourcc, video:fourcc)

tika-core/src/main/java/org/apache/tika/metadata/Audio.java

@@ -84,6 +84,18 @@ public interface Audio {
      */
     Property BITS_PER_SAMPLE = Property.internalInteger("audio:bits-per-sample");
 
+    /**
+     * The audio track's codec as a four-character code (FourCC), as used by
+     * QuickTime/MP4 sample descriptions, AVI and other containers: e.g. "mp4a"
+     * for MPEG-4 audio (AAC), "alac", "ac-3". For protected MP4 streams, where
+     * {@link #HAS_DRM} is also set, this is the original codec named by the
+     * protection scheme info ('sinf'/'frma'), or the protected sample entry format
+     * ("drms"/"enca") if there is none. Trailing padding spaces are trimmed and
+     * non-printable codes are not exposed. A per-stream value: with several audio
+     * tracks or sample entries it reflects the last one.
+     */
+    Property FOURCC = Property.internalText("audio:fourcc");
+
     /**
      * The raw javax.sound encoding name (e.g. "PCM_SIGNED"), as reported by {@code
      * AudioFormat#getEncoding()} (AudioParser). Distinct from {@link XMPDM#AUDIO_SAMPLE_TYPE}

tika-core/src/main/java/org/apache/tika/metadata/Video.java

@@ -37,4 +37,16 @@ public interface Video {
      * reflects the last one.
      */
     Property BITRATE = Property.internalInteger("video:bitrate");
+
+    /**
+     * The video track's codec as a four-character code (FourCC), as used by
+     * QuickTime/MP4 sample descriptions, AVI and other containers: e.g. "avc1"
+     * for H.264, "hvc1"/"hev1" for HEVC. For protected MP4 streams this is the
+     * original codec named by the protection scheme info ('sinf'/'frma'), or the
+     * protected sample entry format ("encv"/"drmi") if there is none. Trailing
+     * padding spaces are trimmed and non-printable codes are not exposed. A
+     * per-stream value: with several video tracks or sample entries it reflects
+     * the last one.
+     */
+    Property FOURCC = Property.internalText("video:fourcc");
 }

tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-key-fields.json

@@ -13,6 +13,7 @@
   {"class":"org.apache.tika.metadata.Audio","field":"CHANNELS","key":"audio:channels"},
   {"class":"org.apache.tika.metadata.Audio","field":"DISC_COUNT","key":"audio:disc-count"},
   {"class":"org.apache.tika.metadata.Audio","field":"ENCODING","key":"audio:encoding"},
+  {"class":"org.apache.tika.metadata.Audio","field":"FOURCC","key":"audio:fourcc"},
   {"class":"org.apache.tika.metadata.Audio","field":"HAS_DRM","key":"audio:has-drm"},
   {"class":"org.apache.tika.metadata.Audio","field":"IS_VARIABLE_BITRATE","key":"audio:is-variable-bitrate"},
   {"class":"org.apache.tika.metadata.Audio","field":"RAW_DISC_NUMBER","key":"audio:raw-disc-number"},
@@ -576,6 +577,7 @@
   {"class":"org.apache.tika.metadata.TikaPagedText","field":"PAGE_NUMBERS","key":"tk:page:numbers"},
   {"class":"org.apache.tika.metadata.TikaPagedText","field":"PAGE_ROTATION","key":"tk:page:rotation"},
   {"class":"org.apache.tika.metadata.Video","field":"BITRATE","key":"video:bitrate"},
+  {"class":"org.apache.tika.metadata.Video","field":"FOURCC","key":"video:fourcc"},
   {"class":"org.apache.tika.metadata.Video","field":"FRAME_RATE","key":"video:frame-rate"},
   {"class":"org.apache.tika.metadata.WARC","field":"WARC_PAYLOAD_CONTENT_TYPE","key":"warc:payload-content-type"},
   {"class":"org.apache.tika.metadata.WARC","field":"WARC_RECORD_CONTENT_TYPE","key":"warc:record-content-type"},

tika-metadata-schema/src/main/resources/org/apache/tika/metadata/metadata-keys.json

@@ -77,6 +77,7 @@
   {"key":"audio:channels","namespace":"audio","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"audio:disc-count","namespace":"audio","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"audio:encoding","namespace":"audio","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
+  {"key":"audio:fourcc","namespace":"audio","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"audio:has-drm","namespace":"audio","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"audio:is-variable-bitrate","namespace":"audio","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"audio:raw-disc-number","namespace":"audio","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
@@ -622,6 +623,7 @@
   {"key":"tk:version-number","namespace":"tk","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"tk:warn:truncated-metadata","namespace":"tk","valueType":"BOOLEAN","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"video:bitrate","namespace":"video","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-core"},
+  {"key":"video:fourcc","namespace":"video","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"video:frame-rate","namespace":"video","valueType":"REAL","cardinality":"SIMPLE","module":"tika-core"},
   {"key":"vlm:completion-tokens","namespace":"vlm","valueType":"INTEGER","cardinality":"SIMPLE","module":"tika-vlm"},
   {"key":"vlm:model","namespace":"vlm","valueType":"TEXT","cardinality":"SIMPLE","module":"tika-vlm"},

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/Mp4SampleEntries.java

@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.parser.mp4;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.tika.io.EndianUtils;
+
+/**
+ * Walks the sample entries of a SampleDescriptionBox ('stsd') payload, shared
+ * by the sound and video handlers. The payload is 4 bytes version and flags,
+ * a 4 byte entry count, then one sample entry per count. Each entry is a box:
+ * a 32-bit size and a FourCC, where a size of 1 announces a 64-bit largesize
+ * and a size of 0 extends the entry to the end of the payload (ISO/IEC
+ * 14496-12, 4.2).
+ */
+final class Mp4SampleEntries {
+
+    /**
+     * Size of the SampleEntry fields that follow the box header in every
+     * entry: 6 reserved bytes and the 2 byte data reference index.
+     */
+    static final int SAMPLE_ENTRY_FIELDS = 8;
+
+    interface Visitor {
+        /**
+         * @param fourCC the entry's FourCC, or null if it is not printable
+         * @param b      the stsd payload
+         * @param start  offset of the first byte after the entry's box header
+         * @param end    offset one past the entry's last byte
+         */
+        void entry(String fourCC, byte[] b, int start, int end);
+    }
+
+    private Mp4SampleEntries() {
+    }
+
+    static void walk(byte[] b, Visitor visitor) {
+        if (b.length < 8) {
+            return;
+        }
+        long entryCount = EndianUtils.getUIntBE(b, 4);
+        int pos = 8;
+        for (long i = 0; i < entryCount && pos + 8 <= b.length; i++) {
+            long size = EndianUtils.getUIntBE(b, pos);
+            int header = 8;
+            if (size == 1) {
+                //largesize: the 64-bit size follows the FourCC
+                if (pos + 16 > b.length) {
+                    return;
+                }
+                //a value beyond 63 bits goes negative and fails the size check below
+                size = (EndianUtils.getUIntBE(b, pos + 8) << 32) | EndianUtils.getUIntBE(b, pos + 12);
+                header = 16;
+            } else if (size == 0) {
+                //the entry extends to the end of the box
+                size = b.length - pos;
+            }
+            if (size < header + SAMPLE_ENTRY_FIELDS || size > b.length - pos) {
+                return;
+            }
+            int end = pos + (int) size;
+            visitor.entry(printableFourCC(b, pos + 4), b, pos + header, end);
+            pos = end;
+        }
+    }
+
+    /**
+     * Returns whether a sample entry FourCC marks a protected stream: 'drms'
+     * and 'drmi' (FairPlay) or 'enca' and 'encv' (ISO common encryption).
+     */
+    static boolean isProtected(String fourCC) {
+        return "drms".equals(fourCC) || "enca".equals(fourCC)
+                || "encv".equals(fourCC) || "drmi".equals(fourCC);
+    }
+
+    /**
+     * Looks up the original (unprotected) format of a protected sample entry:
+     * the ProtectionSchemeInfoBox 'sinf' among the entry's child boxes carries
+     * an OriginalFormatBox 'frma' whose payload is the codec FourCC that the
+     * protected format replaced (ISO/IEC 14496-12, 8.12). Returns null if
+     * there is none or it is not printable.
+     *
+     * @param pos offset of the entry's first child box
+     * @param end offset one past the entry's last byte
+     */
+    static String originalFormat(byte[] b, int pos, int end) {
+        int sinf = findBox(b, pos, end, "sinf");
+        if (sinf < 0) {
+            return null;
+        }
+        int sinfEnd = boxEnd(b, sinf, end);
+        int frma = findBox(b, sinf + 8, sinfEnd, "frma");
+        if (frma < 0 || boxEnd(b, frma, sinfEnd) < frma + 12) {
+            //the box must hold its 4 byte payload, not borrow it from the next box
+            return null;
+        }
+        return printableFourCC(b, frma + 8);
+    }
+
+    /**
+     * Returns the offset of the first box of the given type among the boxes
+     * in [pos, end), or -1.
+     */
+    private static int findBox(byte[] b, int pos, int end, String type) {
+        while (pos >= 0 && pos + 8 <= end) {
+            int boxEnd = boxEnd(b, pos, end);
+            if (boxEnd < 0) {
+                return -1;
+            }
+            if (type.equals(fourCC(b, pos + 4))) {
+                return pos;
+            }
+            pos = boxEnd;
+        }
+        return -1;
+    }
+
+    /**
+     * Returns the offset one past the box starting at pos, or -1 if its size
+     * is invalid or runs past end.
+     */
+    private static int boxEnd(byte[] b, int pos, int end) {
+        long size = EndianUtils.getUIntBE(b, pos);
+        if (size < 8 || size > end - pos) {
+            return -1;
+        }
+        return pos + (int) size;
+    }
+
+    /**
+     * Reads a FourCC as it is, for comparing against known box types.
+     */
+    static String fourCC(byte[] b, int pos) {
+        return new String(b, pos, 4, StandardCharsets.ISO_8859_1);
+    }
+
+    /**
+     * Reads a FourCC for exposing it as a metadata value: null unless all four
+     * bytes are printable ASCII, with trailing spaces trimmed (QuickTime pads
+     * short codes such as 'raw ' and 'rle ' with spaces). Codes that are blank
+     * after trimming are null as well.
+     */
+    static String printableFourCC(byte[] b, int pos) {
+        int len = 4;
+        while (len > 0 && b[pos + len - 1] == ' ') {
+            len--;
+        }
+        if (len == 0) {
+            return null;
+        }
+        for (int i = 0; i < len; i++) {
+            int c = b[pos + i] & 0xFF;
+            if (c < 0x20 || c > 0x7E) {
+                return null;
+            }
+        }
+        return new String(b, pos, len, StandardCharsets.US_ASCII);
+    }
+}

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4SoundHandler.java

@@ -17,7 +17,6 @@
 package org.apache.tika.parser.mp4;
 
 import java.io.IOException;
-import java.nio.charset.StandardCharsets;
 
 import com.drew.imaging.mp4.Mp4Handler;
 import com.drew.metadata.Metadata;
@@ -52,55 +51,54 @@ public Mp4Handler<?> processBox(String type, byte[] payload, long boxSize,
         return super.processBox(type, payload, boxSize, context);
     }
 
-    /**
-     * Walks the sample description entries: 4 bytes version and flags, a
-     * 4 byte entry count, then one sample entry per count, each starting with
-     * its own size and format fourcc.
-     */
     private void extractFromSampleDescriptions(byte[] b) {
-        if (b.length < 8) {
-            return;
+        Mp4SampleEntries.walk(b, this::sampleEntry);
+    }
+
+    private void sampleEntry(String fourCC, byte[] b, int start, int end) {
+        int children = -1;
+        if (start + Mp4SampleEntries.SAMPLE_ENTRY_FIELDS + 2 <= end) {
+            //after the SampleEntry fields come the version-dependent fixed
+            //sound fields, starting with the 2 byte version, then child boxes
+            int version = EndianUtils.getUShortBE(b, start + Mp4SampleEntries.SAMPLE_ENTRY_FIELDS);
+            children = start + soundEntrySize(version);
         }
-        long entryCount = EndianUtils.getUIntBE(b, 4);
-        int pos = 8;
-        for (long i = 0; i < entryCount && pos + 8 <= b.length; i++) {
-            long size = EndianUtils.getUIntBE(b, pos);
-            if (size < 16 || size > b.length - pos) {
-                break;
-            }
-            int end = pos + (int) size;
-            String format = fourCc(b, pos + 4);
-            //protected streams replace the codec fourcc with a protected
-            //entry format: 'drms' (FairPlay) or 'enca' (ISO common encryption)
-            if ("drms".equals(format) || "enca".equals(format)) {
-                tikaMetadata.set(Audio.HAS_DRM, true);
-            }
-            if (pos + 18 <= end) {
-                //sample entry: 8 byte header, 6 reserved, 2 data ref index,
-                //then version-dependent fixed sound fields before child boxes
-                int version = EndianUtils.getUShortBE(b, pos + 16);
-                int bitRate = findEsdsAverageBitRate(b, pos + soundEntrySize(version), end, 0);
-                if (bitRate > 0) {
-                    tikaMetadata.set(Audio.BITRATE, bitRate);
+        //protected streams replace the codec FourCC with a protected sample
+        //entry format, 'drms' (FairPlay) or 'enca' (ISO common encryption),
+        //and keep the original one in a child 'sinf'/'frma' box
+        if (Mp4SampleEntries.isProtected(fourCC)) {
+            tikaMetadata.set(Audio.HAS_DRM, true);
+            if (children >= 0) {
+                String original = Mp4SampleEntries.originalFormat(b, children, end);
+                if (original != null) {
+                    fourCC = original;
                 }
             }
-            pos = end;
+        }
+        if (fourCC != null) {
+            tikaMetadata.set(Audio.FOURCC, fourCC);
+        }
+        if (children >= 0) {
+            int bitRate = findEsdsAverageBitRate(b, children, end, 0);
+            if (bitRate > 0) {
+                tikaMetadata.set(Audio.BITRATE, bitRate);
+            }
         }
     }
 
     /**
-     * Size of the fixed part of a sound sample entry, after which the child
-     * boxes start: 36 bytes for version 0, 52 for version 1 (four extra
-     * 32-bit QuickTime fields), 72 for version 2.
+     * Size of a sound sample entry after its box header, up to where the
+     * child boxes start: 28 bytes for version 0, 44 for version 1 (four extra
+     * 32-bit QuickTime fields), 64 for version 2.
      */
     private static int soundEntrySize(int version) {
         if (version == 1) {
-            return 52;
+            return 44;
         }
         if (version == 2) {
-            return 72;
+            return 64;
         }
-        return 36;
+        return 28;
     }
 
     //real files nest 'wave' at most one level; this only bounds crafted input,
@@ -123,7 +121,7 @@ private static int findEsdsAverageBitRate(byte[] b, int pos, int end, int depth)
             if (size < 8 || size > end - pos) {
                 return 0;
             }
-            String type = fourCc(b, pos + 4);
+            String type = Mp4SampleEntries.fourCC(b, pos + 4);
             if ("esds".equals(type)) {
                 return readEsdsAverageBitRate(b, pos + 8, pos + (int) size);
             }
@@ -195,8 +193,4 @@ private static int skipDescriptorLength(byte[] b, int pos) {
         }
         return pos + 1;
     }
-
-    private static String fourCc(byte[] b, int pos) {
-        return new String(b, pos, 4, StandardCharsets.ISO_8859_1);
-    }
 }