TIKA-2861: Extract QuickTime GPS location and item-list metadata in MP4Parser (#2935)

* [TIKA-2861] Expose QuickTime item-list metadata in MP4Parser

The MP4 handler descends into the moov/meta container but skips the keys
and ilst boxes, so the QuickTime metadata stored there (the com.apple.quicktime.*
keys, e.g. the Apple Live Photo content identifier and the ISO 6709 location)
was dropped for QuickTime .mov files and any .mp4 carrying it.

Parse the keys/ilst boxes in TikaMp4BoxHandler and expose the UTF-8 text values
under their key names. This is additive: the udta path and all existing fields
are untouched.

* [TIKA-2861] Map QuickTime ISO 6709 location to geo:lat/long/alt

In addition to the raw com.apple.quicktime.location.ISO6709 value, parse it
into the standard geo:lat, geo:long and geo:alt properties. This matches the
geo:* output the udta ("(c)xyz") path already produces for other files, and
additionally exposes the altitude, which the udta path drops.

* [TIKA-2861] Expose the udta ISO 6709 altitude as geo:alt

The udta "(c)xyz" box carries an ISO 6709 string whose optional third
component is the altitude (ISO 6709 Annex H). Older iOS versions and
Google Photos write it with altitude, while Android writes lat/long only.
TikaUserDataBox only matched lat/long, so the altitude was dropped.

Capture the optional altitude and set geo:alt directly on the Tika
metadata: the drewnoakes Mp4Directory only defines latitude/longitude
tags, so it cannot carry the altitude to MP4Parser. Lat/long continue
to flow through the directory as before.

The new crafted fixture is the first test coverage for the udta
location path (no existing fixture contained a "(c)xyz" box).

* [TIKA-2861] Handle numeric QuickTime metadata value types

The item-list parser only emitted UTF-8 text values (well-known type 1)
and silently dropped the numeric types. Real iPhone Live Photo videos
carry several of those, e.g. com.apple.quicktime.live-photo.vitality-score
(float32), live-photo.auto (uint8) and
camera.focal_length.35mm_equivalent (int32).

Decode the QTFF well-known types 21 (signed int BE), 22 (unsigned int BE),
23 (float32) and 24 (float64) as well. Integers may be 1 to 8 bytes wide.
Other value types (images, binary plists) are still skipped.

Author: dschmidt

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4BoxHandler.java

@@ -17,6 +17,14 @@
 package org.apache.tika.parser.mp4;
 
 import java.io.IOException;
+import java.math.BigInteger;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import com.drew.imaging.mp4.Mp4Handler;
 import com.drew.lang.annotations.NotNull;
@@ -26,13 +34,31 @@
 import com.drew.metadata.mp4.Mp4Context;
 import org.xml.sax.SAXException;
 
+import org.apache.tika.metadata.TikaCoreProperties;
 import org.apache.tika.parser.mp4.boxes.TikaUserDataBox;
 import org.apache.tika.sax.XHTMLContentHandler;
 
 public class TikaMp4BoxHandler extends Mp4BoxHandler {
 
+    //QTFF "well-known" metadata item value types
+    private static final int QT_TEXT_TYPE = 1;
+    private static final int QT_INT_BE_TYPE = 21;
+    private static final int QT_UINT_BE_TYPE = 22;
+    private static final int QT_FLOAT32_TYPE = 23;
+    private static final int QT_FLOAT64_TYPE = 24;
+
+    //QuickTime stores location as an ISO 6709 string (e.g. +32.4720-084.9952+073.827/)
+    private static final String QT_LOCATION_ISO6709 = "com.apple.quicktime.location.ISO6709";
+    private static final Pattern ISO6709_PATTERN =
+            Pattern.compile("([+-]\\d+(?:\\.\\d+)?)([+-]\\d+(?:\\.\\d+)?)([+-]\\d+(?:\\.\\d+)?)?");
+
     org.apache.tika.metadata.Metadata tikaMetadata;
     final XHTMLContentHandler xhtml;
+
+    //key names for the current 'meta' box, filled from its 'keys' box and consumed
+    //by the following 'ilst' box (e.g. com.apple.quicktime.content.identifier)
+    private final List<String> quickTimeMetadataKeys = new ArrayList<>();
+
     public TikaMp4BoxHandler(Metadata metadata, org.apache.tika.metadata.Metadata tikaMetadata,
                              XHTMLContentHandler xhtml) {
         super(metadata);
@@ -42,7 +68,7 @@ public TikaMp4BoxHandler(Metadata metadata, org.apache.tika.metadata.Metadata ti
 
     @Override
     public boolean shouldAcceptBox(@NotNull String box) {
-        if (box.equals("udta")) {
+        if (box.equals("udta") || box.equals("keys") || box.equals("ilst")) {
             return true;
         }
         return super.shouldAcceptBox(box);
@@ -59,6 +85,12 @@ public Mp4Handler<?> processBox(@NotNull String box, @Nullable byte[] payload,
             throws IOException {
         if (box.equals("udta")) {
             return processUserData(box, payload, context);
+        } else if (box.equals("keys")) {
+            processQuickTimeKeys(payload);
+            return this;
+        } else if (box.equals("ilst")) {
+            processQuickTimeItemList(payload);
+            return this;
         }
 
         return super.processBox(box, payload, size, context);
@@ -76,4 +108,124 @@ private Mp4Handler<?> processUserData(String box, byte[] payload, Mp4Context con
         }
         return this;
     }
+
+    /**
+     * Parses the QuickTime metadata 'keys' box, which maps 1-based indices to key
+     * names such as {@code com.apple.quicktime.content.identifier}. The base MP4
+     * handler descends into the enclosing 'meta' container but skips 'keys'/'ilst',
+     * so this metadata (content identifier, ISO 6709 location, make/model, ...) was
+     * previously dropped for QuickTime .mov (and any .mp4 carrying it).
+     */
+    private void processQuickTimeKeys(@Nullable byte[] payload) {
+        quickTimeMetadataKeys.clear();
+        if (payload == null || payload.length < 8) {
+            return;
+        }
+        //1 byte version + 3 bytes flags, then uint32 entry count
+        int pos = 4;
+        long entryCount = readUInt32(payload, pos);
+        pos += 4;
+        for (long i = 0; i < entryCount && pos + 8 <= payload.length; i++) {
+            long keySize = readUInt32(payload, pos);
+            if (keySize < 8 || pos + keySize > payload.length) {
+                return;
+            }
+            //4 bytes key namespace, then the UTF-8 key name
+            quickTimeMetadataKeys.add(
+                    new String(payload, pos + 8, (int) keySize - 8, StandardCharsets.UTF_8));
+            pos += (int) keySize;
+        }
+    }
+
+    /**
+     * Parses the QuickTime metadata 'ilst' box, whose entries are keyed by the
+     * 1-based index into the preceding 'keys' box. Each entry holds a 'data' box
+     * with the value. UTF-8 text and the numeric "well-known" value types are emitted
+     * under their key name; other types (e.g. images, binary plists) are skipped.
+     */
+    private void processQuickTimeItemList(@Nullable byte[] payload) {
+        if (payload == null) {
+            return;
+        }
+        int pos = 0;
+        while (pos + 8 <= payload.length) {
+            long entrySize = readUInt32(payload, pos);
+            if (entrySize < 8 || pos + entrySize > payload.length) {
+                return;
+            }
+            int index = (int) readUInt32(payload, pos + 4);
+            int entryEnd = (int) (pos + entrySize);
+            int data = pos + 8;
+            //inner 'data' box: size(4) type(4) valueType(4) locale(4) value
+            if (data + 16 <= entryEnd) {
+                long dataSize = readUInt32(payload, data);
+                boolean isData = payload[data + 4] == 'd' && payload[data + 5] == 'a'
+                        && payload[data + 6] == 't' && payload[data + 7] == 'a';
+                if (isData && dataSize >= 16 && data + dataSize <= entryEnd) {
+                    int valueType = (int) readUInt32(payload, data + 8);
+                    int valueLength = (int) dataSize - 16;
+                    if (index >= 1 && index <= quickTimeMetadataKeys.size()) {
+                        String key = quickTimeMetadataKeys.get(index - 1);
+                        String value = decodeValue(payload, data + 16, valueLength, valueType);
+                        if (value != null) {
+                            tikaMetadata.add(key, value);
+                            if (key.equals(QT_LOCATION_ISO6709)) {
+                                addLocation(value);
+                            }
+                        }
+                    }
+                }
+            }
+            pos += (int) entrySize;
+        }
+    }
+
+    /**
+     * Maps an ISO 6709 location string (latitude, longitude, optional altitude) to the
+     * standard {@code geo:lat}/{@code geo:long}/{@code geo:alt} properties, in addition to
+     * the raw value, so QuickTime location matches the {@code geo:*} output of the udta path.
+     */
+    private void addLocation(String iso6709) {
+        Matcher matcher = ISO6709_PATTERN.matcher(iso6709);
+        if (matcher.find()) {
+            tikaMetadata.set(TikaCoreProperties.LATITUDE, Double.parseDouble(matcher.group(1)));
+            tikaMetadata.set(TikaCoreProperties.LONGITUDE, Double.parseDouble(matcher.group(2)));
+            if (matcher.group(3) != null) {
+                tikaMetadata.set(TikaCoreProperties.ALTITUDE, Double.parseDouble(matcher.group(3)));
+            }
+        }
+    }
+
+    /**
+     * Decodes a metadata item value of one of the QTFF "well-known" types to a string,
+     * or returns null for types that are not handled (e.g. images or binary plists).
+     * Integers may be 1 to 8 bytes wide (e.g. the live-photo.auto flag is a single byte).
+     */
+    @Nullable
+    private static String decodeValue(byte[] b, int off, int len, int valueType) {
+        switch (valueType) {
+            case QT_TEXT_TYPE:
+                return new String(b, off, len, StandardCharsets.UTF_8);
+            case QT_INT_BE_TYPE:
+            case QT_UINT_BE_TYPE:
+                if (len < 1 || len > 8) {
+                    return null;
+                }
+                byte[] intBytes = Arrays.copyOfRange(b, off, off + len);
+                return valueType == QT_INT_BE_TYPE
+                        ? new BigInteger(intBytes).toString()
+                        : new BigInteger(1, intBytes).toString();
+            case QT_FLOAT32_TYPE:
+                return len == 4 ? String.valueOf(ByteBuffer.wrap(b, off, len).getFloat()) : null;
+            case QT_FLOAT64_TYPE:
+                return len == 8 ? String.valueOf(ByteBuffer.wrap(b, off, len).getDouble()) : null;
+            default:
+                return null;
+        }
+    }
+
+    private static long readUInt32(byte[] b, int off) {
+        return ((b[off] & 0xFFL) << 24) | ((b[off + 1] & 0xFFL) << 16)
+                | ((b[off + 2] & 0xFFL) << 8) | (b[off + 3] & 0xFFL);
+    }
 }

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/boxes/TikaUserDataBox.java

@@ -44,7 +44,7 @@ public class TikaUserDataBox {
     private static final String HDLR = "hdlr";
     private static final String MDIR = "mdir";//apple metadata itunes reader
     private static final Pattern COORDINATE_PATTERN =
-            Pattern.compile("([+-]\\d+\\.\\d+)([+-]\\d+\\.\\d+)");
+            Pattern.compile("([+-]\\d+\\.\\d+)([+-]\\d+\\.\\d+)([+-]\\d+(?:\\.\\d+)?)?");
 
     @Nullable
     private String coordinateString;
@@ -270,6 +270,11 @@ public void addMetadata(Mp4Directory directory) {
                 double longitude = Double.parseDouble(matcher.group(2));
                 directory.setDouble(8193, latitude);
                 directory.setDouble(8194, longitude);
+                //Mp4Directory has no altitude tag, so set geo:alt directly
+                if (matcher.group(3) != null) {
+                    metadata.set(TikaCoreProperties.ALTITUDE,
+                            Double.parseDouble(matcher.group(3)));
+                }
             }
         }
     }

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/test/java/org/apache/tika/parser/mp4/MP4ParserTest.java

@@ -281,4 +281,42 @@ private Set<String> getVals(Metadata m, String k) {
         }
         return vals;
     } */
+
+    @Test
+    public void testQuickTimeMetadataKeys() throws Exception {
+        //QuickTime item-list metadata (moov/meta/keys+ilst, the com.apple.quicktime.*
+        //keys such as the content identifier and ISO 6709 location) was previously
+        //dropped by the MP4 handler. See TIKA-2861.
+        Metadata metadata = new Metadata();
+        getText("testMP4_QuickTimeMetadata.mov", metadata);
+        assertEquals("TEST-UUID-0001-LIVEPHOTO",
+                metadata.get("com.apple.quicktime.content.identifier"));
+
+        //the raw ISO 6709 location is preserved ...
+        assertEquals("+12.3456-098.7654+010.500/",
+                metadata.get("com.apple.quicktime.location.ISO6709"));
+        //... and also mapped to the standard geo:* properties (incl. altitude)
+        assertEquals(12.3456, Double.parseDouble(metadata.get(TikaCoreProperties.LATITUDE)), 0.00001);
+        assertEquals(-98.7654, Double.parseDouble(metadata.get(TikaCoreProperties.LONGITUDE)), 0.00001);
+        assertEquals(10.5, Double.parseDouble(metadata.get(TikaCoreProperties.ALTITUDE)), 0.00001);
+
+        //numeric well-known value types (uint8, float32, int32, float64)
+        assertEquals("1", metadata.get("com.apple.quicktime.live-photo.auto"));
+        assertEquals("0.75", metadata.get("com.apple.quicktime.live-photo.vitality-score"));
+        assertEquals("-13",
+                metadata.get("com.apple.quicktime.camera.focal_length.35mm_equivalent"));
+        assertEquals("1.5",
+                metadata.get("com.apple.quicktime.full-frame-rate-playback-intent"));
+    }
+
+    @Test
+    public void testUdtaLocation() throws Exception {
+        //the udta "(c)xyz" ISO 6709 location is mapped to geo:lat/geo:long, and its
+        //optional altitude, which was previously dropped, to geo:alt. See TIKA-2861.
+        Metadata metadata = new Metadata();
+        getText("testMP4_udtaLocation.mp4", metadata);
+        assertEquals(12.3456, Double.parseDouble(metadata.get(TikaCoreProperties.LATITUDE)), 0.00001);
+        assertEquals(-98.7654, Double.parseDouble(metadata.get(TikaCoreProperties.LONGITUDE)), 0.00001);
+        assertEquals(10.5, Double.parseDouble(metadata.get(TikaCoreProperties.ALTITUDE)), 0.00001);
+    }
 }