Skip to content

Commit 6ffb5dd

Browse files
authored
TIKA-4779: Add audio:bitrate, audio:is-variable-bitrate and audio:has-drm (#2953)
1 parent e3cd4e5 commit 6ffb5dd

14 files changed

Lines changed: 484 additions & 2 deletions

File tree

tika-core/src/main/java/org/apache/tika/metadata/Audio.java

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,4 +49,26 @@ public interface Audio {
4949
* The disc value exactly as tagged, see {@link #RAW_TRACK_NUMBER}.
5050
*/
5151
Property RAW_DISC_NUMBER = Property.internalText("audio:raw-disc-number");
52+
53+
/**
54+
* Average or nominal bitrate in bits per second (averaged over the MP3
55+
* frames, the Vorbis nominal bitrate, or the MP4 'esds' average bitrate).
56+
* A per-stream value: in a file with several audio tracks it reflects
57+
* the last sound track's sample description.
58+
*/
59+
Property BITRATE = Property.internalInteger("audio:bitrate");
60+
61+
/**
62+
* True if the stream is variable bitrate: the MP3 frames declare differing
63+
* bitrates, or the Vorbis identification header does not declare one fixed
64+
* rate for upper, nominal and lower.
65+
*/
66+
Property IS_VARIABLE_BITRATE = Property.internalBoolean("audio:is-variable-bitrate");
67+
68+
/**
69+
* True if the container declares DRM protection through a protected
70+
* sample entry format such as 'drms' or 'enca'. A file-level flag: any
71+
* protected audio track sets it. Only set when protection is detected.
72+
*/
73+
Property HAS_DRM = Property.internalBoolean("audio:has-drm");
5274
}

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/Mp3Parser.java

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -96,12 +96,30 @@ protected static ID3TagsAndAudio getAllTagHandlers(InputStream tis, ContentHandl
9696
// Now iterate over all audio frames in the file
9797
AudioFrame frame = mpegStream.nextFrame();
9898
float duration = 0;
99+
long bitRateSum = 0;
100+
long frameCount = 0;
101+
int baselineBitRate = -1;
102+
boolean variableBitRate = false;
103+
boolean firstFrame = true;
99104
boolean skipped = true;
100105
while (frame != null && skipped) {
101106
duration += frame.getDuration();
102107
if (firstAudio == null) {
103108
firstAudio = frame;
104109
}
110+
//the Xing/Info/VBRI header is a valid MPEG frame but carries no
111+
//audio and may use a different bitrate, so keep it out of the stats
112+
if (!(firstFrame && isMetadataFrame(mpegStream))) {
113+
int bitRate = frame.getBitRate();
114+
bitRateSum += bitRate;
115+
frameCount++;
116+
if (baselineBitRate < 0) {
117+
baselineBitRate = bitRate;
118+
} else if (bitRate != baselineBitRate) {
119+
variableBitRate = true;
120+
}
121+
}
122+
firstFrame = false;
105123
skipped = mpegStream.skipFrame();
106124
if (skipped) {
107125
frame = mpegStream.nextFrame();
@@ -136,6 +154,12 @@ protected static ID3TagsAndAudio getAllTagHandlers(InputStream tis, ContentHandl
136154
ret.lyrics = lyrics;
137155
ret.tags = tags.toArray(new ID3Tags[0]);
138156
ret.duration = duration;
157+
if (frameCount > 0) {
158+
//MP3 frame duration does not depend on the bitrate, so the plain
159+
//mean over the frames is the true average bitrate
160+
ret.averageBitRate = (int) (bitRateSum / frameCount);
161+
ret.variableBitRate = variableBitRate;
162+
}
139163
return ret;
140164
}
141165

@@ -159,6 +183,10 @@ public void parse(TikaInputStream tis, ContentHandler handler, Metadata metadata
159183
metadata.set(XMPDM.DURATION, audioAndTags.durationSeconds());
160184
}
161185

186+
if (audioAndTags.averageBitRate > 0) {
187+
metadata.set(Audio.BITRATE, audioAndTags.averageBitRate);
188+
metadata.set(Audio.IS_VARIABLE_BITRATE, audioAndTags.variableBitRate);
189+
}
162190
if (audioAndTags.audio != null) {
163191
metadata.set("channels", String.valueOf(audioAndTags.audio.getChannels()));
164192
metadata.set("version", audioAndTags.audio.getVersion());
@@ -278,11 +306,34 @@ public void setMaxRecordSize(int maxRecordSize) {
278306
public int getMaxRecordSize() {
279307
return ID3v2Frame.getMaxRecordSize();
280308
}
309+
/**
310+
* Does the current frame's payload start with a Xing, Info or VBRI
311+
* header? Those tag frames describe the stream rather than carrying
312+
* audio. The markers sit at small fixed offsets that depend on version
313+
* and channel mode, all within the first 40 payload bytes.
314+
*/
315+
private static boolean isMetadataFrame(MpegStream mpegStream) throws IOException {
316+
byte[] payload = mpegStream.peekFramePayload(40);
317+
for (int i = 0; i + 4 <= payload.length; i++) {
318+
if ((payload[i] == 'X' && payload[i + 1] == 'i'
319+
&& payload[i + 2] == 'n' && payload[i + 3] == 'g')
320+
|| (payload[i] == 'I' && payload[i + 1] == 'n'
321+
&& payload[i + 2] == 'f' && payload[i + 3] == 'o')
322+
|| (payload[i] == 'V' && payload[i + 1] == 'B'
323+
&& payload[i + 2] == 'R' && payload[i + 3] == 'I')) {
324+
return true;
325+
}
326+
}
327+
return false;
328+
}
329+
281330
protected static class ID3TagsAndAudio {
282331
private ID3Tags[] tags;
283332
private AudioFrame audio;
284333
private LyricsHandler lyrics;
285334
private float duration; // Milliseconds
335+
private int averageBitRate; // bits per second, 0 if no frame was seen
336+
private boolean variableBitRate;
286337

287338
private float durationSeconds() {
288339
return duration / 1000;

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp3/MpegStream.java

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import java.io.IOException;
2020
import java.io.InputStream;
2121
import java.io.PushbackInputStream;
22+
import java.util.Arrays;
2223

2324
import org.apache.commons.io.IOUtils;
2425

@@ -113,6 +114,12 @@ class MpegStream extends PushbackInputStream {
113114
*/
114115
private static final int HEADER_SIZE = 4;
115116

117+
/**
118+
* Pushback capacity: enough for the header handling plus
119+
* {@link #peekFramePayload(int)} peeks into the frame payload.
120+
*/
121+
private static final int PEEK_BUFFER_SIZE = 48;
122+
116123
/**
117124
* The current MPEG header.
118125
*/
@@ -130,7 +137,7 @@ class MpegStream extends PushbackInputStream {
130137
* @param in the underlying audio stream
131138
*/
132139
public MpegStream(InputStream in) {
133-
super(in, 2 * HEADER_SIZE);
140+
super(in, PEEK_BUFFER_SIZE);
134141
}
135142

136143
/**
@@ -285,7 +292,9 @@ public AudioFrame nextFrame() throws IOException {
285292
public boolean skipFrame() throws IOException {
286293
if (currentHeader != null) {
287294
long toSkip = currentHeader.getLength() - HEADER_SIZE;
288-
long skipped = IOUtils.skip(in, toSkip);
295+
//skip through this stream, not the wrapped one, so bytes pushed
296+
//back by peekFramePayload are honored
297+
long skipped = IOUtils.skip(this, toSkip);
289298
currentHeader = null;
290299
if (skipped < toSkip) {
291300
return false;
@@ -295,6 +304,27 @@ public boolean skipFrame() throws IOException {
295304
return false;
296305
}
297306

307+
/**
308+
* Reads up to {@code count} bytes of the current frame's payload and
309+
* pushes them back, leaving the stream position and the frame accounting
310+
* undisturbed. Used to recognize metadata-only frames (Xing/Info/VBRI).
311+
*/
312+
byte[] peekFramePayload(int count) throws IOException {
313+
byte[] buffer = new byte[count];
314+
int read = 0;
315+
while (read < count) {
316+
int r = read(buffer, read, count - read);
317+
if (r < 0) {
318+
break;
319+
}
320+
read += r;
321+
}
322+
if (read > 0) {
323+
unread(buffer, 0, read);
324+
}
325+
return read == count ? buffer : Arrays.copyOf(buffer, Math.max(read, 0));
326+
}
327+
298328
/**
299329
* Advances the underlying stream until the first byte of frame sync is
300330
* found.

tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-audiovideo-module/src/main/java/org/apache/tika/parser/mp4/TikaMp4BoxHandler.java

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,12 @@ public Mp4Handler<?> processBox(@NotNull String box, @Nullable byte[] payload,
117117
Long movieTimescale = directory.getLongObject(Mp4Directory.TAG_TIME_SCALE);
118118
return new TikaMp4MetaHandler(metadata, context, tikaMetadata,
119119
emptyEditDuration, movieTimescale == null ? 0 : movieTimescale);
120+
} else if (box.equals("hdlr") && payload != null && payload.length >= 12
121+
&& payload[8] == 's' && payload[9] == 'o'
122+
&& payload[10] == 'u' && payload[11] == 'n') {
123+
//sound track: our handler additionally reads DRM markers and the
124+
//esds average bitrate from the sample description
125+
return new TikaMp4SoundHandler(metadata, context, tikaMetadata);
120126
}
121127

122128
return super.processBox(box, payload, size, context);
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.parser.mp4;
18+
19+
import java.io.IOException;
20+
import java.nio.charset.StandardCharsets;
21+
22+
import com.drew.imaging.mp4.Mp4Handler;
23+
import com.drew.metadata.Metadata;
24+
import com.drew.metadata.mp4.Mp4Context;
25+
import com.drew.metadata.mp4.media.Mp4SoundHandler;
26+
27+
import org.apache.tika.io.EndianUtils;
28+
import org.apache.tika.metadata.Audio;
29+
30+
/**
31+
* Extends the sound track handling with what the base handler does not read
32+
* from the sample description: DRM protection markers (protected sample entry
33+
* formats such as 'drms' or 'enca') and the average bitrate from the 'esds'
34+
* elementary stream descriptor. See TIKA-4779.
35+
*/
36+
class TikaMp4SoundHandler extends Mp4SoundHandler {
37+
38+
private final org.apache.tika.metadata.Metadata tikaMetadata;
39+
40+
TikaMp4SoundHandler(Metadata metadata, Mp4Context context,
41+
org.apache.tika.metadata.Metadata tikaMetadata) {
42+
super(metadata, context);
43+
this.tikaMetadata = tikaMetadata;
44+
}
45+
46+
@Override
47+
public Mp4Handler<?> processBox(String type, byte[] payload, long boxSize,
48+
Mp4Context context) throws IOException {
49+
if ("stsd".equals(type) && payload != null) {
50+
extractFromSampleDescriptions(payload);
51+
}
52+
return super.processBox(type, payload, boxSize, context);
53+
}
54+
55+
/**
56+
* Walks the sample description entries: 4 bytes version and flags, a
57+
* 4 byte entry count, then one sample entry per count, each starting with
58+
* its own size and format fourcc.
59+
*/
60+
private void extractFromSampleDescriptions(byte[] b) {
61+
if (b.length < 8) {
62+
return;
63+
}
64+
long entryCount = EndianUtils.getUIntBE(b, 4);
65+
int pos = 8;
66+
for (long i = 0; i < entryCount && pos + 8 <= b.length; i++) {
67+
long size = EndianUtils.getUIntBE(b, pos);
68+
if (size < 16 || size > b.length - pos) {
69+
break;
70+
}
71+
int end = pos + (int) size;
72+
String format = fourCc(b, pos + 4);
73+
//protected streams replace the codec fourcc with a protected
74+
//entry format: 'drms' (FairPlay) or 'enca' (ISO common encryption)
75+
if ("drms".equals(format) || "enca".equals(format)) {
76+
tikaMetadata.set(Audio.HAS_DRM, true);
77+
}
78+
if (pos + 18 <= end) {
79+
//sample entry: 8 byte header, 6 reserved, 2 data ref index,
80+
//then version-dependent fixed sound fields before child boxes
81+
int version = EndianUtils.getUShortBE(b, pos + 16);
82+
int bitRate = findEsdsAverageBitRate(b, pos + soundEntrySize(version), end);
83+
if (bitRate > 0) {
84+
tikaMetadata.set(Audio.BITRATE, bitRate);
85+
}
86+
}
87+
pos = end;
88+
}
89+
}
90+
91+
/**
92+
* Size of the fixed part of a sound sample entry, after which the child
93+
* boxes start: 36 bytes for version 0, 52 for version 1 (four extra
94+
* 32-bit QuickTime fields), 72 for version 2.
95+
*/
96+
private static int soundEntrySize(int version) {
97+
if (version == 1) {
98+
return 52;
99+
}
100+
if (version == 2) {
101+
return 72;
102+
}
103+
return 36;
104+
}
105+
106+
/**
107+
* Scans the child boxes of a sample entry for an 'esds' box and returns
108+
* its average bitrate, or 0 if there is none. QuickTime version 1/2
109+
* entries may nest the 'esds' inside a 'wave' extension box.
110+
*/
111+
private static int findEsdsAverageBitRate(byte[] b, int pos, int end) {
112+
while (pos >= 0 && pos + 8 <= end) {
113+
long size = EndianUtils.getUIntBE(b, pos);
114+
if (size < 8 || size > end - pos) {
115+
return 0;
116+
}
117+
String type = fourCc(b, pos + 4);
118+
if ("esds".equals(type)) {
119+
return readEsdsAverageBitRate(b, pos + 8, pos + (int) size);
120+
}
121+
if ("wave".equals(type)) {
122+
int nested = findEsdsAverageBitRate(b, pos + 8, pos + (int) size);
123+
if (nested > 0) {
124+
return nested;
125+
}
126+
}
127+
pos += (int) size;
128+
}
129+
return 0;
130+
}
131+
132+
/**
133+
* Extracts the average bitrate from an 'esds' box body, or returns 0 if
134+
* the descriptors cannot be walked. The chain is an ES_Descriptor (tag
135+
* 0x03) with three optional fields signalled by its flags byte, followed
136+
* by a DecoderConfigDescriptor (tag 0x04) whose fixed fields end with the
137+
* maximum and average bitrates.
138+
*/
139+
private static int readEsdsAverageBitRate(byte[] b, int pos, int end) {
140+
//4 bytes version and flags, then the ES descriptor
141+
pos += 4;
142+
if (pos >= end || b[pos] != 0x03) {
143+
return 0;
144+
}
145+
pos = skipDescriptorLength(b, pos + 1);
146+
if (pos + 3 > end) {
147+
return 0;
148+
}
149+
//ES_ID (2 bytes), then a flags/priority byte announcing the
150+
//optional stream dependence, URL and OCR fields
151+
int flags = b[pos + 2] & 0xFF;
152+
pos += 3;
153+
if ((flags & 0x80) != 0) {
154+
pos += 2;
155+
}
156+
if ((flags & 0x40) != 0) {
157+
if (pos >= end) {
158+
return 0;
159+
}
160+
pos += 1 + (b[pos] & 0xFF);
161+
}
162+
if ((flags & 0x20) != 0) {
163+
pos += 2;
164+
}
165+
if (pos >= end || b[pos] != 0x04) {
166+
return 0;
167+
}
168+
pos = skipDescriptorLength(b, pos + 1);
169+
//object type (1), stream type (1), buffer size (3), max bitrate (4)
170+
pos += 9;
171+
if (pos + 4 > end) {
172+
return 0;
173+
}
174+
long averageBitRate = EndianUtils.getUIntBE(b, pos);
175+
return averageBitRate > 0 && averageBitRate <= Integer.MAX_VALUE
176+
? (int) averageBitRate : 0;
177+
}
178+
179+
/**
180+
* Skips a descriptor's variable length encoding (bytes with the high bit
181+
* set continue the length) and returns the position of the payload.
182+
*/
183+
private static int skipDescriptorLength(byte[] b, int pos) {
184+
while (pos < b.length && (b[pos] & 0x80) != 0) {
185+
pos++;
186+
}
187+
return pos + 1;
188+
}
189+
190+
private static String fourCc(byte[] b, int pos) {
191+
return new String(b, pos, 4, StandardCharsets.ISO_8859_1);
192+
}
193+
}

0 commit comments

Comments
 (0)