Skip to content

Commit 88a6fc5

Browse files
tballisontballison
authored
TIKA-4752 -- improve zip name detection (#2869)
1 parent ffd7129 commit 88a6fc5

4 files changed

Lines changed: 252 additions & 9 deletions

File tree

‎tika-core/src/main/java/org/apache/tika/detect/MetadataCharsetDetector.java‎

Lines changed: 23 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@
2525
import org.apache.tika.config.TikaComponent;
2626
import org.apache.tika.io.TikaInputStream;
2727
import org.apache.tika.metadata.Metadata;
28+
import org.apache.tika.metadata.TikaCoreProperties;
2829
import org.apache.tika.mime.MediaType;
2930
import org.apache.tika.parser.ParseContext;
3031

@@ -33,10 +34,13 @@
3334
* reading any bytes from the stream. Returns a single
3435
* {@link EncodingResult.ResultType#DECLARATIVE} result when a charset is found.
3536
*
36-
* <p>Two metadata keys are consulted in order:
37+
* <p>Three metadata keys are consulted in order:
3738
* <ol>
3839
* <li>{@link Metadata#CONTENT_TYPE} — the {@code charset} parameter of the
3940
* HTTP/MIME Content-Type header (e.g. {@code text/html; charset=UTF-8}).</li>
41+
* <li>{@link TikaCoreProperties#CONTENT_TYPE_HINT} — the {@code charset} parameter
42+
* of a content-type a source <em>claimed</em> for the bytes (e.g. an HTML
43+
* {@code <meta>} tag, or a zip entry's UTF-8 (EFS) flag). A hint, not a verdict.</li>
4044
* <li>{@link Metadata#CONTENT_ENCODING} — a bare charset label set by parsers
4145
* such as {@code RFC822Parser}, which splits Content-Type into a bare
4246
* media-type key and a separate charset key.</li>
@@ -56,6 +60,9 @@ public class MetadataCharsetDetector implements EncodingDetector {
5660
public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
5761
ParseContext context) throws IOException {
5862
Charset cs = charsetFromContentType(metadata);
63+
if (cs == null) {
64+
cs = charsetFromContentTypeHint(metadata);
65+
}
5966
if (cs == null) {
6067
cs = charsetFromContentEncoding(metadata);
6168
}
@@ -71,16 +78,28 @@ public List<EncodingResult> detect(TikaInputStream tis, Metadata metadata,
7178
* {@link Metadata#CONTENT_TYPE} value, or {@code null} if absent or unparseable.
7279
*/
7380
public static Charset charsetFromContentType(Metadata metadata) {
74-
String contentType = metadata.get(Metadata.CONTENT_TYPE);
81+
return charsetFromMediaType(metadata.get(Metadata.CONTENT_TYPE));
82+
}
83+
84+
/**
85+
* Returns the charset named in the {@code charset} parameter of the
86+
* {@link TikaCoreProperties#CONTENT_TYPE_HINT} value — a content-type a source
87+
* claimed for the bytes (HTML {@code <meta>}, a zip entry's UTF-8 flag, ...) —
88+
* or {@code null} if absent or unparseable.
89+
*/
90+
public static Charset charsetFromContentTypeHint(Metadata metadata) {
91+
return charsetFromMediaType(metadata.get(TikaCoreProperties.CONTENT_TYPE_HINT));
92+
}
93+
94+
private static Charset charsetFromMediaType(String contentType) {
7595
if (contentType == null) {
7696
return null;
7797
}
7898
MediaType mediaType = MediaType.parse(contentType);
7999
if (mediaType == null) {
80100
return null;
81101
}
82-
String label = mediaType.getParameters().get("charset");
83-
return parseCharset(label);
102+
return parseCharset(mediaType.getParameters().get("charset"));
84103
}
85104

86105
/**

‎tika-core/src/test/java/org/apache/tika/detect/MetadataCharsetDetectorTest.java‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.detect;
18+
19+
import static org.junit.jupiter.api.Assertions.assertEquals;
20+
import static org.junit.jupiter.api.Assertions.assertTrue;
21+
22+
import java.io.IOException;
23+
import java.nio.charset.Charset;
24+
import java.nio.charset.StandardCharsets;
25+
import java.util.List;
26+
27+
import org.junit.jupiter.api.Test;
28+
29+
import org.apache.tika.io.TikaInputStream;
30+
import org.apache.tika.metadata.Metadata;
31+
import org.apache.tika.metadata.TikaCoreProperties;
32+
import org.apache.tika.parser.ParseContext;
33+
34+
public class MetadataCharsetDetectorTest {
35+
36+
private final MetadataCharsetDetector detector = new MetadataCharsetDetector();
37+
38+
private Charset detect(Metadata metadata) throws IOException {
39+
try (TikaInputStream tis = TikaInputStream.get(new byte[0])) {
40+
List<EncodingResult> results = detector.detect(tis, metadata, new ParseContext());
41+
if (results.isEmpty()) {
42+
return null;
43+
}
44+
assertEquals(EncodingResult.ResultType.DECLARATIVE, results.get(0).getResultType());
45+
return results.get(0).getCharset();
46+
}
47+
}
48+
49+
@Test
50+
public void testContentTypeHint() throws Exception {
51+
// TIKA-4752: the charset claimed via CONTENT_TYPE_HINT (e.g. a zip entry's
52+
// UTF-8/EFS flag, recorded as text/plain; charset=UTF-8) is consumed.
53+
Metadata m = new Metadata();
54+
m.set(TikaCoreProperties.CONTENT_TYPE_HINT, "text/plain; charset=UTF-8");
55+
assertEquals(StandardCharsets.UTF_8, detect(m));
56+
}
57+
58+
@Test
59+
public void testContentType() throws Exception {
60+
Metadata m = new Metadata();
61+
// ISO-8859-1 normalizes to its windows-1252 superset (WHATWG), existing behavior.
62+
m.set(Metadata.CONTENT_TYPE, "text/html; charset=ISO-8859-1");
63+
assertEquals(Charset.forName("windows-1252"), detect(m));
64+
}
65+
66+
@Test
67+
public void testContentEncoding() throws Exception {
68+
Metadata m = new Metadata();
69+
m.set(Metadata.CONTENT_ENCODING, "Shift_JIS");
70+
assertEquals(Charset.forName("Shift_JIS"), detect(m));
71+
}
72+
73+
@Test
74+
public void testContentTypeWinsOverHint() throws Exception {
75+
Metadata m = new Metadata();
76+
m.set(Metadata.CONTENT_TYPE, "text/plain; charset=UTF-16");
77+
m.set(TikaCoreProperties.CONTENT_TYPE_HINT, "text/plain; charset=UTF-8");
78+
assertEquals(StandardCharsets.UTF_16, detect(m));
79+
}
80+
81+
@Test
82+
public void testHintWinsOverContentEncoding() throws Exception {
83+
Metadata m = new Metadata();
84+
m.set(TikaCoreProperties.CONTENT_TYPE_HINT, "text/plain; charset=UTF-8");
85+
m.set(Metadata.CONTENT_ENCODING, "Shift_JIS");
86+
assertEquals(StandardCharsets.UTF_8, detect(m));
87+
}
88+
89+
@Test
90+
public void testNoDeclarationIsEmpty() throws Exception {
91+
assertEquals(null, detect(new Metadata()));
92+
// A content-type with no charset parameter is not a declaration.
93+
Metadata m = new Metadata();
94+
m.set(Metadata.CONTENT_TYPE, "text/plain");
95+
assertEquals(null, detect(m));
96+
// An unparseable charset label is ignored, not thrown.
97+
Metadata bad = new Metadata();
98+
bad.set(Metadata.CONTENT_ENCODING, "not-a-charset");
99+
assertTrue(detect(bad) == null);
100+
}
101+
}

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/main/java/org/apache/tika/parser/pkg/ZipParser.java‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.io.IOException;
2323
import java.io.InputStream;
2424
import java.nio.charset.Charset;
25+
import java.nio.charset.StandardCharsets;
2526
import java.nio.file.attribute.FileTime;
2627
import java.util.ArrayList;
2728
import java.util.Collections;
@@ -468,7 +469,7 @@ private void parseZipFileEntry(ZipFile zipFile, ZipArchiveEntry entry,
468469
ZipParserConfig config)
469470
throws SAXException, IOException, TikaException {
470471

471-
String name = detectEntryName(entry, parentMetadata, context, config);
472+
String name = detectEntryName(entry, context, config);
472473

473474
if (entry.getGeneralPurposeBit().usesEncryption()) {
474475
handleEncryptedEntry(name, parentMetadata, xhtml);
@@ -513,7 +514,7 @@ private void parseStreamEntry(ZipArchiveInputStream zis, ZipArchiveEntry entry,
513514
ZipParserConfig config)
514515
throws SAXException, IOException, TikaException {
515516

516-
String name = detectEntryName(entry, parentMetadata, context, config);
517+
String name = detectEntryName(entry, context, config);
517518

518519
if (!zis.canReadEntryData(entry)) {
519520
if (entry.getGeneralPurposeBit().usesEncryption()) {
@@ -549,22 +550,37 @@ private void parseStreamEntry(ZipArchiveInputStream zis, ZipArchiveEntry entry,
549550
}
550551
}
551552

552-
private String detectEntryName(ZipArchiveEntry entry, Metadata parentMetadata,
553-
ParseContext context, ZipParserConfig config) throws IOException {
553+
private String detectEntryName(ZipArchiveEntry entry, ParseContext context,
554+
ZipParserConfig config) throws IOException {
554555
// If user specified an encoding, decode raw bytes with that charset
555556
// This avoids needing to reopen the ZipFile with a different charset
556557
if (config.getEntryEncoding() != null) {
557558
return new String(entry.getRawName(), config.getEntryEncoding());
558559
}
559560

561+
// A zip only ever declares a name as UTF-8 (it can't name a legacy charset),
562+
// two ways. The Unicode extra field carries a CRC-validated UTF-8 name -- that
563+
// CRC check is the evaluation, so trust commons-compress's getName().
564+
if (entry.getNameSource() == ZipArchiveEntry.NameSource.UNICODE_EXTRA_FIELD) {
565+
return entry.getName();
566+
}
567+
560568
// If charset detection is enabled, try to detect and decode.
561569
// Mojibuster handles short inputs natively (zip filenames are often
562570
// 9-30 bytes); no byte-extension trick needed.
563571
if (config.isDetectCharsetsInEntryNames()) {
564572
byte[] entryName = entry.getRawName();
573+
// The EFS flag (general purpose bit 11) also declares UTF-8, but is
574+
// unvalidated. Record it as a content-type hint for the detector to
575+
// evaluate against the bytes, not trust outright.
576+
Metadata nameMetadata = new Metadata();
577+
if (entry.getNameSource() == ZipArchiveEntry.NameSource.NAME_WITH_EFS_FLAG) {
578+
nameMetadata.set(TikaCoreProperties.CONTENT_TYPE_HINT,
579+
new MediaType(MediaType.TEXT_PLAIN, StandardCharsets.UTF_8).toString());
580+
}
565581
try (TikaInputStream detectStream = TikaInputStream.get(entryName)) {
566582
List<EncodingResult> encResults =
567-
getEncodingDetector().detect(detectStream, parentMetadata, context);
583+
getEncodingDetector(context).detect(detectStream, nameMetadata, context);
568584
Charset candidate = encResults.isEmpty() ? null : encResults.get(0).getDecodeAs();
569585
if (candidate != null) {
570586
return new String(entry.getRawName(), candidate);

‎tika-parsers/tika-parsers-standard/tika-parsers-standard-modules/tika-parser-pkg-module/src/test/java/org/apache/tika/parser/pkg/ZipEntryNameEncodingTest.java‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
package org.apache.tika.parser.pkg;
18+
19+
import static org.junit.jupiter.api.Assertions.assertEquals;
20+
21+
import java.io.ByteArrayOutputStream;
22+
import java.io.IOException;
23+
import java.nio.charset.Charset;
24+
import java.nio.charset.StandardCharsets;
25+
import java.util.List;
26+
27+
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
28+
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
29+
import org.junit.jupiter.api.Test;
30+
31+
import org.apache.tika.TikaTest;
32+
import org.apache.tika.detect.CompositeEncodingDetector;
33+
import org.apache.tika.detect.EncodingDetector;
34+
import org.apache.tika.detect.MetadataCharsetDetector;
35+
import org.apache.tika.detect.OverrideEncodingDetector;
36+
import org.apache.tika.io.TikaInputStream;
37+
import org.apache.tika.metadata.Metadata;
38+
import org.apache.tika.metadata.TikaCoreProperties;
39+
import org.apache.tika.parser.ParseContext;
40+
41+
/**
42+
* TIKA-4752: a zip can only declare an entry name as UTF-8 (never a legacy charset),
43+
* two ways -- the EFS flag (general purpose bit 11) and the Unicode path extra field.
44+
* ZipParser must honor both.
45+
*/
46+
public class ZipEntryNameEncodingTest extends TikaTest {
47+
48+
private static final String LATIN = "café-Köln-Süß.txt";
49+
private static final String CJK = "日本語.txt";
50+
51+
@Test
52+
public void testEfsFlagHint() throws Exception {
53+
// Deterministic + discriminating: MetadataCharsetDetector consumes the
54+
// EFS->UTF-8 hint; the override garbles anything it doesn't catch. So only the
55+
// hint yields UTF-8 -- an empty-returning detector wouldn't isolate it, because
56+
// ZipParser would fall back to getName(), already UTF-8 for a flagged entry.
57+
ParseContext context = new ParseContext();
58+
context.set(EncodingDetector.class, new CompositeEncodingDetector(List.of(
59+
new MetadataCharsetDetector(),
60+
new OverrideEncodingDetector(Charset.forName("windows-1252")))));
61+
assertEquals(LATIN, entryName(efsZip(LATIN), context));
62+
}
63+
64+
@Test
65+
public void testUnicodeExtraField() throws Exception {
66+
// CRC-validated UTF-8 name in the extra field; the main-header name is a garbled
67+
// CP437 fallback. We must use the extra-field name, not detect the raw bytes.
68+
assertEquals(CJK, entryName(unicodeExtraFieldZip(CJK), new ParseContext()));
69+
}
70+
71+
private String entryName(byte[] zipBytes, ParseContext context) throws Exception {
72+
try (TikaInputStream tis = TikaInputStream.get(zipBytes)) {
73+
List<Metadata> list = getRecursiveMetadata(tis, new Metadata(), context, false);
74+
assertEquals(2, list.size());
75+
return list.get(1).get(TikaCoreProperties.RESOURCE_NAME_KEY);
76+
}
77+
}
78+
79+
private static byte[] efsZip(String name) throws IOException {
80+
ByteArrayOutputStream bos = new ByteArrayOutputStream();
81+
try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(bos)) {
82+
zos.setEncoding("UTF-8");
83+
zos.setUseLanguageEncodingFlag(true);
84+
zos.setCreateUnicodeExtraFields(ZipArchiveOutputStream.UnicodeExtraFieldPolicy.NEVER);
85+
writeEntry(zos, name);
86+
}
87+
return bos.toByteArray();
88+
}
89+
90+
private static byte[] unicodeExtraFieldZip(String name) throws IOException {
91+
ByteArrayOutputStream bos = new ByteArrayOutputStream();
92+
try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(bos)) {
93+
zos.setEncoding("Cp437");
94+
zos.setUseLanguageEncodingFlag(false);
95+
zos.setCreateUnicodeExtraFields(ZipArchiveOutputStream.UnicodeExtraFieldPolicy.ALWAYS);
96+
writeEntry(zos, name);
97+
}
98+
return bos.toByteArray();
99+
}
100+
101+
private static void writeEntry(ZipArchiveOutputStream zos, String name) throws IOException {
102+
ZipArchiveEntry entry = new ZipArchiveEntry(name);
103+
zos.putArchiveEntry(entry);
104+
zos.write("hello".getBytes(StandardCharsets.US_ASCII));
105+
zos.closeArchiveEntry();
106+
}
107+
}

0 commit comments

Comments
 (0)