Skip to content

TIKA-4515 -- add fully recursive extraction #2363

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tballison merged 3 commits into from
Oct 14, 2025
Merged

TIKA-4515 -- add fully recursive extraction #2363

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
38f3fe5
TIKA-4515 -- add fully recursive extraction
tballison Oct 10, 2025
6cf9315
TIKA-4515 -- improve commandline, add unit tests
tballison Oct 14, 2025
6f2b5ac
TIKA-4515 -- fix commandline in integration tests
tballison Oct 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 61 additions & 0 deletions tika-app/src/main/java/org/apache/tika/cli/AsyncHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.tika.cli;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;


public class AsyncHelper {
public static String[] translateArgs(String[] args) {
List<String> argList = new ArrayList<>();
if (args.length == 2) {
if (args[0].startsWith("-Z")) {
argList.add("-Z");
argList.add("-i");
argList.add(args[1]);
argList.add("-o");
argList.add(args[1]);
return argList.toArray(new String[0]);
} else if (args[0].startsWith("-") || args[1].startsWith("-")) {
argList.add(args[0]);
argList.add(args[1]);
return argList.toArray(new String[0]);
} else {
argList.add("-i");
argList.add(args[0]);
argList.add("-o");
argList.add(args[1]);
return argList.toArray(new String[0]);
}
}
if (args.length == 3) {
if (args[0].equals("-Z") && ! args[1].startsWith("-") && ! args[2].startsWith("-")) {
argList.add("-Z");
argList.add("-i");
argList.add(args[1]);
argList.add("-o");
argList.add(args[2]);
return argList.toArray(new String[0]);
}
}
argList.addAll(Arrays.asList(args));
argList.remove("-a");
return argList.toArray(new String[0]);
}
}
107 changes: 50 additions & 57 deletions tika-app/src/main/java/org/apache/tika/cli/TikaCLI.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashMap;
Expand Down Expand Up @@ -79,7 +81,7 @@
import org.apache.tika.io.TikaInputStream;
import org.apache.tika.language.detect.LanguageHandler;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.metadata.Property;
import org.apache.tika.mime.MediaType;
import org.apache.tika.mime.MediaTypeRegistry;
import org.apache.tika.mime.MimeType;
Expand All @@ -104,6 +106,7 @@
import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;
import org.apache.tika.serialization.JsonMetadata;
import org.apache.tika.serialization.JsonMetadataList;
import org.apache.tika.utils.StringUtils;
import org.apache.tika.utils.XMLReaderUtils;
import org.apache.tika.xmp.XMPMetadata;

Expand All @@ -112,6 +115,7 @@
*/
public class TikaCLI {
private static final Logger LOG = LoggerFactory.getLogger(TikaCLI.class);
private static final Property NORMALIZED_EMBEDDED_NAME = Property.externalText("tk:normalized-embedded-name");

private final int MAX_MARK = 20 * 1024 * 1024;//20MB

Expand Down Expand Up @@ -254,16 +258,35 @@ public static void main(String[] args) throws Exception {
}

private static void async(String[] args) throws Exception {
args = AsyncHelper.translateArgs(args);
String tikaConfigPath = "";
String config = "--config=";
for (String arg : args) {
if (arg.startsWith(config)) {
tikaConfigPath = arg.substring(config.length());
TikaAsyncCLI.main(new String[]{tikaConfigPath});
return;
for (int i = 0; i < args.length - 1; i++) {
if (args[i].equals("-c")) {
tikaConfigPath = args[i + 1];
break;
}
}
if (! StringUtils.isBlank(tikaConfigPath)) {
TikaAsyncCLI.main(args);
return;
}
Path tmpConfig = null;
try {
tmpConfig = Files.createTempFile("tika-config-", ".xml");
Files.copy(TikaCLI.class.getResourceAsStream("/tika-config-default-single-file.xml"),
tmpConfig, StandardCopyOption.REPLACE_EXISTING);
List<String> argList = new ArrayList<>();
for (String arg : args) {
argList.add(arg);
}
argList.add("-c");
argList.add(tmpConfig.toAbsolutePath().toString());
TikaAsyncCLI.main(argList.toArray(new String[0]));
} finally {
if (tmpConfig != null) {
Files.delete(tmpConfig);
}
}
TikaAsyncCLI.main(args);
}

/**
Expand Down Expand Up @@ -318,6 +341,7 @@ private static TransformerHandler getTransformerHandler(OutputStream output, Str
}

private boolean testForAsync(String[] args) {

if (args.length == 2) {
if (Files.isDirectory(Paths.get(args[0]))) {
return true;
Expand All @@ -333,6 +357,9 @@ private boolean testForAsync(String[] args) {
if (arg.equals("-o") || arg.startsWith("--output")) {
return true;
}
if (arg.equals("-Z")) {
return true;
}

}
return false;
Expand Down Expand Up @@ -1076,16 +1103,18 @@ public boolean shouldParseEmbedded(Metadata metadata) {

@Override
public void parseEmbedded(TikaInputStream tis, ContentHandler contentHandler, Metadata metadata, boolean outputHtml) throws SAXException, IOException {

MediaType contentType = detector.detect(tis, metadata);

String name = metadata.get(TikaCoreProperties.RESOURCE_NAME_KEY);
Path outputFile = null;
if (name == null) {
name = "file_" + count++;
String contentType = metadata.get(Metadata.CONTENT_TYPE);
if (StringUtils.isBlank(contentType)) {
MediaType mediaType = detector.detect(tis, metadata);
if (mediaType == null) {
mediaType = MediaType.OCTET_STREAM;
}
contentType = mediaType.toString();
metadata.set(Metadata.CONTENT_TYPE, contentType);
}
outputFile = getOutputFile(name, metadata, contentType);

Path outputFile = getOutputFile(metadata);
String name = metadata.get(NORMALIZED_EMBEDDED_NAME);

Path parent = outputFile.getParent();
if (parent != null && ! Files.isDirectory(parent)) {
Expand All @@ -1110,33 +1139,14 @@ public void parseEmbedded(TikaInputStream tis, ContentHandler contentHandler, Me
}
}

private Path getOutputFile(String name, Metadata metadata, MediaType contentType) throws IOException {
String ext = getExtension(contentType);
if (name.indexOf('.') == -1 && contentType != null) {
name += ext;
}

String relID = metadata.get(TikaCoreProperties.EMBEDDED_RELATIONSHIP_ID);
if (relID != null && !name.startsWith(relID)) {
name = relID + "_" + name;
}
//defensively do this so that we don't get an exception
//from FilenameUtils.normalize
name = name.replaceAll("\u0000", " ");
String normalizedName = FilenameUtils.normalize(name);

private Path getOutputFile(Metadata metadata) throws IOException {
String normalizedName = org.apache.tika.io.FilenameUtils.getSanitizedEmbeddedFilePath(metadata, ".bin", 50);
if (normalizedName == null) {
normalizedName = FilenameUtils.getName(name);
String ext = org.apache.tika.io.FilenameUtils.calculateExtension(metadata, ".bin");
normalizedName = "file-" + count++ + ext;
}
metadata.set(NORMALIZED_EMBEDDED_NAME, normalizedName);

if (normalizedName == null) {
normalizedName = "file" + count++ + ext;
}
//strip off initial C:/ or ~/ or /
int prefixLength = FilenameUtils.getPrefixLength(normalizedName);
if (prefixLength > -1) {
normalizedName = normalizedName.substring(prefixLength);
}
Path outputFile = extractDir.resolve(normalizedName);
//if file already exists, prepend uuid
if (Files.exists(outputFile)) {
Expand All @@ -1149,23 +1159,6 @@ private Path getOutputFile(String name, Metadata metadata, MediaType contentType
return outputFile;
}

private String getExtension(MediaType contentType) {
try {
String ext = config
.getMimeRepository()
.forName(contentType.toString())
.getExtension();
if (ext == null) {
return ".bin";
} else {
return ext;
}
} catch (MimeTypeException e) {
LOG.info("bad mime type?", e);
}
return ".bin";

}
}

private class NoDocumentJSONMetHandler extends DefaultHandler {
Expand Down
2 changes: 1 addition & 1 deletion tika-app/src/test/java/org/apache/tika/cli/TikaCLIAsyncTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ private void resetContent() throws Exception {

@Test
public void testAsync() throws Exception {
String content = getParamOutContent("-a", "--config=" + ASYNC_CONFIG.toAbsolutePath());
String content = getParamOutContent("-a", "-c", ASYNC_CONFIG.toAbsolutePath().toString());

int json = 0;
for (File f : ASYNC_OUTPUT_DIR
Expand Down
87 changes: 81 additions & 6 deletions tika-app/src/test/java/org/apache/tika/cli/TikaCLITest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,17 @@
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.HashSet;
import java.util.Set;

import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
Expand Down Expand Up @@ -265,6 +273,19 @@ public void testMacros() throws Exception {
assertTrue(json.contains("Module1"));
}

@Test
public void testRUnpack() throws Exception {
String[] expectedChildren = new String[]{
"testPDFPackage.pdf.json",
//the first two test that the default single file config is working
"testPDFPackage.pdf-embed/00000001-embedded-1",
"testPDFPackage.pdf-embed/00000002-image0.jpg",
"testPDFPackage.pdf-embed/00000003-PDF1.pdf",
"testPDFPackage.pdf-embed/00000004-PDF2.pdf"};
testRecursiveUnpack("testPDFPackage.pdf", expectedChildren, 2);
}


/**
* Tests -l option of the cli
*
Expand Down Expand Up @@ -311,7 +332,7 @@ public void testListSupportedTypes() throws Exception {

@Test
public void testExtractSimple() throws Exception {
String[] expectedChildren = new String[]{"MBD002B040A.cdx", "file_4.png", "MBD002B0FA6.bin", "MBD00262FE3.txt", "file_0.emf"};
String[] expectedChildren = new String[]{"MBD002B040A.cdx", "file-4.png", "MBD002B0FA6.bin", "MBD00262FE3.txt", "file-0.emf"};
testExtract("/coffee.xls", expectedChildren, 8);
}

Expand All @@ -323,7 +344,7 @@ public void testExtractAbsolute() throws Exception {

@Test
public void testExtractRelative() throws Exception {
String[] expectedChildren = new String[]{"touch.pl",};
String[] expectedChildren = new String[]{"dangerous/dont/touch.pl",};
testExtract("testZip_relative.zip", expectedChildren);
}

Expand All @@ -340,6 +361,60 @@ public void testExtract0x00() throws Exception {
testExtract("testZip_zeroByte.zip", expectedChildren);
}


private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFileNames) throws Exception {
testRecursiveUnpack(targetFile, expectedChildrenFileNames, expectedChildrenFileNames.length);
}

private void testRecursiveUnpack(String targetFile, String[] expectedChildrenFileNames, int expectedLength) throws Exception {
Path input = Paths.get(new URI(resourcePrefix + "/" + targetFile));
String[] params = {"-Z",
ProcessUtils.escapeCommandLine(input.toAbsolutePath().toString()),
ProcessUtils.escapeCommandLine(extractDir
.toAbsolutePath()
.toString())};

TikaCLI.main(params);
Set<String> fileNames = getFileNames(extractDir);
String[] jsonFile = extractDir
.toFile()
.list();
assertNotNull(jsonFile);
assertEquals(expectedLength, jsonFile.length);
//assertEquals(fileNames.size(), expectedChildrenFileNames.length);

for (String expectedChildName : expectedChildrenFileNames) {
assertTrue(fileNames.contains(expectedChildName));
}
}

private Set<String> getFileNames(Path extractDir) throws IOException {
final Set<String> names = new HashSet<>();
Files.walkFileTree(extractDir, new FileVisitor<Path>() {
@Override
public @NotNull FileVisitResult preVisitDirectory(Path path, @NotNull BasicFileAttributes basicFileAttributes) throws IOException {
return FileVisitResult.CONTINUE;
}

@Override
public @NotNull FileVisitResult visitFile(Path path, @NotNull BasicFileAttributes basicFileAttributes) throws IOException {
names.add(extractDir.relativize(path).toString());
return FileVisitResult.CONTINUE;
}

@Override
public @NotNull FileVisitResult visitFileFailed(Path path, @NotNull IOException e) throws IOException {
return FileVisitResult.CONTINUE;
}

@Override
public @NotNull FileVisitResult postVisitDirectory(Path path, @Nullable IOException e) throws IOException {
return FileVisitResult.CONTINUE;
}
});
return names;
}

private void testExtract(String targetFile, String[] expectedChildrenFileNames) throws Exception {
testExtract(targetFile, expectedChildrenFileNames, expectedChildrenFileNames.length);
}
Expand Down Expand Up @@ -399,10 +474,10 @@ public void testZipWithSubdirs() throws Exception {
new File("subdir/foo.txt").delete();
new File("subdir").delete();
String content = getParamOutContent("-z", "--extract-dir=target", resourcePrefix + "testWithSubdirs.zip");
assertTrue(content.contains("Extracting 'subdir/foo.txt'"));
//assertTrue(content.contains("Extracting 'subdir/foo.txt'"));
// clean up. TODO: These should be in target.
new File("target/subdir/foo.txt").delete();
new File("target/subdir").delete();
assertTrue(new File("target/subdir/foo.txt").delete());
assertTrue(new File("target/subdir").delete());
}

@Test
Expand All @@ -420,7 +495,7 @@ public void testExtractInlineImages() throws Exception {
Path jpeg = extractDir.resolve("image0.jpg");
//tiff isn't extracted without optional image dependency
// File tiff = new File(tempFile, "image1.tif");
Path jobOptions = extractDir.resolve("Press Quality(1).joboptions");
Path jobOptions = extractDir.resolve("Press Quality(1).joboptions.txt");
Path doc = extractDir.resolve("Unit10.doc");

assertExtracted(jpeg, allFiles);
Expand Down
Loading