Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions CHANGES.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
Release 4.0.0-BETA1 - ???
BREAKING CHANGES

* tika-pipes core classes have been moved to a new module: tika-pipes-core,
and the FileSystem pipes components have moved (TIKA-4334).

* Headers are no longer injected into the body/content of MSG files (TIKA-4345). Please open
a ticket if you need this behavior across email formats.

Expand Down
5 changes: 0 additions & 5 deletions tika-app/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -63,11 +63,6 @@
<artifactId>tika-xmp</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-emitter-fs</artifactId>
<version>${project.version}</version>
</dependency>
<!-- this brings in tika-core -->
<dependency>
<groupId>${project.groupId}</groupId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ public class TikaCLIAsyncTest {
public static void setUpClass() throws Exception {
ASYNC_CONFIG = Files.createTempFile(ASYNC_OUTPUT_DIR, "async-config-", ".xml");
String xml = "<properties>" + "<async>" + "<numClients>3</numClients>" + "<tikaConfig>" + ASYNC_CONFIG.toAbsolutePath() + "</tikaConfig>" + "</async>" + "<fetchers>" +
"<fetcher class=\"org.apache.tika.pipes.fetcher.fs.FileSystemFetcher\">" + "<name>fsf</name>" + "<basePath>" + TEST_DATA_FILE.getAbsolutePath() + "</basePath>" +
"<fetcher class=\"org.apache.tika.pipes.fetcher.fs.FileSystemFetcher\">" + "<name>fsf</name>" + "<basePath>" + TEST_DATA_FILE.getAbsolutePath() +
"</basePath>" +
"</fetcher>" + "</fetchers>" + "<emitters>" + "<emitter class=\"org.apache.tika.pipes.emitter.fs.FileSystemEmitter\">" + "<name>fse</name>" + "<basePath>" +
ASYNC_OUTPUT_DIR.toAbsolutePath() + "</basePath>" + "<prettyPrint>true</prettyPrint>" + "</emitter>" + "</emitters>" +
"<pipesIterator class=\"org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator\">" + "<basePath>" + TEST_DATA_FILE.getAbsolutePath() + "</basePath>" +
Expand Down
2 changes: 1 addition & 1 deletion tika-bom/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -310,7 +310,7 @@
<!-- Tika Pipes modules -->
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-emitter-fs</artifactId>
<artifactId>tika-pipes-core</artifactId>
<version>4.0.0-SNAPSHOT</version>
</dependency>
<dependency>
Expand Down
2 changes: 1 addition & 1 deletion tika-eval/tika-eval-app/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
<dependencies>
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-core</artifactId>
<artifactId>tika-pipes-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@
import org.apache.tika.eval.core.util.ContentTags;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.core.fetcher.FetchKey;

public class ExtractComparer extends ProfilerBase {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@
import org.apache.tika.eval.app.io.ExtractReader;
import org.apache.tika.eval.app.io.ExtractReaderException;
import org.apache.tika.eval.app.io.IDBWriter;
import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.pipesiterator.CallablePipesIterator;
import org.apache.tika.pipes.pipesiterator.PipesIterator;
import org.apache.tika.pipes.core.FetchEmitTuple;
import org.apache.tika.pipes.core.pipesiterator.CallablePipesIterator;
import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
import org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator;

public class ExtractComparerRunner {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@
import org.apache.tika.eval.app.io.ExtractReader;
import org.apache.tika.eval.app.io.ExtractReaderException;
import org.apache.tika.eval.app.io.IDBWriter;
import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.pipesiterator.CallablePipesIterator;
import org.apache.tika.pipes.pipesiterator.PipesIterator;
import org.apache.tika.pipes.core.FetchEmitTuple;
import org.apache.tika.pipes.core.pipesiterator.CallablePipesIterator;
import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
import org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIterator;

public class ExtractProfileRunner {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
import org.apache.tika.eval.core.util.ContentTags;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.core.fetcher.FetchKey;

public class ExtractProfiler extends ProfilerBase {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@
import org.apache.tika.metadata.PagedText;
import org.apache.tika.metadata.Property;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.core.fetcher.FetchKey;
import org.apache.tika.sax.ToXMLContentHandler;
import org.apache.tika.utils.StringUtils;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.tika.pipes.pipesiterator.CallablePipesIterator;
import org.apache.tika.pipes.core.pipesiterator.CallablePipesIterator;
import org.apache.tika.utils.DurationFormatUtils;

public class StatusReporter implements Callable<Long> {
Expand Down
8 changes: 2 additions & 6 deletions tika-fuzzing/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-core</artifactId>
<artifactId>tika-pipes-core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
Expand All @@ -44,11 +44,7 @@
<artifactId>tika-serialization</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-emitter-fs</artifactId>
<version>${project.version}</version>
</dependency>

<dependency>
<groupId>commons-cli</groupId>
<artifactId>commons-cli</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,14 +46,14 @@
import org.apache.tika.fuzzing.general.GeneralTransformer;
import org.apache.tika.fuzzing.general.SpanSwapper;
import org.apache.tika.fuzzing.general.Truncator;
import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.PipesConfig;
import org.apache.tika.pipes.PipesParser;
import org.apache.tika.pipes.PipesResult;
import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.fetcher.FetcherManager;
import org.apache.tika.pipes.pipesiterator.PipesIterator;
import org.apache.tika.pipes.core.FetchEmitTuple;
import org.apache.tika.pipes.core.PipesConfig;
import org.apache.tika.pipes.core.PipesParser;
import org.apache.tika.pipes.core.PipesResult;
import org.apache.tika.pipes.core.emitter.EmitKey;
import org.apache.tika.pipes.core.fetcher.FetchKey;
import org.apache.tika.pipes.core.fetcher.FetcherManager;
import org.apache.tika.pipes.core.pipesiterator.PipesIterator;

public class FuzzingCLI {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.tika.pipes.fetcher.AbstractFetcher;
import org.apache.tika.pipes.fetcher.config.AbstractConfig;
import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;

public class ExpiringFetcherStore implements AutoCloseable {
private static final Logger LOG = LoggerFactory.getLogger(ExpiringFetcherStore.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,15 @@
import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.PipesClient;
import org.apache.tika.pipes.PipesConfig;
import org.apache.tika.pipes.PipesResult;
import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.fetcher.config.AbstractConfig;
import org.apache.tika.pipes.fetcher.config.FetcherConfigContainer;
import org.apache.tika.pipes.core.FetchEmitTuple;
import org.apache.tika.pipes.core.PipesClient;
import org.apache.tika.pipes.core.PipesConfig;
import org.apache.tika.pipes.core.PipesResult;
import org.apache.tika.pipes.core.emitter.EmitKey;
import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
import org.apache.tika.pipes.core.fetcher.FetchKey;
import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
import org.apache.tika.pipes.core.fetcher.config.FetcherConfigContainer;
import org.apache.tika.utils.XMLReaderUtils;

class TikaGrpcServerImpl extends TikaGrpc.TikaImplBase {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@

import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.fetcher.AbstractFetcher;
import org.apache.tika.pipes.fetcher.config.AbstractConfig;
import org.apache.tika.pipes.core.fetcher.AbstractFetcher;
import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;

class ExpiringFetcherStoreTest {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@
import org.apache.tika.SaveFetcherReply;
import org.apache.tika.SaveFetcherRequest;
import org.apache.tika.TikaGrpc;
import org.apache.tika.pipes.PipesResult;
import org.apache.tika.pipes.core.PipesResult;
import org.apache.tika.pipes.fetcher.fs.FileSystemFetcher;

@ExtendWith(GrpcCleanupExtension.class)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@
import org.testcontainers.utility.DockerImageName;

import org.apache.tika.cli.TikaCLI;
import org.apache.tika.pipes.HandlerConfig;
import org.apache.tika.pipes.core.HandlerConfig;
import org.apache.tika.utils.SystemUtils;

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,15 +50,15 @@
import org.apache.tika.exception.TikaConfigException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.HandlerConfig;
import org.apache.tika.pipes.emitter.Emitter;
import org.apache.tika.pipes.emitter.EmitterManager;
import org.apache.tika.pipes.core.HandlerConfig;
import org.apache.tika.pipes.core.emitter.Emitter;
import org.apache.tika.pipes.core.emitter.EmitterManager;
import org.apache.tika.pipes.emitter.opensearch.JsonResponse;
import org.apache.tika.pipes.emitter.opensearch.OpenSearchEmitter;

@Testcontainers(disabledWithoutDocker = true)
public class OpenSearchTest {
private static DockerImageName OPENSEARCH_IMAGE = DockerImageName.parse("opensearchproject/opensearch:2.17.1");
private static DockerImageName OPENSEARCH_IMAGE = DockerImageName.parse("opensearchproject/opensearch:2.19.3");
private static OpensearchContainer<?> CONTAINER;

protected static final String TEST_INDEX = "tika-pipes-index";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,14 @@

import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.emitter.Emitter;
import org.apache.tika.pipes.emitter.EmitterManager;
import org.apache.tika.pipes.core.FetchEmitTuple;
import org.apache.tika.pipes.core.emitter.Emitter;
import org.apache.tika.pipes.core.emitter.EmitterManager;
import org.apache.tika.pipes.core.fetcher.Fetcher;
import org.apache.tika.pipes.core.fetcher.FetcherManager;
import org.apache.tika.pipes.core.pipesiterator.CallablePipesIterator;
import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
import org.apache.tika.pipes.emitter.s3.S3Emitter;
import org.apache.tika.pipes.fetcher.Fetcher;
import org.apache.tika.pipes.fetcher.FetcherManager;
import org.apache.tika.pipes.pipesiterator.CallablePipesIterator;
import org.apache.tika.pipes.pipesiterator.PipesIterator;

@Disabled("turn these into actual tests with mock s3")
public class PipeIntegrationTests {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
import org.testcontainers.shaded.org.hamcrest.Matchers;

import org.apache.tika.cli.TikaCLI;
import org.apache.tika.pipes.HandlerConfig;
import org.apache.tika.pipes.core.HandlerConfig;

@TestInstance(TestInstance.Lifecycle.PER_CLASS)
@Testcontainers(disabledWithoutDocker = true)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ public class TikaPipesSolr9Test extends TikaPipesSolrTestBase {

@Override
public String getSolrImageName() {
return "solr:9.7.0";
return "solr:9.9.0";
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@
import org.testcontainers.utility.DockerImageName;

import org.apache.tika.cli.TikaCLI;
import org.apache.tika.pipes.HandlerConfig;
import org.apache.tika.pipes.core.HandlerConfig;
import org.apache.tika.pipes.emitter.solr.SolrEmitter;
import org.apache.tika.utils.SystemUtils;

Expand Down
1 change: 1 addition & 0 deletions tika-pipes/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
<packaging>pom</packaging>

<modules>
<module>tika-pipes-core</module>
<module>tika-httpclient-commons</module>
<module>tika-fetchers</module>
<module>tika-emitters</module>
Expand Down
8 changes: 1 addition & 7 deletions tika-pipes/tika-async-cli/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-core</artifactId>
<artifactId>tika-pipes-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
Expand All @@ -57,12 +57,6 @@
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-emitter-fs</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-serialization</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,9 +30,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.async.AsyncProcessor;
import org.apache.tika.pipes.pipesiterator.PipesIterator;
import org.apache.tika.pipes.core.FetchEmitTuple;
import org.apache.tika.pipes.core.async.AsyncProcessor;
import org.apache.tika.pipes.core.pipesiterator.PipesIterator;

public class TikaAsyncCLI {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,13 +36,13 @@
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.FetchEmitTuple;
import org.apache.tika.pipes.HandlerConfig;
import org.apache.tika.pipes.async.AsyncProcessor;
import org.apache.tika.pipes.emitter.EmitKey;
import org.apache.tika.pipes.extractor.EmbeddedDocumentBytesConfig;
import org.apache.tika.pipes.fetcher.FetchKey;
import org.apache.tika.pipes.pipesiterator.PipesIterator;
import org.apache.tika.pipes.core.FetchEmitTuple;
import org.apache.tika.pipes.core.HandlerConfig;
import org.apache.tika.pipes.core.async.AsyncProcessor;
import org.apache.tika.pipes.core.emitter.EmitKey;
import org.apache.tika.pipes.core.extractor.EmbeddedDocumentBytesConfig;
import org.apache.tika.pipes.core.fetcher.FetchKey;
import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
import org.apache.tika.serialization.JsonMetadataList;

/**
Expand Down
1 change: 0 additions & 1 deletion tika-pipes/tika-emitters/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
<packaging>pom</packaging>

<modules>
<module>tika-emitter-fs</module>
<module>tika-emitter-s3</module>
<module>tika-emitter-kafka</module>
<module>tika-emitter-solr</module>
Expand Down
2 changes: 1 addition & 1 deletion tika-pipes/tika-emitters/tika-emitter-az-blob/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>tika-core</artifactId>
<artifactId>tika-pipes-core</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
</dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,9 @@
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.emitter.AbstractEmitter;
import org.apache.tika.pipes.emitter.StreamEmitter;
import org.apache.tika.pipes.emitter.TikaEmitterException;
import org.apache.tika.pipes.core.emitter.AbstractEmitter;
import org.apache.tika.pipes.core.emitter.StreamEmitter;
import org.apache.tika.pipes.core.emitter.TikaEmitterException;
import org.apache.tika.serialization.JsonMetadataList;
import org.apache.tika.utils.StringUtils;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,8 @@

import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
import org.apache.tika.pipes.emitter.Emitter;
import org.apache.tika.pipes.emitter.EmitterManager;
import org.apache.tika.pipes.core.emitter.Emitter;
import org.apache.tika.pipes.core.emitter.EmitterManager;

@Disabled("turn into an actual test")
public class TestAZBlobEmitter {
Expand Down
Loading