Example JSON configuration:
+ *
+ * {
+ * "emitters": {
+ * "az-blob-emitter": {
+ * "my-azure": {
+ * "endpoint": "https://myaccount.blob.core.windows.net",
+ * "sasToken": "sv=2020-08-04&ss=b...",
+ * "container": "my-container",
+ * "prefix": "output",
+ * "fileExtension": "json",
+ * "overwriteExisting": false
+ * }
+ * }
+ * }
+ * }
+ *
*/
-
-public class AZBlobEmitter extends AbstractEmitter implements Initializable, StreamEmitter {
+public class AZBlobEmitter extends AbstractStreamEmitter implements StreamEmitter {
private static final Logger LOGGER = LoggerFactory.getLogger(AZBlobEmitter.class);
- private String fileExtension = "json";
- private String prefix = "";
+ private final AZBlobEmitterConfig config;
+ private final BlobContainerClient blobContainerClient;
- private String sasToken;
- private String container;
- private String endpoint;
- private BlobServiceClient blobServiceClient;
- private BlobContainerClient blobContainerClient;
- private boolean overwriteExisting = false;
+ public static AZBlobEmitter build(ExtensionConfig extensionConfig) throws TikaConfigException, IOException {
+ AZBlobEmitterConfig config = AZBlobEmitterConfig.load(extensionConfig.jsonConfig());
+ config.validate();
+ BlobContainerClient containerClient = buildContainerClient(config);
+ return new AZBlobEmitter(extensionConfig, config, containerClient);
+ }
+
+ private AZBlobEmitter(ExtensionConfig extensionConfig, AZBlobEmitterConfig config, BlobContainerClient containerClient) {
+ super(extensionConfig);
+ this.config = config;
+ this.blobContainerClient = containerClient;
+ }
+
+ private static BlobContainerClient buildContainerClient(AZBlobEmitterConfig config) {
+ BlobServiceClient blobServiceClient = new BlobServiceClientBuilder()
+ .endpoint(config.endpoint())
+ .credential(new AzureSasCredential(config.sasToken()))
+ .buildClient();
+ return blobServiceClient.getBlobContainerClient(config.container());
+ }
- /**
- * Requires the src-bucket/path/to/my/file.txt in the {@link TikaCoreProperties#SOURCE_PATH}.
- *
- * @param emitKey
- * @param metadataList
- * @param parseContext
- * @throws IOException
- * @throws TikaEmitterException
- */
@Override
- public void emit(String emitKey, ListExample JSON configuration: + *
+ * "emitters": {
+ * "az-blob-emitter": {
+ * "my-az-emitter": {
+ * "sasToken": "your-sas-token",
+ * "endpoint": "https://account.blob.core.windows.net",
+ * "container": "my-container",
+ * "prefix": "output/",
+ * "fileExtension": "json"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class AZBlobEmitterFactory implements EmitterFactory {
+
+ private static final String NAME = "az-blob-emitter";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Emitter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return AZBlobEmitter.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/java/org/apache/tika/pipes/emitter/azblob/AZBlobEmitterPlugin.java b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/java/org/apache/tika/pipes/emitter/azblob/AZBlobEmitterPlugin.java
new file mode 100644
index 00000000000..9f2fdf5f861
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/java/org/apache/tika/pipes/emitter/azblob/AZBlobEmitterPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter.azblob;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AZBlobEmitterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(AZBlobEmitterPlugin.class);
+
+ public AZBlobEmitterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting Azure Blob Emitter Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping Azure Blob Emitter Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting Azure Blob Emitter Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/resources/plugin.properties b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..b4080d9bcd3
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=az-blob-emitter
+plugin.class=org.apache.tika.pipes.emitter.azblob.AZBlobEmitterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Azure Blob Storage Emitter
+plugin.description=Capable of emitting to Azure Blob Storage
diff --git a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java
index bed2367a084..c4af50a7c20 100644
--- a/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-az-blob/src/test/java/org/apache/tika/pipes/emitter/azblob/TestAZBlobEmitter.java
@@ -16,27 +16,40 @@
*/
package org.apache.tika.pipes.emitter.azblob;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.Emitter;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
+import org.apache.tika.plugins.ExtensionConfig;
+/**
+ * This is meant only for one off development tests with a locally
+ * configured Azure Blob Storage instance. Please add unit tests to the
+ * appropriate integration test module.
+ */
@Disabled("turn into an actual test")
public class TestAZBlobEmitter {
@Test
public void testBasic() throws Exception {
- EmitterManager emitterManager = EmitterManager.load(getConfig("tika-config-az-blob.xml"));
- Emitter emitter = emitterManager.getEmitter("az-blob");
+ ObjectMapper mapper = new ObjectMapper();
+ ObjectNode configNode = mapper.createObjectNode();
+ configNode.put("endpoint", "https://myaccount.blob.core.windows.net");
+ configNode.put("sasToken", "sv=2020-08-04&ss=b...");
+ configNode.put("container", "my-container");
+ configNode.put("prefix", "output");
+ configNode.put("fileExtension", "json");
+
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-az-blob", "az-blob-emitter",
+ mapper.writeValueAsString(configNode));
+ AZBlobEmitter emitter = AZBlobEmitter.build(extensionConfig);
+
List+ * This calculates the path to write to based on the {@link FileSystemEmitterConfig#basePath()} + * and the value of the {@link TikaCoreProperties#SOURCE_PATH} value. + * + *
+ *+ */ +public class FileSystemEmitter extends AbstractStreamEmitter { + + private static final Logger LOG = LoggerFactory.getLogger(FileSystemEmitter.class); + + public static FileSystemEmitter build(ExtensionConfig pluginConfig) throws TikaConfigException, IOException { + FileSystemEmitter emitter = new FileSystemEmitter(pluginConfig); + emitter.configure(); + return emitter; + } + + private FileSystemEmitterConfig fileSystemEmitterConfig; + + public FileSystemEmitter(ExtensionConfig pluginConfig) { + super(pluginConfig); + } + + private void configure() throws TikaConfigException, IOException { + fileSystemEmitterConfig = FileSystemEmitterConfig.load(pluginConfig.jsonConfig()); + checkConfig(fileSystemEmitterConfig); + } + + private void checkConfig(FileSystemEmitterConfig fileSystemEmitterConfig) { + if (fileSystemEmitterConfig.onExists() == null) { + throw new IllegalArgumentException("Must configure 'onExists' as 'skip', 'exception' or 'replace'"); + } + } + + @Override + public void emit(String emitKey, List
Example JSON configuration: + *
+ * "emitters": {
+ * "file-system-emitter": {
+ * "my-emitter": {
+ * "basePath": "/path/to/output",
+ * "fileExtension": "json",
+ * "onExists": "SKIP",
+ * "prettyPrint": true
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class FileSystemEmitterFactory implements EmitterFactory {
+
+ private static final String NAME = "file-system-emitter";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Emitter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return FileSystemEmitter.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitterPlugin.java b/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitterPlugin.java
new file mode 100644
index 00000000000..e8bec02a50b
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/java/org/apache/tika/pipes/emitter/fs/FileSystemEmitterPlugin.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter.fs;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileSystemEmitterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(FileSystemEmitterPlugin.class);
+
+ public FileSystemEmitterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting");
+ super.delete();
+ }
+
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/resources/plugin.properties b/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..a85876524c7
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-file-system/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=file-system-emitter
+plugin.class=org.apache.tika.pipes.emitter.fs.FileSystemEmitterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Local File System Emitter
+plugin.description=Capable of emitting the local file system
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml b/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml
index db18c00d2ef..ed3d2f9ab5c 100644
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-gcs/pom.xml
@@ -27,12 +27,36 @@
Example JSON configuration:
+ *
+ * {
+ * "emitters": {
+ * "gcs-emitter": {
+ * "my-gcs": {
+ * "projectId": "my-project-id",
+ * "bucket": "my-bucket",
+ * "prefix": "output",
+ * "fileExtension": "json"
+ * }
+ * }
+ * }
+ * }
+ *
+ */
+public class GCSEmitter extends AbstractStreamEmitter {
private static final Logger LOGGER = LoggerFactory.getLogger(GCSEmitter.class);
- private String projectId;
- private String bucket;
- private String fileExtension = "json";
- private String prefix = null;
- private Storage storage;
-
- /**
- * Requires the src-bucket/path/to/my/file.txt in the {@link TikaCoreProperties#SOURCE_PATH}.
- *
- * @param metadataList
- * @throws IOException
- * @throws TikaException
- */
+
+ private final GCSEmitterConfig config;
+ private final Storage storage;
+
+ public static GCSEmitter build(ExtensionConfig extensionConfig) throws TikaConfigException, IOException {
+ GCSEmitterConfig config = GCSEmitterConfig.load(extensionConfig.jsonConfig());
+ config.validate();
+ Storage storage = buildStorage(config);
+ return new GCSEmitter(extensionConfig, config, storage);
+ }
+
+ private GCSEmitter(ExtensionConfig extensionConfig, GCSEmitterConfig config, Storage storage) {
+ super(extensionConfig);
+ this.config = config;
+ this.storage = storage;
+ }
+
+ private static Storage buildStorage(GCSEmitterConfig config) {
+ return StorageOptions.newBuilder()
+ .setProjectId(config.projectId())
+ .build()
+ .getService();
+ }
+
@Override
- public void emit(String emitKey, ListExample JSON configuration: + *
+ * "emitters": {
+ * "gcs-emitter": {
+ * "my-gcs-emitter": {
+ * "projectId": "my-project",
+ * "bucket": "my-bucket",
+ * "prefix": "output/",
+ * "fileExtension": "json"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class GCSEmitterFactory implements EmitterFactory {
+
+ private static final String NAME = "gcs-emitter";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Emitter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return GCSEmitter.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterPlugin.java b/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterPlugin.java
new file mode 100644
index 00000000000..d68a718aa13
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/java/org/apache/tika/pipes/emitter/gcs/GCSEmitterPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter.gcs;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GCSEmitterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(GCSEmitterPlugin.class);
+
+ public GCSEmitterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting GCS Emitter Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping GCS Emitter Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting GCS Emitter Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/resources/plugin.properties b/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..32029e81103
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-gcs/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=gcs-emitter
+plugin.class=org.apache.tika.pipes.emitter.gcs.GCSEmitterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Google Cloud Storage Emitter
+plugin.description=Capable of emitting to Google Cloud Storage
diff --git a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java b/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java
index 23c42aa0fb6..54f3d95b09e 100644
--- a/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java
+++ b/tika-pipes/tika-emitters/tika-emitter-gcs/src/test/java/org/apache/tika/pipes/emitter/gcs/TestGCSEmitter.java
@@ -16,27 +16,39 @@
*/
package org.apache.tika.pipes.emitter.gcs;
-import java.net.URISyntaxException;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.Emitter;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
+import org.apache.tika.plugins.ExtensionConfig;
+/**
+ * This is meant only for one off development tests with a locally
+ * configured GCS instance. Please add unit tests to the appropriate
+ * integration test module.
+ */
@Disabled("turn into an actual test")
public class TestGCSEmitter {
@Test
public void testBasic() throws Exception {
- EmitterManager emitterManager = EmitterManager.load(getConfig("tika-config-gcs.xml"));
- Emitter emitter = emitterManager.getEmitter("gcs");
+ ObjectMapper mapper = new ObjectMapper();
+ ObjectNode configNode = mapper.createObjectNode();
+ configNode.put("projectId", "my-project");
+ configNode.put("bucket", "my-bucket");
+ configNode.put("prefix", "output");
+ configNode.put("fileExtension", "json");
+
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-gcs", "gcs-emitter",
+ mapper.writeValueAsString(configNode));
+ GCSEmitter emitter = GCSEmitter.build(extensionConfig);
+
ListExample JSON configuration:
+ *
+ * {
+ * "emitters": {
+ * "jdbc-emitter": {
+ * "my-db": {
+ * "connection": "jdbc:postgresql://localhost/mydb",
+ * "createTable": "CREATE TABLE IF NOT EXISTS docs (path VARCHAR(1024), content TEXT)",
+ * "insert": "INSERT INTO docs (path, content) VALUES (?, ?)",
+ * "keys": {
+ * "tika:content": "string"
+ * },
+ * "attachmentStrategy": "FIRST_ONLY",
+ * "multivaluedFieldStrategy": "CONCATENATE"
+ * }
+ * }
+ * }
+ * }
+ *
* + * This is only an initial, basic implementation of an emitter for JDBC. * It is currently NOT thread safe because of the shared prepared statement, * and depending on the jdbc implementation because of the shared connection. - *
- * As of the 2.5.0 release, this is ALPHA version. There may be breaking changes - * in the future. + *
*/ -public class JDBCEmitter extends AbstractEmitter implements Initializable, Closeable { +public class JDBCEmitter extends AbstractEmitter implements Closeable { private static final Logger LOGGER = LoggerFactory.getLogger(JDBCEmitter.class); - public enum AttachmentStrategy { - FIRST_ONLY, ALL - //anything else? - } - - public enum MultivaluedFieldStrategy { - FIRST_ONLY, CONCATENATE - //anything else? - } - - //some file formats do not have time zones... - //try both private static final String[] TIKA_DATE_PATTERNS = - new String[] {"yyyy-MM-dd'T'HH:mm:ss'Z'", "yyyy-MM-dd'T'HH:mm:ss"}; - //the "write" lock is used for creating the table - private static ReadWriteLock READ_WRITE_LOCK = new ReentrantReadWriteLock(); - //this keeps track of which table + connection string have been created - //so that only one table is created per table + connection string. - //This is necessary for testing and if someone specifies multiple - //different jdbc emitters. - private static Set- * The default values as of 2.6.1 are {@link MultivaluedFieldStrategy#CONCATENATE} - * and the default delimiter is ", " - * - * @param strategy - * @throws TikaConfigException - */ - @Field - public void setMultivaluedFieldStrategy(String strategy) throws TikaConfigException { - String lc = strategy.toLowerCase(Locale.US); - if (lc.equals("first_only")) { - setMultivaluedFieldStrategy(MultivaluedFieldStrategy.FIRST_ONLY); - } else if (lc.equals("concatenate")) { - setMultivaluedFieldStrategy(MultivaluedFieldStrategy.CONCATENATE); - } else { - throw new TikaConfigException("I'm sorry, I only recogize 'first_only' and " + - "'concatenate'. I don't mind '" + strategy + "'"); + private void initialize() throws TikaConfigException { + try { + createConnection(); + } catch (SQLException e) { + throw new TikaConfigException("couldn't open connection: " + config.connection(), e); } - } - - public void setMultivaluedFieldStrategy(MultivaluedFieldStrategy multivaluedFieldStrategy) { - this.multivaluedFieldStrategy = multivaluedFieldStrategy; - } - /** - * See {@link JDBCEmitter#setMultivaluedFieldDelimiter(String)} - * - * @param delimiter - */ - @Field - public void setMultivaluedFieldDelimiter(String delimiter) { - this.multivaluedFieldDelimiter = delimiter; - } + if (!StringUtils.isBlank(config.createTable())) { + READ_WRITE_LOCK.writeLock().lock(); + try { + String tableCreationString = config.connection() + " " + config.createTable(); + if (!TABLES_CREATED.contains(tableCreationString)) { + try (Statement st = connection.createStatement()) { + st.execute(config.createTable()); + if (!StringUtils.isBlank(config.alterTable())) { + st.execute(config.alterTable()); + } + TABLES_CREATED.add(tableCreationString); + } catch (SQLException e) { + throw new TikaConfigException("can't create table", e); + } + } + } finally { + READ_WRITE_LOCK.writeLock().unlock(); + } + } - /** - * The implementation of keys should be a LinkedHashMap because - * order matters! - *
- * Key is the name of the metadata field, value is the type of column:
- * boolean, string, int, long
- *
- * @param keys
- */
- @Field
- public void setKeys(Map Example JSON configuration:
+ * Example JSON configuration: Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration: Example JSON configuration:
+ * Example JSON configuration: Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ *
- * Default size is -1.
- *
- * @param maxSpoolSize
- */
- @Field
- public void setMaxSpoolSize(long maxSpoolSize) {
- httpFetcherConfig.setMaxSpoolSize(maxSpoolSize);
- }
-
- @Field
- public void setMaxRedirects(int maxRedirects) {
- httpFetcherConfig.setMaxRedirects(maxRedirects);
- }
-
- /**
- * Which http request headers should we send in the http fetch requests.
- *
- * @param headers The headers to add to the HTTP GET requests.
- */
- @Field
- public void setHttpRequestHeaders(List Example JSON configuration:
+ *
+ * "emitters": {
+ * "jdbc-emitter": {
+ * "my-jdbc-emitter": {
+ * "connection": "jdbc:postgresql://localhost/mydb",
+ * "insert": "insert into docs (id, content) values (?, ?)",
+ * "keys": {"id": "X-TIKA:content", "content": "content"}
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class JDBCEmitterFactory implements EmitterFactory {
+
+ private static final String NAME = "jdbc-emitter";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Emitter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return JDBCEmitter.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterPlugin.java b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterPlugin.java
new file mode 100644
index 00000000000..fb55f3e7bb8
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter.jdbc;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JDBCEmitterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(JDBCEmitterPlugin.class);
+
+ public JDBCEmitterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting JDBC Emitter Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping JDBC Emitter Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting JDBC Emitter Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/resources/plugin.properties b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..2c7bcaca7bf
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=jdbc-emitter
+plugin.class=org.apache.tika.pipes.emitter.jdbc.JDBCEmitterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=JDBC Emitter
+plugin.description=Capable of emitting to JDBC databases
diff --git a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
index 44848419748..a5195e79022 100644
--- a/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
+++ b/tika-pipes/tika-emitters/tika-emitter-jdbc/src/test/java/org/apache/tika/pipes/emitter/jdbc/JDBCEmitterTest.java
@@ -20,8 +20,6 @@
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
@@ -32,32 +30,81 @@
import java.time.ZoneId;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
+import java.util.Map;
-import org.apache.commons.io.IOUtils;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.emitter.Emitter;
-import org.apache.tika.pipes.core.emitter.EmitterManager;
+import org.apache.tika.plugins.ExtensionConfig;
public class JDBCEmitterTest {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ /**
+ * Helper method to create a config ObjectNode with properly ordered keys.
+ * Uses valueToTree to preserve LinkedHashMap ordering.
+ */
+ private ObjectNode createConfigNode(String connection, String insert, String createTable,
+ String alterTable, String attachmentStrategy, Map
+ * {
+ * "emitters": {
+ * "kafka-emitter": {
+ * "my-kafka": {
+ * "topic": "tika-output",
+ * "bootstrapServers": "localhost:9092",
+ * "acks": "all",
+ * "lingerMs": 5000
+ * }
+ * }
+ * }
+ * }
+ *
*/
-public class KafkaEmitter extends AbstractEmitter implements Initializable {
+public class KafkaEmitter extends AbstractEmitter {
private static final Logger LOGGER = LoggerFactory.getLogger(KafkaEmitter.class);
-
private static final ObjectMapper OM = new ObjectMapper();
- String topic;
- String bootstrapServers;
-
- String acks = "all";
- int lingerMs = 5000;
- int batchSize = 16384;
- int bufferMemory = 32 * 1024 * 1024;
- String compressionType = "none";
- int connectionsMaxIdleMs = 9 * 60 * 1000;
- int deliveryTimeoutMs = 120 * 1000;
- boolean enableIdempotence = false;
- String interceptorClasses;
- int maxBlockMs = 60 * 1000;
- int maxInFlightRequestsPerConnection = 5;
- int maxRequestSize = 1024 * 1024;
- int metadataMaxAgeMs = 5 * 60 * 1000;
- int requestTimeoutMs = 30 * 1000;
- int retries = Integer.MAX_VALUE;
- int retryBackoffMs = 100;
- int transactionTimeoutMs = 60000;
- String transactionalId;
- String clientId;
- String keySerializer;
- String valueSerializer;
-
- private Producer
+ * "emitters": {
+ * "kafka-emitter": {
+ * "my-kafka-emitter": {
+ * "topic": "my-topic",
+ * "bootstrapServers": "localhost:9092",
+ * "acks": "all",
+ * "lingerMs": 5000
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class KafkaEmitterFactory implements EmitterFactory {
+
+ private static final String NAME = "kafka-emitter";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Emitter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return KafkaEmitter.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/java/org/apache/tika/pipes/emitter/kafka/KafkaEmitterPlugin.java b/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/java/org/apache/tika/pipes/emitter/kafka/KafkaEmitterPlugin.java
new file mode 100644
index 00000000000..57c5cb27a33
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/java/org/apache/tika/pipes/emitter/kafka/KafkaEmitterPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter.kafka;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class KafkaEmitterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaEmitterPlugin.class);
+
+ public KafkaEmitterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting Kafka Emitter Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping Kafka Emitter Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting Kafka Emitter Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/resources/plugin.properties b/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..4bbbcaf33f6
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-kafka/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=kafka-emitter
+plugin.class=org.apache.tika.pipes.emitter.kafka.KafkaEmitterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Apache Kafka Emitter
+plugin.description=Capable of emitting to Apache Kafka topics
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml b/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml
index afba8919d55..00137848172 100644
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-opensearch/pom.xml
@@ -27,11 +27,17 @@
+ * "emitters": {
+ * "opensearch-emitter": {
+ * "my-opensearch-emitter": {
+ * "openSearchUrl": "http://localhost:9200/my-index",
+ * "idField": "id",
+ * "attachmentStrategy": "PARENT_CHILD",
+ * "updateStrategy": "UPSERT",
+ * "commitWithin": 1000
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class OpenSearchEmitterFactory implements EmitterFactory {
+
+ public static final String NAME = "opensearch-emitter";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Emitter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return OpenSearchEmitter.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchEmitterPlugin.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchEmitterPlugin.java
new file mode 100644
index 00000000000..3b07102bf50
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchEmitterPlugin.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter.opensearch;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OpenSearchEmitterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(OpenSearchEmitterPlugin.class);
+
+ public OpenSearchEmitterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting");
+ super.delete();
+ }
+
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/resources/plugin.properties b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..29ab46e97d3
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=opensearch-emitter
+plugin.class=org.apache.tika.pipes.emitter.opensearch.OpenSearchEmitterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=OpenSearch emitter
+plugin.description=Capable of emitting to OpenSearch
diff --git a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java
index 994ff00440e..b097c3bd8a1 100644
--- a/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java
+++ b/tika-pipes/tika-emitters/tika-emitter-opensearch/src/test/java/org/apache/tika/pipes/emitter/opensearch/OpenSearchClientTest.java
@@ -29,8 +29,8 @@ public void testSerialization() throws Exception {
metadata.add("authors", "author1");
metadata.add("authors", "author2");
metadata.add("title", "title1");
- for (OpenSearchEmitter.AttachmentStrategy strategy :
- OpenSearchEmitter.AttachmentStrategy.values()) {
+ for (OpenSearchEmitterConfig.AttachmentStrategy strategy :
+ OpenSearchEmitterConfig.AttachmentStrategy.values()) {
String json = OpenSearchClient.metadataToJsonContainerInsert(metadata,
strategy);
assertContains("author1", json);
@@ -38,8 +38,8 @@ public void testSerialization() throws Exception {
assertContains("authors", json);
assertContains("title1", json);
}
- for (OpenSearchEmitter.AttachmentStrategy strategy :
- OpenSearchEmitter.AttachmentStrategy.values()) {
+ for (OpenSearchEmitterConfig.AttachmentStrategy strategy :
+ OpenSearchEmitterConfig.AttachmentStrategy.values()) {
String json = OpenSearchClient.metadataToJsonEmbeddedInsert(metadata, strategy,
"myEmitKey", OpenSearchEmitter.DEFAULT_EMBEDDED_FILE_FIELD_NAME);
assertContains("author1", json);
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml b/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
index f8acd036dd3..e5b29c2f6de 100644
--- a/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-s3/pom.xml
@@ -28,14 +28,31 @@
- * <properties>
- * <emitters>
- * <emitter class="org.apache.tika.pipes.emitter.s3.S3Emitter>
- * <params>
- * <!-- required -->
- * <param name="name" type="string">s3e</param>
- * <!-- required -->
- * <param name="region" type="string">us-east-1</param>
- * <!-- required -->
- * <param name="credentialsProvider"
- * type="string">(profile|instance)</param>
- * <!-- required if credentialsProvider=profile-->
- * <param name="profile" type="string">my-profile</param>
- * <!-- required -->
- * <param name="bucket" type="string">my-bucket</param>
- * <!-- optional; prefix to add to the path before emitting;
- * default is no prefix -->
- * <param name="prefix" type="string">my-prefix</param>
- * <!-- optional; default is 'json' this will be added to the SOURCE_PATH
- * if no emitter key is specified. Do not add a "."
- * before the extension -->
- * <param name="fileExtension" type="string">json</param>
- * <!-- optional; default is 'true'-- whether to copy the
- * json to a local file before putting to s3 -->
- * <param name="spoolToTemp" type="bool">true</param>
- * </params>
- * </emitter>
- * </emitters>
- * </properties>
+ * Emitter to write to an existing S3 bucket.
+ *
+ *
+ * {
+ * "emitters": {
+ * "s3-emitter": {
+ * "my-s3": {
+ * "region": "us-east-1",
+ * "bucket": "my-bucket",
+ * "credentialsProvider": "profile",
+ * "profile": "my-profile",
+ * "prefix": "output",
+ * "fileExtension": "json",
+ * "spoolToTemp": true
+ * }
+ * }
+ * }
+ * }
+ *
*/
-public class S3Emitter extends AbstractEmitter implements Initializable, StreamEmitter {
+public class S3Emitter extends AbstractStreamEmitter {
private static final Logger LOGGER = LoggerFactory.getLogger(S3Emitter.class);
- private String region;
- private String profile;
- private String bucket;
- private String credentialsProvider;
- private String accessKey;
- private String secretKey;
- private String endpointConfigurationService;
- private String fileExtension = "json";
- private boolean spoolToTemp = true;
- private String prefix = null;
- private int maxConnections = SdkHttpConfigurationOption.GLOBAL_HTTP_DEFAULTS.get(SdkHttpConfigurationOption.MAX_CONNECTIONS);
- private boolean pathStyleAccessEnabled = false;
- private S3Client s3Client;
- /**
- * Requires the src-bucket/path/to/my/file.txt in the {@link TikaCoreProperties#SOURCE_PATH}.
- *
- * @param emitKey
- * @param metadataList
- * @param parseContext
- * @throws IOException
- * @throws TikaEmitterException
- */
+ private final S3EmitterConfig config;
+ private final S3Client s3Client;
+
+ public static S3Emitter build(ExtensionConfig extensionConfig) throws TikaConfigException, IOException {
+ S3EmitterConfig config = S3EmitterConfig.load(extensionConfig.jsonConfig());
+ config.validate();
+ S3Client s3Client = buildS3Client(config);
+ return new S3Emitter(extensionConfig, config, s3Client);
+ }
+
+ private S3Emitter(ExtensionConfig extensionConfig, S3EmitterConfig config, S3Client s3Client) {
+ super(extensionConfig);
+ this.config = config;
+ this.s3Client = s3Client;
+ }
+
+ private static S3Client buildS3Client(S3EmitterConfig config) throws TikaConfigException {
+ AwsCredentialsProvider provider = buildCredentialsProvider(config);
+ SdkHttpClient httpClient = ApacheHttpClient.builder()
+ .maxConnections(config.maxConnections())
+ .build();
+ S3Configuration clientConfig = S3Configuration.builder()
+ .pathStyleAccessEnabled(config.pathStyleAccessEnabled())
+ .build();
+
+ try {
+ S3ClientBuilder s3ClientBuilder = S3Client.builder()
+ .httpClient(httpClient)
+ .requestChecksumCalculation(RequestChecksumCalculation.WHEN_REQUIRED)
+ .serviceConfiguration(clientConfig)
+ .credentialsProvider(provider);
+
+ if (!StringUtils.isBlank(config.endpointConfigurationService())) {
+ try {
+ s3ClientBuilder.endpointOverride(new URI(config.endpointConfigurationService()))
+ .region(Region.of(config.region()));
+ } catch (URISyntaxException ex) {
+ throw new TikaConfigException("bad endpointConfigurationService: " +
+ config.endpointConfigurationService(), ex);
+ }
+ } else {
+ s3ClientBuilder.region(Region.of(config.region()));
+ }
+ return s3ClientBuilder.build();
+ } catch (SdkClientException e) {
+ throw new TikaConfigException("can't initialize s3 emitter", e);
+ }
+ }
+
+ private static AwsCredentialsProvider buildCredentialsProvider(S3EmitterConfig config) throws TikaConfigException {
+ switch (config.credentialsProvider()) {
+ case "instance":
+ return InstanceProfileCredentialsProvider.builder().build();
+ case "profile":
+ return ProfileCredentialsProvider.builder()
+ .profileName(config.profile())
+ .build();
+ case "key_secret":
+ AwsBasicCredentials awsCreds = AwsBasicCredentials.create(
+ config.accessKey(), config.secretKey());
+ return StaticCredentialsProvider.create(awsCreds);
+ default:
+ throw new TikaConfigException("credentialsProvider must be 'instance', 'profile' or 'key_secret'");
+ }
+ }
+
@Override
- public void emit(String emitKey, Listtrue. If this is set to false,
- * this emitter writes the json object to memory and then puts that into s3.
- *
- * @param spoolToTemp
- */
- @Field
- public void setSpoolToTemp(boolean spoolToTemp) {
- this.spoolToTemp = spoolToTemp;
- }
-
- @Field
- public void setRegion(String region) {
- this.region = region;
- }
-
- @Field
- public void setProfile(String profile) {
- this.profile = profile;
- }
-
- @Field
- public void setBucket(String bucket) {
- this.bucket = bucket;
- }
-
- @Field
- public void setPrefix(String prefix) {
- //strip final "/" if it exists
- if (prefix.endsWith("/")) {
- this.prefix = prefix.substring(0, prefix.length() - 1);
- } else {
- this.prefix = prefix;
- }
- }
-
- @Field
- public void setCredentialsProvider(String credentialsProvider) {
- if (!credentialsProvider.equals("profile") && !credentialsProvider.equals("instance") && !credentialsProvider.equals("key_secret")) {
- throw new IllegalArgumentException("credentialsProvider must be either 'profile', 'instance' or 'key_secret'");
- }
- this.credentialsProvider = credentialsProvider;
- }
-
- /**
- * If you want to customize the output file's file extension.
- * Do not include the "."
- *
- * @param fileExtension
- */
- @Field
- public void setFileExtension(String fileExtension) {
- this.fileExtension = fileExtension;
- }
-
- @Field
- public void setAccessKey(String accessKey) {
- this.accessKey = accessKey;
- }
-
- @Field
- public void setSecretKey(String secretKey) {
- this.secretKey = secretKey;
- }
-
- /**
- * maximum number of http connections allowed. This should be
- * greater than or equal to the number of threads emitting to S3.
- *
- * @param maxConnections
- */
- @Field
- public void setMaxConnections(int maxConnections) {
- this.maxConnections = maxConnections;
- }
-
- @Field
- public void setEndpointConfigurationService(String endpointConfigurationService) {
- this.endpointConfigurationService = endpointConfigurationService;
- }
-
- /**
- * This initializes the s3 client. Note, we wrap S3's RuntimeExceptions,
- * e.g. SdkClientException in a TikaConfigException.
- *
- * @param params params to use for initialization
- * @throws TikaConfigException
- */
- @Override
- public void initialize(Map
+ * "emitters": {
+ * "s3-emitter": {
+ * "my-s3-emitter": {
+ * "region": "us-east-1",
+ * "bucket": "my-bucket",
+ * "credentialsProvider": "profile",
+ * "profile": "default",
+ * "prefix": "output/",
+ * "fileExtension": "json"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class S3EmitterFactory implements EmitterFactory {
+
+ private static final String NAME = "s3-emitter";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Emitter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return S3Emitter.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3EmitterPlugin.java b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3EmitterPlugin.java
new file mode 100644
index 00000000000..22e9b43a213
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/java/org/apache/tika/pipes/emitter/s3/S3EmitterPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter.s3;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class S3EmitterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(S3EmitterPlugin.class);
+
+ public S3EmitterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting S3 Emitter Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping S3 Emitter Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting S3 Emitter Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-s3/src/main/resources/plugin.properties b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..8fd980ec8a4
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-s3/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=s3-emitter
+plugin.class=org.apache.tika.pipes.emitter.s3.S3EmitterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Amazon S3 Emitter
+plugin.description=Capable of emitting to Amazon S3
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
index a840895ae93..4fcb6c5ffc7 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/pom.xml
@@ -27,11 +27,25 @@
+ * {
+ * "emitters": {
+ * "solr-emitter": {
+ * "my-solr": {
+ * "solrCollection": "my-collection",
+ * "solrUrls": ["http://localhost:8983/solr"],
+ * "idField": "id",
+ * "commitWithin": 1000,
+ * "attachmentStrategy": "PARENT_CHILD",
+ * "updateStrategy": "ADD"
+ * }
+ * }
+ * }
+ * }
+ *
+ */
+public class SolrEmitter extends AbstractEmitter {
-public class SolrEmitter extends AbstractEmitter implements Initializable {
-
- public static String DEFAULT_EMBEDDED_FILE_FIELD_NAME = "embedded";
+ public static final String DEFAULT_EMBEDDED_FILE_FIELD_NAME = "embedded";
private static final Logger LOG = LoggerFactory.getLogger(SolrEmitter.class);
- private final HttpClientFactory httpClientFactory;
- private AttachmentStrategy attachmentStrategy = AttachmentStrategy.PARENT_CHILD;
- private UpdateStrategy updateStrategy = UpdateStrategy.ADD;
- private String solrCollection;
- /**
- * You can specify solrUrls, or you can specify solrZkHosts and use use zookeeper to determine the solr server urls.
- */
- private List
+ * "emitters": {
+ * "solr-emitter": {
+ * "my-solr-emitter": {
+ * "solrCollection": "my-collection",
+ * "solrUrls": ["http://localhost:8983/solr"],
+ * "idField": "id",
+ * "commitWithin": 1000,
+ * "attachmentStrategy": "PARENT_CHILD"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class SolrEmitterFactory implements EmitterFactory {
+
+ private static final String NAME = "solr-emitter";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Emitter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return SolrEmitter.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitterPlugin.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitterPlugin.java
new file mode 100644
index 00000000000..f14624b44ee
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/java/org/apache/tika/pipes/emitter/solr/SolrEmitterPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.emitter.solr;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SolrEmitterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(SolrEmitterPlugin.class);
+
+ public SolrEmitterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting Solr Emitter Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping Solr Emitter Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting Solr Emitter Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/main/resources/plugin.properties b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..b45385375f6
--- /dev/null
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=solr-emitter
+plugin.class=org.apache.tika.pipes.emitter.solr.SolrEmitterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Apache Solr Emitter
+plugin.description=Capable of emitting to Apache Solr
diff --git a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java
index 1e68b1d4e4b..a0fba59e9ae 100644
--- a/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java
+++ b/tika-pipes/tika-emitters/tika-emitter-solr/src/test/java/org/apache/tika/pipes/emitter/solr/SolrEmitterDevTest.java
@@ -21,6 +21,9 @@
import java.util.HashMap;
import java.util.Map;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@@ -28,6 +31,7 @@
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.metadata.filter.FieldNameMappingFilter;
import org.apache.tika.parser.ParseContext;
+import org.apache.tika.plugins.ExtensionConfig;
/**
* This is meant only for one off development tests with a locally
@@ -42,16 +46,22 @@ public void oneOff() throws Exception {
String core = "tika-example";
String url = "http://localhost:8983/solr";
String emitKey = "one-off-test-doc";
- SolrEmitter solrEmitter = new SolrEmitter();
- solrEmitter.setSolrUrls(Collections.singletonList(url));
- solrEmitter.setSolrCollection(core);
- solrEmitter.initialize(Collections.EMPTY_MAP);
+
+ ObjectMapper mapper = new ObjectMapper();
+ ObjectNode configNode = mapper.createObjectNode();
+ configNode.put("solrCollection", core);
+ ArrayNode urlsNode = configNode.putArray("solrUrls");
+ urlsNode.add(url);
+
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-solr", "solr-emitter",
+ mapper.writeValueAsString(configNode));
+ SolrEmitter solrEmitter = SolrEmitter.build(extensionConfig);
Metadata metadata = new Metadata();
metadata.set(TikaCoreProperties.CREATED, new Date());
metadata.set(TikaCoreProperties.TIKA_CONTENT, "the quick brown fox");
- Map
+ * "fetchers": {
+ * "az-blob-fetcher": {
+ * "my-az-fetcher": {
+ * "sasToken": "your-sas-token",
+ * "endpoint": "https://account.blob.core.windows.net",
+ * "container": "my-container",
+ * "extractUserMetadata": true
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class AZBlobFetcherFactory implements FetcherFactory {
+
+ public static final String NAME = "az-blob-fetcher";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Fetcher buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return AZBlobFetcher.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcherPlugin.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcherPlugin.java
new file mode 100644
index 00000000000..38490200758
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/AZBlobFetcherPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher.azblob;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AZBlobFetcherPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(AZBlobFetcherPlugin.class);
+
+ public AZBlobFetcherPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting Azure Blob Fetcher Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping Azure Blob Fetcher Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting Azure Blob Fetcher Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java
index e29e623198a..d62a512a058 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/java/org/apache/tika/pipes/fetcher/azblob/config/AZBlobFetcherConfig.java
@@ -16,57 +16,68 @@
*/
package org.apache.tika.pipes.fetcher.azblob.config;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
-public class AZBlobFetcherConfig extends AbstractConfig {
- private boolean spoolToTemp;
+import org.apache.tika.exception.TikaConfigException;
+
+public class AZBlobFetcherConfig {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public static AZBlobFetcherConfig load(final String json)
+ throws TikaConfigException {
+ try {
+ return OBJECT_MAPPER.readValue(json, AZBlobFetcherConfig.class);
+ } catch (JsonProcessingException e) {
+ throw new TikaConfigException(
+ "Failed to parse AZBlobFetcherConfig from JSON", e);
+ }
+ }
+
+ private boolean spoolToTemp = true;
private String sasToken;
private String endpoint;
private String container;
- private boolean extractUserMetadata;
+ private boolean extractUserMetadata = true;
public boolean isSpoolToTemp() {
return spoolToTemp;
}
- public AZBlobFetcherConfig setSpoolToTemp(boolean spoolToTemp) {
+ public void setSpoolToTemp(boolean spoolToTemp) {
this.spoolToTemp = spoolToTemp;
- return this;
}
public String getSasToken() {
return sasToken;
}
- public AZBlobFetcherConfig setSasToken(String sasToken) {
+ public void setSasToken(String sasToken) {
this.sasToken = sasToken;
- return this;
}
public String getEndpoint() {
return endpoint;
}
- public AZBlobFetcherConfig setEndpoint(String endpoint) {
+ public void setEndpoint(String endpoint) {
this.endpoint = endpoint;
- return this;
}
public String getContainer() {
return container;
}
- public AZBlobFetcherConfig setContainer(String container) {
+ public void setContainer(String container) {
this.container = container;
- return this;
}
public boolean isExtractUserMetadata() {
return extractUserMetadata;
}
- public AZBlobFetcherConfig setExtractUserMetadata(boolean extractUserMetadata) {
+ public void setExtractUserMetadata(boolean extractUserMetadata) {
this.extractUserMetadata = extractUserMetadata;
- return this;
}
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/resources/plugin.properties b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..4cfb4b8f247
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=az-blob-fetcher
+plugin.class=org.apache.tika.pipes.fetcher.azblob.AZBlobFetcherPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Azure Blob Fetcher
+plugin.description=Capable of fetching files from Azure Blob Storage
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java
index bbfc63c65ae..2b7e39d8083 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-az-blob/src/test/java/org/apache/tika/pipes/fetcher/azblob/TestAZBlobFetcher.java
@@ -16,40 +16,43 @@
*/
package org.apache.tika.pipes.fetcher.azblob;
-
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
-import java.nio.file.Paths;
import java.util.List;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.apache.tika.TikaTest;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.Fetcher;
-import org.apache.tika.pipes.core.fetcher.FetcherManager;
+import org.apache.tika.plugins.ExtensionConfig;
import org.apache.tika.serialization.JsonMetadataList;
@Disabled("write actual unit tests")
public class TestAZBlobFetcher extends TikaTest {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String FETCH_STRING = "something-or-other/test-out.json";
@Test
public void testConfig() throws Exception {
- FetcherManager fetcherManager = FetcherManager.load(Paths.get(this
- .getClass()
- .getResource("/tika-config-az-blob.xml")
- .toURI()));
- Fetcher fetcher = fetcherManager.getFetcher("az-blob");
+ ObjectNode jsonConfig = OBJECT_MAPPER.createObjectNode();
+ jsonConfig.put("endpoint", "https://myaccount.blob.core.windows.net");
+ jsonConfig.put("container", "my-container");
+ jsonConfig.put("sasToken", "my-sas-token");
+
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-az-blob-fetcher", "az-blob-fetcher",
+ OBJECT_MAPPER.writeValueAsString(jsonConfig));
+ AZBlobFetcher fetcher = AZBlobFetcher.build(extensionConfig);
+
List{@code
+ * "file-system-fetcher": {
+ * "basePath": "BASE_PATH",
+ * "extractFileSystemMetadata": false
+ * }
+ * }
+ *
+ */
- public FileSystemFetcher(FileSystemFetcherConfig fileSystemFetcherConfig) {
- setBasePath(fileSystemFetcherConfig.getBasePath());
- setExtractFileSystemMetadata(fileSystemFetcherConfig.isExtractFileSystemMetadata());
- }
+public class FileSystemFetcher extends AbstractTikaExtension implements Fetcher {
private static final Logger LOG = LoggerFactory.getLogger(FileSystemFetcher.class);
- //Warning! basePath can be null!
- private Path basePath = null;
+ private FileSystemFetcherConfig defaultFileSystemFetcherConfig;
- private boolean extractFileSystemMetadata = false;
-
- static boolean isDescendant(Path root, Path descendant) {
- return descendant.toAbsolutePath().normalize()
- .startsWith(root.toAbsolutePath().normalize());
+ public FileSystemFetcher(ExtensionConfig pluginConfig) throws TikaConfigException {
+ super(pluginConfig);
+ defaultFileSystemFetcherConfig = FileSystemFetcherConfig.load(pluginConfig.jsonConfig());
+ checkConfig(defaultFileSystemFetcherConfig);
}
@Override
@@ -73,33 +75,42 @@ public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseC
"Please review the life decisions that led you to requesting " +
"a file name with this character in it.");
}
+ FileSystemFetcherConfig config = defaultFileSystemFetcherConfig;
+ ExtensionConfigs pluginConfigManager = parseContext.get(ExtensionConfigs.class);
+ if (pluginConfigManager != null) {
+ Optionalnull if no base path was set
- */
- public Path getBasePath() {
- return basePath;
- }
-
- /**
- * Default behavior si that clients will send in relative paths, this
- * must be set to allow this fetcher to fetch the
- * full path.
- *
- * @param basePath
- */
- @Field
- public void setBasePath(String basePath) {
- this.basePath = Paths.get(basePath);
- }
-
- /**
- * Extract file system metadata (created, modified, accessed) when fetching file.
- * The default is false.
- *
- * @param extractFileSystemMetadata
- */
- @Field
- public void setExtractFileSystemMetadata(boolean extractFileSystemMetadata) {
- this.extractFileSystemMetadata = extractFileSystemMetadata;
- }
-
- @Override
- public void initialize(Map
+ * "fetchers": {
+ * "file-system-fetcher": {
+ * "my-fetcher": {
+ * "basePath": "/path/to/files",
+ * "extractFileSystemMetadata": true
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class FileSystemFetcherFactory implements FetcherFactory {
+
+ private static final String NAME = "file-system-fetcher";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Fetcher buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return new FileSystemFetcher(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherPlugin.java b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherPlugin.java
new file mode 100644
index 00000000000..f6a457e0a70
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherPlugin.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher.fs;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileSystemFetcherPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(FileSystemFetcherPlugin.class);
+
+ public FileSystemFetcherPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting");
+ super.delete();
+ }
+
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/resources/plugin.properties b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..a317ce13771
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=file-system-fetcher
+plugin.class=org.apache.tika.pipes.fetcher.fs.FileSystemFetcherPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Local File System Fetcher
+plugin.description=Capable of emitting the local file system
diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java
similarity index 79%
rename from tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java
rename to tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java
index 5c493da5915..8c32545034e 100644
--- a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/FileSystemFetcherTest.java
@@ -20,14 +20,14 @@
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import java.nio.file.InvalidPathException;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.junit.jupiter.api.Test;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.pipes.fetcher.fs.FileSystemFetcher;
+import org.apache.tika.exception.TikaConfigException;
+import org.apache.tika.pipes.api.fetcher.Fetcher;
+import org.apache.tika.plugins.ExtensionConfig;
public class FileSystemFetcherTest {
@@ -48,11 +48,10 @@ public void testDescendant() throws Exception {
@Test
public void testNullByte() throws Exception {
- FileSystemFetcher f = new FileSystemFetcher();
- assertThrows(InvalidPathException.class, () -> {
- f.setBasePath("bad\u0000path");
- f.setName("fs");
- f.checkInitialization(InitializableProblemHandler.IGNORE);
+ assertThrows(TikaConfigException.class, () -> {
+ ExtensionConfig pluginConfig = new ExtensionConfig("test", "test",
+ "{ \"basePath\":\"bad\\u0000path\"}");
+ Fetcher f = new FileSystemFetcherFactory().buildExtension(pluginConfig);
});
}
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/config/FileSystemFetcherConfigTest.java b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/config/FileSystemFetcherConfigTest.java
new file mode 100644
index 00000000000..9277adedb56
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-file-system/src/test/java/org/apache/tika/pipes/fetcher/fs/config/FileSystemFetcherConfigTest.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher.fs.config;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import org.junit.jupiter.api.Test;
+
+import org.apache.tika.pipes.fetcher.fs.FileSystemFetcherConfig;
+
+public class FileSystemFetcherConfigTest {
+
+ @Test
+ public void testBasic() throws Exception {
+ String json = """
+ {
+ "basePath":"/some/base/path",
+ "extractFileSystemMetadata":true
+ }
+ """;
+
+ FileSystemFetcherConfig config = FileSystemFetcherConfig.load(json);
+ assertEquals("/some/base/path", config.getBasePath());
+ assertTrue(config.isExtractFileSystemMetadata());
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml
index 3305e03b4c9..e4befeef390 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-gcs/pom.xml
@@ -28,10 +28,22 @@
+ * "fetchers": {
+ * "gcs-fetcher": {
+ * "my-gcs-fetcher": {
+ * "projectId": "my-project",
+ * "bucket": "my-bucket",
+ * "spoolToTemp": true,
+ * "extractUserMetadata": true
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class GCSFetcherFactory implements FetcherFactory {
+
+ public static final String NAME = "gcs-fetcher";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Fetcher buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return GCSFetcher.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcherPlugin.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcherPlugin.java
new file mode 100644
index 00000000000..06f1448b6bc
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/GCSFetcherPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher.gcs;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GCSFetcherPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(GCSFetcherPlugin.class);
+
+ public GCSFetcherPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting GCS Fetcher Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping GCS Fetcher Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting GCS Fetcher Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java
index 6a808f1ec5f..4c25d37987b 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/java/org/apache/tika/pipes/fetcher/gcs/config/GCSFetcherConfig.java
@@ -16,47 +16,59 @@
*/
package org.apache.tika.pipes.fetcher.gcs.config;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
-public class GCSFetcherConfig extends AbstractConfig {
- private boolean spoolToTemp;
+import org.apache.tika.exception.TikaConfigException;
+
+public class GCSFetcherConfig {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public static GCSFetcherConfig load(final String json)
+ throws TikaConfigException {
+ try {
+ return OBJECT_MAPPER.readValue(json, GCSFetcherConfig.class);
+ } catch (JsonProcessingException e) {
+ throw new TikaConfigException(
+ "Failed to parse GCSFetcherConfig from JSON", e);
+ }
+ }
+
+ private boolean spoolToTemp = true;
private String projectId;
private String bucket;
- private boolean extractUserMetadata;
+ private boolean extractUserMetadata = true;
public boolean isSpoolToTemp() {
return spoolToTemp;
}
- public GCSFetcherConfig setSpoolToTemp(boolean spoolToTemp) {
+ public void setSpoolToTemp(boolean spoolToTemp) {
this.spoolToTemp = spoolToTemp;
- return this;
}
public String getProjectId() {
return projectId;
}
- public GCSFetcherConfig setProjectId(String projectId) {
+ public void setProjectId(String projectId) {
this.projectId = projectId;
- return this;
}
public String getBucket() {
return bucket;
}
- public GCSFetcherConfig setBucket(String bucket) {
+ public void setBucket(String bucket) {
this.bucket = bucket;
- return this;
}
public boolean isExtractUserMetadata() {
return extractUserMetadata;
}
- public GCSFetcherConfig setExtractUserMetadata(boolean extractUserMetadata) {
+ public void setExtractUserMetadata(boolean extractUserMetadata) {
this.extractUserMetadata = extractUserMetadata;
- return this;
}
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/resources/plugin.properties b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..e220dd80212
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=gcs-fetcher
+plugin.class=org.apache.tika.pipes.fetcher.gcs.GCSFetcherPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=GCS Fetcher
+plugin.description=Capable of fetching files from Google Cloud Storage
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java
index 3844337f3a9..f227037b369 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-gcs/src/test/java/org/apache/tika/pipes/fetcher/s3/TestGCSFetcher.java
@@ -21,9 +21,10 @@
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
@@ -31,12 +32,13 @@
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.Fetcher;
-import org.apache.tika.pipes.core.fetcher.FetcherManager;
+import org.apache.tika.pipes.fetcher.gcs.GCSFetcher;
+import org.apache.tika.plugins.ExtensionConfig;
@Disabled("write actual unit tests")
public class TestGCSFetcher {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String FETCH_STRING = "testExtraSpaces.pdf";
@TempDir
@@ -48,12 +50,16 @@ public static void setUp() throws Exception {
outputFile = Files.createTempFile(TEMP_DIR, "tika-test", ".pdf");
}
-
@Test
public void testConfig() throws Exception {
- FetcherManager fetcherManager = FetcherManager.load(
- Paths.get(this.getClass().getResource("/tika-config-gcs.xml").toURI()));
- Fetcher fetcher = fetcherManager.getFetcher("gcs");
+ ObjectNode jsonConfig = OBJECT_MAPPER.createObjectNode();
+ jsonConfig.put("projectId", "my-project");
+ jsonConfig.put("bucket", "my-bucket");
+
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-gcs-fetcher", "gcs-fetcher",
+ OBJECT_MAPPER.writeValueAsString(jsonConfig));
+ GCSFetcher fetcher = GCSFetcher.build(extensionConfig);
+
Metadata metadata = new Metadata();
try (InputStream is = fetcher.fetch(FETCH_STRING, metadata, new ParseContext())) {
Files.copy(is, outputFile, StandardCopyOption.REPLACE_EXISTING);
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
index 10c4e5ee135..3eaba79b2c2 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/pom.xml
@@ -17,7 +17,8 @@
specific language governing permissions and limitations
under the License.
-->
--1, the full stream will be spooled to a temp file
- *
+ * "fetchers": {
+ * "http-fetcher": {
+ * "my-http-fetcher": {
+ * "userName": "user",
+ * "password": "pass",
+ * "connectTimeout": 30000,
+ * "socketTimeout": 120000,
+ * "maxConnections": 200
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class HttpFetcherFactory implements FetcherFactory {
+ private static final String NAME = "http-fetcher";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Fetcher buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return HttpFetcher.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/LoggingPipesReporter.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcherPlugin.java
similarity index 62%
rename from tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/LoggingPipesReporter.java
rename to tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcherPlugin.java
index 795db75906f..ac12dc8a7e2 100644
--- a/tika-pipes/tika-pipes-core/src/main/java/org/apache/tika/pipes/core/LoggingPipesReporter.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/HttpFetcherPlugin.java
@@ -14,30 +14,36 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.tika.pipes.core;
-
+package org.apache.tika.pipes.fetcher.http;
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-/**
- * Simple PipesReporter that logs everything at the debug level.
- */
-public class LoggingPipesReporter extends PipesReporter {
- Logger LOGGER = LoggerFactory.getLogger(LoggingPipesReporter.class);
+public class HttpFetcherPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(HttpFetcherPlugin.class);
+
+ public HttpFetcherPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
@Override
- public void report(FetchEmitTuple t, PipesResult result, long elapsed) {
- LOGGER.debug("{} {} {}", t, result, elapsed);
+ public void start() {
+ LOG.info("Starting");
+ super.start();
}
@Override
- public void error(Throwable t) {
- LOGGER.error("pipes error", t);
+ public void stop() {
+ LOG.info("Stopping");
+ super.stop();
}
@Override
- public void error(String msg) {
- LOGGER.error("error {}", msg);
+ public void delete() {
+ LOG.info("Deleting");
+ super.delete();
}
+
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java
index 61d768e9c3c..36fcb82161c 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpFetcherConfig.java
@@ -19,9 +19,25 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.tika.exception.TikaConfigException;
+
+public class HttpFetcherConfig {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public static HttpFetcherConfig load(final String json)
+ throws TikaConfigException {
+ try {
+ return OBJECT_MAPPER.readValue(json, HttpFetcherConfig.class);
+ } catch (JsonProcessingException e) {
+ throw new TikaConfigException(
+ "Failed to parse HttpFetcherConfig from JSON", e);
+ }
+ }
-public class HttpFetcherConfig extends AbstractConfig {
private String userName;
private String password;
private String ntDomain;
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpHeaders.java b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpHeaders.java
index d3f27111f65..da50ad7d0a4 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpHeaders.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-http/src/main/java/org/apache/tika/pipes/fetcher/http/config/HttpHeaders.java
@@ -17,14 +17,18 @@
package org.apache.tika.pipes.fetcher.http.config;
import java.util.Collection;
+import java.util.List;
import java.util.Map;
import java.util.Objects;
+import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
+import com.fasterxml.jackson.annotation.JsonProperty;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
public class HttpHeaders {
+
@JsonIgnore
private Multimap
Example JSON configuration: + *
+ * "fetchers": {
+ * "microsoft-graph-fetcher": {
+ * "my-graph-fetcher": {
+ * "spoolToTemp": true,
+ * "scopes": ["https://graph.microsoft.com/.default"],
+ * "clientSecretCredentialsConfig": {
+ * "tenantId": "tenant-id",
+ * "clientId": "client-id",
+ * "clientSecret": "client-secret"
+ * }
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class MicrosoftGraphFetcherFactory implements FetcherFactory {
+
+ public static final String NAME = "microsoft-graph-fetcher";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Fetcher buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return MicrosoftGraphFetcher.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcherPlugin.java b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcherPlugin.java
new file mode 100644
index 00000000000..89002b6fbc0
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/MicrosoftGraphFetcherPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetchers.microsoftgraph;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MicrosoftGraphFetcherPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(MicrosoftGraphFetcherPlugin.class);
+
+ public MicrosoftGraphFetcherPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting Microsoft Graph Fetcher Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping Microsoft Graph Fetcher Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting Microsoft Graph Fetcher Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/config/MicrosoftGraphFetcherConfig.java b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/config/MicrosoftGraphFetcherConfig.java
index d2970773c6f..c4e8fd0bdac 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/config/MicrosoftGraphFetcherConfig.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/java/org/apache/tika/pipes/fetchers/microsoftgraph/config/MicrosoftGraphFetcherConfig.java
@@ -19,9 +19,27 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import org.apache.tika.exception.TikaConfigException;
+
+public class MicrosoftGraphFetcherConfig {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public static MicrosoftGraphFetcherConfig load(final String json)
+ throws TikaConfigException {
+ try {
+ return OBJECT_MAPPER.readValue(json,
+ MicrosoftGraphFetcherConfig.class);
+ } catch (JsonProcessingException e) {
+ throw new TikaConfigException(
+ "Failed to parse MicrosoftGraphFetcherConfig from JSON",
+ e);
+ }
+ }
-public class MicrosoftGraphFetcherConfig extends AbstractConfig {
private long[] throttleSeconds;
private boolean spoolToTemp;
private ClientSecretCredentialsConfig clientSecretCredentialsConfig;
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/resources/plugin.properties b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..e098a475b95
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-microsoft-graph/src/main/resources/plugin.properties
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=microsoft-graph-fetcher
+plugin.class=org.apache.tika.pipes.fetchers.microsoftgraph.MicrosoftGraphFetcherPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Microsoft Graph Fetcher
+plugin.description=Capable of fetching files from Microsoft Graph API
+
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml b/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
index 91f8323366f..1b5a333da1b 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/pom.xml
@@ -17,54 +17,114 @@
specific language governing permissions and limitations
under the License.
-->
-Example JSON configuration: + *
+ * "fetchers": {
+ * "s3-fetcher": {
+ * "my-s3-fetcher": {
+ * "region": "us-east-1",
+ * "bucket": "my-bucket",
+ * "credentialsProvider": "profile",
+ * "profile": "default",
+ * "extractUserMetadata": true
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class S3FetcherFactory implements FetcherFactory {
+
+ public static final String NAME = "s3-fetcher";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Fetcher buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return S3Fetcher.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3FetcherPlugin.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3FetcherPlugin.java
new file mode 100644
index 00000000000..c9ec0a5dbb3
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/S3FetcherPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.fetcher.s3;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class S3FetcherPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(S3FetcherPlugin.class);
+
+ public S3FetcherPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting S3 Fetcher Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping S3 Fetcher Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting S3 Fetcher Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/config/S3FetcherConfig.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/config/S3FetcherConfig.java
index 8fa70bfa018..ff443384285 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/config/S3FetcherConfig.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/java/org/apache/tika/pipes/fetcher/s3/config/S3FetcherConfig.java
@@ -16,19 +16,34 @@
*/
package org.apache.tika.pipes.fetcher.s3.config;
-import org.apache.tika.pipes.core.fetcher.config.AbstractConfig;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
-public class S3FetcherConfig extends AbstractConfig {
- private boolean spoolToTemp;
+import org.apache.tika.exception.TikaConfigException;
+
+public class S3FetcherConfig {
+
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ public static S3FetcherConfig load(final String json)
+ throws TikaConfigException {
+ try {
+ return OBJECT_MAPPER.readValue(json, S3FetcherConfig.class);
+ } catch (JsonProcessingException e) {
+ throw new TikaConfigException(
+ "Failed to parse S3FetcherConfig from JSON", e);
+ }
+ }
+
+ private boolean spoolToTemp = true;
private String region;
private String profile;
private String bucket;
- private String commaDelimitedLongs;
private String prefix;
- private boolean extractUserMetadata;
+ private boolean extractUserMetadata = true;
private int maxConnections;
private String credentialsProvider;
- private long maxLength;
+ private long maxLength = -1;
private String accessKey;
private String secretKey;
private String endpointConfigurationService;
@@ -71,15 +86,6 @@ public S3FetcherConfig setBucket(String bucket) {
return this;
}
- public String getCommaDelimitedLongs() {
- return commaDelimitedLongs;
- }
-
- public S3FetcherConfig setCommaDelimitedLongs(String commaDelimitedLongs) {
- this.commaDelimitedLongs = commaDelimitedLongs;
- return this;
- }
-
public String getPrefix() {
return prefix;
}
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/resources/plugin.properties b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..64452ca7d11
--- /dev/null
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/main/resources/plugin.properties
@@ -0,0 +1,22 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=s3-fetcher
+plugin.class=org.apache.tika.pipes.fetcher.s3.S3FetcherPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=S3 Fetcher
+plugin.description=Capable of fetching files from Amazon S3
+
diff --git a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
index d492f9a4122..7f7d912727f 100644
--- a/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
+++ b/tika-pipes/tika-fetchers/tika-fetcher-s3/src/test/java/org/apache/tika/pipes/fetcher/s3/TestS3Fetcher.java
@@ -21,18 +21,19 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
-import java.util.Collections;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
-import org.apache.tika.pipes.core.fetcher.Fetcher;
-import org.apache.tika.pipes.core.fetcher.FetcherManager;
+import org.apache.tika.plugins.ExtensionConfig;
@Disabled("write actual unit tests")
public class TestS3Fetcher {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String FETCH_STRING = "";
private final Path outputFile = Paths.get("");
private final String region = "us-east-1";
@@ -40,22 +41,15 @@ public class TestS3Fetcher {
@Test
public void testBasic() throws Exception {
- S3Fetcher fetcher = new S3Fetcher();
- fetcher.setProfile(profile);
- fetcher.setRegion(region);
- fetcher.initialize(Collections.EMPTY_MAP);
+ ObjectNode jsonConfig = OBJECT_MAPPER.createObjectNode();
+ jsonConfig.put("region", region);
+ jsonConfig.put("profile", profile);
+ jsonConfig.put("credentialsProvider", "profile");
- Metadata metadata = new Metadata();
- try (InputStream is = fetcher.fetch(FETCH_STRING, metadata, new ParseContext())) {
- Files.copy(is, outputFile, StandardCopyOption.REPLACE_EXISTING);
- }
- }
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-s3-fetcher", "s3-fetcher",
+ OBJECT_MAPPER.writeValueAsString(jsonConfig));
+ S3Fetcher fetcher = S3Fetcher.build(extensionConfig);
- @Test
- public void testConfig() throws Exception {
- FetcherManager fetcherManager = FetcherManager.load(
- Paths.get(this.getClass().getResource("/tika-config-s3.xml").toURI()));
- Fetcher fetcher = fetcherManager.getFetcher("s3");
Metadata metadata = new Metadata();
try (InputStream is = fetcher.fetch(FETCH_STRING, metadata, new ParseContext())) {
Files.copy(is, outputFile, StandardCopyOption.REPLACE_EXISTING);
diff --git a/tika-pipes/tika-pipes-api/pom.xml b/tika-pipes/tika-pipes-api/pom.xml
new file mode 100644
index 00000000000..1cfbb1e5072
--- /dev/null
+++ b/tika-pipes/tika-pipes-api/pom.xml
@@ -0,0 +1,68 @@
+
+
+
* Implementations of Fetcher must be thread safe.
*/
-public interface Fetcher {
-
- String getName();
+public interface Fetcher extends TikaExtension, ExtensionPoint {
InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws TikaException, IOException;
}
diff --git a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/FetcherFactory.java b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/FetcherFactory.java
new file mode 100644
index 00000000000..4fef27c4b78
--- /dev/null
+++ b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/fetcher/FetcherFactory.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.api.fetcher;
+
+import org.apache.tika.plugins.TikaExtensionFactory;
+
+public interface FetcherFactory extends TikaExtensionFactory
- * This does not allow multiple fetchers supporting the same prefix.
+ * This does not allow multiple emitters supporting the same prefix.
*/
-public class EmitterManager extends ConfigBase {
+public class EmitterManager {
+ public static final String CONFIG_KEY = "emitters";
+
+ private static final Logger LOG = LoggerFactory.getLogger(EmitterManager.class);
private final Map
- * This forbids multiple fetchers supporting the same name.
+ * This forbids multiple fetchers with the same pluginId
*/
-public class FetcherManager extends ConfigBase {
+public class FetcherManager {
- public static FetcherManager load(Path p) throws IOException, TikaConfigException {
- try (InputStream is =
- Files.newInputStream(p)) {
- return FetcherManager.buildComposite("fetchers", FetcherManager.class,
- "fetcher", Fetcher.class, is);
- }
+ public static final String CONFIG_KEY = "fetchers";
+ private static final Logger LOG = LoggerFactory.getLogger(FetcherManager.class);
+
+
+ public static FetcherManager load(PluginManager pluginManager, TikaConfigs tikaConfigs) throws TikaConfigException, IOException {
+ JsonNode fetchersNode = tikaConfigs.getRoot().get(CONFIG_KEY);
+ Map
+ * This forbids multiple fetchers with the same pluginId
+ */
+public class PipesIteratorManager {
+
+ public static final String CONFIG_KEY = "pipes-iterator";
+
+ public static Optional
+ * This forbids multiple fetchers with the same pluginId
+ */
+public class ReporterManager {
+
+ public static final String CONFIG_KEY = "pipes-reporters";
+
+ public static PipesReporter load(PluginManager pluginManager, TikaConfigs tikaConfigs) throws IOException, TikaConfigException {
+
+ JsonNode node = tikaConfigs.getRoot().get(CONFIG_KEY);
+
+ List
- * This calculates the path to write to based on the {@link #basePath}
- * and the value of the {@link TikaCoreProperties#SOURCE_PATH} value.
- *
- *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.core;
+
+import org.apache.tika.config.AbstractTikaConfigTest;
+
+public class TikaPipesConfigTest extends AbstractTikaConfigTest {
+ //this handles tests for the newer pipes type configs.
+/*
+ @Test
+ public void testFetchers() throws Exception {
+ FetcherManager m = FetcherManager.load(getConfigFilePath("fetchers-config.xml"));
+ Fetcher f1 = m.getFetcher("fs1");
+ assertEquals(Paths.get("/my/base/path1"), ((FileSystemFetcher) f1).getBasePath());
+
+ Fetcher f2 = m.getFetcher("fs2");
+ assertEquals(Paths.get("/my/base/path2"), ((FileSystemFetcher) f2).getBasePath());
+ }
+
+ @Test
+ public void testDuplicateFetchers() throws Exception {
+ //can't have two fetchers with the same name
+ assertThrows(TikaConfigException.class, () -> {
+ FetcherManager.load(getConfigFilePath("fetchers-duplicate-config.xml"));
+ });
+ }
+
+ @Test
+ public void testNoNameFetchers() throws Exception {
+ //can't have two fetchers with an empty name
+ assertThrows(TikaConfigException.class, () -> {
+ FetcherManager.load(getConfigFilePath("fetchers-noname-config.xml"));
+ });
+ }
+
+ @Test
+ public void testNoBasePathFetchers() throws Exception {
+ //no basepath is allowed as of > 2.3.0
+ //test that this does not throw an exception.
+
+ FetcherManager fetcherManager = FetcherManager.load(
+ getConfigFilePath("fetchers-nobasepath-config.xml"));
+ }
+
+ @Test
+ public void testEmitters() throws Exception {
+ EmitterManager emitterManager =
+ EmitterManager.load(getConfigFilePath("emitters-config.xml"));
+ Emitter em1 = emitterManager.getEmitter("file-system-emitter-1");
+ assertNotNull(em1);
+ Emitter em2 = emitterManager.getEmitter("file-system-emitter-2");
+ assertNotNull(em2);
+ }
+
+ @Test
+ public void testDuplicateEmitters() throws Exception {
+ assertThrows(TikaConfigException.class, () -> {
+ EmitterManager.load(getConfigFilePath("emitters-duplicate-config.xml"));
+ });
+ }
+
+ @Test
+ public void testPipesIterator() throws Exception {
+ PipesIteratorBase it =
+ PipesIteratorBase.build(getConfigFilePath("pipes-iterator-config.xml"));
+ assertEquals("fsf1", it.getFetcherId());
+ }
+
+ @Test
+ public void testMultiplePipesIterators() throws Exception {
+ assertThrows(TikaConfigException.class, () -> {
+ PipesIteratorBase it =
+ PipesIteratorBase.build(getConfigFilePath("pipes-iterator-multiple-config.xml"));
+ assertEquals("fsf1", it.getFetcherId());
+ });
+ }
+*/
+}
diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/async/AsyncChaosMonkeyTest.java b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/async/AsyncChaosMonkeyTest.java
similarity index 65%
rename from tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/async/AsyncChaosMonkeyTest.java
rename to tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/async/AsyncChaosMonkeyTest.java
index 87a2749aace..db14a7c3fb6 100644
--- a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/core/async/AsyncChaosMonkeyTest.java
+++ b/tika-pipes/tika-pipes-integration-tests/src/test/java/org/apache/tika/pipes/core/async/AsyncChaosMonkeyTest.java
@@ -18,12 +18,13 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
-import java.io.IOException;
+import java.io.BufferedReader;
+import java.io.File;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.sql.SQLException;
import java.util.HashSet;
+import java.util.List;
import java.util.Random;
import java.util.Set;
@@ -32,16 +33,18 @@
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.PipesResult;
-import org.apache.tika.pipes.core.emitter.EmitData;
-import org.apache.tika.pipes.core.emitter.EmitKey;
-import org.apache.tika.pipes.core.fetcher.FetchKey;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
-import org.apache.tika.utils.ProcessUtils;
+import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.pipes.api.emitter.EmitKey;
+import org.apache.tika.pipes.api.fetcher.FetchKey;
+import org.apache.tika.pipes.api.pipesiterator.PipesIterator;
+import org.apache.tika.pipes.core.PluginsTestHelper;
+import org.apache.tika.serialization.JsonMetadataList;
public class AsyncChaosMonkeyTest {
+ String fetcherPluginId = "fsf";
+ String emitterPluginId = "fse";
+
private final String OOM = "" + " Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ * Example JSON configuration:
+ *
+ * JSON structure: { "typeName": { config } }
+ *
+ * JSON structure:
+ *
+ * JSON structure: { "typeName": { config }, "typeName2": { config2 }, ... }
+ *
+ * Use this for composite components like reporters where each type appears once
+ * and instances don't need individual names.
+ *
+ * Keys prefixed with "x-" are allowed for custom extensions.
+ *
+ * @throws TikaConfigException if unknown keys are found
+ */
+ private void validateNoUnknownKeys() throws TikaConfigException {
+ Iteratortrue
* @param totalCountResult
*/
- public void report(TotalCountResult totalCountResult) {
-
- }
+ void report(TotalCountResult totalCountResult);
/**
* Override this if your reporter supports total count.
* @return false as the baseline implementation
*/
- public boolean supportsTotalCount() {
- return false;
- }
- /**
- * No-op implementation. Override for custom behavior
- * @throws IOException
- */
- @Override
- public void close() throws IOException {
- //no-op
- }
+ boolean supportsTotalCount();
/**
* This is called if the process has crashed.
* Implementers should not rely on close() to be called after this.
* @param t
*/
- public abstract void error(Throwable t);
+ void error(Throwable t);
/**
* This is called if the process has crashed.
* Implementers should not rely on close() to be called after this.
* @param msg
*/
- public abstract void error(String msg);
+ void error(String msg);
}
diff --git a/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/reporter/PipesReporterFactory.java b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/reporter/PipesReporterFactory.java
new file mode 100644
index 00000000000..6bccb484f85
--- /dev/null
+++ b/tika-pipes/tika-pipes-api/src/main/java/org/apache/tika/pipes/api/reporter/PipesReporterFactory.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.api.reporter;
+
+import org.apache.tika.plugins.TikaExtensionFactory;
+
+public interface PipesReporterFactory extends TikaExtensionFactory
- * <properties>
- * <emitters>
- * <emitter class="org.apache.tika.pipes.emitter.fs.FileSystemEmitter>
- * <params>
- * <!-- required -->
- * <param name="name" type="string">fs</param>
- * <!-- required -->
- * <param name="basePath" type="string">/path/to/output</param>
- * <!-- optional; default is 'json' -->
- * <param name="fileExtension" type="string">json</param>
- * <!-- optional; if the file already exists,
- * options ('skip', 'replace', 'exception')
- * default is 'exception' -->
- * <param name="onExists" type="string">skip</param>
- * <!-- optional; whether or not to pretty print the output
- * default is false -->
- * <param name="prettyPrint" type="boolean">true</param>
- * </params>
- * </emitter>
- * </emitters>
- * </properties>
- */
-public class FileSystemEmitter extends AbstractEmitter implements StreamEmitter {
-
- private Path basePath = null;
- private String fileExtension = "json";
- private ON_EXISTS onExists = ON_EXISTS.EXCEPTION;
-
- private boolean prettyPrint = false;
-
- @Override
- public void emit(String emitKey, List
+ * "pipes-iterator": {
+ * "az-blob-pipes-iterator": {
+ * "sasToken": "your-sas-token",
+ * "endpoint": "https://account.blob.core.windows.net",
+ * "container": "my-container",
+ * "prefix": "documents/",
+ * "baseConfig": {
+ * "fetcherId": "my-fetcher",
+ * "emitterId": "my-emitter"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class AZBlobPipesIteratorFactory implements PipesIteratorFactory {
+
+ public static final String NAME = "az-blob-pipes-iterator";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public AZBlobPipesIterator buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return AZBlobPipesIterator.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/main/java/org/apache/tika/pipes/pipesiterator/azblob/AZBlobPipesIteratorPlugin.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/main/java/org/apache/tika/pipes/pipesiterator/azblob/AZBlobPipesIteratorPlugin.java
new file mode 100644
index 00000000000..af350c26c10
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/main/java/org/apache/tika/pipes/pipesiterator/azblob/AZBlobPipesIteratorPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.pipesiterator.azblob;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class AZBlobPipesIteratorPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(AZBlobPipesIteratorPlugin.class);
+
+ public AZBlobPipesIteratorPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting Azure Blob Pipes Iterator Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping Azure Blob Pipes Iterator Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting Azure Blob Pipes Iterator Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..acb245b1ee1
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=az-blob-pipes-iterator
+plugin.class=org.apache.tika.pipes.pipesiterator.azblob.AZBlobPipesIteratorPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Azure Blob Storage Pipes Iterator
+plugin.description=Capable of iterating over Azure Blob Storage containers
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/test/java/org/apache/tika/pipes/pipesiterator/azblob/TestAZBlobPipesIterator.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/test/java/org/apache/tika/pipes/pipesiterator/azblob/TestAZBlobPipesIterator.java
index a393fee9289..fa64c3270e4 100644
--- a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/test/java/org/apache/tika/pipes/pipesiterator/azblob/TestAZBlobPipesIterator.java
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-az-blob/src/test/java/org/apache/tika/pipes/pipesiterator/azblob/TestAZBlobPipesIterator.java
@@ -19,7 +19,6 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
@@ -29,27 +28,41 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
+import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.pipes.pipesiterator.PipesIteratorBase;
+import org.apache.tika.plugins.ExtensionConfig;
@Disabled("turn into an actual unit test")
public class TestAZBlobPipesIterator {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
@Test
public void testSimple() throws Exception {
- AZBlobPipesIterator it = new AZBlobPipesIterator();
- it.setContainer("");
- it.setEndpoint("");
- it.setSasToken("");
- it.initialize(Collections.EMPTY_MAP);
+ ObjectNode configNode = MAPPER.createObjectNode();
+ configNode.put("container", ""); // select one
+ configNode.put("endpoint", ""); // use one
+ configNode.put("sasToken", ""); // find one
+
+ ObjectNode baseConfigNode = MAPPER.createObjectNode();
+ baseConfigNode.put("fetcherId", "az-blob");
+ baseConfigNode.put("emitterId", "test-emitter");
+ configNode.set("baseConfig", baseConfigNode);
+
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-az-blob", "az-blob-pipes-iterator",
+ MAPPER.writeValueAsString(configNode));
+ AZBlobPipesIterator it = AZBlobPipesIterator.build(extensionConfig);
+
int numConsumers = 2;
ArrayBlockingQueue
+ * "pipes-iterator": {
+ * "csv-pipes-iterator": {
+ * "csvPath": "/path/to/files.csv",
+ * "fetchKeyColumn": "path",
+ * "emitKeyColumn": "id",
+ * "baseConfig": {
+ * "fetcherId": "my-fetcher",
+ * "emitterId": "my-emitter"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class CSVPipesIteratorFactory implements PipesIteratorFactory {
+
+ public static final String NAME = "csv-pipes-iterator";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public CSVPipesIterator buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return CSVPipesIterator.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/main/java/org/apache/tika/pipes/pipesiterator/csv/CSVPipesIteratorPlugin.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/main/java/org/apache/tika/pipes/pipesiterator/csv/CSVPipesIteratorPlugin.java
new file mode 100644
index 00000000000..4bc0291bac5
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/main/java/org/apache/tika/pipes/pipesiterator/csv/CSVPipesIteratorPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.pipesiterator.csv;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CSVPipesIteratorPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(CSVPipesIteratorPlugin.class);
+
+ public CSVPipesIteratorPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting CSV Pipes Iterator Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping CSV Pipes Iterator Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting CSV Pipes Iterator Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..c5d3f641107
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=csv-pipes-iterator
+plugin.class=org.apache.tika.pipes.pipesiterator.csv.CSVPipesIteratorPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=CSV Pipes Iterator
+plugin.description=Capable of iterating over CSV files
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/test/java/TestCSVPipesIterator.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/test/java/TestCSVPipesIterator.java
index dccc1f70ccf..6dc4523f516 100644
--- a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/test/java/TestCSVPipesIterator.java
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-csv/src/test/java/TestCSVPipesIterator.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-import static org.apache.tika.pipes.core.pipesiterator.PipesIterator.COMPLETED_SEMAPHORE;
+import static org.apache.tika.pipes.pipesiterator.PipesIteratorBase.COMPLETED_SEMAPHORE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -31,22 +31,22 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Test;
-import org.apache.tika.pipes.core.FetchEmitTuple;
+import org.apache.tika.pipes.api.FetchEmitTuple;
import org.apache.tika.pipes.pipesiterator.csv.CSVPipesIterator;
+import org.apache.tika.plugins.ExtensionConfig;
public class TestCSVPipesIterator {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@Test
public void testSimple() throws Exception {
Path p = get("test-simple.csv");
- CSVPipesIterator it = new CSVPipesIterator();
- it.setFetcherName("fsf");
- it.setEmitterName("fse");
- it.setCsvPath(p);
- it.setFetchKeyColumn("fetchKey");
+ CSVPipesIterator it = createIterator(p, "fsf", "fse", "fetchKey", null, null);
int numConsumers = 2;
ExecutorService es = Executors.newFixedThreadPool(numConsumers);
ExecutorCompletionService c = new ExecutorCompletionService(es);
@@ -92,17 +92,39 @@ public void testSimple() throws Exception {
@Test
public void testBadFetchKeyCol() throws Exception {
Path p = get("test-simple.csv");
- CSVPipesIterator it = new CSVPipesIterator();
- it.setFetcherName("fs");
- it.setCsvPath(p);
assertThrows(RuntimeException.class, () -> {
- it.setFetchKeyColumn("fetchKeyDoesntExist");
+ CSVPipesIterator it = createIterator(p, "fs", "fse", "fetchKeyDoesntExist", null, null);
for (FetchEmitTuple t : it) {
}
});
}
+ private CSVPipesIterator createIterator(Path csvPath, String fetcherName, String emitterName,
+ String fetchKeyColumn, String emitKeyColumn, String idColumn) throws Exception {
+ ObjectNode jsonConfig = OBJECT_MAPPER.createObjectNode();
+ jsonConfig.put("csvPath", csvPath.toAbsolutePath().toString());
+ if (fetchKeyColumn != null) {
+ jsonConfig.put("fetchKeyColumn", fetchKeyColumn);
+ }
+ if (emitKeyColumn != null) {
+ jsonConfig.put("emitKeyColumn", emitKeyColumn);
+ }
+ if (idColumn != null) {
+ jsonConfig.put("idColumn", idColumn);
+ }
+
+ // Add baseConfig
+ ObjectNode baseConfig = OBJECT_MAPPER.createObjectNode();
+ baseConfig.put("fetcherId", fetcherName);
+ baseConfig.put("emitterId", emitterName);
+ jsonConfig.set("baseConfig", baseConfig);
+
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-csv-iterator", "csv-pipes-iterator",
+ OBJECT_MAPPER.writeValueAsString(jsonConfig));
+ return CSVPipesIterator.build(extensionConfig);
+ }
+
private Path get(String testFileName) throws Exception {
return Paths.get(TestCSVPipesIterator.class
.getResource("/" + testFileName)
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/pom.xml b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/pom.xml
new file mode 100644
index 00000000000..7571857fcc0
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/pom.xml
@@ -0,0 +1,119 @@
+
+
+
+ * "pipes-iterator": {
+ * "file-system-pipes-iterator": {
+ * "basePath": "/path/to/files",
+ * "countTotal": true,
+ * "baseConfig": {
+ * "fetcherId": "my-fetcher",
+ * "emitterId": "my-emitter"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class FileSystemPipesIteratorFactory implements PipesIteratorFactory {
+
+ public static final String NAME = "file-system-pipes-iterator";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public FileSystemPipesIterator buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return FileSystemPipesIterator.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/src/main/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorPlugin.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/src/main/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorPlugin.java
new file mode 100644
index 00000000000..486def5a9db
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/src/main/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorPlugin.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.pipesiterator.fs;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileSystemPipesIteratorPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(FileSystemPipesIteratorPlugin.class);
+
+ public FileSystemPipesIteratorPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting");
+ super.delete();
+ }
+
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..71c66a7515a
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=file-system-pipes-iterator
+plugin.class=org.apache.tika.pipes.pipesiterator.fs.FileSystemPipesIteratorPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Local File System Fetcher
+plugin.description=Capable of emitting the local file system
diff --git a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java
similarity index 71%
rename from tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java
rename to tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java
index cfcb1231885..c7ead9ff23f 100644
--- a/tika-pipes/tika-pipes-core/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-file-system/src/test/java/org/apache/tika/pipes/pipesiterator/fs/FileSystemPipesIteratorTest.java
@@ -16,24 +16,18 @@
*/
package org.apache.tika.pipes.pipesiterator.fs;
-import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
-import java.net.URL;
+import java.io.StringWriter;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.Timeout;
-
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
public class FileSystemPipesIteratorTest {
@@ -48,6 +42,19 @@ public static List
+ * "pipes-iterator": {
+ * "gcs-pipes-iterator": {
+ * "projectId": "my-project",
+ * "bucket": "my-bucket",
+ * "prefix": "documents/",
+ * "baseConfig": {
+ * "fetcherId": "my-fetcher",
+ * "emitterId": "my-emitter"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class GCSPipesIteratorFactory implements PipesIteratorFactory {
+
+ public static final String NAME = "gcs-pipes-iterator";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public GCSPipesIterator buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return GCSPipesIterator.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/main/java/org/apache/tika/pipes/pipesiterator/gcs/GCSPipesIteratorPlugin.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/main/java/org/apache/tika/pipes/pipesiterator/gcs/GCSPipesIteratorPlugin.java
new file mode 100644
index 00000000000..cedab601755
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/main/java/org/apache/tika/pipes/pipesiterator/gcs/GCSPipesIteratorPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.pipesiterator.gcs;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GCSPipesIteratorPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(GCSPipesIteratorPlugin.class);
+
+ public GCSPipesIteratorPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting GCS Pipes Iterator Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping GCS Pipes Iterator Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting GCS Pipes Iterator Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..d0338d830a1
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=gcs-pipes-iterator
+plugin.class=org.apache.tika.pipes.pipesiterator.gcs.GCSPipesIteratorPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=GCS Pipes Iterator
+plugin.description=Capable of iterating over Google Cloud Storage buckets
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/test/java/org/apache/tika/pipes/pipesiterator/gcs/TestGCSPipesIterator.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/test/java/org/apache/tika/pipes/pipesiterator/gcs/TestGCSPipesIterator.java
index e240d1d4f9c..d759489aef5 100644
--- a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/test/java/org/apache/tika/pipes/pipesiterator/gcs/TestGCSPipesIterator.java
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-gcs/src/test/java/org/apache/tika/pipes/pipesiterator/gcs/TestGCSPipesIterator.java
@@ -19,7 +19,6 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
@@ -29,23 +28,23 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
+import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.pipes.pipesiterator.PipesIteratorBase;
+import org.apache.tika.plugins.ExtensionConfig;
@Disabled("turn into an actual unit test")
public class TestGCSPipesIterator {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
@Test
public void testSimple() throws Exception {
- GCSPipesIterator it = new GCSPipesIterator();
- it.setFetcherName("gcs");
- it.setBucket("tika-tallison-test-bucket");
- it.setProjectId("My First Project");
- it.setPrefix("pdfs");
- it.initialize(Collections.EMPTY_MAP);
+ GCSPipesIterator it = createIterator("tika-tallison-test-bucket", "My First Project", "pdfs", "gcs", "gcs-emitter");
int numConsumers = 6;
ArrayBlockingQueue
+ * "pipes-iterator": {
+ * "jdbc-pipes-iterator": {
+ * "connection": "jdbc:postgresql://localhost/mydb",
+ * "select": "select id, path from documents",
+ * "fetchKeyColumn": "path",
+ * "idColumn": "id",
+ * "baseConfig": {
+ * "fetcherId": "my-fetcher",
+ * "emitterId": "my-emitter"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class JDBCPipesIteratorFactory implements PipesIteratorFactory {
+
+ public static final String NAME = "jdbc-pipes-iterator";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public JDBCPipesIterator buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return JDBCPipesIterator.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/java/org/apache/tika/pipes/pipesiterator/jdbc/JDBCPipesIteratorPlugin.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/java/org/apache/tika/pipes/pipesiterator/jdbc/JDBCPipesIteratorPlugin.java
new file mode 100644
index 00000000000..0635550da8e
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/java/org/apache/tika/pipes/pipesiterator/jdbc/JDBCPipesIteratorPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.pipesiterator.jdbc;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JDBCPipesIteratorPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(JDBCPipesIteratorPlugin.class);
+
+ public JDBCPipesIteratorPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting JDBC Pipes Iterator Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping JDBC Pipes Iterator Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting JDBC Pipes Iterator Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..0aac1b93afd
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=jdbc-pipes-iterator
+plugin.class=org.apache.tika.pipes.pipesiterator.jdbc.JDBCPipesIteratorPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=JDBC Pipes Iterator
+plugin.description=Capable of iterating over JDBC result sets
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/test/java/org/apache/tika/pipes/pipesiterator/jdbc/TestJDBCPipesIterator.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/test/java/org/apache/tika/pipes/pipesiterator/jdbc/TestJDBCPipesIterator.java
index 818e6f8ac97..ffaa31d8e60 100644
--- a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/test/java/org/apache/tika/pipes/pipesiterator/jdbc/TestJDBCPipesIterator.java
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-jdbc/src/test/java/org/apache/tika/pipes/pipesiterator/jdbc/TestJDBCPipesIterator.java
@@ -20,8 +20,6 @@
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.fail;
-import java.nio.charset.StandardCharsets;
-import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
import java.sql.DriverManager;
@@ -38,19 +36,23 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
+import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.pipes.pipesiterator.PipesIteratorBase;
+import org.apache.tika.plugins.ExtensionConfig;
public class TestJDBCPipesIterator {
static final String TABLE = "fetchkeys";
static final String db = "mydb";
private static final int NUM_ROWS = 1000;
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
static Connection CONNECTION;
@TempDir
@@ -90,7 +92,7 @@ public static void tearDown() throws Exception {
public void testSimple() throws Exception {
int numConsumers = 5;
- PipesIterator pipesIterator = getConfig();
+ JDBCPipesIterator pipesIterator = createIterator();
ExecutorService es = Executors.newFixedThreadPool(numConsumers);
ExecutorCompletionService
+ * "pipes-iterator": {
+ * "json-pipes-iterator": {
+ * "jsonPath": "/path/to/files.json",
+ * "baseConfig": {
+ * "fetcherId": "my-fetcher",
+ * "emitterId": "my-emitter"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class JsonPipesIteratorFactory implements PipesIteratorFactory {
+
+ public static final String NAME = "json-pipes-iterator";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public JsonPipesIterator buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return JsonPipesIterator.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/main/java/org/apache/tika/pipes/pipesiterator/json/JsonPipesIteratorPlugin.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/main/java/org/apache/tika/pipes/pipesiterator/json/JsonPipesIteratorPlugin.java
new file mode 100644
index 00000000000..796b2c9af22
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/main/java/org/apache/tika/pipes/pipesiterator/json/JsonPipesIteratorPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.pipesiterator.json;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JsonPipesIteratorPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(JsonPipesIteratorPlugin.class);
+
+ public JsonPipesIteratorPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting JSON Pipes Iterator Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping JSON Pipes Iterator Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting JSON Pipes Iterator Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..c2836e25a97
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=json-pipes-iterator
+plugin.class=org.apache.tika.pipes.pipesiterator.json.JsonPipesIteratorPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=JSON Pipes Iterator
+plugin.description=Capable of iterating over JSON files with FetchEmitTuple entries
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/test/java/org/apache/tika/pipes/pipesiterator/json/TestJsonPipesIterator.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/test/java/org/apache/tika/pipes/pipesiterator/json/TestJsonPipesIterator.java
index d08be1ded80..9e443d8eea4 100644
--- a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/test/java/org/apache/tika/pipes/pipesiterator/json/TestJsonPipesIterator.java
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-json/src/test/java/org/apache/tika/pipes/pipesiterator/json/TestJsonPipesIterator.java
@@ -17,27 +17,32 @@
package org.apache.tika.pipes.pipesiterator.json;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Iterator;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import org.apache.tika.pipes.core.FetchEmitTuple;
+import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.plugins.ExtensionConfig;
@Disabled("until we can write actual tests")
public class TestJsonPipesIterator {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
@Test
public void testBasic() throws Exception {
- JsonPipesIterator pipesIterator = new JsonPipesIterator();
- pipesIterator.setJsonPath(Paths
+ Path jsonPath = Paths
.get(this
.getClass()
.getResource("/test-documents/test.json")
.toURI())
- .toAbsolutePath()
- .toString());
+ .toAbsolutePath();
+ JsonPipesIterator pipesIterator = createIterator(jsonPath);
Iterator
+ * "pipes-iterator": {
+ * "kafka-pipes-iterator": {
+ * "topic": "my-topic",
+ * "bootstrapServers": "localhost:9092",
+ * "groupId": "my-group",
+ * "autoOffsetReset": "earliest",
+ * "baseConfig": {
+ * "fetcherId": "my-fetcher",
+ * "emitterId": "my-emitter"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class KafkaPipesIteratorFactory implements PipesIteratorFactory {
+
+ public static final String NAME = "kafka-pipes-iterator";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public KafkaPipesIterator buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return KafkaPipesIterator.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/main/java/org/apache/tika/pipes/pipesiterator/kafka/KafkaPipesIteratorPlugin.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/main/java/org/apache/tika/pipes/pipesiterator/kafka/KafkaPipesIteratorPlugin.java
new file mode 100644
index 00000000000..728fb26349b
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/main/java/org/apache/tika/pipes/pipesiterator/kafka/KafkaPipesIteratorPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.pipesiterator.kafka;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class KafkaPipesIteratorPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaPipesIteratorPlugin.class);
+
+ public KafkaPipesIteratorPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting Kafka Pipes Iterator Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping Kafka Pipes Iterator Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting Kafka Pipes Iterator Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..e387e88f50b
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=kafka-pipes-iterator
+plugin.class=org.apache.tika.pipes.pipesiterator.kafka.KafkaPipesIteratorPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Apache Kafka Pipes Iterator
+plugin.description=Capable of iterating over Apache Kafka topics
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/test/java/org/apache/tika/pipes/pipesiterator/kafka/TestKafkaPipesIterator.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/test/java/org/apache/tika/pipes/pipesiterator/kafka/TestKafkaPipesIterator.java
index b9f8662a13c..26be328d0d2 100644
--- a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/test/java/org/apache/tika/pipes/pipesiterator/kafka/TestKafkaPipesIterator.java
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-kafka/src/test/java/org/apache/tika/pipes/pipesiterator/kafka/TestKafkaPipesIterator.java
@@ -19,7 +19,6 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
@@ -29,28 +28,41 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
+import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.pipes.pipesiterator.PipesIteratorBase;
+import org.apache.tika.plugins.ExtensionConfig;
@Disabled("turn into an actual unit test")
public class TestKafkaPipesIterator {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
@Test
public void testSimple() throws Exception {
- KafkaPipesIterator it = new KafkaPipesIterator();
- it.setFetcherName("kafka");
- it.setGroupId("");//find one
- it.setBootstrapServers("");//use one
- it.setTopic("");//select one
- it.initialize(Collections.EMPTY_MAP);
+ ObjectNode configNode = MAPPER.createObjectNode();
+ configNode.put("topic", ""); // select one
+ configNode.put("bootstrapServers", ""); // use one
+ configNode.put("groupId", ""); // find one
+
+ ObjectNode baseConfigNode = MAPPER.createObjectNode();
+ baseConfigNode.put("fetcherId", "kafka");
+ baseConfigNode.put("emitterId", "test-emitter");
+ configNode.set("baseConfig", baseConfigNode);
+
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-kafka", "kafka-pipes-iterator",
+ MAPPER.writeValueAsString(configNode));
+ KafkaPipesIterator it = KafkaPipesIterator.build(extensionConfig);
+
int numConsumers = 6;
ArrayBlockingQueue
+ * "pipes-iterator": {
+ * "s3-pipes-iterator": {
+ * "region": "us-east-1",
+ * "bucket": "my-bucket",
+ * "prefix": "documents/",
+ * "credentialsProvider": "profile",
+ * "profile": "default",
+ * "baseConfig": {
+ * "fetcherId": "my-fetcher",
+ * "emitterId": "my-emitter"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class S3PipesIteratorFactory implements PipesIteratorFactory {
+
+ public static final String NAME = "s3-pipes-iterator";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public S3PipesIterator buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return S3PipesIterator.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/java/org/apache/tika/pipes/pipesiterator/s3/S3PipesIteratorPlugin.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/java/org/apache/tika/pipes/pipesiterator/s3/S3PipesIteratorPlugin.java
new file mode 100644
index 00000000000..ed90ce12f4e
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/java/org/apache/tika/pipes/pipesiterator/s3/S3PipesIteratorPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.pipesiterator.s3;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class S3PipesIteratorPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(S3PipesIteratorPlugin.class);
+
+ public S3PipesIteratorPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting S3 Pipes Iterator Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping S3 Pipes Iterator Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting S3 Pipes Iterator Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..f3c176ddf6f
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=s3-pipes-iterator
+plugin.class=org.apache.tika.pipes.pipesiterator.s3.S3PipesIteratorPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=S3 Pipes Iterator
+plugin.description=Capable of iterating over AWS S3 buckets
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/test/java/org/apache/tika/pipes/pipesiterator/s3/TestS3PipesIterator.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/test/java/org/apache/tika/pipes/pipesiterator/s3/TestS3PipesIterator.java
index 82dbdcffe02..d2bd20d32df 100644
--- a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/test/java/org/apache/tika/pipes/pipesiterator/s3/TestS3PipesIterator.java
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-s3/src/test/java/org/apache/tika/pipes/pipesiterator/s3/TestS3PipesIterator.java
@@ -19,7 +19,6 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Callable;
@@ -29,24 +28,37 @@
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.pipesiterator.PipesIterator;
+import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.pipes.pipesiterator.PipesIteratorBase;
+import org.apache.tika.plugins.ExtensionConfig;
@Disabled("turn into an actual unit test")
public class TestS3PipesIterator {
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
@Test
public void testSimple() throws Exception {
- S3PipesIterator it = new S3PipesIterator();
- it.setFetcherName("s3");
- it.setBucket("");//find one
- it.setProfile("");//use one
- it.setRegion("");//select one
- it.initialize(Collections.EMPTY_MAP);
+ ObjectNode jsonConfig = OBJECT_MAPPER.createObjectNode();
+ jsonConfig.put("bucket", ""); // find one
+ jsonConfig.put("region", ""); // select one
+ jsonConfig.put("profile", ""); // use one
+ jsonConfig.put("credentialsProvider", "profile");
+
+ ObjectNode baseConfig = OBJECT_MAPPER.createObjectNode();
+ baseConfig.put("fetcherId", "s3");
+ baseConfig.put("emitterId", "fs");
+ jsonConfig.set("baseConfig", baseConfig);
+
+ ExtensionConfig extensionConfig = new ExtensionConfig("test-s3-iterator", "s3-pipes-iterator",
+ OBJECT_MAPPER.writeValueAsString(jsonConfig));
+ S3PipesIterator it = S3PipesIterator.build(extensionConfig);
+
int numConsumers = 6;
ArrayBlockingQueue
+ * "pipes-iterator": {
+ * "solr-pipes-iterator": {
+ * "solrCollection": "my-collection",
+ * "solrUrls": ["http://localhost:8983/solr"],
+ * "idField": "id",
+ * "rows": 5000,
+ * "baseConfig": {
+ * "fetcherId": "my-fetcher",
+ * "emitterId": "my-emitter"
+ * }
+ * }
+ * }
+ *
+ */
+@Extension
+public class SolrPipesIteratorFactory implements PipesIteratorFactory {
+
+ public static final String NAME = "solr-pipes-iterator";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public SolrPipesIterator buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return SolrPipesIterator.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-solr/src/main/java/org/apache/tika/pipes/pipesiterator/solr/SolrPipesIteratorPlugin.java b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-solr/src/main/java/org/apache/tika/pipes/pipesiterator/solr/SolrPipesIteratorPlugin.java
new file mode 100644
index 00000000000..c9d20ed35c1
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-solr/src/main/java/org/apache/tika/pipes/pipesiterator/solr/SolrPipesIteratorPlugin.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.pipesiterator.solr;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class SolrPipesIteratorPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(SolrPipesIteratorPlugin.class);
+
+ public SolrPipesIteratorPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting Solr Pipes Iterator Plugin");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping Solr Pipes Iterator Plugin");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting Solr Pipes Iterator Plugin");
+ super.delete();
+ }
+}
diff --git a/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-solr/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-solr/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..99198d9beab
--- /dev/null
+++ b/tika-pipes/tika-pipes-iterators/tika-pipes-iterator-solr/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=solr-pipes-iterator
+plugin.class=org.apache.tika.pipes.pipesiterator.solr.SolrPipesIteratorPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=Solr Pipes Iterator
+plugin.description=Capable of iterating over Solr query results
diff --git a/tika-pipes/tika-pipes-reporters/pom.xml b/tika-pipes/tika-pipes-reporters/pom.xml
index 90e1e4b5962..d347ac80b58 100644
--- a/tika-pipes/tika-pipes-reporters/pom.xml
+++ b/tika-pipes/tika-pipes-reporters/pom.xml
@@ -32,11 +32,50 @@
+ * "reporter": {
+ * "file-system-reporter": {
+ * "statusFile": "/path/to/status.json",
+ * "reportUpdateMs": 1000
+ * }
+ * }
+ *
+ */
+@Extension
+public class FileSystemReporterFactory implements PipesReporterFactory {
+
+ public static final String NAME = "file-system-reporter";
@Override
public String getName() {
- return name;
- }
-
- public void setName(String name) {
- this.name = name;
+ return NAME;
}
- /**
- * The default behavior is to call {@link #emit(String, List, ParseContext)} on each item.
- * Some implementations, e.g. Solr/ES/vespa, can benefit from subclassing this and
- * emitting a bunch of docs at once.
- *
- * @param emitData
- * @throws IOException
- * @throws TikaEmitterException
- */
@Override
- public void emit(List emitData) throws IOException, TikaEmitterException {
- for (EmitData d : emitData) {
- emit(d.getEmitKey().getEmitKey(), d.getMetadataList(), d.getParseContext());
- }
+ public FileSystemStatusReporter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return FileSystemStatusReporter.build(extensionConfig);
}
}
diff --git a/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-fs-status/src/main/java/org/apache/tika/pipes/reporters/fs/FileSystemReporterPlugin.java b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-fs-status/src/main/java/org/apache/tika/pipes/reporters/fs/FileSystemReporterPlugin.java
new file mode 100644
index 00000000000..c1f4fb274e8
--- /dev/null
+++ b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-fs-status/src/main/java/org/apache/tika/pipes/reporters/fs/FileSystemReporterPlugin.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.reporters.fs;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FileSystemReporterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(FileSystemReporterPlugin.class);
+
+ public FileSystemReporterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting");
+ super.delete();
+ }
+
+}
diff --git a/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-fs-status/src/main/java/org/apache/tika/pipes/reporters/fs/FileSystemStatusReporter.java b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-fs-status/src/main/java/org/apache/tika/pipes/reporters/fs/FileSystemStatusReporter.java
index 64a30082e27..9f42f81fd9a 100644
--- a/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-fs-status/src/main/java/org/apache/tika/pipes/reporters/fs/FileSystemStatusReporter.java
+++ b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-fs-status/src/main/java/org/apache/tika/pipes/reporters/fs/FileSystemStatusReporter.java
@@ -20,8 +20,6 @@
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
@@ -34,16 +32,13 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.apache.tika.config.Field;
-import org.apache.tika.config.Initializable;
-import org.apache.tika.config.InitializableProblemHandler;
-import org.apache.tika.config.Param;
import org.apache.tika.exception.TikaConfigException;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.PipesReporter;
-import org.apache.tika.pipes.core.PipesResult;
-import org.apache.tika.pipes.core.async.AsyncStatus;
-import org.apache.tika.pipes.core.pipesiterator.TotalCountResult;
+import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.pipes.api.PipesResult;
+import org.apache.tika.pipes.api.pipesiterator.TotalCountResult;
+import org.apache.tika.pipes.api.reporter.PipesReporter;
+import org.apache.tika.plugins.AbstractTikaExtension;
+import org.apache.tika.plugins.ExtensionConfig;
import org.apache.tika.utils.ExceptionUtils;
/**
@@ -58,16 +53,21 @@
* the unit tests for how to deserialize AsyncStatus.
*
*/
-public class FileSystemStatusReporter extends PipesReporter
- implements Initializable {
+public class FileSystemStatusReporter extends AbstractTikaExtension implements PipesReporter {
+
+ public static FileSystemStatusReporter build(ExtensionConfig pluginConfig) throws TikaConfigException, IOException {
+ FileSystemReporterConfig config = FileSystemReporterConfig.load(pluginConfig.jsonConfig());
+
+ FileSystemStatusReporter fileSystemStatusReporter = new FileSystemStatusReporter(pluginConfig, config);
+ fileSystemStatusReporter.configure();
+ return fileSystemStatusReporter;
+ }
private static final Logger LOG = LoggerFactory.getLogger(FileSystemStatusReporter.class);
ObjectMapper objectMapper;
- private Path statusFile;
-
- private long reportUpdateMillis = 1000;
+ private final FileSystemReporterConfig config;
private volatile boolean crashed = false;
Thread reporterThread;
@@ -76,18 +76,25 @@ public class FileSystemStatusReporter extends PipesReporter
private TotalCountResult totalCountResult = new TotalCountResult(0,
TotalCountResult.STATUS.NOT_COMPLETED);
- @Field
- public void setStatusFile(String path) {
- this.statusFile = Paths.get(path);
- }
- @Field
- public void setReportUpdateMillis(long millis) {
- this.reportUpdateMillis = millis;
+ private FileSystemStatusReporter(ExtensionConfig pluginConfig, FileSystemReporterConfig config) {
+ super(pluginConfig);
+ this.config = config;
}
- @Override
- public void initialize(Mapinsert into tika_status (id, status, timestamp) values (?,?,?)
- *
- * This can be modified for specific dialects of SQL or to run an upsert, merge or update
- * instead of the default insert.
- *
- * Users need to coordinate this with {@link #setReportVariables(List)}
- * @param reportSql
- */
- @Field
- public void setReportSql(String reportSql) {
- this.reportSql = reportSql;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public Listinsert into tika_status (id, status, timestamp) values (?,?,?)
+ * @param tableName table name or defaults to 'tika_status'
+ * @param createTable whether or not to create the table NOTE The default behavior is to drop the table if it exists and
+ * then create it. Make sure to set this to false if you do not want to drop the table.
+ *
+ * @param postConnectionSql This sql will be called immediately after the connection is made. This was
+ * initially added for setting pragmas on sqlite3, but may be used for other connection configuration in other dbs.
+ * Note: This is called before the table is created if it needs to be created.
+ * @param reportVariables ADVANCED: This is used to set the variables in the prepared statement for the report. This needs to be coordinated
+ * with {@link #reportSql}. The available variables are "id, status, timestamp". If you're modifying to an update
+ * statement like "update table tika_status set status=?, timestamp=? where id = ?"
+ * then the values for this would be ["status", "timestamp", "id"].
+ * @param reportWithinMs
+ * @param cacheSize
+ */
+public record JDBCPipesReporterConfig(String connectionString, Set
+ * "reporter": {
+ * "jdbc-reporter": {
+ * "connectionString": "jdbc:postgresql://localhost/mydb",
+ * "tableName": "tika_status",
+ * "createTable": true
+ * }
+ * }
+ *
+ */
+@Extension
+public class JDBCPipesReporterFactory implements PipesReporterFactory {
+
+ public static final String NAME = "jdbc-reporter";
@Override
public String getName() {
- return "mock";
+ return NAME;
}
+
@Override
- public InputStream fetch(String fetchKey, Metadata metadata, ParseContext parseContext) throws TikaException, IOException {
- return new ByteArrayInputStream(BYTES);
+ public JDBCPipesReporter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return JDBCPipesReporter.build(extensionConfig);
}
}
diff --git a/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/main/java/org/apache/tika/pipes/reporters/jdbc/JDBCReporterPlugin.java b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/main/java/org/apache/tika/pipes/reporters/jdbc/JDBCReporterPlugin.java
new file mode 100644
index 00000000000..d9ed7b9cd71
--- /dev/null
+++ b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/main/java/org/apache/tika/pipes/reporters/jdbc/JDBCReporterPlugin.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.reporters.jdbc;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class JDBCReporterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(JDBCReporterPlugin.class);
+
+ public JDBCReporterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting");
+ super.delete();
+ }
+
+}
diff --git a/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..4406e2e3897
--- /dev/null
+++ b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=jdbc-reporter
+plugin.class=org.apache.tika.pipes.reporters.fs.FileSystemReporterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=FileSystemReporterPlugin
+plugin.description=Capable of reporting status to a file system
diff --git a/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/test/java/org/apache/tika/pipes/reporters/jdbc/TestJDBCPipesReporter.java b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/test/java/org/apache/tika/pipes/reporters/jdbc/TestJDBCPipesReporter.java
index 533fa6dbdb0..ad8b9078776 100644
--- a/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/test/java/org/apache/tika/pipes/reporters/jdbc/TestJDBCPipesReporter.java
+++ b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-jdbc/src/test/java/org/apache/tika/pipes/reporters/jdbc/TestJDBCPipesReporter.java
@@ -16,14 +16,12 @@
*/
package org.apache.tika.pipes.reporters.jdbc;
-import static org.apache.tika.pipes.core.PipesResult.STATUS.PARSE_SUCCESS;
-import static org.apache.tika.pipes.core.PipesResult.STATUS.PARSE_SUCCESS_WITH_EXCEPTION;
+import static org.apache.tika.pipes.api.PipesResult.STATUS.PARSE_SUCCESS;
+import static org.apache.tika.pipes.api.PipesResult.STATUS.PARSE_SUCCESS_WITH_EXCEPTION;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.sql.Connection;
@@ -44,21 +42,44 @@
import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.io.IOUtils;
+import com.fasterxml.jackson.databind.ObjectMapper;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
-import org.apache.tika.pipes.core.FetchEmitTuple;
-import org.apache.tika.pipes.core.PipesReporter;
-import org.apache.tika.pipes.core.PipesResult;
-import org.apache.tika.pipes.core.async.AsyncConfig;
-import org.apache.tika.pipes.core.emitter.EmitKey;
-import org.apache.tika.pipes.core.fetcher.FetchKey;
-import org.apache.tika.pipes.core.pipesiterator.TotalCountResult;
+import org.apache.tika.pipes.api.FetchEmitTuple;
+import org.apache.tika.pipes.api.PipesResult;
+import org.apache.tika.pipes.api.emitter.EmitKey;
+import org.apache.tika.pipes.api.fetcher.FetchKey;
+import org.apache.tika.pipes.api.pipesiterator.TotalCountResult;
+import org.apache.tika.pipes.api.reporter.PipesReporter;
+import org.apache.tika.plugins.ExtensionConfig;
public class TestJDBCPipesReporter {
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ private static final String JSON_TEMPLATE = """
+ {
+ "connectionString":"CONNECTION_STRING"
+ }
+ """;
+
+ private static final String JSON_TEMPLATE_INCLUDES = """
+ {
+ "connectionString":"CONNECTION_STRING",
+ "includes": ["PARSE_SUCCESS", "PARSE_SUCCESS_WITH_EXCEPTION"]
+ }
+ """;
+
+ private static final String JSON_TEMPLATE_EXCLUDES = """
+ {
+ "connectionString":"CONNECTION_STRING",
+ "excludes": ["PARSE_SUCCESS", "PARSE_SUCCESS_WITH_EXCEPTION"]
+ }
+ """;
+
+
@Test
public void testBasic(@TempDir Path tmpDir) throws Exception {
Files.createDirectories(tmpDir.resolve("db"));
@@ -67,9 +88,8 @@ public void testBasic(@TempDir Path tmpDir) throws Exception {
int numThreads = 10;
int numIterations = 200;
- JDBCPipesReporter reporter = new JDBCPipesReporter();
- reporter.setConnection(connectionString);
- reporter.initialize(new HashMap<>());
+ String json = JSON_TEMPLATE.replace("CONNECTION_STRING", connectionString);
+ JDBCPipesReporter reporter = JDBCPipesReporter.build(new ExtensionConfig("test-jdbc", "jdbc-reporter", json));
Map
+ * "reporter": {
+ * "opensearch-pipes-reporter": {
+ * "openSearchUrl": "http://localhost:9200/tika-status",
+ * "includes": ["PARSE_SUCCESS", "PARSE_EXCEPTION"],
+ * "keyPrefix": "status_"
+ * }
+ * }
+ *
+ */
+@Extension
+public class OpenSearchReporterFactory implements PipesReporterFactory {
+
+ public static final String NAME = "opensearch-pipes-reporter";
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public OpenSearchPipesReporter buildExtension(ExtensionConfig extensionConfig) throws IOException, TikaConfigException {
+ return OpenSearchPipesReporter.build(extensionConfig);
+ }
+}
diff --git a/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-opensearch/src/main/java/org/apache/tika/pipes/reporters/opensearch/OpenSearchReporterPlugin.java b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-opensearch/src/main/java/org/apache/tika/pipes/reporters/opensearch/OpenSearchReporterPlugin.java
new file mode 100644
index 00000000000..21fa72e9e60
--- /dev/null
+++ b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-opensearch/src/main/java/org/apache/tika/pipes/reporters/opensearch/OpenSearchReporterPlugin.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tika.pipes.reporters.opensearch;
+
+import org.pf4j.Plugin;
+import org.pf4j.PluginWrapper;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OpenSearchReporterPlugin extends Plugin {
+ private static final Logger LOG = LoggerFactory.getLogger(OpenSearchReporterPlugin.class);
+
+ public OpenSearchReporterPlugin(PluginWrapper wrapper) {
+ super(wrapper);
+ }
+
+ @Override
+ public void start() {
+ LOG.info("Starting");
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping");
+ super.stop();
+ }
+
+ @Override
+ public void delete() {
+ LOG.info("Deleting");
+ super.delete();
+ }
+
+}
diff --git a/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-opensearch/src/main/resources/plugin.properties b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-opensearch/src/main/resources/plugin.properties
new file mode 100644
index 00000000000..782986ba27d
--- /dev/null
+++ b/tika-pipes/tika-pipes-reporters/tika-pipes-reporter-opensearch/src/main/resources/plugin.properties
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+plugin.id=opensearch-pipes-reporter
+plugin.class=org.apache.tika.pipes.reporters.opensearch.OpenSearchReporterPlugin
+plugin.version=4.0.0-SNAPSHOT
+plugin.provider=OpenSearch pipes reporter
+plugin.description=Capable of reporting status to an opensearch instance
diff --git a/tika-plugins-core/README.md b/tika-plugins-core/README.md
new file mode 100644
index 00000000000..13fcef2a3db
--- /dev/null
+++ b/tika-plugins-core/README.md
@@ -0,0 +1,38 @@
+# Tika's use of pf4j -- Informal Notes
+
+As of the initial `pf4j` contribution, we aren't really using the plugin functionality. We only
+need the Extension functionality for now. We can add the plugin lifecycle items later if needed.
+
+At a high level, we needed to be able to inject configurations into Extensions, and we needed to be able to
+create multiple instances of the same Extension with different configurations.
+
+We tried to get this to work with subclassing the PluginManager, but it was cleaner to
+create Extensions that were factories for the Extensions that we wanted, rather than Extensions that
+were directly the Fetchers, etc. This adds a bit to code bloat, but it is much, much cleaner.
+
+We do have a custom TikaPluginManager that performs the plugin unzipping in a multithreaded/multiprocess safe way.
+Given that we're heading towards avoiding fat jars in 4.x, it felt like we should prefer the zip approach
+to our own plugins. When we did that though, in the pipes framework, we ran into race conditions where
+different PipesServers were all trying to unzip the plugins at the same time, or perhaps read from a
+plugin directory that was only half-unzipped.
+
+And, we added `buildConfiguredExtensions(...)`, which is the main functionality that we needed.
+
+
+## Thoughts for the future...
+We didn't have much luck building unit tests for the plugins within the plugin modules. So,
+we opted for integration tests, which are critical for making sure that the plugin zip contains
+all that it needs to, etc. There are probably better ways to manage this.
+
+If this all works out, this is a really elegant way of modularizing and packaging plugins and extensions.
+It would be great to do the same for detectors and parsers.
+
+We should add filters so that we're only loading the plugins we need based on the extensions that
+are configured in the json config file.
+
+We may want to move to a resource based organization for pipes so that we'd have, say, a `tika-pipes-s3`
+plugin that has a fetcher, emitter and iterator. Some plugins would only have a fetcher or an emitter, but
+some would have all three. If we did this, the heavy s3 dependencies for all three components would be
+packaged only once in a zip.
+
+## Configuration generally
\ No newline at end of file
diff --git a/tika-plugins-core/pom.xml b/tika-plugins-core/pom.xml
new file mode 100644
index 00000000000..c6fc4368ac1
--- /dev/null
+++ b/tika-plugins-core/pom.xml
@@ -0,0 +1,68 @@
+
+
+
+ * {
+ * "typeName": {
+ * "instanceId1": { config },
+ * "instanceId2": { config }
+ * },
+ * "typeName2": {
+ * "instanceId3": { config }
+ * }
+ * }
+ *
+ *