diff --git a/CHANGES.txt b/CHANGES.txt index bf320ab3af8..06d73730b4f 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -13,6 +13,8 @@ Release 4.0.0-BETA1 - ??? * Removed the dotnet module (TIKA-4332). + * Removed the advanced media module (TIKA-4500). + * Remove the tika-dl module (TIKA-4499). OTHER CHANGES diff --git a/assembly.xml b/assembly.xml index 4095cac5617..f69a4af6238 100644 --- a/assembly.xml +++ b/assembly.xml @@ -28,7 +28,6 @@ **/.*/** **/opennlp/ner-*.bin **/opennlp/en-*.bin - **/recognition/*.bin **/*.releaseBackup diff --git a/tika-parsers/tika-parsers-ml/pom.xml b/tika-parsers/tika-parsers-ml/pom.xml index 0f9a5f04803..8f2a98b0ed8 100644 --- a/tika-parsers/tika-parsers-ml/pom.xml +++ b/tika-parsers/tika-parsers-ml/pom.xml @@ -93,19 +93,6 @@ - - - sandbox - - - tika-age-recogniser - tika-parser-advancedmedia-module - tika-parser-advancedmedia-package - - - - 3.0.0-rc1 diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/pom.xml b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/pom.xml deleted file mode 100644 index 73211f6cb12..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/pom.xml +++ /dev/null @@ -1,85 +0,0 @@ - - - - - tika-parsers-ml - org.apache.tika - 4.0.0-SNAPSHOT - - 4.0.0 - - tika-parser-advancedmedia-module - Apache Tika advanced media module - - - - ${project.groupId} - tika-parser-audiovideo-module - ${project.version} - - - com.googlecode.json-simple - json-simple - - - org.apache.httpcomponents - httpclient - - - org.apache.httpcomponents - httpcore - - - commons-codec - commons-codec - - - org.apache.httpcomponents - httpmime - - - com.github.openjson - openjson - ${openjson.version} - - - jakarta.ws.rs - jakarta.ws.rs-api - - - org.apache.commons - commons-exec - - - org.apache.commons - commons-lang3 - test - - - org.apache.cxf - cxf-rt-rs-client - test - - - - - 3.0.0-rc1 - - \ No newline at end of file diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/captioning/CaptionObject.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/captioning/CaptionObject.java deleted file mode 100644 index 26c19901a54..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/captioning/CaptionObject.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tika.parser.captioning; - -import org.apache.tika.parser.recognition.RecognisedObject; - -/** - * A model for caption objects from graphics and texts typically includes - * human readable sentence, language of the sentence and confidence score. - * - * @since Apache Tika 1.16 - */ -public class CaptionObject extends RecognisedObject { - - public CaptionObject(String sentence, String sentenceLang, double confidence) { - super(sentence, sentenceLang, null, confidence); - } - - @Override - public String toString() { - return "Caption{" + "sentence='" + label + "\' (" + labelLang + ')' + ", confidence=" + - confidence + '}'; - } - -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/captioning/tf/TensorflowRESTCaptioner.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/captioning/tf/TensorflowRESTCaptioner.java deleted file mode 100644 index 8d3320dea1e..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/captioning/tf/TensorflowRESTCaptioner.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tika.parser.captioning.tf; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; - -import org.apache.tika.config.Field; -import org.apache.tika.config.InitializableProblemHandler; -import org.apache.tika.config.Param; -import org.apache.tika.exception.TikaConfigException; -import org.apache.tika.exception.TikaException; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.mime.MediaType; -import org.apache.tika.parser.ParseContext; -import org.apache.tika.parser.captioning.CaptionObject; -import org.apache.tika.parser.recognition.ObjectRecogniser; - -/** - * Tensorflow image captioner. - * This implementation uses Tensorflow via REST API. - *

- * NOTE : https://wiki.apache.org/tika/ImageCaption - * - * @since Apache Tika 1.17 - */ -public class TensorflowRESTCaptioner implements ObjectRecogniser { - private static final Logger LOG = LoggerFactory.getLogger(TensorflowRESTCaptioner.class); - - private static final Set SUPPORTED_MIMES = Collections.unmodifiableSet(new HashSet<>( - Arrays.asList(new MediaType[]{MediaType.image("jpeg"), MediaType.image("png"), - MediaType.image("gif")}))); - - private static final String LABEL_LANG = "eng"; - - @Field - private URI apiBaseUri = URI.create("http://localhost:8764/inception/v3"); - - @Field - private int captions = 5; - - @Field - private int maxCaptionLength = 15; - - private URI apiUri; - - private URI healthUri; - - private boolean available; - - protected URI getApiUri(Metadata metadata) { - return apiUri; - } - - @Override - public Set getSupportedMimes() { - return SUPPORTED_MIMES; - } - - @Override - public boolean isAvailable() { - return available; - } - - @Override - public void initialize(Map params) throws TikaConfigException { - healthUri = URI.create(apiBaseUri + "/ping"); - apiUri = URI.create(apiBaseUri + String.format(Locale.getDefault(), - "/caption/image?beam_size=%1$d&max_caption_length=%2$d", captions, - maxCaptionLength)); - - try (CloseableHttpClient client = HttpClientBuilder.create().build()) { - HttpResponse response = client.execute(new HttpGet(healthUri)); - available = response.getStatusLine().getStatusCode() == 200; - - LOG.info("Available = {}, API Status = {}", available, response.getStatusLine()); - LOG.info("Captions = {}, MaxCaptionLength = {}", captions, maxCaptionLength); - } catch (Exception e) { - available = false; - throw new TikaConfigException(e.getMessage(), e); - } - } - - @Override - public void checkInitialization(InitializableProblemHandler handler) - throws TikaConfigException { - //TODO -- what do we want to check? - } - - @Override - public List recognise(InputStream stream, ContentHandler handler, - Metadata metadata, ParseContext context) - throws IOException, SAXException, TikaException { - List capObjs = new ArrayList<>(); - try (CloseableHttpClient client = HttpClientBuilder.create().build()) { - - HttpPost request = new HttpPost(getApiUri(metadata)); - - try (UnsynchronizedByteArrayOutputStream byteStream = UnsynchronizedByteArrayOutputStream.builder().get()) { - //TODO: convert this to stream, this might cause OOM issue - // InputStreamEntity is not working - // request.setEntity(new InputStreamEntity(stream, -1)); - IOUtils.copy(stream, byteStream); - request.setEntity(new ByteArrayEntity(byteStream.toByteArray())); - } - - HttpResponse response = client.execute(request); - try (InputStream reply = response.getEntity().getContent()) { - String replyMessage = IOUtils.toString(reply, StandardCharsets.UTF_8); - if (response.getStatusLine().getStatusCode() == 200) { - JSONObject jReply = (JSONObject) new JSONParser().parse(replyMessage); - JSONArray jCaptions = (JSONArray) jReply.get("captions"); - for (Object caption : jCaptions) { - JSONObject jCaption = (JSONObject) caption; - String sentence = (String) jCaption.get("sentence"); - Double confidence = (Double) jCaption.get("confidence"); - capObjs.add(new CaptionObject(sentence, LABEL_LANG, confidence)); - } - } else { - LOG.warn("Status = {}", response.getStatusLine()); - LOG.warn("Response = {}", replyMessage); - } - } - } catch (Exception e) { - LOG.warn(e.getMessage(), e); - } - return capObjs; - } -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/pot/PooledTimeSeriesParser.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/pot/PooledTimeSeriesParser.java deleted file mode 100644 index ceaec071da6..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/pot/PooledTimeSeriesParser.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tika.parser.pot; - -import static java.nio.charset.StandardCharsets.UTF_8; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DefaultExecutor; -import org.apache.commons.exec.ExecuteWatchdog; -import org.apache.commons.exec.PumpStreamHandler; -import org.apache.commons.exec.environment.EnvironmentUtils; -import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; -import org.xml.sax.helpers.AttributesImpl; - -import org.apache.tika.exception.TikaException; -import org.apache.tika.io.TemporaryResources; -import org.apache.tika.io.TikaInputStream; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.mime.MediaType; -import org.apache.tika.mime.MediaTypeRegistry; -import org.apache.tika.parser.CompositeParser; -import org.apache.tika.parser.ParseContext; -import org.apache.tika.parser.Parser; -import org.apache.tika.parser.external.ExternalParser; -import org.apache.tika.parser.mp4.MP4Parser; -import org.apache.tika.sax.XHTMLContentHandler; - -/** - * Uses the Pooled Time Series algorithm + command line tool, to - * generate a numeric representation of the video suitable for - * similarity searches. - *

See https://wiki.apache.org/tika/PooledTimeSeriesParser for - * more details and setup instructions. - */ -public class PooledTimeSeriesParser implements Parser { - - static final boolean isAvailable = - ExternalParser.check(new String[]{"pooled-time-series", "--help"}, -1); - private static final long serialVersionUID = -2855917932512164988L; - private static final Set SUPPORTED_TYPES = isAvailable ? Collections.unmodifiableSet( - new HashSet<>(Arrays.asList( - new MediaType[]{MediaType.video("avi"), MediaType.video("mp4")}))) : - Collections.emptySet(); - // TODO: Add all supported video types - - private static final Logger LOG = LoggerFactory.getLogger(PooledTimeSeriesParser.class); - // TIKA-1445 workaround parser - private static Parser _TMP_VIDEO_METADATA_PARSER = new CompositeVideoParser(); - - /** - * Returns the set of media types supported by this parser when used with the - * given parse context. - * - * @param context parse context - * @return immutable set of media types - * @since Apache Tika 0.7 - */ - @Override - public Set getSupportedTypes(ParseContext context) { - return SUPPORTED_TYPES; - } - - /** - * Parses a document stream into a sequence of XHTML SAX events. Fills in - * related document metadata in the given metadata object. - *

- * The given document stream is consumed but not closed by this method. The - * responsibility to close the stream remains on the caller. - *

- * Information about the parsing context can be passed in the context - * parameter. See the parser implementations for the kinds of context - * information they expect. - * - * @param stream the document stream (input) - * @param handler handler for the XHTML SAX events (output) - * @param metadata document metadata (input and output) - * @param context parse context - * @throws IOException if the document stream could not be read - * @throws SAXException if the SAX events could not be processed - * @throws TikaException if the document could not be parsed - * @since Apache Tika 0.5 - */ - @Override - public void parse(InputStream stream, ContentHandler handler, Metadata metadata, - ParseContext context) throws IOException, SAXException, TikaException { - - if (!isAvailable) { - LOG.warn("PooledTimeSeries not installed!"); - return; - } - - XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); - - TemporaryResources tmp = new TemporaryResources(); - try { - TikaInputStream tikaStream = TikaInputStream.get(stream, tmp, metadata); - File input = tikaStream.getFile(); - String cmdOutput = computePoT(input); - try (InputStream ofStream = new FileInputStream( - new File(input.getAbsoluteFile() + ".of.txt"))) { - try (InputStream ogStream = new FileInputStream( - new File(input.getAbsoluteFile() + ".hog.txt"))) { - - extractHeaderOutput(ofStream, metadata, "of"); - extractHeaderOutput(ogStream, metadata, "og"); - xhtml.startDocument(); - doExtract(ofStream, xhtml, "Histogram of Optical Flows (HOF)", - metadata.get("of_frames"), metadata.get("of_vecSize")); - doExtract(ogStream, xhtml, "Histogram of Oriented Gradients (HOG)", - metadata.get("og_frames"), metadata.get("og_vecSize")); - xhtml.endDocument(); - } - } - // Temporary workaround for TIKA-1445 - until we can specify - // composite parsers with strategies (eg Composite, Try In Turn), - // always send the image onwards to the regular parser to have - // the metadata for them extracted as well - _TMP_VIDEO_METADATA_PARSER.parse(tikaStream, handler, metadata, context); - - } finally { - tmp.dispose(); - } - } - - private String computePoT(File input) throws IOException { - - CommandLine cmdLine = new CommandLine("pooled-time-series"); - try (UnsynchronizedByteArrayOutputStream outputStream = UnsynchronizedByteArrayOutputStream.builder().get()) { - cmdLine.addArgument("-f"); - cmdLine.addArgument(input.getAbsolutePath()); - LOG.trace("Executing: {}", cmdLine); - DefaultExecutor exec = DefaultExecutor.builder().get(); - exec.setExitValue(0); - ExecuteWatchdog watchdog = ExecuteWatchdog.builder().setTimeout(Duration.ofMillis(60000)).get(); - exec.setWatchdog(watchdog); - PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream); - exec.setStreamHandler(streamHandler); - int exitValue = exec.execute(cmdLine, EnvironmentUtils.getProcEnvironment()); - return outputStream.toString(UTF_8); - } - } - - /** - * Reads the contents of the given stream and write it to the given XHTML - * content handler. The stream is closed once fully processed. - * - * @param stream Stream where is the result of ocr - * @param xhtml XHTML content handler - * @param tableTitle The name of the matrix/table to display. - * @param frames Number of frames read from the video. - * @param vecSize Size of the OF or HOG vector. - * @throws SAXException if the XHTML SAX events could not be handled - * @throws IOException if an input error occurred - */ - private void doExtract(InputStream stream, XHTMLContentHandler xhtml, String tableTitle, - String frames, String vecSize) throws SAXException, IOException { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, UTF_8))) { - String line = null; - AttributesImpl attributes = new AttributesImpl(); - attributes.addAttribute("", "", "rows", "CDATA", frames); - attributes.addAttribute("", "", "cols", "CDATA", vecSize); - - xhtml.startElement("h3"); - xhtml.characters(tableTitle); - xhtml.endElement("h3"); - xhtml.startElement("table", attributes); - while ((line = reader.readLine()) != null) { - xhtml.startElement("tr"); - for (String val : line.split(" ")) { - xhtml.startElement("td"); - xhtml.characters(val); - xhtml.endElement("td"); - } - xhtml.endElement("tr"); - } - xhtml.endElement("table"); - } - } - - private void extractHeaderOutput(InputStream stream, Metadata metadata, String prefix) - throws IOException { - try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, UTF_8))) { - String line = reader.readLine(); - String[] firstLine = line.split(" "); - String frames = firstLine[0]; - String vecSize = firstLine[1]; - - if (prefix == null) { - prefix = ""; - } - metadata.add(prefix + "_frames", frames); - metadata.add(prefix + "_vecSize", vecSize); - } - } - - private static class CompositeVideoParser extends CompositeParser { - private static final long serialVersionUID = -2398203965206381382L; - private static List videoParsers = Arrays.asList(new Parser[]{new MP4Parser()}); - - CompositeVideoParser() { - super(new MediaTypeRegistry(), videoParsers); - } - } - -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/ObjectRecogniser.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/ObjectRecogniser.java deleted file mode 100644 index 97b4d4d67e7..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/ObjectRecogniser.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tika.parser.recognition; - -import java.io.IOException; -import java.io.InputStream; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; - -import org.apache.tika.config.Initializable; -import org.apache.tika.config.Param; -import org.apache.tika.exception.TikaConfigException; -import org.apache.tika.exception.TikaException; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.mime.MediaType; -import org.apache.tika.parser.ParseContext; -import org.apache.tika.parser.recognition.tf.TensorflowImageRecParser; - -/** - * This is a contract for object recognisers used by {@link ObjectRecognitionParser} - * - * @see TensorflowImageRecParser - */ -public interface ObjectRecogniser extends Initializable { - - /** - * The mimes supported by this recogniser - * - * @return set of mediatypes - */ - Set getSupportedMimes(); - - /** - * Is this service available - * - * @return {@code true} when the service is available, {@code false} otherwise - */ - boolean isAvailable(); - - /** - * This is the hook for configuring the recogniser - * - * @param params configuration instance in the form of context - * @throws TikaConfigException when there is an issue with configuration - */ - void initialize(Map params) throws TikaConfigException; - - /** - * Recognise the objects in the stream - * - * @param stream content stream - * @param handler tika's content handler - * @param metadata metadata instance - * @param context parser context - * @return List of {@link RecognisedObject}s - * @throws IOException when an I/O error occurs - * @throws SAXException when an issue with XML occurs - * @throws TikaException any generic error - */ - List recognise(InputStream stream, ContentHandler handler, - Metadata metadata, ParseContext context) - throws IOException, SAXException, TikaException; -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/ObjectRecognitionParser.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/ObjectRecognitionParser.java deleted file mode 100644 index 289289a74e3..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/ObjectRecognitionParser.java +++ /dev/null @@ -1,170 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tika.parser.recognition; - -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; - -import org.apache.tika.config.Field; -import org.apache.tika.config.Initializable; -import org.apache.tika.config.InitializableProblemHandler; -import org.apache.tika.config.Param; -import org.apache.tika.exception.TikaConfigException; -import org.apache.tika.exception.TikaException; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.mime.MediaType; -import org.apache.tika.parser.ParseContext; -import org.apache.tika.parser.Parser; -import org.apache.tika.parser.captioning.CaptionObject; -import org.apache.tika.sax.XHTMLContentHandler; -import org.apache.tika.utils.AnnotationUtils; -import org.apache.tika.utils.ServiceLoaderUtils; - - -/** - * This parser recognises objects from Images. - * The Object Recognition implementation can be switched using 'class' argument. - *

- * Example Usage : - * 

- * <properties>
- *  <parsers>
- *   <parser class="org.apache.tika.parser.recognition.ObjectRecognitionParser">
- *    <params>
- *      <param name="class" type="string">
- *      org.apache.tika.parser.recognition.tf.TensorflowRESTRecogniser</param>
- *      <param name="class" type="string">
- *      org.apache.tika.parser.captioning.tf.TensorflowRESTCaptioner</param>
- *    </params>
- *   </parser>
- *  </parsers>
- * </properties>
- *
- * - * @since Apache Tika 1.14 - */ -public class ObjectRecognitionParser implements Parser, Initializable { - public static final String MD_KEY_OBJ_REC = "OBJECT"; - public static final String MD_KEY_IMG_CAP = "CAPTION"; - public static final String MD_REC_IMPL_KEY = - ObjectRecognitionParser.class.getPackage().getName() + ".object.rec.impl"; - private static final Logger LOG = LoggerFactory.getLogger(ObjectRecognitionParser.class); - private ObjectRecogniser recogniser; - - @Field(name = "class") - public void setRecogniser(String recogniserClass) { - this.recogniser = ServiceLoaderUtils.newInstance(recogniserClass); - } - - @Override - public void initialize(Map params) throws TikaConfigException { - AnnotationUtils.assignFieldParams(recogniser, params); - recogniser.initialize(params); - LOG.info("Recogniser = {}", recogniser.getClass().getName()); - LOG.info("Recogniser Available = {}", recogniser.isAvailable()); - } - - @Override - public void checkInitialization(InitializableProblemHandler handler) - throws TikaConfigException { - //TODO -- what do we want to check? - } - - @Override - public Set getSupportedTypes(ParseContext context) { - return recogniser.isAvailable() ? recogniser.getSupportedMimes() : - Collections.emptySet(); - } - - @Override - public synchronized void parse(InputStream stream, ContentHandler handler, Metadata metadata, - ParseContext context) - throws IOException, SAXException, TikaException { - if (!recogniser.isAvailable()) { - LOG.warn("{} is not available for service", recogniser.getClass()); - return; - } - metadata.set(MD_REC_IMPL_KEY, recogniser.getClass().getName()); - long start = System.currentTimeMillis(); - List objects = - recogniser.recognise(stream, handler, metadata, context); - - LOG.debug("Found {} objects", objects != null ? objects.size() : 0); - LOG.info("Time taken {}ms", System.currentTimeMillis() - start); - - if (objects != null && !objects.isEmpty()) { - int count; - List acceptedObjects = new ArrayList<>(); - List xhtmlIds = new ArrayList<>(); - String xhtmlStartVal = null; - count = 0; - objects.sort((o1, o2) -> Double.compare(o2.getConfidence(), o1.getConfidence())); - // first process all the MD objects - for (RecognisedObject object : objects) { - if (object instanceof CaptionObject) { - if (xhtmlStartVal == null) { - xhtmlStartVal = "captions"; - } - String labelAndConfidence = - String.format(Locale.ENGLISH, "%s (%.5f)", object.getLabel(), - object.getConfidence()); - metadata.add(MD_KEY_IMG_CAP, labelAndConfidence); - xhtmlIds.add(String.valueOf(count++)); - } else { - if (xhtmlStartVal == null) { - xhtmlStartVal = "objects"; - } - String labelAndConfidence = - String.format(Locale.ENGLISH, "%s (%.5f)", object.getLabel(), - object.getConfidence()); - metadata.add(MD_KEY_OBJ_REC, labelAndConfidence); - xhtmlIds.add(object.getId()); - } - LOG.info("Add {}", object); - acceptedObjects.add(object); - } - XHTMLContentHandler xhtml = new XHTMLContentHandler(handler, metadata); - xhtml.startDocument(); - xhtml.startElement("ol", "id", xhtmlStartVal); - count = 0; - for (RecognisedObject object : acceptedObjects) { - //writing to handler - xhtml.startElement("li", "id", xhtmlIds.get(count++)); - String text = String.format(Locale.ENGLISH, " %s [%s](confidence = %f)", - object.getLabel(), object.getLabelLang(), object.getConfidence()); - xhtml.characters(text); - xhtml.endElement("li"); - } - xhtml.endElement("ol"); - xhtml.endDocument(); - } else { - LOG.warn("NO objects"); - metadata.add("no.objects", Boolean.TRUE.toString()); - } - } -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/RecognisedObject.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/RecognisedObject.java deleted file mode 100644 index b2b841dbaad..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/RecognisedObject.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tika.parser.recognition; - -/** - * A model for recognised objects from graphics and texts typically includes - * human readable label for the object, language of the label, id and confidence score. - * - * @since Apache Tika 1.14 - */ -public class RecognisedObject { - - /** - * Label of this object. Usually the name given to this object by humans - */ - protected String label; - /** - * Language of label, Example : english - */ - protected String labelLang; - /** - * Identifier for this object - */ - protected String id; - /** - * Confidence score - */ - protected double confidence; - - public RecognisedObject(String label, String labelLang, String id, double confidence) { - this.label = label; - this.labelLang = labelLang; - this.id = id; - this.confidence = confidence; - } - - public String getLabel() { - return label; - } - - public void setLabel(String label) { - this.label = label; - } - - public String getLabelLang() { - return labelLang; - } - - public void setLabelLang(String labelLang) { - this.labelLang = labelLang; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public double getConfidence() { - return confidence; - } - - public void setConfidence(double confidence) { - this.confidence = confidence; - } - - @Override - public String toString() { - return "RecognisedObject{" + "label='" + label + "\' (" + labelLang + ')' + ", id='" + id + - '\'' + ", confidence=" + confidence + '}'; - } -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/tf/TensorflowImageRecParser.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/tf/TensorflowImageRecParser.java deleted file mode 100644 index fd989cbf033..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/tf/TensorflowImageRecParser.java +++ /dev/null @@ -1,174 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tika.parser.recognition.tf; - -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; - -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; - -import org.apache.tika.config.Field; -import org.apache.tika.config.InitializableProblemHandler; -import org.apache.tika.config.Param; -import org.apache.tika.exception.TikaConfigException; -import org.apache.tika.exception.TikaException; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.mime.MediaType; -import org.apache.tika.parser.ParseContext; -import org.apache.tika.parser.external.ExternalParser; -import org.apache.tika.parser.recognition.ObjectRecogniser; -import org.apache.tika.parser.recognition.RecognisedObject; - -/** - * This is an implementation of {@link ObjectRecogniser} powered by - * Tensorflow - * convolutional neural network (CNN). This implementation binds to - * Python API using {@link ExternalParser}. - *
- * // NOTE: This is a proof of concept for an efficient implementation using JNI binding to - * Tensorflow's C++ api. - *

- *
- *

- * b>Environment Setup: - *

    - *
  1. Python must be available
    2. - *
  2. Tensorflow must be available for import by the python script. - * Setup Instructions here
    3. - *
  3. All dependencies of tensor flow (such as numpy) must also be available. - * Follow the image recognition - * guide and make sure it works
    4. - *
- *

- * - * @see TensorflowRESTRecogniser - * @since Apache Tika 1.14 - */ -public class TensorflowImageRecParser extends ExternalParser implements ObjectRecogniser { - static final Set SUPPORTED_MIMES = Collections.singleton(MediaType.image("jpeg")); - private static final Logger LOG = LoggerFactory.getLogger(TensorflowImageRecParser.class); - private static final String SCRIPT_FILE_NAME = "classify_image.py"; - private static final File DEFAULT_SCRIPT_FILE = - new File("tensorflow" + File.separator + SCRIPT_FILE_NAME); - private static final File DEFAULT_MODEL_FILE = - new File("tensorflow" + File.separator + "tf-objectrec-model"); - private static final LineConsumer IGNORED_LINE_LOGGER = LOG::debug; - - @Field - private String executor = "python"; - @Field - private File scriptFile = DEFAULT_SCRIPT_FILE; - @Field - private String modelArg = "--model_dir"; - @Field - private File modelFile = DEFAULT_MODEL_FILE; - @Field - private String imageArg = "--image_file"; - @Field - private String outPattern = "(.*) \\(score = ([0-9]+\\.[0-9]+)\\)$"; - @Field - private String availabilityTestArgs = ""; //when no args are given, the script will test itself! - - private boolean available = false; - - public Set getSupportedMimes() { - return SUPPORTED_MIMES; - } - - @Override - public boolean isAvailable() { - return available; - } - - @Override - public void initialize(Map params) throws TikaConfigException { - try { - if (!modelFile.exists()) { - modelFile.getParentFile().mkdirs(); - LOG.warn("Model doesn't exist at {}. Expecting the script to download it.", - modelFile); - } - if (!scriptFile.exists()) { - scriptFile.getParentFile().mkdirs(); - LOG.info("Copying script to : {}", scriptFile); - try (InputStream sourceStream = getClass().getResourceAsStream(SCRIPT_FILE_NAME)) { - try (OutputStream destStream = new FileOutputStream(scriptFile)) { - IOUtils.copy(sourceStream, destStream); - } - } - LOG.debug("Copied.."); - } - String[] availabilityCheckArgs = - {executor, scriptFile.getAbsolutePath(), modelArg, modelFile.getAbsolutePath(), - availabilityTestArgs}; - available = ExternalParser.check(availabilityCheckArgs); - LOG.debug("Available? {}", available); - if (!available) { - return; - } - String[] parseCmd = - {executor, scriptFile.getAbsolutePath(), modelArg, modelFile.getAbsolutePath(), - imageArg, INPUT_FILE_TOKEN, "--out_file", - OUTPUT_FILE_TOKEN}; //inserting output token to let - // external parser parse metadata - setCommand(parseCmd); - HashMap patterns = new HashMap<>(); - patterns.put(Pattern.compile(outPattern), null); - setMetadataExtractionPatterns(patterns); - setIgnoredLineConsumer(IGNORED_LINE_LOGGER); - } catch (Exception e) { - throw new TikaConfigException(e.getMessage(), e); - } - } - - @Override - public void checkInitialization(InitializableProblemHandler handler) - throws TikaConfigException { - //TODO -- what do we want to check? - } - - @Override - public List recognise(InputStream stream, ContentHandler handler, - Metadata metadata, ParseContext context) - throws IOException, SAXException, TikaException { - Metadata md = new Metadata(); - parse(stream, handler, md, context); - List objects = new ArrayList<>(); - for (String key : md.names()) { - double confidence = Double.parseDouble(md.get(key)); - objects.add(new RecognisedObject(key, "eng", key, confidence)); - } - return objects; - } -} - diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/tf/TensorflowRESTRecogniser.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/tf/TensorflowRESTRecogniser.java deleted file mode 100644 index fa07ef4e259..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/tf/TensorflowRESTRecogniser.java +++ /dev/null @@ -1,178 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tika.parser.recognition.tf; - -import java.io.IOException; -import java.io.InputStream; -import java.net.URI; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -import com.github.openjson.JSONArray; -import com.github.openjson.JSONObject; -import org.apache.commons.io.IOUtils; -import org.apache.commons.io.output.UnsynchronizedByteArrayOutputStream; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.ContentHandler; -import org.xml.sax.SAXException; - -import org.apache.tika.config.Field; -import org.apache.tika.config.InitializableProblemHandler; -import org.apache.tika.config.Param; -import org.apache.tika.exception.TikaConfigException; -import org.apache.tika.exception.TikaException; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.mime.MediaType; -import org.apache.tika.parser.ParseContext; -import org.apache.tika.parser.recognition.ObjectRecogniser; -import org.apache.tika.parser.recognition.RecognisedObject; - -/** - * Tensor Flow image recogniser which has high performance. - * This implementation uses Tensorflow via REST API. - *

- * NOTE : https://wiki.apache.org/tika/TikaAndVision - * - * @since Apache Tika 1.14 - */ -public class TensorflowRESTRecogniser implements ObjectRecogniser { - - protected static final String LABEL_LANG = "eng"; - /** - * Some variables are protected, because this class is extended by - * TensorflowRESTVideoRecognizer class - */ - - private static final Logger LOG = LoggerFactory.getLogger(TensorflowRESTRecogniser.class); - private static final Set SUPPORTED_MIMES = Collections.unmodifiableSet(new HashSet<>( - Arrays.asList(new MediaType[]{MediaType.image("jpeg"), MediaType.image("png"), - MediaType.image("gif")}))); - @Field - protected URI apiBaseUri = URI.create("http://localhost:8764/inception/v4"); - - @Field - protected int topN = 2; - - @Field - protected double minConfidence = 0.015; - - protected URI apiUri; - - protected URI healthUri; - - protected boolean available; - - protected URI getApiUri(Metadata metadata) { - return apiUri; - } - - @Override - public Set getSupportedMimes() { - return SUPPORTED_MIMES; - } - - @Override - public boolean isAvailable() { - return available; - } - - @Override - public void initialize(Map params) throws TikaConfigException { - healthUri = URI.create(apiBaseUri + "/ping"); - apiUri = URI.create(apiBaseUri + String.format(Locale.getDefault(), - "/classify/image?topn=%1$d&min_confidence=%2$f", topN, minConfidence)); - - try (CloseableHttpClient client = HttpClientBuilder.create().build()) { - HttpResponse response = client.execute(new HttpGet(healthUri)); - available = response.getStatusLine().getStatusCode() == 200; - - LOG.info("Available = {}, API Status = {}", available, response.getStatusLine()); - LOG.info("topN = {}, minConfidence = {}", topN, minConfidence); - } catch (Exception e) { - available = false; - throw new TikaConfigException(e.getMessage(), e); - } - } - - @Override - public void checkInitialization(InitializableProblemHandler handler) - throws TikaConfigException { - //TODO -- what do we want to check? - } - - @Override - public List recognise(InputStream stream, ContentHandler handler, - Metadata metadata, ParseContext context) - throws IOException, SAXException, TikaException { - List recObjs = new ArrayList<>(); - try (CloseableHttpClient client = HttpClientBuilder.create().build()) { - - HttpPost request = new HttpPost(getApiUri(metadata)); - - try (UnsynchronizedByteArrayOutputStream byteStream = UnsynchronizedByteArrayOutputStream.builder().get()) { - //TODO: convert this to stream, this might cause OOM issue - // InputStreamEntity is not working - // request.setEntity(new InputStreamEntity(stream, -1)); - IOUtils.copy(stream, byteStream); - request.setEntity(new ByteArrayEntity(byteStream.toByteArray())); - } - - HttpResponse response = client.execute(request); - try (InputStream reply = response.getEntity().getContent()) { - String replyMessage = IOUtils.toString(reply, StandardCharsets.UTF_8); - if (response.getStatusLine().getStatusCode() == 200) { - JSONObject jReply = new JSONObject(replyMessage); - JSONArray jClasses = jReply.getJSONArray("classnames"); - JSONArray jConfidence = jReply.getJSONArray("confidence"); - if (jClasses.length() != jConfidence.length()) { - LOG.warn("Classes of size {} is not equal to confidence of size {}", - jClasses.length(), jConfidence.length()); - } - assert jClasses.length() == jConfidence.length(); - for (int i = 0; i < jClasses.length(); i++) { - RecognisedObject recObj = - new RecognisedObject(jClasses.getString(i), LABEL_LANG, - jClasses.getString(i), jConfidence.getDouble(i)); - recObjs.add(recObj); - } - } else { - LOG.warn("Status = {}", response.getStatusLine()); - LOG.warn("Response = {}", replyMessage); - } - } - } catch (Exception e) { - LOG.warn(e.getMessage(), e); - } - LOG.debug("Num Objects found {}", recObjs.size()); - return recObjs; - } -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/tf/TensorflowRESTVideoRecogniser.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/tf/TensorflowRESTVideoRecogniser.java deleted file mode 100644 index 4094bc416a7..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/java/org/apache/tika/parser/recognition/tf/TensorflowRESTVideoRecogniser.java +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tika.parser.recognition.tf; - -import java.net.URI; -import java.util.Collections; -import java.util.Locale; -import java.util.Map; -import java.util.Set; - -import jakarta.ws.rs.core.UriBuilder; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import org.apache.tika.config.Field; -import org.apache.tika.config.Param; -import org.apache.tika.config.TikaConfig; -import org.apache.tika.exception.TikaConfigException; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.mime.MediaType; -import org.apache.tika.mime.MimeType; -import org.apache.tika.mime.MimeTypeException; - -/** - * Tensor Flow video recogniser which has high performance. - * This implementation uses Tensorflow via REST API. - *

- * NOTE : https://wiki.apache.org/tika/TikaAndVisionVideo - * - * @since Apache Tika 1.15 - */ -public class TensorflowRESTVideoRecogniser extends TensorflowRESTRecogniser { - - private static final Logger LOG = LoggerFactory.getLogger(TensorflowRESTVideoRecogniser.class); - - private static final Set SUPPORTED_MIMES = - Collections.singleton(MediaType.video("mp4")); - - @Field - private String mode = "fixed"; - - @Override - protected URI getApiUri(Metadata metadata) { - TikaConfig config = TikaConfig.getDefaultConfig(); - String ext = null; - //Find extension for video. It's required for OpenCV in InceptionAPI to decode video - try { - MimeType mimeType = config.getMimeRepository().forName(metadata.get("Content-Type")); - ext = mimeType.getExtension(); - return UriBuilder.fromUri(apiUri).queryParam("ext", ext).build(); - } catch (MimeTypeException e) { - LOG.error("Can't find extension from metadata"); - return apiUri; - } - } - - @Override - public void initialize(Map params) throws TikaConfigException { - healthUri = URI.create(apiBaseUri + "/ping"); - apiUri = URI.create(apiBaseUri + String.format(Locale.getDefault(), - "/classify/video?topn=%1$d&min_confidence=%2$f&mode=%3$s", topN, minConfidence, - mode)); - - try (CloseableHttpClient client = HttpClientBuilder.create().build()) { - HttpResponse response = client.execute(new HttpGet(healthUri)); - available = response.getStatusLine().getStatusCode() == 200; - - LOG.info("Available = {}, API Status = {}", available, response.getStatusLine()); - LOG.info("topN = {}, minConfidence = {}", topN, minConfidence); - } catch (Exception e) { - available = false; - throw new TikaConfigException(e.getMessage(), e); - } - } -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/caption_generator.py b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/caption_generator.py deleted file mode 100644 index 9dc13e4b695..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/caption_generator.py +++ /dev/null @@ -1,179 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import heapq -import math -import numpy as np - - -class Caption(object): - """ - A complete or partial caption object - """ - - def __init__(self, sentence, state, logprob, score): - """Initializes the Caption""" - - # list of word_ids in the caption - self.sentence = sentence - # model state after generating the previous word - self.state = state - # log probability of the caption - self.logprob = logprob - # score of the caption - self.score = score - - def __cmp__(self, other): - """Compares Captions by score""" - - assert isinstance(other, Caption) - if self.score == other.score: - return 0 - elif self.score < other.score: - return -1 - else: - return 1 - - # for Python 3 compatibility (__cmp__ is deprecated). - def __lt__(self, other): - assert isinstance(other, Caption) - return self.score < other.score - - # also for Python 3 compatibility. - def __eq__(self, other): - assert isinstance(other, Caption) - return self.score == other.score - - -class TopN(object): - """Maintains the top N elements of an incrementally provided set""" - - def __init__(self, n): - self._n = n - self._data = [] - - def size(self): - assert self._data is not None - return len(self._data) - - def push(self, x): - """Pushes a new element""" - - assert self._data is not None - if len(self._data) < self._n: - heapq.heappush(self._data, x) - else: - heapq.heappushpop(self._data, x) - - def extract(self, sort=False): - """ - Extracts all elements from the TopN. This is a destructive operation, - The only method that can be called immediately after extract() is reset() - """ - assert self._data is not None - data = self._data - self._data = None - if sort: - data.sort(reverse=True) - return data - - def reset(self): - """Returns the TopN to an empty state""" - - self._data = [] - - -class CaptionGenerator(object): - """ - Class to generate captions from an image-to-text model - """ - - def __init__(self, - model, - vocab, - beam_size, - max_caption_length, - length_normalization_factor=0.0): - - self.vocab = vocab - self.model = model - - self.beam_size = beam_size - self.max_caption_length = max_caption_length - self.length_normalization_factor = length_normalization_factor - - def beam_search(self, sess, encoded_image): - """Runs beam search caption generation on a single image""" - - # feed in the image to get the initial state. - initial_state = self.model.feed_image(sess, encoded_image) - - initial_beam = Caption( - sentence=[self.vocab.start_id], - state=initial_state[0], - logprob=0.0, - score=0.0) - partial_captions = TopN(self.beam_size) - partial_captions.push(initial_beam) - complete_captions = TopN(self.beam_size) - - # run beam search. - for _ in range(self.max_caption_length - 1): - partial_captions_list = partial_captions.extract() - partial_captions.reset() - input_feed = np.array([c.sentence[-1] for c in partial_captions_list]) - state_feed = np.array([c.state for c in partial_captions_list]) - - softmax, new_states = self.model.inference_step(sess, input_feed, state_feed) - - for i, partial_caption in enumerate(partial_captions_list): - word_probabilities = softmax[i] - state = new_states[i] - # for this partial caption, get the beam_size most probable next words. - words_and_probs = list(enumerate(word_probabilities)) - words_and_probs.sort(key=lambda x: -x[1]) - words_and_probs = words_and_probs[0:self.beam_size] - # each next word gives a new partial caption. - for w, p in words_and_probs: - if p < 1e-12: - continue # avoid log(0). - sentence = partial_caption.sentence + [w] - logprob = partial_caption.logprob + math.log(p) - score = logprob - - if w == self.vocab.end_id: - if self.length_normalization_factor > 0: - score /= len(sentence) ** self.length_normalization_factor - beam = Caption(sentence, state, logprob, score) - complete_captions.push(beam) - else: - beam = Caption(sentence, state, logprob, score) - partial_captions.push(beam) - if partial_captions.size() == 0: - # we have run out of partial candidates; happens when beam_size = 1. - break - - # if we have no complete captions then fall back to the partial captions, - # but never output a mixture of complete and partial captions because a - # partial caption could have a higher score than all the complete captions - if not complete_captions.size(): - complete_captions = partial_captions - - return complete_captions.extract(sort=True) diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/im2txtapi.py b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/im2txtapi.py deleted file mode 100644 index 97f1f2afd64..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/im2txtapi.py +++ /dev/null @@ -1,266 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -""" - This script exposes image captioning service over a REST API. Image captioning implementation based on the paper, - - "Show and Tell: A Neural Image Caption Generator" - Oriol Vinyals, Alexander Toshev, Samy Bengio, Dumitru Erhan - - For more details, please visit : - http://arxiv.org/abs/1411.4555 - Requirements : - Flask - tensorflow - numpy - requests -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import json -import logging -import math -import requests -import sys - -from flask import Flask, request, Response, jsonify -from io import BytesIO -from PIL import Image -from time import time - -import tensorflow as tf -import xml.etree.ElementTree as ET - -import model_wrapper -import vocabulary -import caption_generator - -# turning off the traceback by limiting its depth -sys.tracebacklimit = 0 - -# informative log messages for advanced users to troubleshoot errors when modifying model_info.xml -try: - info = ET.parse('/usr/share/apache-tika/models/dl/image/caption/model_info.xml').getroot() -except IOError: - logging.exception('model_info.xml is not found') - sys.exit(1) - -model_main = info.find('model_main') -if model_main is None: - logging.exception(' tag under tag in model_info.xml is not found') - sys.exit(1) - -checkpoint_path = model_main.find('checkpoint_path') -if checkpoint_path is None: - logging.exception(' tag under tag in model_info.xml is not found') - sys.exit(1) -else: - checkpoint_path = checkpoint_path.text - -vocab_file = model_main.find('vocab_file') -if vocab_file is None: - logging.exception(' tag under tag in model_info.xml is not found') - sys.exit(1) -else: - vocab_file = vocab_file.text - -port = info.get('port') -if port is None: - logging.exception('port attribute in tag in model_info.xml is not found') - sys.exit(1) - -# turning on the traceback by setting it to default -sys.tracebacklimit = 1000 - -FLAGS = tf.flags.FLAGS -tf.flags.DEFINE_string("checkpoint_path", checkpoint_path, """Directory containing the model checkpoint file.""") -tf.flags.DEFINE_string('vocab_file', vocab_file, """Text file containing the vocabulary.""") -tf.flags.DEFINE_integer('port', port, """Server PORT, default:8764""") - -tf.logging.set_verbosity(tf.logging.INFO) - - -class Initializer(Flask): - """ - Class to initialize the REST API, this class loads the model from the given checkpoint path in model_info.xml - and prepares a caption_generator object - """ - - def __init__(self, name): - super(Initializer, self).__init__(name) - # build the inference graph - g = tf.Graph() - with g.as_default(): - model = model_wrapper.ModelWrapper() - restore_fn = model.build_graph(FLAGS.checkpoint_path) - g.finalize() - # make the model globally available - self.model = model - # create the vocabulary - self.vocab = vocabulary.Vocabulary(FLAGS.vocab_file) - self.sess = tf.Session(graph=g) - # load the model from checkpoint - restore_fn(self.sess) - - -def current_time(): - """Returns current time in milli seconds""" - - return int(1000 * time()) - - -app = Initializer(__name__) - - -def get_remote_file(url, success=200, timeout=10): - """ - Given HTTP URL, this api gets the content of it - returns (Content-Type, image_content) - """ - try: - app.logger.info("GET: %s" % url) - auth = None - res = requests.get(url, stream=True, timeout=timeout, auth=auth) - if res.status_code == success: - return res.headers.get('Content-Type', 'application/octet-stream'), res.raw.data - except: - pass - return None, None - - -@app.route("/") -def index(): - """The index page which provide information about other API end points""" - - return """ -

-

Image Captioning REST API

-

The following API end points are valid

-
    -

    Inception V3

    -
  • /inception/v3/ping -
    - Description : checks availability of the service. returns "pong" with status 200 when it is available -
    • -
  • /inception/v3/caption/image -
    - - - - - -
    Description This is a service that can caption images
    How to supply Image Content
    With HTTP GET : - Include a query parameter url which is an http url of JPEG image
    - Example: curl "localhost:8764/inception/v3/caption/image?url=http://xyz.com/example.jpg" -
    With HTTP POST : - POST JPEG image content as binary data in request body.
    - Example: curl -X POST "localhost:8764/inception/v3/caption/image" --data-binary @example.jpg -
    -
    • -
      -
- """ - - -@app.route("/inception/v3/ping", methods=["GET"]) -def ping_pong(): - """API to do health check. If this says status code 200, then healthy""" - - return "pong" - - -@app.route("/inception/v3/caption/image", methods=["GET", "POST"]) -def caption_image(): - """API to caption images""" - image_format = "not jpeg" - - st = current_time() - # get beam_size - beam_size = int(request.args.get("beam_size", "3")) - # get max_caption_length - max_caption_length = int(request.args.get("max_caption_length", "20")) - # get image_data - if request.method == 'POST': - image_data = request.get_data() - else: - url = request.args.get("url") - c_type, image_data = get_remote_file(url) - if not image_data: - return Response(status=400, response=jsonify(error="Could not HTTP GET %s" % url)) - if 'image/jpeg' in c_type: - image_format = "jpeg" - - # use c_type to find whether image_format is jpeg or not - # if jpeg, don't convert - if image_format == "jpeg": - jpg_image = image_data - # if not jpeg - else: - # open the image from raw bytes - image = Image.open(BytesIO(image_data)) - # convert the image to RGB format, otherwise will give errors when converting to jpeg, if the image isn't RGB - rgb_image = image.convert("RGB") - # convert the RGB image to jpeg - image_bytes = BytesIO() - rgb_image.save(image_bytes, format="jpeg", quality=95) - jpg_image = image_bytes.getvalue() - image_bytes.close() - - read_time = current_time() - st - # restart counter - st = current_time() - - generator = caption_generator.CaptionGenerator(app.model, - app.vocab, - beam_size=beam_size, - max_caption_length=max_caption_length) - captions = generator.beam_search(app.sess, jpg_image) - - captioning_time = current_time() - st - app.logger.info("Captioning time : %d" % captioning_time) - - array_captions = [] - for caption in captions: - sentence = [app.vocab.id_to_word(w) for w in caption.sentence[1:-1]] - sentence = " ".join(sentence) - array_captions.append({ - 'sentence': sentence, - 'confidence': math.exp(caption.logprob) - }) - - response = { - 'beam_size': beam_size, - 'max_caption_length': max_caption_length, - 'captions': array_captions, - 'time': { - 'read': read_time, - 'captioning': captioning_time, - 'units': 'ms' - } - } - return Response(response=json.dumps(response), status=200, mimetype="application/json") - - -def main(_): - if not app.debug: - print("Serving on port %d" % FLAGS.port) - app.run(host="0.0.0.0", port=FLAGS.port) - - -if __name__ == '__main__': - tf.app.run() diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/model_info.xml b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/model_info.xml deleted file mode 100644 index da838170288..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/model_info.xml +++ /dev/null @@ -1,22 +0,0 @@ - - - - - /usr/share/apache-tika/models/dl/image/caption/1M_iters_ckpt/model.ckpt-1000000 - - /usr/share/apache-tika/models/dl/image/caption/1M_iters_ckpt/word_counts.txt - - \ No newline at end of file diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/model_wrapper.py b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/model_wrapper.py deleted file mode 100644 index a542e7b9a6f..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/model_wrapper.py +++ /dev/null @@ -1,340 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os.path - -import tensorflow as tf -from tensorflow.contrib.slim.python.slim.nets.inception_v3 import inception_v3_base - -slim = tf.contrib.slim - - -class ModelWrapper(object): - """ - Model wrapper class to perform image captioning with a ShowAndTellModel - """ - - def __init__(self): - super(ModelWrapper, self).__init__() - - def build_graph(self, checkpoint_path): - """Builds the inference graph""" - - tf.logging.info("Building model.") - ShowAndTellModel().build() - saver = tf.train.Saver() - - return self._create_restore_fn(checkpoint_path, saver) - - def _create_restore_fn(self, checkpoint_path, saver): - """Creates a function that restores a model from checkpoint file""" - - if tf.gfile.IsDirectory(checkpoint_path): - checkpoint_path = tf.train.latest_checkpoint(checkpoint_path) - if not checkpoint_path: - raise ValueError("No checkpoint file found in: %s" % checkpoint_path) - - def _restore_fn(sess): - tf.logging.info("Loading model from checkpoint: %s", checkpoint_path) - saver.restore(sess, checkpoint_path) - tf.logging.info("Successfully loaded checkpoint: %s", - os.path.basename(checkpoint_path)) - - return _restore_fn - - def feed_image(self, sess, encoded_image): - initial_state = sess.run(fetches="lstm/initial_state:0", - feed_dict={"image_feed:0": encoded_image}) - return initial_state - - def inference_step(self, sess, input_feed, state_feed): - softmax_output, state_output = sess.run( - fetches=["softmax:0", "lstm/state:0"], - feed_dict={ - "input_feed:0": input_feed, - "lstm/state_feed:0": state_feed, - }) - return softmax_output, state_output - - -class ShowAndTellModel(object): - """ - Image captioning implementation based on the paper, - - "Show and Tell: A Neural Image Caption Generator" - Oriol Vinyals, Alexander Toshev, Samy Bengio, Dumitru Erhan - - For more details, please visit : http://arxiv.org/abs/1411.4555 - """ - - def __init__(self): - - # scale used to initialize model variables - self.initializer_scale = 0.08 - - # dimensions of Inception v3 input images - self.image_height = 299 - self.image_width = 299 - - # LSTM input and output dimensionality, respectively - self.embedding_size = 512 - self.num_lstm_units = 512 - - # number of unique words in the vocab (plus 1, for ) - # the default value is larger than the expected actual vocab size to allow - # for differences between tokenizer versions used in preprocessing, there is - # no harm in using a value greater than the actual vocab size, but using a - # value less than the actual vocab size will result in an error - self.vocab_size = 12000 - - # reader for the input data - self.reader = tf.TFRecordReader() - - # to match the "Show and Tell" paper we initialize all variables with a - # random uniform initializer - self.initializer = tf.random_uniform_initializer( - minval=-self.initializer_scale, - maxval=self.initializer_scale) - - # a float32 Tensor with shape [batch_size, height, width, channels] - self.images = None - - # an int32 Tensor with shape [batch_size, padded_length] - self.input_seqs = None - - # an int32 Tensor with shape [batch_size, padded_length] - self.target_seqs = None - - # an int32 0/1 Tensor with shape [batch_size, padded_length] - self.input_mask = None - - # a float32 Tensor with shape [batch_size, embedding_size] - self.image_embeddings = None - - # a float32 Tensor with shape [batch_size, padded_length, embedding_size] - self.seq_embeddings = None - - # collection of variables from the inception submodel - self.inception_variables = [] - - # global step Tensor - self.global_step = None - - def process_image(self, encoded_image, resize_height=346, resize_width=346, thread_id=0): - """Decodes and processes an image string""" - - # helper function to log an image summary to the visualizer. Summaries are - # only logged in thread 0 - def image_summary(name, img): - if not thread_id: - tf.summary.image(name, tf.expand_dims(img, 0)) - - # decode image into a float32 Tensor of shape [?, ?, 3] with values in [0, 1) - with tf.name_scope("decode", values=[encoded_image]): - image = tf.image.decode_jpeg(encoded_image, channels=3) - - image = tf.image.convert_image_dtype(image, dtype=tf.float32) - image_summary("original_image", image) - - # resize image - assert (resize_height > 0) == (resize_width > 0) - if resize_height: - image = tf.image.resize_images(image, - size=[resize_height, resize_width], - method=tf.image.ResizeMethod.BILINEAR) - - # central crop, assuming resize_height > height, resize_width > width - image = tf.image.resize_image_with_crop_or_pad(image, self.image_height, self.image_width) - - image_summary("resized_image", image) - - image_summary("final_image", image) - - # rescale to [-1,1] instead of [0, 1] - image = tf.subtract(image, 0.5) - image = tf.multiply(image, 2.0) - return image - - def build_inputs(self): - """Input prefetching, preprocessing and batching""" - - image_feed = tf.placeholder(dtype=tf.string, shape=[], name="image_feed") - input_feed = tf.placeholder(dtype=tf.int64, - shape=[None], # batch_size - name="input_feed") - - # process image and insert batch dimensions - images = tf.expand_dims(self.process_image(image_feed), 0) - input_seqs = tf.expand_dims(input_feed, 1) - - # no target sequences or input mask in inference mode - target_seqs = None - input_mask = None - - self.images = images - self.input_seqs = input_seqs - self.target_seqs = target_seqs - self.input_mask = input_mask - - def build_image_embeddings(self): - """Builds the image model(Inception V3) subgraph and generates image embeddings""" - - # parameter initialization - batch_norm_params = { - "is_training": False, - "trainable": False, - # decay for the moving averages - "decay": 0.9997, - # epsilon to prevent 0s in variance - "epsilon": 0.001, - # collection containing the moving mean and moving variance - "variables_collections": { - "beta": None, - "gamma": None, - "moving_mean": ["moving_vars"], - "moving_variance": ["moving_vars"], - } - } - - stddev = 0.1, - dropout_keep_prob = 0.8 - - with tf.variable_scope("InceptionV3", "InceptionV3", [self.images]) as scope: - with slim.arg_scope( - [slim.conv2d, slim.fully_connected], - weights_regularizer=None, - trainable=False): - with slim.arg_scope( - [slim.conv2d], - weights_initializer=tf.truncated_normal_initializer(stddev=stddev), - activation_fn=tf.nn.relu, - normalizer_fn=slim.batch_norm, - normalizer_params=batch_norm_params): - net, end_points = inception_v3_base(self.images, scope=scope) - with tf.variable_scope("logits"): - shape = net.get_shape() - net = slim.avg_pool2d(net, shape[1:3], padding="VALID", scope="pool") - net = slim.dropout( - net, - keep_prob=dropout_keep_prob, - is_training=False, - scope="dropout") - net = slim.flatten(net, scope="flatten") - - # add summaries - for v in end_points.values(): - tf.contrib.layers.summaries.summarize_activation(v) - - self.inception_variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="InceptionV3") - - # map inception output(net) into embedding space - with tf.variable_scope("image_embedding") as scope: - image_embeddings = tf.contrib.layers.fully_connected( - inputs=net, - num_outputs=self.embedding_size, - activation_fn=None, - weights_initializer=self.initializer, - biases_initializer=None, - scope=scope) - - # save the embedding size in the graph - tf.constant(self.embedding_size, name="embedding_size") - - self.image_embeddings = image_embeddings - - def build_seq_embeddings(self): - """Builds the input sequence embeddings""" - - with tf.variable_scope("seq_embedding"), tf.device("/cpu:0"): - embedding_map = tf.get_variable( - name="map", - shape=[self.vocab_size, self.embedding_size], - initializer=self.initializer) - seq_embeddings = tf.nn.embedding_lookup(embedding_map, self.input_seqs) - - self.seq_embeddings = seq_embeddings - - def build_model(self): - - # this LSTM cell has biases and outputs tanh(new_c) * sigmoid(o), but the - # modified LSTM in the "Show and Tell" paper has no biases and outputs - # new_c * sigmoid(o). - - lstm_cell = tf.contrib.rnn.BasicLSTMCell( - num_units=self.num_lstm_units, state_is_tuple=True) - - with tf.variable_scope("lstm", initializer=self.initializer) as lstm_scope: - # feed the image embeddings to set the initial LSTM state - zero_state = lstm_cell.zero_state( - batch_size=self.image_embeddings.get_shape()[0], dtype=tf.float32) - _, initial_state = lstm_cell(self.image_embeddings, zero_state) - - # allow the LSTM variables to be reused - lstm_scope.reuse_variables() - - # because this is inference mode, - # use concatenated states for convenient feeding and fetching - tf.concat(axis=1, values=initial_state, name="initial_state") - - # placeholder for feeding a batch of concatenated states - state_feed = tf.placeholder(dtype=tf.float32, - shape=[None, sum(lstm_cell.state_size)], - name="state_feed") - state_tuple = tf.split(value=state_feed, num_or_size_splits=2, axis=1) - - # run a single LSTM step - lstm_outputs, state_tuple = lstm_cell( - inputs=tf.squeeze(self.seq_embeddings, axis=[1]), - state=state_tuple) - - # concatentate the resulting state - tf.concat(axis=1, values=state_tuple, name="state") - - # stack batches vertically - lstm_outputs = tf.reshape(lstm_outputs, [-1, lstm_cell.output_size]) - - with tf.variable_scope("logits") as logits_scope: - logits = tf.contrib.layers.fully_connected( - inputs=lstm_outputs, - num_outputs=self.vocab_size, - activation_fn=None, - weights_initializer=self.initializer, - scope=logits_scope) - - tf.nn.softmax(logits, name="softmax") - - def setup_global_step(self): - """Sets up the global step Tensor""" - - global_step = tf.Variable( - initial_value=0, - name="global_step", - trainable=False, - collections=[tf.GraphKeys.GLOBAL_STEP, tf.GraphKeys.GLOBAL_VARIABLES]) - - self.global_step = global_step - - def build(self): - self.build_inputs() - self.build_image_embeddings() - self.build_seq_embeddings() - self.build_model() - self.setup_global_step() diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/vocabulary.py b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/vocabulary.py deleted file mode 100644 index 566b034cb0a..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/captioning/tf/vocabulary.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - - -class Vocabulary(object): - """ - Vocabulary class for an image-to-text model - """ - - def __init__(self, - vocab_file, - start_word="", - end_word="", - unk_word=""): - """Initializes the vocabulary""" - - if not tf.gfile.Exists(vocab_file): - tf.logging.fatal("Vocab file %s not found.", vocab_file) - tf.logging.info("Initializing vocabulary from file: %s", vocab_file) - - with tf.gfile.GFile(vocab_file, mode="r") as f: - reverse_vocab = list(f.readlines()) - reverse_vocab = [line.split()[0] for line in reverse_vocab] - assert start_word in reverse_vocab - assert end_word in reverse_vocab - if unk_word not in reverse_vocab: - reverse_vocab.append(unk_word) - vocab = dict([(x, y) for (y, x) in enumerate(reverse_vocab)]) - - tf.logging.info("Created vocabulary with %d words" % len(vocab)) - - # vocab[word] = id - self.vocab = vocab - # reverse_vocab[id] = word - self.reverse_vocab = reverse_vocab - - # save special word ids - self.start_id = vocab[start_word] - self.end_id = vocab[end_word] - self.unk_id = vocab[unk_word] - - def id_to_word(self, word_id): - """Returns the word string of an integer word id""" - - if word_id >= len(self.reverse_vocab): - return self.reverse_vocab[self.unk_id] - else: - return self.reverse_vocab[word_id] diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tf/inception_v4.py b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tf/inception_v4.py deleted file mode 100644 index b6644160fed..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tf/inception_v4.py +++ /dev/null @@ -1,362 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Contains the definition of the Inception V4 architecture. - -As described in http://arxiv.org/abs/1602.07261. - - Inception-v4, Inception-ResNet and the Impact of Residual Connections - on Learning - Christian Szegedy, Sergey Ioffe, Vincent Vanhoucke, Alex Alemi -""" -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import tensorflow as tf - -slim = tf.contrib.slim - - -def block_inception_a(inputs, scope=None, reuse=None): - """Builds Inception-A block for Inception v4 network.""" - # By default use stride=1 and SAME padding - with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], - stride=1, padding='SAME'): - with tf.variable_scope(scope, 'BlockInceptionA', [inputs], reuse=reuse): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(inputs, 96, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(inputs, 64, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 96, [3, 3], scope='Conv2d_0b_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(inputs, 64, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 96, [3, 3], scope='Conv2d_0b_3x3') - branch_2 = slim.conv2d(branch_2, 96, [3, 3], scope='Conv2d_0c_3x3') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(inputs, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 96, [1, 1], scope='Conv2d_0b_1x1') - return tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) - - -def block_reduction_a(inputs, scope=None, reuse=None): - """Builds Reduction-A block for Inception v4 network.""" - # By default use stride=1 and SAME padding - with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], - stride=1, padding='SAME'): - with tf.variable_scope(scope, 'BlockReductionA', [inputs], reuse=reuse): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(inputs, 384, [3, 3], stride=2, padding='VALID', - scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(inputs, 192, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 224, [3, 3], scope='Conv2d_0b_3x3') - branch_1 = slim.conv2d(branch_1, 256, [3, 3], stride=2, - padding='VALID', scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.max_pool2d(inputs, [3, 3], stride=2, padding='VALID', - scope='MaxPool_1a_3x3') - return tf.concat(axis=3, values=[branch_0, branch_1, branch_2]) - - -def block_inception_b(inputs, scope=None, reuse=None): - """Builds Inception-B block for Inception v4 network.""" - # By default use stride=1 and SAME padding - with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], - stride=1, padding='SAME'): - with tf.variable_scope(scope, 'BlockInceptionB', [inputs], reuse=reuse): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(inputs, 384, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(inputs, 192, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 224, [1, 7], scope='Conv2d_0b_1x7') - branch_1 = slim.conv2d(branch_1, 256, [7, 1], scope='Conv2d_0c_7x1') - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(inputs, 192, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 192, [7, 1], scope='Conv2d_0b_7x1') - branch_2 = slim.conv2d(branch_2, 224, [1, 7], scope='Conv2d_0c_1x7') - branch_2 = slim.conv2d(branch_2, 224, [7, 1], scope='Conv2d_0d_7x1') - branch_2 = slim.conv2d(branch_2, 256, [1, 7], scope='Conv2d_0e_1x7') - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(inputs, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 128, [1, 1], scope='Conv2d_0b_1x1') - return tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) - - -def block_reduction_b(inputs, scope=None, reuse=None): - """Builds Reduction-B block for Inception v4 network.""" - # By default use stride=1 and SAME padding - with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], - stride=1, padding='SAME'): - with tf.variable_scope(scope, 'BlockReductionB', [inputs], reuse=reuse): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(inputs, 192, [1, 1], scope='Conv2d_0a_1x1') - branch_0 = slim.conv2d(branch_0, 192, [3, 3], stride=2, - padding='VALID', scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(inputs, 256, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 256, [1, 7], scope='Conv2d_0b_1x7') - branch_1 = slim.conv2d(branch_1, 320, [7, 1], scope='Conv2d_0c_7x1') - branch_1 = slim.conv2d(branch_1, 320, [3, 3], stride=2, - padding='VALID', scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_2'): - branch_2 = slim.max_pool2d(inputs, [3, 3], stride=2, padding='VALID', - scope='MaxPool_1a_3x3') - return tf.concat(axis=3, values=[branch_0, branch_1, branch_2]) - - -def block_inception_c(inputs, scope=None, reuse=None): - """Builds Inception-C block for Inception v4 network.""" - # By default use stride=1 and SAME padding - with slim.arg_scope([slim.conv2d, slim.avg_pool2d, slim.max_pool2d], - stride=1, padding='SAME'): - with tf.variable_scope(scope, 'BlockInceptionC', [inputs], reuse=reuse): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(inputs, 256, [1, 1], scope='Conv2d_0a_1x1') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(inputs, 384, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = tf.concat(axis=3, values=[ - slim.conv2d(branch_1, 256, [1, 3], scope='Conv2d_0b_1x3'), - slim.conv2d(branch_1, 256, [3, 1], scope='Conv2d_0c_3x1')]) - with tf.variable_scope('Branch_2'): - branch_2 = slim.conv2d(inputs, 384, [1, 1], scope='Conv2d_0a_1x1') - branch_2 = slim.conv2d(branch_2, 448, [3, 1], scope='Conv2d_0b_3x1') - branch_2 = slim.conv2d(branch_2, 512, [1, 3], scope='Conv2d_0c_1x3') - branch_2 = tf.concat(axis=3, values=[ - slim.conv2d(branch_2, 256, [1, 3], scope='Conv2d_0d_1x3'), - slim.conv2d(branch_2, 256, [3, 1], scope='Conv2d_0e_3x1')]) - with tf.variable_scope('Branch_3'): - branch_3 = slim.avg_pool2d(inputs, [3, 3], scope='AvgPool_0a_3x3') - branch_3 = slim.conv2d(branch_3, 256, [1, 1], scope='Conv2d_0b_1x1') - return tf.concat(axis=3, values=[branch_0, branch_1, branch_2, branch_3]) - - -def inception_v4_base(inputs, final_endpoint='Mixed_7d', scope=None): - """Creates the Inception V4 network up to the given final endpoint. - - Args: - inputs: a 4-D tensor of size [batch_size, height, width, 3]. - final_endpoint: specifies the endpoint to construct the network up to. - It can be one of [ 'Conv2d_1a_3x3', 'Conv2d_2a_3x3', 'Conv2d_2b_3x3', - 'Mixed_3a', 'Mixed_4a', 'Mixed_5a', 'Mixed_5b', 'Mixed_5c', 'Mixed_5d', - 'Mixed_5e', 'Mixed_6a', 'Mixed_6b', 'Mixed_6c', 'Mixed_6d', 'Mixed_6e', - 'Mixed_6f', 'Mixed_6g', 'Mixed_6h', 'Mixed_7a', 'Mixed_7b', 'Mixed_7c', - 'Mixed_7d'] - scope: Optional variable_scope. - - Returns: - logits: the logits outputs of the model. - end_points: the set of end_points from the inception model. - - Raises: - ValueError: if final_endpoint is not set to one of the predefined values, - """ - end_points = {} - - def add_and_check_final(name, net): - end_points[name] = net - return name == final_endpoint - - with tf.variable_scope(scope, 'InceptionV4', [inputs]): - with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], - stride=1, padding='SAME'): - # 299 x 299 x 3 - net = slim.conv2d(inputs, 32, [3, 3], stride=2, - padding='VALID', scope='Conv2d_1a_3x3') - if add_and_check_final('Conv2d_1a_3x3', net): return net, end_points - # 149 x 149 x 32 - net = slim.conv2d(net, 32, [3, 3], padding='VALID', - scope='Conv2d_2a_3x3') - if add_and_check_final('Conv2d_2a_3x3', net): return net, end_points - # 147 x 147 x 32 - net = slim.conv2d(net, 64, [3, 3], scope='Conv2d_2b_3x3') - if add_and_check_final('Conv2d_2b_3x3', net): return net, end_points - # 147 x 147 x 64 - with tf.variable_scope('Mixed_3a'): - with tf.variable_scope('Branch_0'): - branch_0 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', - scope='MaxPool_0a_3x3') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 96, [3, 3], stride=2, padding='VALID', - scope='Conv2d_0a_3x3') - net = tf.concat(axis=3, values=[branch_0, branch_1]) - if add_and_check_final('Mixed_3a', net): return net, end_points - - # 73 x 73 x 160 - with tf.variable_scope('Mixed_4a'): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 64, [1, 1], scope='Conv2d_0a_1x1') - branch_0 = slim.conv2d(branch_0, 96, [3, 3], padding='VALID', - scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_1'): - branch_1 = slim.conv2d(net, 64, [1, 1], scope='Conv2d_0a_1x1') - branch_1 = slim.conv2d(branch_1, 64, [1, 7], scope='Conv2d_0b_1x7') - branch_1 = slim.conv2d(branch_1, 64, [7, 1], scope='Conv2d_0c_7x1') - branch_1 = slim.conv2d(branch_1, 96, [3, 3], padding='VALID', - scope='Conv2d_1a_3x3') - net = tf.concat(axis=3, values=[branch_0, branch_1]) - if add_and_check_final('Mixed_4a', net): return net, end_points - - # 71 x 71 x 192 - with tf.variable_scope('Mixed_5a'): - with tf.variable_scope('Branch_0'): - branch_0 = slim.conv2d(net, 192, [3, 3], stride=2, padding='VALID', - scope='Conv2d_1a_3x3') - with tf.variable_scope('Branch_1'): - branch_1 = slim.max_pool2d(net, [3, 3], stride=2, padding='VALID', - scope='MaxPool_1a_3x3') - net = tf.concat(axis=3, values=[branch_0, branch_1]) - if add_and_check_final('Mixed_5a', net): return net, end_points - - # 35 x 35 x 384 - # 4 x Inception-A blocks - for idx in range(4): - block_scope = 'Mixed_5' + chr(ord('b') + idx) - net = block_inception_a(net, block_scope) - if add_and_check_final(block_scope, net): return net, end_points - - # 35 x 35 x 384 - # Reduction-A block - net = block_reduction_a(net, 'Mixed_6a') - if add_and_check_final('Mixed_6a', net): return net, end_points - - # 17 x 17 x 1024 - # 7 x Inception-B blocks - for idx in range(7): - block_scope = 'Mixed_6' + chr(ord('b') + idx) - net = block_inception_b(net, block_scope) - if add_and_check_final(block_scope, net): return net, end_points - - # 17 x 17 x 1024 - # Reduction-B block - net = block_reduction_b(net, 'Mixed_7a') - if add_and_check_final('Mixed_7a', net): return net, end_points - - # 8 x 8 x 1536 - # 3 x Inception-C blocks - for idx in range(3): - block_scope = 'Mixed_7' + chr(ord('b') + idx) - net = block_inception_c(net, block_scope) - if add_and_check_final(block_scope, net): return net, end_points - raise ValueError('Unknown final endpoint %s' % final_endpoint) - - -def inception_v4(inputs, num_classes=1001, is_training=True, - dropout_keep_prob=0.8, - reuse=None, - scope='InceptionV4', - create_aux_logits=True): - """Creates the Inception V4 model. - - Args: - inputs: a 4-D tensor of size [batch_size, height, width, 3]. - num_classes: number of predicted classes. - is_training: whether is training or not. - dropout_keep_prob: float, the fraction to keep before final layer. - reuse: whether or not the network and its variables should be reused. To be - able to reuse 'scope' must be given. - scope: Optional variable_scope. - create_aux_logits: Whether to include the auxiliary logits. - - Returns: - logits: the logits outputs of the model. - end_points: the set of end_points from the inception model. - """ - end_points = {} - with tf.variable_scope(scope, 'InceptionV4', [inputs], reuse=reuse) as scope: - with slim.arg_scope([slim.batch_norm, slim.dropout], - is_training=is_training): - net, end_points = inception_v4_base(inputs, scope=scope) - - with slim.arg_scope([slim.conv2d, slim.max_pool2d, slim.avg_pool2d], - stride=1, padding='SAME'): - # Auxiliary Head logits - if create_aux_logits: - with tf.variable_scope('AuxLogits'): - # 17 x 17 x 1024 - aux_logits = end_points['Mixed_6h'] - aux_logits = slim.avg_pool2d(aux_logits, [5, 5], stride=3, - padding='VALID', - scope='AvgPool_1a_5x5') - aux_logits = slim.conv2d(aux_logits, 128, [1, 1], - scope='Conv2d_1b_1x1') - aux_logits = slim.conv2d(aux_logits, 768, - aux_logits.get_shape()[1:3], - padding='VALID', scope='Conv2d_2a') - aux_logits = slim.flatten(aux_logits) - aux_logits = slim.fully_connected(aux_logits, num_classes, - activation_fn=None, - scope='Aux_logits') - end_points['AuxLogits'] = aux_logits - - # Final pooling and prediction - with tf.variable_scope('Logits'): - # 8 x 8 x 1536 - net = slim.avg_pool2d(net, net.get_shape()[1:3], padding='VALID', - scope='AvgPool_1a') - # 1 x 1 x 1536 - net = slim.dropout(net, dropout_keep_prob, scope='Dropout_1b') - net = slim.flatten(net, scope='PreLogitsFlatten') - end_points['PreLogitsFlatten'] = net - # 1536 - logits = slim.fully_connected(net, num_classes, activation_fn=None, - scope='Logits') - end_points['Logits'] = logits - end_points['Predictions'] = tf.nn.softmax(logits, name='Predictions') - return logits, end_points - - -def inception_v4_arg_scope(weight_decay=0.00004, - use_batch_norm=True, - batch_norm_decay=0.9997, - batch_norm_epsilon=0.001): - """Defines the default arg scope for inception models. - Args: - weight_decay: The weight decay to use for regularizing the model. - use_batch_norm: "If `True`, batch_norm is applied after each convolution. - batch_norm_decay: Decay for batch norm moving average. - batch_norm_epsilon: Small float added to variance to avoid dividing by zero - in batch norm. - Returns: - An `arg_scope` to use for the inception models. - """ - batch_norm_params = { - # Decay for the moving averages. - 'decay': batch_norm_decay, - # epsilon to prevent 0s in variance. - 'epsilon': batch_norm_epsilon, - # collection containing update_ops. - 'updates_collections': tf.GraphKeys.UPDATE_OPS, - } - if use_batch_norm: - normalizer_fn = slim.batch_norm - normalizer_params = batch_norm_params - else: - normalizer_fn = None - normalizer_params = {} - # Set weight_decay for weights in Conv and FC layers. - with slim.arg_scope([slim.conv2d, slim.fully_connected], - weights_regularizer=slim.l2_regularizer(weight_decay)): - with slim.arg_scope( - [slim.conv2d], - weights_initializer=slim.variance_scaling_initializer(), - activation_fn=tf.nn.relu, - normalizer_fn=normalizer_fn, - normalizer_params=normalizer_params) as sc: - return sc - - -default_image_size = 299 diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tf/inceptionapi.py b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tf/inceptionapi.py deleted file mode 100755 index 09d830c0ddc..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tf/inceptionapi.py +++ /dev/null @@ -1,483 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" - Image classification with Inception. - - This script exposes the tensorflow's inception classification service over REST API. - - For more details, visit: - https://tensorflow.org/tutorials/image_recognition/ - - Requirements : - Flask - tensorflow - numpy - requests - pillow -""" - -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function - -import os -import tempfile -import json -import logging -import requests - -from flask import Flask, request, Response, jsonify -from io import BytesIO -from logging.handlers import RotatingFileHandler -from PIL import Image -from time import time - -import tensorflow as tf - -from inception_v4 import default_image_size, inception_v4_arg_scope, inception_v4 - -try: - # This import is placed inside here to ensure that video_util and OpenCV is not required for image recognition APIs - from video_util import get_center_frame, get_frames_interval, get_n_frames -except: - print("Can't import video libraries, No video functionality is available") - -json.encoder.FLOAT_REPR = lambda o: format(o, '.2f') # JSON serialization of floats -slim = tf.contrib.slim -FLAGS = tf.app.flags.FLAGS - -tf.app.flags.DEFINE_string('model_dir', - '/usr/share/apache-tika/models/dl/image-video/recognition/', - """Path to inception_v4.ckpt & meta files""") -tf.app.flags.DEFINE_integer('port', - '8764', - """Server PORT, default:8764""") -tf.app.flags.DEFINE_string('log', - 'inception.log', - """Log file name, default: inception.log""") - - -def preprocess_image(image, height, width, central_fraction=0.875, scope=None): - """Prepare one image for evaluation. - If height and width are specified it would output an image with that size by - applying resize_bilinear. - If central_fraction is specified it would crop the central fraction of the - input image. - Args: - image: 3-D Tensor of image. If dtype is tf.float32 then the range should be - [0, 1], otherwise it would converted to tf.float32 assuming that the range - is [0, MAX], where MAX is largest positive representable number for - int(8/16/32) data type (see `tf.image.convert_image_dtype` for details). - height: integer - width: integer - central_fraction: Optional Float, fraction of the image to crop. - scope: Optional scope for name_scope. - Returns: - 3-D float Tensor of prepared image. - """ - with tf.name_scope(scope, 'eval_image', [image, height, width]): - if image.dtype != tf.float32: - image = tf.image.convert_image_dtype(image, dtype=tf.float32) - # Crop the central region of the image with an area containing 87.5% of - # the original image. - if central_fraction: - image = tf.image.central_crop(image, central_fraction=central_fraction) - - if height and width: - # Resize the image to the specified height and width. - image = tf.expand_dims(image, 0) - image = tf.image.resize_bilinear(image, [height, width], - align_corners=False) - image = tf.squeeze(image, [0]) - image = tf.subtract(image, 0.5) - image = tf.multiply(image, 2.0) - return image - - -def create_readable_names_for_imagenet_labels(): - """ - Create a dict mapping label id to human readable string. - Returns: - labels_to_names: dictionary where keys are integers from to 1000 - and values are human-readable names. - - We retrieve a synset file, which contains a list of valid synset labels used - by ILSVRC competition. There is one synset one per line, eg. - # n01440764 - # n01443537 - We also retrieve a synset_to_human_file, which contains a mapping from synsets - to human-readable names for every synset in Imagenet. These are stored in a - tsv format, as follows: - # n02119247 black fox - # n02119359 silver fox - We assign each synset (in alphabetical order) an integer, starting from 1 - (since 0 is reserved for the background class). - - Code is based on - https://github.com/tensorflow/models/blob/master/inception/inception/data/build_imagenet_data.py - """ - - dest_directory = FLAGS.model_dir - - synset_list = [s.strip() for s in open(os.path.join(dest_directory, 'imagenet_lsvrc_2015_synsets.txt')).readlines()] - num_synsets_in_ilsvrc = len(synset_list) - assert num_synsets_in_ilsvrc == 1000 - - synset_to_human_list = open(os.path.join(dest_directory, 'imagenet_metadata.txt')).readlines() - num_synsets_in_all_imagenet = len(synset_to_human_list) - assert num_synsets_in_all_imagenet == 21842 - - synset_to_human = {} - for s in synset_to_human_list: - parts = s.strip().split('\t') - assert len(parts) == 2 - synset = parts[0] - human = parts[1] - synset_to_human[synset] = human - - label_index = 1 - labels_to_names = {0: 'background'} - for synset in synset_list: - name = synset_to_human[synset] - labels_to_names[label_index] = name - label_index += 1 - - return labels_to_names - - -def get_remote_file(url, success=200, timeout=10): - """ - Given HTTP URL, this api gets the content of it - returns (Content-Type, image_content) - """ - try: - app.logger.info("GET: %s" % url) - auth = None - res = requests.get(url, stream=True, timeout=timeout, auth=auth) - if res.status_code == success: - return res.headers.get('Content-Type', 'application/octet-stream'), res.raw.data - except: - pass - return None, None - - -def current_time(): - """Returns current time in milli seconds""" - - return int(1000 * time()) - - -class Classifier(Flask): - """Classifier Service class""" - - def __init__(self, name): - super(Classifier, self).__init__(name) - file_handler = RotatingFileHandler(FLAGS.log, maxBytes=1024 * 1024 * 100, backupCount=20) - file_handler.setLevel(logging.INFO) - formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") - file_handler.setFormatter(formatter) - self.logger.addHandler(file_handler) - self.names = create_readable_names_for_imagenet_labels() - self.image_size = default_image_size - - self.image_str_placeholder = tf.placeholder(tf.string) - image = tf.image.decode_jpeg(self.image_str_placeholder, channels=3) - processed_image = preprocess_image(image, self.image_size, self.image_size) - processed_images = tf.expand_dims(processed_image, 0) - # create the model, use the default arg scope to configure the batch norm parameters. - with slim.arg_scope(inception_v4_arg_scope()): - logits, _ = inception_v4(processed_images, num_classes=1001, is_training=False) - self.probabilities = tf.nn.softmax(logits) - - dest_directory = FLAGS.model_dir - init_fn = slim.assign_from_checkpoint_fn( - os.path.join(dest_directory, 'inception_v4.ckpt'), - slim.get_model_variables('InceptionV4')) - - self.sess = tf.Session() - init_fn(self.sess) - - def classify(self, image_string, topn, min_confidence): - eval_probabilities = self.sess.run(self.probabilities, feed_dict={self.image_str_placeholder: image_string}) - eval_probabilities = eval_probabilities[0, 0:] - sorted_inds = [i[0] for i in sorted(enumerate(-eval_probabilities), key=lambda x: x[1])] - - if topn is None: - topn = len(sorted_inds) - - res = [] - for i in range(topn): - index = sorted_inds[i] - score = float(eval_probabilities[index]) - if min_confidence is None: - res.append((index, self.names[index], score)) - else: - if score >= min_confidence: - res.append((index, self.names[index], score)) - else: - # the scores are in sorted order, so we can break the loop whenever we get a low score object - break - return res - - -app = Classifier(__name__) - - -@app.route("/") -def index(): - """The index page which provide information about other API end points""" - - return """ -
-

Inception REST API

-

The following API end points are valid

-
    -

    Inception V4

    -
  • /inception/v4/ping -
    - Description : checks availability of the service. returns "pong" with status 200 when it is available -
    • -
  • /inception/v4/classify/image -
    - - - - - - -
    Description This is a classifier service that can classify images
    Query Params :
    - topn : type = int : top classes to get; default : 5
    - min_confidence : type = float : minimum confidence that a label should have to exist in topn; default : 0.015
    - human : type = boolean : human readable class names; default : true
    -
    How to supply Image Content
    With HTTP GET : - Include a query parameter url which is an http url of JPEG image
    - Example: curl "localhost:8764/inception/v4/classify/image?url=http://xyz.com/example.jpg" -
    With HTTP POST : - POST JPEG image content as binary data in request body.
    - Example: curl -X POST "localhost:8764/inception/v4/classify/image?topn=5&min_confidence=0.015&human=false" --data-binary @example.jpg -
    -
    • -
  • /inception/v4/classify/video -
    - - - - -
    - -
    Description This is a classifier service that can classify videos
    Query Params :
    - topn : type = int : top classes to get; default : 5
    - min_confidence : type = float : minimum confidence that a label should have to exist in topn; default : 0.015
    - human : type = boolean : human readable class names; default : true
    - mode : options = {"center", "interval", "fixed"} : Modes of frame extraction; default : center
    -   "center" - Just one frame in center.
    -   "interval" - Extracts frames after fixed interval.
    -   "fixed" - Extract fixed number of frames.
    - frame-interval : type = int : Interval for frame extraction to be used with INTERVAL mode. If frame_interval=10 then every 10th frame will be extracted; default : 10
    - num-frame : type = int : Number of frames to be extracted from video while using FIXED model. If num_frame=10 then 10 frames equally distant from each other will be extracted; default : 10
    - -
    How to supply Video Content
    With HTTP GET : - Include a query parameter url which is path on file system
    - Example: curl "localhost:8764/inception/v4/classify/video?url=filesystem/path/to/video"
    -
    With HTTP POST : - POST video content as binary data in request body. If video can be decoded by OpenCV it should be fine. It's tested on mp4 and avi on mac
    - Include a query parameter ext this extension is needed to tell OpenCV which decoder to use, default is ".mp4"
    - Example: curl -X POST "localhost:8764/inception/v4/classify/video?topn=5&min_confidence=0.015&human=false" --data-binary @example.mp4 -
    -
    • -
      -
- """ - - -@app.route("/inception/v4/ping", methods=["GET"]) -def ping_pong(): - """API to do health check. If this says status code 200, then healthy""" - - return "pong" - - -@app.route("/inception/v4/classify/image", methods=["GET", "POST"]) -def classify_image(): - """API to classify images""" - - image_format = "not jpeg" - - st = current_time() - topn = int(request.args.get("topn", "5")) - min_confidence = float(request.args.get("min_confidence", "0.015")) - human = request.args.get("human", "true").lower() in ("true", "1", "yes") - if request.method == 'POST': - image_data = request.get_data() - else: - url = request.args.get("url") - c_type, image_data = get_remote_file(url) - if not image_data: - return Response(status=400, response=jsonify(error="Could not HTTP GET %s" % url)) - if 'image/jpeg' in c_type: - image_format = "jpeg" - - # use c_type to find whether image_format is jpeg or not - # if jpeg, don't convert - if image_format == "jpeg": - jpg_image = image_data - # if not jpeg - else: - # open the image from raw bytes - image = Image.open(BytesIO(image_data)) - # convert the image to RGB format, otherwise will give errors when converting to jpeg, if the image isn't RGB - rgb_image = image.convert("RGB") - # convert the RGB image to jpeg - image_bytes = BytesIO() - rgb_image.save(image_bytes, format="jpeg", quality=95) - jpg_image = image_bytes.getvalue() - image_bytes.close() - - read_time = current_time() - st - st = current_time() # reset start time - try: - classes = app.classify(image_string=jpg_image, topn=topn, min_confidence=min_confidence) - except Exception as e: - app.logger.error(e) - return Response(status=400, response=str(e)) - classids, classnames, confidence = zip(*classes) - - print(classnames, confidence) - - classifier_time = current_time() - st - app.logger.info("Classifier time : %d" % classifier_time) - res = { - 'classids': classids, - 'confidence': confidence, - 'time': { - 'read': read_time, - 'classification': classifier_time, - 'units': 'ms' - } - } - if human: - res['classnames'] = classnames - return Response(response=json.dumps(res), status=200, mimetype="application/json") - - -@app.route("/inception/v4/classify/video", methods=["GET", "POST"]) -def classify_video(): - """ - API to classify videos - Request args - - url - PATH of file - topn - number of top scoring labels - min_confidence - minimum confidence that a label should have to exist in topn - human - human readable or not - mode - Modes of frame extraction {"center", "interval", "fixed"} - "center" - Just one frame in center. - "interval" - Extracts frames after fixed interval. - "fixed" - Extract fixed number of frames. - frame-interval - Interval for frame extraction to be used with INTERVAL mode. If frame_interval=10 then every 10th frame will be extracted. - num-frame - Number of frames to be extracted from video while using FIXED model. If num_frame=10 then 10 frames equally distant from each other will be extracted - - ext - If video is sent in binary format, then ext is needed to tell OpenCV which decoder to use. eg ".mp4" - """ - - st = current_time() - topn = int(request.args.get("topn", "5")) - min_confidence = float(request.args.get("min_confidence", "0.015")) - human = request.args.get("human", "true").lower() in ("true", "1", "yes") - - mode = request.args.get("mode", "center").lower() - if mode not in {"center", "interval", "fixed"}: - ''' - Throw invalid request error - ''' - return Response(status=400, response=jsonify(error="not a valid mode. Available mode %s" % str(ALLOWED_MODE))) - - frame_interval = int(request.args.get("frame-interval", "10")) - num_frame = int(request.args.get("num-frame", "10")) - - if request.method == 'POST': - video_data = request.get_data() - ext = request.args.get("ext", ".mp4").lower() - - temp_file = tempfile.NamedTemporaryFile(suffix=ext) - temp_file.file.write(video_data) - temp_file.file.close() - - url = temp_file.name - else: - url = request.args.get("url") - - read_time = current_time() - st - st = current_time() # reset start time - - if mode == "center": - image_data_arr = [get_center_frame(url)] - elif mode == "interval": - image_data_arr = get_frames_interval(url, frame_interval) - else: - image_data_arr = get_n_frames(url, num_frame) - - classes = [] - for image_data in image_data_arr: - try: - _classes = app.classify(image_data, topn=None, min_confidence=None) - except Exception as e: - app.logger.error(e) - return Response(status=400, response=str(e)) - - _classes.sort() - if len(classes) == 0: - classes = _classes - else: - for idx, _c in enumerate(_classes): - c = list(classes[idx]) - c[2] += _c[2] - classes[idx] = tuple(c) - - top_classes = [] - for c in classes: - c = list(c) - # avg out confidence score - avg_score = c[2] / len(image_data_arr) - c[2] = avg_score - if avg_score >= min_confidence: - top_classes.append(tuple(c)) - - top_classes = sorted(top_classes, key=lambda tup: tup[2])[-topn:][::-1] - - classids, classnames, confidence = zip(*top_classes) - - classifier_time = current_time() - st - app.logger.info("Classifier time : %d" % classifier_time) - res = { - 'classids': classids, - 'confidence': confidence, - 'time': { - 'read': read_time, - 'classification': classifier_time, - 'units': 'ms' - } - } - if human: - res['classnames'] = classnames - return Response(response=json.dumps(res), status=200, mimetype="application/json") - - -def main(_): - if not app.debug: - print("Serving on port %d" % FLAGS.port) - app.run(host="0.0.0.0", port=FLAGS.port) - - -if __name__ == '__main__': - tf.app.run() diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tf/video_util.py b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tf/video_util.py deleted file mode 100644 index a4c208bf9ca..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tf/video_util.py +++ /dev/null @@ -1,113 +0,0 @@ -#!/usr/bin/env python -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -import cv2 -import ntpath -import numpy as np - -print("cv2.__version__", cv2.__version__) - -CV_FRAME_COUNT = None - -if hasattr(cv2, "cv"): - CV_FRAME_COUNT = cv2.cv.CV_CAP_PROP_FRAME_COUNT -else: - CV_FRAME_COUNT = cv2.CAP_PROP_FRAME_COUNT - - -def _get_image_from_array(image_array): - # JPG to support tensorflow - byte_arr = cv2.imencode(".jpg", image_array)[1] - return "".join(map(chr, byte_arr)) - - -def _path_leaf(path): - """ - Returns file name from path. Path should not end with slash(/) - """ - head, tail = ntpath.split(path) - return tail or ntpath.basename(head) - - -def get_center_frame(video_path): - """ - Traverse till half of video and saves center snapshot - @param video_path: Path to video file on system - """ - cap = cv2.VideoCapture(video_path) - - length = int(cap.get(CV_FRAME_COUNT)) - - success, image = cap.read() - count = 0 - - while success and count < length / 2: - success, image = cap.read() - count += 1 - - return _get_image_from_array(image) - - -def get_frames_interval(video_path, frame_interval): - """ - Selects one frames after every frame_interval - @param video_path: Path to video file on system - @param frame_interval: Interval after which frame should be picked. If frame_interval=10 then every 10th frame will be extracted - """ - cap = cv2.VideoCapture(video_path) - - length = int(cap.get(CV_FRAME_COUNT)) - - success, image = cap.read() - count = 0 - - image_arr = [] - while success and count < length: - success, image = cap.read() - if count % frame_interval == 0: - image = _get_image_from_array(image) - image_arr.append(image) - - count += 1 - - return image_arr - - -def get_n_frames(video_path, num_frame): - """ - Get N frames equidistant to each other in a video - @param video_path: Path to video file on system - @param num_frame: Number of frames to be extracted from video. If num_frame=10 then 10 frames equally distant from each other will be extracted - """ - cap = cv2.VideoCapture(video_path) - - length = int(cap.get(CV_FRAME_COUNT)) - - op_frame_idx = set(np.linspace(0, length - 2, num_frame, dtype=int)) - - success, image = cap.read() - count = 0 - - image_arr = [] - while success and count < length: - success, image = cap.read() - if success and count in op_frame_idx: - image = _get_image_from_array(image) - image_arr.append(image) - - count += 1 - - return image_arr diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow-im2txt-rest.xml b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow-im2txt-rest.xml deleted file mode 100644 index ce85f7b9860..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow-im2txt-rest.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - image/jpeg - image/png - image/gif - - http://localhost:8764/inception/v3 - 5 - 15 - org.apache.tika.parser.captioning.tf.TensorflowRESTCaptioner - - - - diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow-rest.xml b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow-rest.xml deleted file mode 100644 index 69a65d0765a..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow-rest.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - image/jpeg - image/png - image/gif - - http://localhost:8764/inception/v4 - 2 - 0.015 - org.apache.tika.parser.recognition.tf.TensorflowRESTRecogniser - - - - diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow-video-rest.xml b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow-video-rest.xml deleted file mode 100644 index 3096745402d..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow-video-rest.xml +++ /dev/null @@ -1,33 +0,0 @@ - - - - - - - video/mp4 - - http://localhost:8764/inception/v4 - 2 - 0.015 - fixed - org.apache.tika.parser.recognition.tf.TensorflowRESTVideoRecogniser - - - - - diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow.xml b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow.xml deleted file mode 100644 index f848d15dca1..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/main/resources/org/apache/tika/parser/recognition/tika-config-tflow.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - image/jpeg - - 2 - 0.015 - org.apache.tika.parser.recognition.tf.TensorflowImageRecParser - - - - \ No newline at end of file diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/test/java/org/apache/tika/parser/recognition/ObjectRecognitionParserTest.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/test/java/org/apache/tika/parser/recognition/ObjectRecognitionParserTest.java deleted file mode 100644 index 272a0daffb6..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/test/java/org/apache/tika/parser/recognition/ObjectRecognitionParserTest.java +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.tika.parser.recognition; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -import java.io.IOException; -import java.io.InputStream; -import java.io.Reader; -import java.util.List; - -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.cxf.jaxrs.client.WebClient; -import org.junit.jupiter.api.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import org.apache.tika.Tika; -import org.apache.tika.config.TikaConfig; -import org.apache.tika.exception.TikaException; -import org.apache.tika.metadata.Metadata; -import org.apache.tika.parser.recognition.tf.TensorflowImageRecParser; - -/** - * Testcases for Object Recognition Parser - */ -public class ObjectRecognitionParserTest { - - // Config files - private static final String CONFIG_FILE_OBJ_REC = - "org/apache/tika/parser/recognition/tika-config-tflow.xml"; - private static final String CONFIG_REST_FILE_OBJ_REC = - "org/apache/tika/parser/recognition/tika-config-tflow-rest.xml"; - private static final String CONFIG_REST_FILE_IM2TXT = - "org/apache/tika/parser/recognition/tika-config-tflow-im2txt-rest.xml"; - - // Test images - private static final String CAT_IMAGE_JPEG = "test-documents/testJPEG.jpg"; - private static final String CAT_IMAGE_PNG = "test-documents/testPNG.png"; - private static final String CAT_IMAGE_GIF = "test-documents/testGIF.gif"; - - private static final String BASEBALL_IMAGE_JPEG = "test-documents/baseball.jpg"; - private static final String BASEBALL_IMAGE_PNG = "test-documents/baseball.png"; - private static final String BASEBALL_IMAGE_GIF = "test-documents/baseball.gif"; - - private static final ClassLoader loader = ObjectRecognitionParserTest.class.getClassLoader(); - - private static final Logger LOG = LoggerFactory.getLogger(ObjectRecognitionParserTest.class); - - @Test - public void jpegTFObjRecTest() throws IOException, TikaException, SAXException { - TensorflowImageRecParser p = new TensorflowImageRecParser(); - assumeTrue(p.isAvailable()); - try (InputStream stream = loader.getResourceAsStream(CONFIG_FILE_OBJ_REC)) { - assert stream != null; - Tika tika = new Tika(new TikaConfig(stream)); - Metadata metadata = new Metadata(); - try (InputStream imageStream = loader.getResourceAsStream(CAT_IMAGE_JPEG)) { - Reader reader = tika.parse(imageStream, metadata); - List lines = IOUtils.readLines(reader); - String text = StringUtils.join(lines, " "); - String[] expectedObjects = {"Egyptian cat", "tabby, tabby cat"}; - String metaValues = StringUtils - .join(metadata.getValues(ObjectRecognitionParser.MD_KEY_OBJ_REC), " "); - for (String expectedObject : expectedObjects) { - String message = "'" + expectedObject + "' must have been detected"; - assertTrue(text.contains(expectedObject), message); - assertTrue(metaValues.contains(expectedObject), message); - } - } - } - } - - @Test - public void jpegRESTObjRecTest() throws Exception { - String apiUrl = "http://localhost:8764/inception/v4/ping"; - boolean available = false; - int status = 500; - try { - status = WebClient.create(apiUrl).get().getStatus(); - available = status == 200; - } catch (Exception ignore) { - } - assumeTrue(available); - String[] expectedObjects = {"Egyptian cat", "tabby, tabby cat"}; - doRecognize(CONFIG_REST_FILE_OBJ_REC, CAT_IMAGE_JPEG, - ObjectRecognitionParser.MD_KEY_OBJ_REC, expectedObjects); - } - - @Test - public void pngRESTObjRecTest() throws Exception { - String apiUrl = "http://localhost:8764/inception/v4/ping"; - boolean available = false; - int status = 500; - try { - status = WebClient.create(apiUrl).get().getStatus(); - available = status == 200; - } catch (Exception ignore) { - } - assumeTrue(available); - String[] expectedObjects = {"Egyptian cat", "tabby, tabby cat"}; - doRecognize(CONFIG_REST_FILE_OBJ_REC, CAT_IMAGE_PNG, ObjectRecognitionParser.MD_KEY_OBJ_REC, - expectedObjects); - } - - @Test - public void gifRESTObjRecTest() throws Exception { - String apiUrl = "http://localhost:8764/inception/v4/ping"; - boolean available = false; - int status = 500; - try { - status = WebClient.create(apiUrl).get().getStatus(); - available = status == 200; - } catch (Exception ignore) { - } - assumeTrue(available); - String[] expectedObjects = {"Egyptian cat"}; - doRecognize(CONFIG_REST_FILE_OBJ_REC, CAT_IMAGE_GIF, ObjectRecognitionParser.MD_KEY_OBJ_REC, - expectedObjects); - } - - @Test - public void jpegRESTim2txtTest() throws Exception { - String apiUrl = "http://localhost:8764/inception/v3/ping"; - boolean available = false; - int status = 500; - try { - status = WebClient.create(apiUrl).get().getStatus(); - available = status == 200; - } catch (Exception ignore) { - } - assumeTrue(available); - String[] expectedCaption = {"a baseball player holding a bat on a field"}; - doRecognize(CONFIG_REST_FILE_IM2TXT, BASEBALL_IMAGE_JPEG, - ObjectRecognitionParser.MD_KEY_IMG_CAP, expectedCaption); - } - - @Test - public void pngRESTim2txtTest() throws Exception { - String apiUrl = "http://localhost:8764/inception/v3/ping"; - boolean available = false; - int status = 500; - try { - status = WebClient.create(apiUrl).get().getStatus(); - available = status == 200; - } catch (Exception ignore) { - } - assumeTrue(available); - String[] expectedCaption = {"a baseball player holding a bat on a field"}; - doRecognize(CONFIG_REST_FILE_IM2TXT, BASEBALL_IMAGE_PNG, - ObjectRecognitionParser.MD_KEY_IMG_CAP, expectedCaption); - } - - @Test - public void gifRESTim2txtTest() throws Exception { - String apiUrl = "http://localhost:8764/inception/v3/ping"; - boolean available = false; - int status = 500; - try { - status = WebClient.create(apiUrl).get().getStatus(); - available = status == 200; - } catch (Exception ignore) { - } - assumeTrue(available); - String[] expectedCaption = {"a baseball player pitching a ball on top of a field"}; - doRecognize(CONFIG_REST_FILE_IM2TXT, BASEBALL_IMAGE_GIF, - ObjectRecognitionParser.MD_KEY_IMG_CAP, expectedCaption); - } - - private void doRecognize(String configFile, String testImg, String mdKey, - String[] expectedObjects) throws Exception { - try (InputStream stream = loader.getResourceAsStream(configFile)) { - assert stream != null; - Tika tika = new Tika(new TikaConfig(stream)); - Metadata metadata = new Metadata(); - try (InputStream imageStream = loader.getResourceAsStream(testImg)) { - Reader reader = tika.parse(imageStream, metadata); - String text = IOUtils.toString(reader); - String metaValues = StringUtils.join(metadata.getValues(mdKey), " "); - LOG.info("MetaValues = {}", metaValues); - for (String expectedObject : expectedObjects) { - String message = "'" + expectedObject + "' must have been detected"; - assertTrue(text.contains(expectedObject), message); - assertTrue(metaValues.contains(expectedObject), message); - } - } - } - } -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/test/java/org/apache/tika/parser/recognition/tf/TensorflowImageRecParserTest.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/test/java/org/apache/tika/parser/recognition/tf/TensorflowImageRecParserTest.java deleted file mode 100644 index 28e5b6fe02a..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/test/java/org/apache/tika/parser/recognition/tf/TensorflowImageRecParserTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tika.parser.recognition.tf; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.InputStream; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.xml.sax.helpers.DefaultHandler; - -import org.apache.tika.metadata.Metadata; -import org.apache.tika.parser.ParseContext; -import org.apache.tika.parser.recognition.RecognisedObject; - - -@Disabled -public class TensorflowImageRecParserTest { - - @Test - public void recognise() throws Exception { - TensorflowImageRecParser recogniser = new TensorflowImageRecParser(); - recogniser.initialize(new HashMap<>()); - try (InputStream stream = getClass().getClassLoader() - .getResourceAsStream("test-documents/testJPEG.jpg")) { - List objects = recogniser - .recognise(stream, new DefaultHandler(), new Metadata(), new ParseContext()); - assertTrue(5 == objects.size()); - Set objectLabels = new HashSet<>(); - for (RecognisedObject object : objects) { - objectLabels.add(object.getLabel()); - } - System.out.println(objectLabels); - String[] expected = {"Egyptian cat", "tabby, tabby cat"}; - for (String label : expected) { - assertTrue(objectLabels.contains(label), label + " is expected"); - } - } - } -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/test/java/org/apache/tika/parser/recognition/tf/TensorflowVideoRecParserTest.java b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/test/java/org/apache/tika/parser/recognition/tf/TensorflowVideoRecParserTest.java deleted file mode 100644 index c77f9f7bb60..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-module/src/test/java/org/apache/tika/parser/recognition/tf/TensorflowVideoRecParserTest.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tika.parser.recognition.tf; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.InputStream; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.xml.sax.helpers.DefaultHandler; - -import org.apache.tika.metadata.Metadata; -import org.apache.tika.parser.ParseContext; -import org.apache.tika.parser.recognition.RecognisedObject; - - -@Disabled -public class TensorflowVideoRecParserTest { - - @Test - public void recognise() throws Exception { - TensorflowRESTVideoRecogniser recogniser = new TensorflowRESTVideoRecogniser(); - recogniser.initialize(new HashMap<>()); - try (InputStream stream = getClass().getClassLoader() - .getResourceAsStream("test-documents/testVideoMp4.mp4")) { - List objects = recogniser - .recognise(stream, new DefaultHandler(), new Metadata(), new ParseContext()); - - assertTrue(objects.size() > 0); - Set objectLabels = new HashSet<>(); - for (RecognisedObject object : objects) { - objectLabels.add(object.getLabel()); - } - assertTrue(objectLabels.size() > 0); - } - } -} diff --git a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-package/pom.xml b/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-package/pom.xml deleted file mode 100644 index 17998c7f4ca..00000000000 --- a/tika-parsers/tika-parsers-ml/tika-parser-advancedmedia-package/pom.xml +++ /dev/null @@ -1,115 +0,0 @@ - - - - - tika-parsers-ml - org.apache.tika - 4.0.0-SNAPSHOT - - 4.0.0 - - tika-parser-advancedmedia-package - Apache Tika advanced media package - - - - - - - - ${project.groupId} - tika-parser-advancedmedia-module - ${project.version} - - - - - - - org.apache.maven.plugins - maven-jar-plugin - - - - org.apache.tika.parser.advancedmedia - - - - - - - test-jar - - - - - - org.apache.maven.plugins - maven-shade-plugin - ${maven.shade.version} - - - package - - shade - - - - false - - - - *:* - - module-info.class - META-INF/*.SF - META-INF/*.DSA - META-INF/*.RSA - META-INF/DEPENDENCIES - META-INF/MANIFEST.MF - META-INF/LICENSE.md - META-INF/NOTICE.md - - - - - - false - - - - - - META-INF/LICENSE - target/classes/META-INF/LICENSE - - - - - - - - - - - - 3.0.0-rc1 - - \ No newline at end of file