Merge branch 'master' of github.com:aleks-v-k/textract

Author: aleks-v-k

.gitignore

@@ -25,6 +25,9 @@ var/
 pip-log.txt
 pip-delete-this-directory.txt
 
+# Virtual environments
+**/venv*
+
 # Unit test / coverage reports
 htmlcov/
 .tox/

.pyup.yml

@@ -1,7 +1,7 @@
 update: all
 branch: master
 schedule: "every two weeks"
-pin: True
+pin: False
 requirements:
   - requirements/python:
       updates: all

.travis.yml

@@ -1,5 +1,5 @@
-sudo: required
-dist: bionic
+dist: focal
+os: linux
 
 language: python
 python:
@@ -9,6 +9,7 @@ python:
 # install system dependencies here with apt-get.
 before_install:
   - sudo ./provision/debian.sh
+  - python -m pip install --upgrade pip
 
 # install python dependencies including this package in the travis
 # virtualenv
@@ -27,7 +28,7 @@ script:
   - cd tests && make && cd -
   - nosetests --with-coverage --cover-package=textract
   - cd tests && pytest && cd -
-  - pycodestyle textract/ bin/textract
+#  - pycodestyle textract/ bin/textract
   - if [[ $TRAVIS_PYTHON_VERSION == 3.7 ]];
         then cd docs && make html && cd -;
         fi

README.rst

@@ -2,7 +2,7 @@
 ..
 ..   * bumpversion {major|minor|patch}
 ..   * git push && git push --tags
-..   * python setup.py sdist upload
+..   * twine upload -r textract dist/*
 ..   * convert into release https://github.com/deanmalmgren/textract/releases
 
 textract

docs/changelog.rst

@@ -10,6 +10,14 @@ latest changes in development for next release
 ----------------------------------------------
 
 .. THANKS FOR CONTRIBUTING; ADD YOUR UNRELEASED CHANGES HERE!
+1.6.5
+-------------------
+
+* switched epub parsing to MIT license compatible package (`#411`_ by
+  `@jhale1805`_)
+
+1.6.4
+-------------------
 
 * several bug fixes, including:
 
@@ -276,6 +284,7 @@ latest changes in development for next release
 .. _@eiotec: https://github.com/eiotec
 .. _@evfredericksen: https://github.com/evfredericksen
 .. _@jaraco: https://github.com/jaraco
+.. _@jhale1805: https://github.com/jhale1805
 .. _@jsmith-mploir: https://github.com/jsmith-mploir
 .. _@kokxx: https://github.com/Kokxx
 .. _@levivm: https://github.com/levivm
@@ -356,3 +365,4 @@ latest changes in development for next release
 .. _#149: https://github.com/deanmalmgren/textract/issues/149
 .. _#150: https://github.com/deanmalmgren/textract/issues/150
 .. _#162: https://github.com/deanmalmgren/textract/issues/162
+.. _#411: https://github.com/deanmalmgren/textract/issues/411

docs/conf.py

@@ -58,7 +58,7 @@
 # built documents.
 #
 # The short X.Y version.
-release = version = "1.6.3"
+release = version = "1.6.5"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/index.rst

@@ -6,7 +6,7 @@
 textract
 ================================
 
-As undesireable as it might be, more often than not there is extremely
+As undesirable as it might be, more often than not there is extremely
 useful information embedded in Word documents, PowerPoint
 presentations, PDFs, etc---so-called "dark data"---that would be
 valuable for further textual analysis and visualization. While
@@ -44,6 +44,8 @@ file types by either mentioning them on the `issue tracker
 
 * ``.csv`` via python builtins
 
+* ``.tsv`` and ``.tab`` via python builtins
+
 * ``.doc`` via `antiword`_
 
 * ``.docx`` via `python-docx2txt`_

docs/installation.rst

@@ -43,7 +43,7 @@ pypi.
 
 .. code-block:: bash
 
-    brew cask install xquartz
+    brew install --cask xquartz
     brew install poppler antiword unrtf tesseract swig
     pip install textract
 
@@ -62,6 +62,18 @@ pypi.
     homebrew, you may also need to install the python
     development header files for textract to properly install.
 
+FreeBSD
+-------
+
+Setting up this package on FreeBSD pretty much follows the steps for
+Ubuntu / Debian while using ``pkg`` as package manager.
+
+.. code-block:: bash
+
+    pkg install lang/python38 devel/py-pip textproc/libxml2 textproc/libxslt textproc/antiword textproc/unrtf \
+    graphics/poppler print/pstotext graphics/tesseract audio/flac multimedia/ffmpeg audio/lame audio/sox \
+    graphics/jpeg-turbo
+    pip install textract
 
 Don't see your operating system installation instructions here?
 ---------------------------------------------------------------

provision/debian.sh

@@ -16,6 +16,5 @@ base=$(pwd)
 
 # Install all of the dependencies required in the examples.
 # http://docs.travis-ci.com/user/installing-dependencies/#Installing-Ubuntu-packages
-add-apt-repository ppa:mc3man/trusty-media -y
 apt-get update -qq
 sed 's/\(.*\)\#.*/\1/' < $base/requirements/debian | xargs apt-get install -y --fix-missing

requirements/debian

@@ -10,7 +10,6 @@ make
 # these packages are required by python-docx, which depends on lxml
 # and requires these things
 python-dev
-python-pip
 libxml2-dev
 libxslt1-dev
 

requirements/freebsd

@@ -0,0 +1,36 @@
+# required packages
+audio/pulseaudio
+devel/git
+
+# these packages are required by python-docx, which depends on lxml
+# and requires these things
+lang/python38
+devel/py-pippython-pip
+textproc/libxml2
+textproc/libxslt
+
+# parse word documents
+textproc/antiword
+
+# parse rtf documents
+textproc/unrtf
+
+# parse image files
+graphics/tesseract
+graphics/jpeg-turbo
+
+# parse pdfs
+graphics/poppler
+
+# parse postscript files
+print/pstotext
+
+# parse audio files, with SpeechRecognition
+audio/flac 
+
+# filetype conversion libs
+multimedia/ffmpeg
+audio/lame
+
+# convert audio files
+audio/sox

requirements/python

@@ -1,14 +1,13 @@
 # This file contains all python dependencies that are required by the textract
 # package in order for it to properly work.
 
-argcomplete==1.10.0
-beautifulsoup4==4.8.0
-chardet==3.0.4
-docx2txt==0.8
-EbookLib==0.17.1
-extract-msg==0.23.1
-pdfminer.six==20181108
-python-pptx==0.6.18
-six==1.12.0
-SpeechRecognition==3.8.1
-xlrd==1.2.0
+argcomplete~=1.10.0
+beautifulsoup4~=4.8.0
+chardet==3.*
+docx2txt~=0.8
+extract-msg<=0.29.* #Last with python2 support
+pdfminer.six==20191110 #Last with python2 support
+python-pptx~=0.6.18
+six~=1.12.0
+SpeechRecognition~=3.8.1
+xlrd~=1.2.0

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.6.3
+current_version = 1.6.5
 commit = True
 tag = True
 
@@ -20,4 +20,3 @@ search = THANKS FOR CONTRIBUTING; ADD YOUR UNRELEASED CHANGES HERE!
 replace = THANKS FOR CONTRIBUTING; ADD YOUR UNRELEASED CHANGES HERE!
 	{new_version}
 	-------------------
-

setup.py

@@ -42,7 +42,7 @@ def parse_requirements(requirements_filename):
 
 setup(
     name=textract.__name__,
-    version="1.6.3",
+    version="1.6.5",
     description="extract text from any document. no muss. no fuss.",
     long_description=long_description,
     url=github_url,

tests/base.py

@@ -68,18 +68,6 @@ def get_filename(self, filename_root, default_filename_root):
             return filename
         return self.get_filename(default_filename_root, default_filename_root)
 
-    def download_file(self, url, filename):
-        if not os.path.exists(filename):
-
-            # stream the request to make sure it works correctly
-            # http://stackoverflow.com/a/16696317/564709
-            response = requests.get(url, stream=True)
-            with open(filename, 'wb') as stream:
-                for chunk in response.iter_content(chunk_size=1024):
-                    if chunk: # filter out keep-alive new chunks
-                        stream.write(chunk)
-                        stream.flush()
-
     @property
     def raw_text_filename(self):
         return self.get_filename(self.raw_text_filename_root,

tests/epub/raw_text.txt

@@ -1,11 +1,7 @@
-
 Epub testing
 With subtitle...
-
 Introduction
 Welcome here! All the text have ben generate with the Samuel L lorem ipsum.
-
-
 We happy?
 Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
 We happy?
@@ -16,7 +12,6 @@ No man, I don't eat pork
 Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
 Is she dead, yes or no?
 The path of the righteous man is beset on all sides by the iniquities of the selfish and the tyranny of evil men. Blessed is he who, in the name of charity and good will, shepherds the weak through the valley of darkness, for he is truly his brother's keeper and the finder of lost children. And I will strike down upon thee with great vengeance and furious anger those who would attempt to poison and destroy My brothers. And you will know My name is the Lord when I lay My vengeance upon thee.
-
 We happy?
 Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
 We happy?
@@ -27,7 +22,6 @@ No man, I don't eat pork
 Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
 Is she dead, yes or no?
 The path of the righteous man is beset on all sides by the iniquities of the selfish and the tyranny of evil men. Blessed is he who, in the name of charity and good will, shepherds the weak through the valley of darkness, for he is truly his brother's keeper and the finder of lost children. And I will strike down upon thee with great vengeance and furious anger those who would attempt to poison and destroy My brothers. And you will know My name is the Lord when I lay My vengeance upon thee.
-
 We happy?
 Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
 We happy?
@@ -38,7 +32,6 @@ No man, I don't eat pork
 Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
 Is she dead, yes or no?
 The path of the righteous man is beset on all sides by the iniquities of the selfish and the tyranny of evil men. Blessed is he who, in the name of charity and good will, shepherds the weak through the valley of darkness, for he is truly his brother's keeper and the finder of lost children. And I will strike down upon thee with great vengeance and furious anger those who would attempt to poison and destroy My brothers. And you will know My name is the Lord when I lay My vengeance upon thee.
-
 We happy?
 Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
 We happy?
@@ -49,18 +42,6 @@ No man, I don't eat pork
 Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
 Is she dead, yes or no?
 The path of the righteous man is beset on all sides by the iniquities of the selfish and the tyranny of evil men. Blessed is he who, in the name of charity and good will, shepherds the weak through the valley of darkness, for he is truly his brother's keeper and the finder of lost children. And I will strike down upon thee with great vengeance and furious anger those who would attempt to poison and destroy My brothers. And you will know My name is the Lord when I lay My vengeance upon thee.
-
-We happy?
-Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
-We happy?
-The lysine contingency - it's intended to prevent the spread of the animals is case they ever got off the island. Dr. Wu inserted a gene that makes a single faulty enzyme in protein metabolism. The animals can't manufacture the amino acid lysine. Unless they're continually supplied with lysine by us, they'll slip into a coma and die.
-Oh... what I'm gon' do?
-The path of the righteous man is beset on all sides by the iniquities of the selfish and the tyranny of evil men. Blessed is he who, in the name of charity and good will, shepherds the weak through the valley of darkness, for he is truly his brother's keeper and the finder of lost children. And I will strike down upon thee with great vengeance and furious anger those who would attempt to poison and destroy My brothers. And you will know My name is the Lord when I lay My vengeance upon thee.
-No man, I don't eat pork
-Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
-Is she dead, yes or no?
-The path of the righteous man is beset on all sides by the iniquities of the selfish and the tyranny of evil men. Blessed is he who, in the name of charity and good will, shepherds the weak through the valley of darkness, for he is truly his brother's keeper and the finder of lost children. And I will strike down upon thee with great vengeance and furious anger those who would attempt to poison and destroy My brothers. And you will know My name is the Lord when I lay My vengeance upon thee.
-
 We happy?
 Well, the way they make shows is, they make one show. That show's called a pilot. Then they show that show to the people who make shows, and on the strength of that one show they decide if they're going to make more shows. Some pilots get picked and become television programs. Some don't, become nothing. She starred in one of the ones that became nothing.
 We happy?

tests/ps/raw_text.txt

@@ -0,0 +1,2 @@
+           Narrow Text
+    How exciting!

tests/test_pdf.py

@@ -25,19 +25,6 @@ def test_tesseract_cli(self):
             method='tesseract',
         )
 
-    def test_large_pdf(self):
-        """Make sure extraction does not hang (issue #33)"""
-
-        # download the file
-        filename = os.path.join(self.get_extension_directory(), "large.pdf")
-        self.download_file(
-            "https://openknowledge.worldbank.org/bitstream/handle/10986/16091/9780821399378.pdf",
-            filename,
-        )
-
-        # make sure textract can successfully run
-        self.assertSuccessfulTextract(filename)
-
     def test_two_column(self):
         """Preserve two column layout in extraction"""
         filename = os.path.join(self.get_extension_directory(), 'two_column.pdf')

textract/__init__.py

@@ -1,3 +1,3 @@
 from .parsers import process
 
-VERSION = "1.6.3"
+VERSION = "1.6.5"

textract/parsers/__init__.py

@@ -17,12 +17,14 @@
     ".htm": ".html",
     "": ".txt",
     ".log": ".txt",
+    ".tab": ".tsv",
 }
 
 # default encoding that is returned by the process method. specify it
 # here so the default is used on both the process function and also by
 # the command line interface
 DEFAULT_OUTPUT_ENCODING = 'utf_8'
+DEFAULT_ENCODING = 'utf_8'
 
 # filename format
 _FILENAME_SUFFIX = '_parser'

textract/parsers/audio.py

@@ -45,8 +45,6 @@ def extract(self, filename, method='', **kwargs):
                 speech = ''
             except sr.UnknownValueError:
                 speech = ''
-            except sr.RequestError as e:
-                speech = ''
 
             # add a newline, to make output cleaner
             speech += '\n'