build(deps): split up dependencies by document type (#986)

* split dependencies by document type

* make pip-compile with new requirements

* add extra requirements to setup.py

* add in all docs; re pip-compile

* extra for all docs

* add pandas to xlsx

* dependency requires for tsv and csv

* handling for doc, docx and odt

* dependency check for pypandoc

* required dependencies for pandoc files

* xml and html

* markdown

* msg

* add in pdf

* add in pptx

* add in excel

* add lxml as base req

* extra all docs for local inference

* local inference installs all

* pin pillow version

* fixes for plain text tests

* fixes for doc

* update make commands

* changelog and version

* add xlrd

* update pip-compile

* pin numpy for python 3.8 support

* more constraints

* contraint on scipy

* update install docs

* constrain ipython

* add outlook to pip-compile

* more ipython constraints

* add extras to dockerfile

* pin office365 client

* few doc tweaks

* types as strings

* last pip-compile

* re pip-comple

* make tidy

* make tidy

Author: MthwRobinson

CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.9.0
+
+### Enhancements
+
+* Dependencies are now split by document type, creating a slimmer base installation.
+
 ## 0.8.8
 
 ### Enhancements
@@ -6,6 +12,7 @@
 
 ### Fixes
 
+
 * Rename "date" field to "last_modified"
 * Adds Box connector
 

Dockerfile

@@ -30,7 +30,15 @@ RUN python3.8 -m pip install pip==${PIP_VERSION} && \
   pip install --no-cache -r requirements/ingest-s3.txt && \
   pip install --no-cache -r requirements/ingest-slack.txt && \
   pip install --no-cache -r requirements/ingest-wikipedia.txt && \
-  pip install --no-cache -r requirements/local-inference.txt && \
+  pip install --no-cache -r requirements/extra-csv.txt && \
+  pip install --no-cache -r requirements/extra-docx.txt && \
+  pip install --no-cache -r requirements/extra-markdown.txt && \
+  pip install --no-cache -r requirements/extra-msg.txt && \
+  pip install --no-cache -r requirements/extra-odt.txt && \
+  pip install --no-cache -r requirements/extra-pandoc.txt && \
+  pip install --no-cache -r requirements/extra-pdf-image.txt && \
+  pip install --no-cache -r requirements/extra-pptx.txt && \
+  pip install --no-cache -r requirements/extra-xlsx.txt && \
   dnf -y groupremove "Development Tools" && \
   dnf clean all
 

Makefile

@@ -18,10 +18,10 @@ install-base: install-base-pip-packages install-nltk-models
 
 ## install:                 installs all test, dev, and experimental requirements
 .PHONY: install
-install: install-base-pip-packages install-dev install-nltk-models install-test install-huggingface install-unstructured-inference
+install: install-base-pip-packages install-dev install-nltk-models install-test install-huggingface install-all-docs
 
 .PHONY: install-ci
-install-ci: install-base-pip-packages install-nltk-models install-huggingface install-unstructured-inference install-test
+install-ci: install-base-pip-packages install-nltk-models install-huggingface install-all-docs install-test
 
 .PHONY: install-base-pip-packages
 install-base-pip-packages:
@@ -53,6 +53,45 @@ install-dev:
 install-build:
 	python3 -m pip install -r requirements/build.txt
 
+.PHONY: install-csv
+install-csv:
+	python3 -m pip install -r requirements/extra-csv.txt
+
+.PHONY: install-docx
+install-docx:
+	python3 -m pip install -r requirements/extra-docx.txt
+
+.PHONY: install-odt
+install-odt:
+	python3 -m pip install -r requirements/extra-odt.txt
+
+.PHONY: install-pypandoc
+install-pypandoc:
+	python3 -m pip install -r requirements/extra-pandoc.txt
+
+.PHONY: install-markdown
+install-markdown:
+	python3 -m pip install -r requirements/extra-markdown.txt
+
+.PHONY: install-msg
+install-msg:
+	python3 -m pip install -r requirements/extra-msg.txt
+
+.PHONY: install-pdf-image
+install-pdf-image:
+	python3 -m pip install -r requirements/extra-pdf-image.txt
+
+.PHONY: install-pptx
+install-pptx:
+	python3 -m pip install -r requirements/extra-pptx.txt
+
+.PHONY: install-xlsx
+install-xlsx:
+	python3 -m pip install -r requirements/extra-xlsx.txt
+
+.PHONY: install-all-docs
+install-all-docs: install-base install-csv install-docx install-docx install-odt install-pypandoc install-markdown install-msg install-pdf-image install-pptx install-xlsx
+
 .PHONY: install-ingest-google-drive
 install-ingest-google-drive:
 	python3 -m pip install -r requirements/ingest-google-drive.txt
@@ -124,7 +163,7 @@ install-unstructured-inference:
 
 ## install-local-inference: installs requirements for local inference
 .PHONY: install-local-inference
-install-local-inference: install install-unstructured-inference
+install-local-inference: install install-all-docs
 
 .PHONY: install-pandoc
 install-pandoc:
@@ -135,12 +174,23 @@ install-pandoc:
 .PHONY: pip-compile
 pip-compile:
 	pip-compile --upgrade requirements/base.in
+
+	# Extra requirements that are specific to document types
+	pip-compile --upgrade requirements/extra-csv.in
+	pip-compile --upgrade requirements/extra-docx.in
+	pip-compile --upgrade requirements/extra-pandoc.in
+	pip-compile --upgrade requirements/extra-markdown.in
+	pip-compile --upgrade requirements/extra-msg.in
+	pip-compile --upgrade requirements/extra-odt.in
+	pip-compile --upgrade requirements/extra-pdf-image.in
+	pip-compile --upgrade requirements/extra-pptx.in
+	pip-compile --upgrade requirements/extra-xlsx.in
+
 	# Extra requirements for huggingface staging functions
 	pip-compile --upgrade requirements/huggingface.in
 	pip-compile --upgrade requirements/test.in
 	pip-compile --upgrade requirements/dev.in
 	pip-compile --upgrade requirements/build.in
-	pip-compile --upgrade requirements/local-inference.in
 	# NOTE(robinson) - doc/requirements.txt is where the GitHub action for building
 	# sphinx docs looks for additional requirements
 	cp requirements/build.txt docs/requirements.txt
@@ -158,6 +208,7 @@ pip-compile:
 	pip-compile --upgrade requirements/ingest-google-drive.in
 	pip-compile --upgrade requirements/ingest-elasticsearch.in
 	pip-compile --upgrade requirements/ingest-onedrive.in
+	pip-compile --upgrade requirements/ingest-outlook.in
 	pip-compile --upgrade requirements/ingest-confluence.in
 
 ## install-project-local:   install unstructured into your local python environment

docs/requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile requirements/build.in
@@ -12,22 +12,26 @@ beautifulsoup4==4.12.2
     # via furo
 certifi==2023.7.22
     # via
+    #   -c requirements/base.txt
+    #   -c requirements/constraints.in
     #   -r requirements/build.in
     #   requests
 charset-normalizer==3.2.0
-    # via requests
+    # via
+    #   -c requirements/base.txt
+    #   requests
 docutils==0.18.1
     # via
     #   sphinx
     #   sphinx-rtd-theme
 furo==2023.7.26
     # via -r requirements/build.in
 idna==3.4
-    # via requests
+    # via
+    #   -c requirements/base.txt
+    #   requests
 imagesize==1.4.1
     # via sphinx
-importlib-metadata==6.8.0
-    # via sphinx
 jinja2==3.1.2
     # via sphinx
 markupsafe==2.1.3
@@ -38,10 +42,10 @@ pygments==2.15.1
     # via
     #   furo
     #   sphinx
-pytz==2023.3
-    # via babel
 requests==2.31.0
-    # via sphinx
+    # via
+    #   -c requirements/base.txt
+    #   sphinx
 snowballstemmer==2.2.0
     # via sphinx
 soupsieve==2.4.1
@@ -71,7 +75,8 @@ sphinxcontrib-qthelp==1.0.3
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.5
     # via sphinx
-urllib3==2.0.4
-    # via requests
-zipp==3.16.2
-    # via importlib-metadata
+urllib3==1.26.16
+    # via
+    #   -c requirements/base.txt
+    #   -c requirements/constraints.in
+    #   requests

docs/source/installing.rst

@@ -7,8 +7,15 @@ Quick Start
 Use the following instructions to get up and running with ``unstructured`` and test your
 installation.
 
-* Install the Python SDK with ``pip install "unstructured[local-inference]"``
-	* If you do not need to process PDFs or images, you can run ``pip install unstructured``
+* Install the Python SDK with ``pip install unstructured``
+	* Plain text files, HTML, XML, JSON and Emails do not require any extra dependencies.
+	* If you need to process other document types, you can install the extras required for those documents
+		with ``pip install "unstructured[docx,pptx]"``.
+	* To install the extras for every document type, use ``pip install "unstructured[all-docs]"``.
+	* For ``unstructured<0.9.0``, you can install the extras for all document types with
+		``pip install "unstructured[local-inference]"``. The ``local-inference`` extra is still
+		supported in newer versions for backward compatibility, but may be deprecated in a future version.
+		The ``all-docs`` extra is the officially supported installation pattern.
 
 * Install the following system dependencies if they are not already available on your system. Depending on what document types you're parsing, you may not need all of these.
 	* ``libmagic-dev`` (filetype detection)

requirements/base.in

@@ -1,19 +1,8 @@
 -c "constraints.in"
 chardet
 filetype
+python-magic
 lxml
-msg_parser
 nltk
-openpyxl
-pandas
-pdf2image
-pdfminer.six
-pillow
-pypandoc
-python-docx
-python-pptx
-python-magic
-markdown
-requests
 tabulate
-xlrd
+requests

requirements/base.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.8
+# This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
 #    pip-compile requirements/base.in
@@ -8,89 +8,33 @@ certifi==2023.7.22
     # via
     #   -c requirements/constraints.in
     #   requests
-cffi==1.15.1
-    # via cryptography
 chardet==5.1.0
     # via -r requirements/base.in
 charset-normalizer==3.2.0
-    # via
-    #   pdfminer-six
-    #   requests
+    # via requests
 click==8.1.6
     # via nltk
-cryptography==41.0.2
-    # via pdfminer-six
-et-xmlfile==1.1.0
-    # via openpyxl
 filetype==1.2.0
     # via -r requirements/base.in
 idna==3.4
     # via requests
-importlib-metadata==6.8.0
-    # via markdown
 joblib==1.3.1
     # via nltk
 lxml==4.9.3
-    # via
-    #   -r requirements/base.in
-    #   python-docx
-    #   python-pptx
-markdown==3.4.4
-    # via -r requirements/base.in
-msg-parser==1.2.0
     # via -r requirements/base.in
 nltk==3.8.1
     # via -r requirements/base.in
-numpy==1.24.4
-    # via pandas
-olefile==0.46
-    # via msg-parser
-openpyxl==3.1.2
-    # via -r requirements/base.in
-pandas==2.0.3
-    # via -r requirements/base.in
-pdf2image==1.16.3
-    # via -r requirements/base.in
-pdfminer-six==20221105
-    # via -r requirements/base.in
-pillow==10.0.0
-    # via
-    #   -r requirements/base.in
-    #   pdf2image
-    #   python-pptx
-pycparser==2.21
-    # via cffi
-pypandoc==1.11
-    # via -r requirements/base.in
-python-dateutil==2.8.2
-    # via pandas
-python-docx==0.8.11
-    # via -r requirements/base.in
 python-magic==0.4.27
     # via -r requirements/base.in
-python-pptx==0.6.21
-    # via -r requirements/base.in
-pytz==2023.3
-    # via pandas
 regex==2023.6.3
     # via nltk
 requests==2.31.0
     # via -r requirements/base.in
-six==1.16.0
-    # via python-dateutil
 tabulate==0.9.0
     # via -r requirements/base.in
 tqdm==4.65.0
     # via nltk
-tzdata==2023.3
-    # via pandas
 urllib3==1.26.16
     # via
     #   -c requirements/constraints.in
     #   requests
-xlrd==2.0.1
-    # via -r requirements/base.in
-xlsxwriter==3.1.2
-    # via python-pptx
-zipp==3.16.2
-    # via importlib-metadata

requirements/build.in

@@ -1,3 +1,6 @@
+-c base.txt
+-c constraints.in
+
 sphinx
 # NOTE(alan) - Pinning to resolve a conflict with sphinx. We can unpin on next sphinx_rtd_theme release.
 sphinx_rtd_theme==1.2.2