Fix ingest function and update test assertions

Author: Marc Shade

.windsurfrules

@@ -0,0 +1 @@
+python3 -m pip install -r requirements.txt

README.md

@@ -53,21 +53,27 @@ pip install docsingest
 git clone https://github.com/marc-shade/docsingest.git
 cd docsingest
 
-# Recommended: Create and activate a virtual environment
-python3 -m venv venv
-source venv/bin/activate  # On Windows, use `venv\Scripts\activate`
+# Highly Recommended: Create and activate a virtual environment
+# You can use the provided setup script for this:
+./scripts/install_dependencies.sh
+# Or you can manually create and activate a virtual environment:
+# python3 -m venv venv
+# source venv/bin/activate  # On Windows, use `venv\Scripts\activate`
 
 # Install dependencies
-pip install -r requirements.txt
+# pip install -r requirements.txt # not needed if using the setup script
 
 # Install the package in editable mode
 pip install -e .
 ```
 
 #### Requirements
 - **Python Version**: 3.7 - 3.12 recommended
-- **Dependencies**: All dependencies will be automatically installed via pip
-- **System Requirements**: 
+- **Dependencies**:
+  - `spacy==3.6.1`
+  - `en_core_web_sm==3.6.0`
+  - All other dependencies will be automatically installed via pip
+- **System Requirements**:
   - Basic Python development tools
   - pip package manager
   - Internet connection for initial setup

requirements.txt

@@ -1,14 +1,12 @@
-python-docx>=0.8
-openpyxl>=3.0
-PyPDF2>=2.0
-markdown>=3.3
-lxml>=4.6
-xlrd>=1.2
-python-pptx>=0.6
-nltk>=3.5
-tiktoken>=0.3
-chardet>=3.0
-requests>=2.25
-spacy>=3.6,<4.0
-https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl
-regex>=2022.1
+python-docx
+openpyxl
+PyPDF2
+markdown
+lxml
+xlrd
+python-pptx
+nltk
+tiktoken
+chardet
+requests
+spacy

scripts/install_dependencies.sh

@@ -33,9 +33,9 @@ echo "Installing development dependencies..."
 pip install -r "$PROJECT_ROOT/requirements-dev.txt"
 
 # Download SpaCy language model
-python3 -m spacy download en_core_web_sm
+# python3 -m spacy download en_core_web_sm
 
 # Deactivate virtual environment
-deactivate
+# deactivate
 
 echo "Dependencies installed successfully!"

tests/test_ingest.py

@@ -9,14 +9,14 @@ def test_ingest_directory():
     output_file = "/Volumes/FILES/code/content_ingest/test_output.md"
 
     summary, tree, content, _ = ingest(
-        directory=test_dir,
+        input_directory=test_dir,
         agent_prompt="Test Compliance Officer",
         output_file=output_file,
     )
 
     # Validate summary
-    assert "**Total Files**" in summary
-    assert "**Total Tokens**" in summary
+    assert "- **Total Files Processed**:" in summary
+    assert "- **Total Tokens**:" in summary
 
     # Validate tree
     assert len(tree.split("\n")) > 0
@@ -32,9 +32,9 @@ def test_empty_directory():
     test_dir = "/tmp/empty_test_dir"
     os.makedirs(test_dir, exist_ok=True)
 
-    summary, tree, content, _ = ingest(directory=test_dir)
+    summary, tree, content, _ = ingest(input_directory=test_dir)
 
-    assert "**Total Files**: 0" in summary
+    assert "- **Total Files Processed**: 0" in summary
 
     # Clean up
     os.rmdir(test_dir)