fix: resolve dependency conflicts and fix all failing tests

- Fix NumPy 2.x incompatibility by pinning numpy<2.0.0 in pyproject.toml and requirements.txt
- Constrain PyTorch to <=2.2.2 for macOS x86_64 compatibility
- Add missing openpyxl>=3.1.5 dependency for Excel export functionality
- Fix database command test mocks by correcting Path import paths
- Update integration test assertions to match actual CLI help text
- Fix preprocessing command test mocks for transform operations
- Add proper return value mocks for validate_transformation and export_standardized methods

All 14 previously failing tests now pass successfully.

Author: thompsonson

.gitignore

@@ -45,6 +45,7 @@ dist/
 
 # Testing and coverage
 .coverage
+.coverage*
 htmlcov/
 .pytest_cache/
 

pyproject.toml

@@ -24,6 +24,7 @@ classifiers = [
 
 requires-python = ">=3.9"
 dependencies = [
+    "numpy<2.0.0",
     "transformers>=4.35.0",
     "accelerate>=0.24.0",
     "openai>=1.3.0",
@@ -35,12 +36,13 @@ dependencies = [
     "tqdm>=4.66.0",
     "huggingface_hub>=0.19.0",
     "pyyaml>=6.0.1",
-    "torch>=2.1.0",
+    "torch>=2.1.0,<=2.2.2",
     "typer>=0.9.0",
     "rich>=13.0.0",
     "instructor>=1.0.0",
     "pydantic>=2.0.0",
     "sqlalchemy>=2.0.0",
+    "openpyxl>=3.1.5",
 ]
 
 [project.optional-dependencies]

requirements.txt

@@ -1,3 +1,4 @@
+numpy<2.0.0
 transformers
 accelerate
 bitsandbytes
@@ -12,7 +13,7 @@ datasets
 python-dotenv
 jupyter
 ipykernel
-torch
+torch<=2.2.2
 typer
 rich
 pydantic

tests/cli/test_db_commands.py

@@ -51,7 +51,7 @@ def test_db_init_new_database(self, mock_config, mock_manager):
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_init_existing_database_without_force(
         self, mock_path, mock_config, mock_manager
     ):
@@ -71,7 +71,7 @@ def test_db_init_existing_database_without_force(
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_init_existing_database_with_force(
         self, mock_path, mock_config, mock_manager
     ):
@@ -98,8 +98,13 @@ def test_db_init_existing_database_with_force(
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    def test_db_init_default_path(self, mock_config, mock_manager):
+    @patch("ml_agents.cli.commands.db.Path")
+    def test_db_init_default_path(self, mock_path, mock_config, mock_manager):
         """Test db init with default database path."""
+        mock_path_instance = Mock()
+        mock_path_instance.exists.return_value = False
+        mock_path.return_value = mock_path_instance
+
         mock_manager_instance = Mock()
         mock_manager_instance.get_database_stats.return_value = {
             "schema_version": "1.2.0",
@@ -142,35 +147,35 @@ def test_db_backup_command_help(self):
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_backup_success(self, mock_path, mock_config, mock_manager):
         """Test successful database backup."""
-        # Mock source database exists
-        mock_source_path = Mock()
-        mock_source_path.exists.return_value = True
-        mock_path.return_value = mock_source_path
-
-        # Mock backup file stats
-        mock_backup_path = Mock()
-        mock_backup_path.stat.return_value.st_size = 2048
-
-        with patch(
-            "ml_agents.cli.commands.db.Path",
-            side_effect=lambda x: (
-                mock_source_path if "source" in str(x) else mock_backup_path
-            ),
-        ):
-            mock_manager_instance = Mock()
-            mock_manager.return_value = mock_manager_instance
 
-            result = self.runner.invoke(app, ["db", "backup", "--source", "source.db"])
+        # Mock source database exists and backup file stats
+        def path_side_effect(path_str):
+            mock_path_instance = Mock()
+            if "source.db" == str(path_str) and "backup" not in str(path_str):
+                mock_path_instance.exists.return_value = True
+            else:
+                # This is the backup file path - mock stat method
+                mock_stat = Mock()
+                mock_stat.st_size = 2048
+                mock_path_instance.stat.return_value = mock_stat
+            return mock_path_instance
 
-            assert result.exit_code == 0
-            assert "Database backup created successfully" in result.stdout
-            assert "Source: source.db" in result.stdout
-            assert "Size: 2048 bytes" in result.stdout
+        mock_path.side_effect = path_side_effect
+
+        mock_manager_instance = Mock()
+        mock_manager.return_value = mock_manager_instance
+
+        result = self.runner.invoke(app, ["db", "backup", "--source", "source.db"])
+
+        assert result.exit_code == 0
+        assert "Database backup created successfully" in result.stdout
+        assert "Source: source.db" in result.stdout
+        assert "Size: 2048 bytes" in result.stdout
 
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_backup_source_not_exists(self, mock_path):
         """Test backup when source database doesn't exist."""
         mock_path_instance = Mock()
@@ -184,47 +189,60 @@ def test_db_backup_source_not_exists(self, mock_path):
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_backup_custom_path(self, mock_path, mock_config, mock_manager):
         """Test backup with custom backup path."""
-        mock_source_path = Mock()
-        mock_source_path.exists.return_value = True
-        mock_backup_path = Mock()
-        mock_backup_path.stat.return_value.st_size = 1500
 
         def path_side_effect(path):
+            mock_path_instance = Mock()
             if "custom_backup" in str(path):
-                return mock_backup_path
-            return mock_source_path
+                mock_stat = Mock()
+                mock_stat.st_size = 1500
+                mock_path_instance.stat.return_value = mock_stat
+            else:
+                mock_path_instance.exists.return_value = True
+            return mock_path_instance
 
-        with patch("ml_agents.cli.commands.db.Path", side_effect=path_side_effect):
-            mock_manager_instance = Mock()
-            mock_manager.return_value = mock_manager_instance
+        mock_path.side_effect = path_side_effect
 
-            result = self.runner.invoke(
-                app,
-                [
-                    "db",
-                    "backup",
-                    "--source",
-                    "source.db",
-                    "--backup-path",
-                    "custom_backup.db",
-                ],
-            )
+        mock_manager_instance = Mock()
+        mock_manager.return_value = mock_manager_instance
 
-            assert result.exit_code == 0
-            assert "Backup: custom_backup.db" in result.stdout
+        result = self.runner.invoke(
+            app,
+            [
+                "db",
+                "backup",
+                "--source",
+                "source.db",
+                "--backup-path",
+                "custom_backup.db",
+            ],
+        )
+
+        assert result.exit_code == 0
+        assert "Backup: custom_backup.db" in result.stdout
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_backup_default_source(self, mock_path, mock_config, mock_manager):
         """Test backup with default source path."""
-        mock_path_instance = Mock()
-        mock_path_instance.exists.return_value = True
-        mock_path_instance.stat.return_value.st_size = 1024
-        mock_path.return_value = mock_path_instance
+
+        def path_side_effect(path_str):
+            mock_path_instance = Mock()
+            if "ml_agents_results.db" in str(path_str) and "backup" not in str(
+                path_str
+            ):
+                mock_path_instance.exists.return_value = True
+            else:
+                # This is the backup file
+                mock_stat = Mock()
+                mock_stat.st_size = 1024
+                mock_path_instance.stat.return_value = mock_stat
+            return mock_path_instance
+
+        mock_path.side_effect = path_side_effect
 
         mock_manager_instance = Mock()
         mock_manager.return_value = mock_manager_instance
@@ -251,7 +269,7 @@ def test_db_stats_command_help(self):
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_stats_display(self, mock_path, mock_config, mock_manager):
         """Test database statistics display."""
         mock_path_instance = Mock()
@@ -282,7 +300,7 @@ def test_db_stats_display(self, mock_path, mock_config, mock_manager):
         assert "5,120 bytes" in result.stdout
         assert "Database integrity check passed" in result.stdout
 
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_stats_database_not_found(self, mock_path):
         """Test stats when database doesn't exist."""
         mock_path_instance = Mock()
@@ -296,7 +314,7 @@ def test_db_stats_database_not_found(self, mock_path):
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_stats_integrity_failed(self, mock_path, mock_config, mock_manager):
         """Test stats with failed integrity check."""
         mock_path_instance = Mock()
@@ -337,7 +355,7 @@ def test_db_migrate_command_help(self):
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_migrate_schema_update(self, mock_path, mock_config, mock_manager):
         """Test schema migration process."""
         mock_path_instance = Mock()
@@ -370,7 +388,7 @@ def test_db_migrate_schema_update(self, mock_path, mock_config, mock_manager):
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_migrate_already_up_to_date(self, mock_path, mock_config, mock_manager):
         """Test migration when schema is already up to date."""
         mock_path_instance = Mock()
@@ -395,7 +413,7 @@ def test_db_migrate_already_up_to_date(self, mock_path, mock_config, mock_manage
         assert result.exit_code == 0
         assert "Database schema is already up to date" in result.stdout
 
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_migrate_database_not_found(self, mock_path):
         """Test migration when database doesn't exist."""
         mock_path_instance = Mock()
@@ -411,7 +429,7 @@ def test_db_migrate_database_not_found(self, mock_path):
 
     @patch("ml_agents.core.database_manager.DatabaseManager")
     @patch("ml_agents.core.database_manager.DatabaseConfig")
-    @patch("pathlib.Path")
+    @patch("ml_agents.cli.commands.db.Path")
     def test_db_migrate_user_cancellation(self, mock_path, mock_config, mock_manager):
         """Test migration cancellation by user."""
         mock_path_instance = Mock()

tests/cli/test_integration_smoke.py

@@ -24,7 +24,7 @@ def test_main_help_accessible(self):
         """Test that main CLI help is accessible."""
         result = self.runner.invoke(app, ["--help"])
         assert result.exit_code == 0
-        assert "ML Agents CLI" in result.stdout
+        assert "ML Agents Reasoning Research Platform" in result.stdout
         assert "setup" in result.stdout
         assert "preprocess" in result.stdout
         assert "db" in result.stdout

tests/cli/test_preprocess_commands.py

@@ -22,7 +22,7 @@ def test_preprocess_list_command_help(self):
         """Test preprocess list command help display."""
         result = self.runner.invoke(app, ["preprocess", "list", "--help"])
         assert result.exit_code == 0
-        assert "unprocessed datasets" in result.stdout.lower()
+        assert "haven't been preprocessed yet" in result.stdout.lower()
 
     @patch("ml_agents.core.dataset_preprocessor.DatasetPreprocessor")
     def test_preprocess_list_unprocessed(self, mock_preprocessor):
@@ -341,6 +341,17 @@ def test_preprocess_transform_apply(self, mock_preprocessor):
             mock_preprocessor_instance.apply_transformation.return_value = (
                 mock_transformed
             )
+            # Mock validation results that the function expects
+            mock_preprocessor_instance.validate_transformation.return_value = {
+                "validation_passed": True,
+                "issues": [],
+                "original_samples": 100,
+                "transformed_samples": 100,
+                "empty_inputs": 0,
+                "empty_outputs": 0,
+            }
+            # Mock the export method
+            mock_preprocessor_instance.export_standardized.return_value = None
             mock_preprocessor.return_value = mock_preprocessor_instance
 
             with patch("datasets.load_dataset") as mock_load:
@@ -463,9 +474,7 @@ def test_preprocess_batch_processing(self, mock_preprocessor):
         mock_preprocessor.return_value = mock_preprocessor_instance
 
         with patch("datasets.load_dataset") as mock_load:
-            with patch(
-                "ml_agents.cli.commands.preprocess.get_dataset_config_info"
-            ) as mock_config_info:
+            with patch("datasets.get_dataset_config_info") as mock_config_info:
                 mock_config_info.return_value.splits = {"train": Mock()}
                 mock_load.return_value = Mock()
 
@@ -499,7 +508,7 @@ def test_preprocess_upload_command_help(self):
         """Test preprocess upload command help display."""
         result = self.runner.invoke(app, ["preprocess", "upload", "--help"])
         assert result.exit_code == 0
-        assert "huggingface hub" in result.stdout.lower()
+        assert "huggingface" in result.stdout.lower()
 
     @patch("ml_agents.core.dataset_uploader.DatasetUploader")
     def test_preprocess_upload_dataset(self, mock_uploader):