Merge pull request #41 from obeone/reorganize-project

Reorganize project

Author: obeone

.dockerignore

@@ -7,3 +7,6 @@
 /output
 __pycache__
 .git
+/pytest_cahe
+/tests
+/.github

main.py

@@ -1,17 +1,16 @@
-from turtle import title
-import log_setup
+import src.log_setup as log_setup
 import os
 import argparse
-import utils
+import src.utils as utils
 import sys
 
 # Setup logging based on environment variable or default to WARN level before importing other modules.
 log_level = os.getenv("LOG_LEVEL", "WARN")
 log_setup.setup_logging(log_level)
 
-from database_manager import DatabaseManager
-from export_manager import ExportManager
-from scraper import Scraper
+from src.database_manager import DatabaseManager
+from src.export_manager import ExportManager
+from src.scraper import Scraper
 
 logger = log_setup.get_logger()
 logger.name = "main"

src/__init__.py

@@ -0,0 +1 @@
+# Package initialization file

database_manager.py src/database_manager.pydatabase_manager.py renamed to src/database_manager.py

@@ -1,6 +1,6 @@
 import json
-from database_manager import DatabaseManager
-import log_setup
+from . import log_setup
+from .database_manager import DatabaseManager
 import os
 from tqdm import tqdm
 

export_manager.py src/export_manager.pyexport_manager.py renamed to src/export_manager.py

@@ -2,11 +2,11 @@
 import requests
 from bs4 import BeautifulSoup
 from urllib.parse import urljoin, urldefrag
-import log_setup
+from . import log_setup
 import trafilatura
 import mdformat
 import json
-from database_manager import DatabaseManager
+from .database_manager import DatabaseManager
 from tqdm import tqdm
 import coloredlogs
 import time

log_setup.py src/log_setup.pylog_setup.py renamed to src/log_setup.py

@@ -1,4 +1,4 @@
-import log_setup  # Add log messages
+from . import log_setup  # Add log messages
 from urllib.parse import urlparse, urlunparse
 
 logger = log_setup.get_logger()
@@ -99,4 +99,3 @@ def deduplicate_list(input_list):
     seen = set()
     deduplicated_list = [x for x in input_list if not (x in seen or seen.add(x))]
     return deduplicated_list
-