Merge pull request #2 from gambolputty/next

Next version

Author: gambolputty

.env.example

@@ -2,4 +2,7 @@ PYTHON_ENV=development
 MONGO_USER=user
 MONGO_PASSWORD=password
 MONGO_DB_NAME=somedb
-MONGO_CREATE_TEXT_INDEX=False
+MONGO_CREATE_TEXT_INDEX=False
+MONGO_OUTSIDE_PORT=1234
+MAX_WORKERS=4
+KEEP_DAYS=2

CHANGELOG.md

@@ -0,0 +1,18 @@
+# Changelog
+All notable changes to this project will be documented in this file.
+
+The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
+and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
+
+## [1.1.0] - 2021-02-11
+### Added
+- Three new config variables (`MONGO_OUTSIDE_PORT`, `MAX_WORKERS`, `KEEP_DAYS`). See [readme](README.md) for details.
+- Network name for this project (`network name`)
+- Changelog file
+### Changed
+- Config variables are now uppercase
+- Readme
+
+## [1.0.0] - 2021-02-07
+### Added
+- Initial project release

HOWTOs.md

@@ -8,22 +8,26 @@ Description=Newscorpus
 After=docker.service
 
 [Service]
-User=newscorpus
+Restart=on-failure
+RestartSec=5s
+User=username
 
 Type=oneshot
 RemainAfterExit=yes
+StandardOutput=file:/var/log/newscorpus.log
+StandardError=file:/var/log/newscorpus_error.log
 
 WorkingDirectory=/path/to/newscorpus
-ExecStart=/usr/local/bin/docker-compose up -d
-ExecStop=/usr/local/bin/docker-compose down
+ExecStart=/path/to/docker-compose/docker-compose up -d
+ExecStop=/path/to/docker-compose/docker-compose down
 
 [Install]
 WantedBy=multi-user.target
 ```
 
 3. Commands:
-1. `systemctl enable newscorpus`
-2. `systemctl start newscorpus`
+Enable the service:
+- `systemctl enable newscorpus`
 
 Then:
 - `service newscorpus status`

README.md

@@ -21,7 +21,10 @@ Environment variables in `.env`:
 | MONGO_USER              | MongoDB user name                                                                                                                  |
 | MONGO_PASSWORD          | MongoDB password                                                                                                                   |
 | MONGO_DB_NAME           | MongoDB database name                                                                                                              |
-| MONGO_CREATE_TEXT_INDEX | `true` or `1` to let MongoDB create a text index (helpful for [text search](https://docs.mongodb.com/manual/text-search/)) |   |
+| MONGO_CREATE_TEXT_INDEX | `true` or `1` to let MongoDB create a text index (helpful for [text search](https://docs.mongodb.com/manual/text-search/))         |
+| MONGO_OUTSIDE_PORT      | Exposed MongoDB port, accessible on your host machine.                                                                                         |
+| MAX_WORKERS             | Number of worker threads for the crawler. Remove for auto assignment.                                                         |
+| KEEP_DAYS               | Discard articles older than **n** days. Default is "2".                                                                            |
 
 At the moment, there are no other options. If you want to change the sources being crawled, take a look at [sources.json](crawler/app/assets/sources.json).
 

crawler/app/__main__.py

@@ -17,6 +17,3 @@
 if cmd == 'crawl':
     from app import crawler
     crawler.init()
-# elif cmd == 'dups':
-#     from .database.remove_duplicates import remove_duplicates
-#     remove_duplicates()

crawler/app/config.py

@@ -1,19 +1,23 @@
 import os
 from pathlib import Path
 
-env = os.getenv('PYTHON_ENV', 'development')
+ENV = os.getenv('PYTHON_ENV', 'development')
 
-root = Path.cwd()
-root = root.joinpath('app')
+# paths
+ROOT_PATH = Path.cwd()
+ROOT_PATH = ROOT_PATH.joinpath('app')
 
-sources_path = root.joinpath('assets', 'sources.json')
-assets_dir_path = root.joinpath('assets')
-logs_dir_path = root.joinpath('logs')
+SOURCES_PATH = ROOT_PATH.joinpath('assets', 'sources.json')
+LOGS_PATH = ROOT_PATH.joinpath('logs')
+
+# general
+MAX_WORKERS = int(os.getenv('MAX_WORKERS', '0')) or None
+KEEP_DAYS = int(os.getenv('KEEP_DAYS', '2'))
 
 # MongoDB
-mongo_host = 'mongo'
-mongo_port = 27017
-mongo_username = os.getenv('MONGO_USER', 'admin')
-mongo_password = os.getenv('MONGO_PASSWORD', '')
-mongo_dbname = os.getenv("MONGO_DB_NAME", '')
-create_mongo_text_index = os.getenv("MONGO_CREATE_TEXT_INDEX", 'False').lower() in ['true', '1']
+MONGO_HOST = 'mongo'
+MONGO_PORT = 27017
+MONGO_USERNAME = os.getenv('MONGO_USER', 'admin')
+MONGO_PASSWORD = os.getenv('MONGO_PASSWORD', '')
+MONGO_DB_NAME = os.getenv("MONGO_DB_NAME", '')
+MONGO_CREATE_TEXT_INDEX = os.getenv("MONGO_CREATE_TEXT_INDEX", 'False').lower() in ['true', '1']

crawler/app/crawler/__init__.py

@@ -16,23 +16,25 @@
 from app.crawler.random_user_agent import random_user_agent
 
 
-create_rotating_log(config.logs_dir_path.joinpath("crawlerlog"))
+create_rotating_log(config.LOGS_PATH.joinpath("crawlerlog"))
 logger = logging.getLogger("rotating_log")
 
 MIN_TEXT_LENGTH = 350
-DISCARD_OLDER_THAN_DAYS = 2
-MAX_WORKERS = 4
 
 
 def init():
     ts = time()
     logger.info('Downloading new articles')
+    logger.info(f'Ignoring articles older than {config.KEEP_DAYS} days')
+
+    if config.MAX_WORKERS:
+        logger.info(f'Maximum crawler workers: {config.MAX_WORKERS}')
 
     # load sources
-    with open(config.sources_path, encoding='utf-8') as f:
+    with open(config.SOURCES_PATH, encoding='utf-8') as f:
         sources = json.load(f)
 
-    with ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
+    with ThreadPoolExecutor(max_workers=config.MAX_WORKERS) as executor:
         executor.map(scrape_source, sources)
 
     logger.info(f"Downloading done in {time() - ts}")
@@ -107,9 +109,9 @@ def process_feed_item(feed_item, source, articles_in_memory, db):
 
     # date must be withing last n days
     difference = datetime.now() - published_at
-    if difference.days > DISCARD_OLDER_THAN_DAYS:
+    if difference.days > config.KEEP_DAYS:
         logger.debug(
-            f'Skip: Article older than {DISCARD_OLDER_THAN_DAYS} days ({published_at})'
+            f'Skip: Article older than {config.KEEP_DAYS} days ({published_at})'
         )
         return False
 

crawler/app/database/create_client.py

@@ -4,16 +4,16 @@
 
 
 def create_client():
-    client = MongoClient(config.mongo_host, config.mongo_port,
-                         username=config.mongo_username, password=config.mongo_password)
+    client = MongoClient(config.MONGO_HOST, config.MONGO_PORT,
+                         username=config.MONGO_USERNAME, password=config.MONGO_PASSWORD)
 
-    db = client[config.mongo_dbname]
+    db = client[config.MONGO_DB_NAME]
 
     # create indexes
     db.articles.create_index('url', unique=True)
     db.articles.create_index([('created_at', DESCENDING)])
 
-    if config.create_mongo_text_index is True:
+    if config.MONGO_CREATE_TEXT_INDEX is True:
         db.articles.create_index([('text', TEXT)], default_language='german')
 
     return [client, db]

crawler/app/logger.py

@@ -8,16 +8,16 @@
 # logger examples: https://www.programcreek.com/python/example/1475/logging.handlers.RotatingFileHandler
 
 # create log folder if not exist
-if os.path.isdir(config.logs_dir_path) is False:
-    os.mkdir(config.logs_dir_path)
+if os.path.isdir(config.LOGS_PATH) is False:
+    os.mkdir(config.LOGS_PATH)
 
 
 def create_rotating_log(path):
     """
     Creates a rotating log
     """
     logger = logging.getLogger('rotating_log')
-    logger.setLevel(logging.DEBUG if config.env == 'development' else logging.INFO)
+    logger.setLevel(logging.DEBUG if config.ENV == 'development' else logging.INFO)
     rotation_handler = RotatingFileHandler(path, maxBytes=1000000, backupCount=4, encoding='utf-8')
     rotation_handler.setFormatter(logging.Formatter('%(levelname)s:%(asctime)s: %(message)s'))
     logger.addHandler(rotation_handler)