No need to add imported items to the check queue, the scheduler will do this #3762

changedetectionio/api/Watch.py

@@ -438,7 +438,8 @@ def post(self):
 
         new_uuid = self.datastore.add_watch(url=url, extras=extras, tag=tags)
         if new_uuid:
-            worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
+# Dont queue because the scheduler will check that it hasnt been checked before anyway
+#            worker_handler.queue_item_async_safe(self.update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': new_uuid}))
             return {'uuid': new_uuid}, 201
         else:
             return "Invalid or unsupported URL", 400

changedetectionio/blueprint/imports/__init__.py

@@ -1,13 +1,8 @@
 from flask import Blueprint, request, redirect, url_for, flash, render_template
+from loguru import logger
+
 from changedetectionio.store import ChangeDetectionStore
 from changedetectionio.auth_decorator import login_optionally_required
-from changedetectionio import worker_handler
-from changedetectionio.blueprint.imports.importer import (
-    import_url_list, 
-    import_distill_io_json, 
-    import_xlsx_wachete, 
-    import_xlsx_custom
-)
 
 def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData):
     import_blueprint = Blueprint('imports', __name__, template_folder="templates")
@@ -17,15 +12,26 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMe
     def import_page():
         remaining_urls = []
         from changedetectionio import forms
-
+#
         if request.method == 'POST':
+#            from changedetectionio import worker_handler
+
+            from changedetectionio.blueprint.imports.importer import (
+                import_url_list,
+                import_distill_io_json,
+                import_xlsx_wachete,
+                import_xlsx_custom
+            )
+
             # URL List import
             if request.values.get('urls') and len(request.values.get('urls').strip()):
                 # Import and push into the queue for immediate update check
                 importer_handler = import_url_list()
                 importer_handler.run(data=request.values.get('urls'), flash=flash, datastore=datastore, processor=request.values.get('processor', 'text_json_diff'))
-                for uuid in importer_handler.new_uuids:
-                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                logger.debug(f"Imported {len(importer_handler.new_uuids)} new UUIDs")
+                # Dont' add to queue because scheduler can see that they haven't been checked and will add them to the queue
+#                for uuid in importer_handler.new_uuids:
+#                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
 
                 if len(importer_handler.remaining_data) == 0:
                     return redirect(url_for('watchlist.index'))
@@ -37,8 +43,10 @@ def import_page():
                 # Import and push into the queue for immediate update check
                 d_importer = import_distill_io_json()
                 d_importer.run(data=request.values.get('distill-io'), flash=flash, datastore=datastore)
-                for uuid in d_importer.new_uuids:
-                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                # Dont' add to queue because scheduler can see that they haven't been checked and will add them to the queue
+#                for uuid in importer_handler.new_uuids:
+#                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
 
             # XLSX importer
             if request.files and request.files.get('xlsx_file'):
@@ -60,8 +68,10 @@ def import_page():
                     w_importer.import_profile = map
                     w_importer.run(data=file, flash=flash, datastore=datastore)
 
-                for uuid in w_importer.new_uuids:
-                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+                # Dont' add to queue because scheduler can see that they haven't been checked and will add them to the queue
+#                for uuid in importer_handler.new_uuids:
+#                    worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+
 
         # Could be some remaining, or we could be on GET
         form = forms.importForm(formdata=request.form if request.method == 'POST' else None)

changedetectionio/blueprint/ui/__init__.py

@@ -227,12 +227,9 @@ def form_watch_checknow():
 
         i = 0
 
-        running_uuids = worker_handler.get_running_uuids()
-
         if uuid:
-            if uuid not in running_uuids:
-                worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
-                i += 1
+            worker_handler.queue_item_async_safe(update_q, queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid}))
+            i += 1
 
         else:
             # Recheck all, including muted
@@ -241,7 +238,7 @@ def form_watch_checknow():
                 watch_uuid = k[0]
                 watch = k[1]
                 if not watch['paused']:
-                    if watch_uuid not in running_uuids:
+                    if watch_uuid:
                         if with_errors and not watch.get('last_error'):
                             continue
 

changedetectionio/flask_app.py

@@ -27,9 +27,7 @@
     session,
     url_for,
 )
-from urllib.parse import urlparse
 from flask_compress import Compress as FlaskCompress
-from flask_login import current_user
 from flask_restful import abort, Api
 from flask_cors import CORS
 
@@ -46,6 +44,7 @@
 from changedetectionio.api.Search import Search
 from .time_handler import is_within_schedule
 from changedetectionio.languages import get_available_languages, get_language_codes, get_flag_for_locale, get_timeago_locale
+IN_PYTEST = "pytest" in sys.modules or "PYTEST_CURRENT_TEST" in os.environ
 
 datastore = None
 
@@ -979,6 +978,10 @@ def ticker_thread_check_time_launch_checks():
     logger.debug(f"System env MINIMUM_SECONDS_RECHECK_TIME {recheck_time_minimum_seconds}")
 
     # Workers are now started during app initialization, not here
+    WAIT_TIME_BETWEEN_LOOP = 1.0 if not IN_PYTEST else 0.01
+    if IN_PYTEST:
+        # The time between loops should be less than the first .sleep/wait in def wait_for_all_checks() of tests/util.py
+        logger.warning(f"Looks like we're in PYTEST! Setting time between searching for items to add to the queue to {WAIT_TIME_BETWEEN_LOOP}s")
 
     while not app.config.exit.is_set():
 
@@ -1120,4 +1123,4 @@ def ticker_thread_check_time_launch_checks():
                     watch.jitter_seconds = 0
 
         # Should be low so we can break this out in testing
-        app.config.exit.wait(1)
+        app.config.exit.wait(WAIT_TIME_BETWEEN_LOOP)

changedetectionio/run_basic_tests.sh

@@ -82,20 +82,14 @@ echo "RUNNING WITH BASE_URL SET"
 # Re #65 - Ability to include a link back to the installation, in the notification.
 export BASE_URL="https://really-unique-domain.io"
 
-REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 tests/test_notification.py
-
-
 # Re-run with HIDE_REFERER set - could affect login
 export HIDE_REFERER=True
-pytest -vv -s --maxfail=1 tests/test_access_control.py
+REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 tests/test_notification.py tests/test_access_control.py
+
 
 # Re-run a few tests that will trigger brotli based storage
-export SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD=5
-pytest -vv -s --maxfail=1 tests/test_access_control.py
-REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest tests/test_notification.py
-pytest -vv -s --maxfail=1 tests/test_backend.py
-pytest -vv -s --maxfail=1 tests/test_rss.py
-pytest -vv -s --maxfail=1 tests/test_unique_lines.py
+# And again with brotli+screenshot attachment
+SNAPSHOT_BROTLI_COMPRESSION_THRESHOLD=5 REMOVE_REQUESTS_OLD_SCREENSHOTS=false pytest -vv -s --maxfail=1 --dist=load tests/test_backend.py tests/test_rss.py tests/test_unique_lines.py tests/test_notification.py  tests/test_access_control.py
 
 # Try high concurrency
 FETCH_WORKERS=50 pytest  tests/test_history_consistency.py -vv -l -s