Feature/add better scheduling via croniter

GeoHealthCheck/migrations/versions/2f9f07aebc78_add_cron_column_to_resource_table.py

@@ -0,0 +1,25 @@
+"""Add cron column to Resource table
+
+Revision ID: 2f9f07aebc78
+Revises: 933717a14052
+Create Date: 2024-08-28 04:00:34.863188
+
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '2f9f07aebc78'
+down_revision = '933717a14052'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('resource', sa.Column('cron', sa.Text(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('resource', 'cron')

GeoHealthCheck/models.py

@@ -35,7 +35,7 @@
 from itsdangerous import TimedJSONWebSignatureSerializer as Serializer
 from sqlalchemy import func, and_
 
-from sqlalchemy.orm import deferred
+from sqlalchemy.orm import deferred, validates
 from sqlalchemy.orm.exc import MultipleResultsFound, NoResultFound
 
 import util
@@ -45,6 +45,7 @@
 from resourceauth import ResourceAuth
 from wtforms.validators import Email, ValidationError
 from owslib.util import bind_url
+from croniter import croniter, CroniterBadCronError
 
 APP = App.get_app()
 DB = App.get_db()
@@ -402,8 +403,24 @@ class Resource(DB.Model):
     owner = DB.relationship('User',
                             backref=DB.backref('username2', lazy='dynamic'))
     tags = DB.relationship('Tag', secondary=resource_tags, backref='resource')
-    run_frequency = DB.Column(DB.Integer, default=60)
+    run_frequency = DB.Column(DB.Integer, default=60, nullable=False)
     _auth = DB.Column('auth', DB.Text, nullable=True, default=None)
+    cron = DB.Column(DB.Text, nullable=True, default=None)
+
+    @validates('cron')
+    def validate_cron(self, key, cron):
+        if cron == "":
+            # set null over an empty string
+            return None
+
+        try:
+            croniter(cron)
+        except CroniterBadCronError as error:
+            raise ValueError(
+                f"Bad cron pattern '{cron}': {str(error)}"
+            ) from error
+
+        return cron
 
     def __init__(self, owner, resource_type, title, url, tags, auth=None):
         self.resource_type = resource_type
@@ -653,6 +670,7 @@ def for_json(self):
             'owner': self.owner.username,
             'owner_identifier': self.owner.identifier,
             'run_frequency': self.run_frequency,
+            'cron': self.cron,
             'reliability': round(self.reliability, 1)
         }
 
@@ -664,7 +682,8 @@ class ResourceLock(DB.Model):
                            primary_key=True, autoincrement=False, unique=True)
     resource_identifier = DB.Column(DB.Integer,
                                     DB.ForeignKey('resource.identifier'),
-                                    unique=True)
+                                    unique=True,
+                                    nullable=False)
     resource = DB.relationship('Resource',
                                backref=DB.backref('locks', lazy='dynamic',
                                                   cascade="all,delete"))

GeoHealthCheck/scheduler.py

@@ -40,6 +40,8 @@
 from apscheduler.events import \
     EVENT_SCHEDULER_STARTED, EVENT_SCHEDULER_SHUTDOWN, \
     EVENT_JOB_MISSED, EVENT_JOB_ERROR
+from apscheduler.triggers.cron import CronTrigger
+from croniter import croniter
 from init import App
 
 LOGGER = logging.getLogger(__name__)
@@ -243,14 +245,38 @@ def update_job(resource):
 
 def add_job(resource):
     LOGGER.info('Starting job for resource=%d' % resource.identifier)
-    freq = resource.run_frequency
-    next_run_time = datetime.now() + timedelta(
-        seconds=random.randint(0, freq * 60))
-    scheduler.add_job(
-        run_job, 'interval', args=[resource.identifier, freq],
-        minutes=freq, next_run_time=next_run_time, max_instances=1,
-        misfire_grace_time=round((freq * 60) / 2), coalesce=True,
-        id=str(resource.identifier))
+
+    if resource.cron:
+        # determine the frequency. Used as input to ``run_job`` and to
+        # determine misfire grace time
+        cron = croniter(resource.cron)
+        # Use the next two execution times to determine time difference
+        seconds = abs(cron.get_next() - cron.get_next())
+
+        LOGGER.info(f'scheduling job with cron pattern ({seconds})')
+
+        scheduler.add_job(
+            run_job,
+            CronTrigger.from_crontab(resource.cron),
+            args=[resource.identifier, round(seconds / 60)],
+            max_instances=1,
+            misfire_grace_time=round(seconds / 2),
+            coalesce=True,
+            id=str(resource.identifier)
+        )
+
+    else:
+        freq = resource.run_frequency
+        next_run_time = datetime.now() + timedelta(
+            seconds=random.randint(0, freq * 60))
+
+        LOGGER.info('scheduling job with frequency')
+
+        scheduler.add_job(
+            run_job, 'interval', args=[resource.identifier, freq],
+            minutes=freq, next_run_time=next_run_time, max_instances=1,
+            misfire_grace_time=round((freq * 60) / 2), coalesce=True,
+            id=str(resource.identifier))
 
 
 def stop_job(resource_id):

GeoHealthCheck/templates/edit_resource.html

@@ -138,6 +138,11 @@ <h1 class="page-header">[{{ _('Edit') }}] <span id="resource_title_h1">{{ resour
                     <input type="number" id="input_resource_frequency" name="resource_frequency_value" min="{{ config['GHC_MINIMAL_RUN_FREQUENCY_MINS'] }}" value="{{ resource.run_frequency }}" style="width: 5%;"/> <label>{{ _('minutes') }}</label>
                 </td>
             </tr>
+            <tr>
+              <th>{{ _('crontab pattern') }}</th>
+              <td>
+                <input type="text" class="form-control" id="input_resource_cron" name="resource_cron" value="{{ resource.cron if resource.cron }}" />
+              </td>
             <tr>
                 <th>Probes</th>
 
@@ -317,7 +322,7 @@ <h1 class="page-header">[{{ _('Edit') }}] <span id="resource_title_h1">{{ resour
 
     $('#save').click(function() {
         $('#toggle-status-detail').hide();
-        
+
         // Resource auth: get and map input elements for auth type
         var auth_type = $('select[name="select_resource_auth_type_val"]').val();
         var new_auth = {
@@ -335,7 +340,7 @@ <h1 class="page-header">[{{ _('Edit') }}] <span id="resource_title_h1">{{ resour
             }
             new_auth_data[elm.data('auth-param')] = elm.val();
         });
-        
+
         // Collect title
         var new_title = $('input[name="resource_title_value"]').val();
 
@@ -345,6 +350,9 @@ <h1 class="page-header">[{{ _('Edit') }}] <span id="resource_title_h1">{{ resour
         // Collect test_frequency
         var new_frequency = $('input[name="resource_frequency_value"]').val();
 
+        // Collect cron pattern
+        var new_cron = $('input[name="resource_cron"]').val();
+
         // Collect active
         var new_active = $('#input_resource_active').prop('checked');
 
@@ -425,6 +433,7 @@ <h1 class="page-header">[{{ _('Edit') }}] <span id="resource_title_h1">{{ resour
                 active: new_active,
                 tags: new_tags,
                 run_frequency: new_frequency,
+                cron: new_cron,
                 probes: new_probes,
                 notify_emails: new_notify_emails,
                 notify_webhooks: new_notify_webhooks
@@ -543,7 +552,7 @@ <h1 class="page-header">[{{ _('Edit') }}] <span id="resource_title_h1">{{ resour
                      'focus': function(){return false}}
     $('#input_resource_notify_emails').autocomplete(ac_params);
 
-    // quick'n'dirty autoheight for text fields, 
+    // quick'n'dirty autoheight for text fields,
     // https://stackoverflow.com/a/10080841/35735
     // http://jsfiddle.net/SpYk3/m8Qk9/
     $(function() {

GeoHealthCheck/templates/resource.html

@@ -21,13 +21,13 @@
       /*displays pop-up when "active" class is present*/
       visibility:visible;
     }
-    
+
     .run-popup-content {
       /*Hides pop-up content when there is no "active" class */
      margin: 12px;
      visibility:hidden;
     }
-    
+
     .run-popup-content.active {
       /*Shows pop-up content when "active" class is present */
       visibility:visible;
@@ -83,10 +83,15 @@ <h1 class="page-header">
                 </td>
             </tr>
             <tr>
+              {% if resource.cron %}
+              <th>{{ _('crontab pattern') }}</th>
+              <td>{{ resource.cron }}</td>
+              {% else %}
               <th>{{ _('Run Every') }}</th>
               <td>
               {{ resource.run_frequency }} {{ _('minutes') }}
               </td>
+              {% endif %}
             </tr>
             <tr>
                 <th>Probes</th>
@@ -263,7 +268,7 @@ <h4>{{ _('Download') }}:  <a class="btn btn-default btn-xs" href="{{ url_for('js
 
         {% endif %}
     });
-    
+
 </script>
 
 {% endblock %}

requirements.txt

@@ -22,3 +22,4 @@ APScheduler==3.6.1
 passlib==1.7.1
 Werkzeug==0.16.1
 tzlocal<3.0 # Fix based on https://github.com/Yelp/elastalert/issues/2968
+croniter