Adaptation of the overpass scripts to deploy the OSM-FR server

overpass-api.yml

@@ -1,8 +1,11 @@
 - name: overpass-api.yml
-  hosts: overpass-api
+  hosts: maelito
+  remote_user: maelito
   roles:
     - role: overpass-api
+      vars:
+        overpass_version: v0.7.62.7
+        overpass_database_dir: /data/work/overpass
       when:
         - overpass_version != "skip"
-        - ansible_distribution_release == "stretch"
       become: yes

roles/overpass-api/defaults/main.yml

@@ -2,6 +2,7 @@
 overpass_user: overpass
 overpass_version: skip
 overpass_pbf_region: planet
+overpass_database_dir: /data/work/overpass
 overpass_pbf_root_url: https://planet.osm.org/pbf
 overpass_replication_url: https://planet.osm.org/replication/minute
 overpass_replication_interval: minute

roles/overpass-api/tasks/compile.yml

@@ -23,7 +23,6 @@
   become: yes
   become_user: "{{ overpass_user }}"
 
-
 - name: download {{ overpass_version }} release of overpass-api
   unarchive:
     src: "http://dev.overpass-api.de/releases/osm-3s_{{ overpass_version }}.tar.gz"
@@ -55,6 +54,7 @@
     strip: 1
   become: yes
   become_user: "{{ overpass_user }}"
+  when: false # Looks to me that the diff file used to patch used to target another script and fails in the following versions. Disabling it an hoping it was fixed / not so important :)
 
 - name: install overpass-api executables
   shell: make install
@@ -78,6 +78,8 @@
 
 - name: install rules for areas
   command: cp -r {{ project_dir }}/osm-3s_{{ overpass_version }}/rules {{ overpass_database_dir }}/
+  become: yes
+  become_user: "{{ overpass_user }}"
 
 - name: change delay to fetch diff
   replace:

roles/overpass-api/tasks/main.yml

@@ -1,15 +1,14 @@
 - name: check ansible version
   assert:
     that:
-      - "ansible_version.full | version_compare('2.3', '>=')"
+      - "ansible_version.full is version_compare('2.3', '>=')"
     msg: "ansible version must be >= 2.3, please update ansible."
 
 - include_tasks: ../../../shared/project-account.yml
   vars:
     user: "{{ overpass_user }}"
     user_system: "yes"
 
-
 - name: set overpass_database_dir
   set_fact:
     overpass_database_dir: "{{ ssd_dir|default(work_dir) }}/database"
@@ -29,7 +28,21 @@
     - "{{ overpass_webroot_dir }}"
 
 - import_tasks: compile.yml
-- import_tasks: set_up_database.yml
+# Deactivated, it looks like the "Fast Startup" option here https://overpass-api.de/no_frills.html#startup did remove the need for this...
+# ... as long as the .de instance is online
+#
+#- import_tasks: set_up_database.yml
+
+# TODO meta option could be set in general options
+- name: Run "Fast Startup" scenario
+  tags: fast-startup
+  shell: "/usr/local/bin/download_clone.sh --source=https://dev.overpass-api.de/api_drolbr/ --db-dir={{ overpass_database_dir }} --meta=yes &"
+  become_user: "{{ overpass_user }}"
+  become: yes
+  become_method: sudo
+  args:
+    chdir: "{{ project_dir }}/osm-3s_{{ overpass_version }}"
+
 - import_tasks: services.yml
 - import_tasks: web.yml
 - import_tasks: munin.yml

roles/overpass-api/tasks/set_up_database.yml

@@ -12,6 +12,7 @@
     state: directory
 
 - name: install overpass-setup.py
+  tags: overpass-setup
   template:
     src: templates/overpass-setup.py
     dest: "/usr/local/bin/"

roles/overpass-api/templates/overpass-dispatcher.service

@@ -5,7 +5,7 @@ AssertPathExists={{ overpass_database_dir }}/replicate_id
 [Service]
 Type=simple
 User={{ overpass_user }}
-ExecStart=/usr/local/bin/dispatcher --osm-base --db-dir={{ overpass_database_dir }} --meta
+ExecStart=/usr/local/bin/dispatcher --osm-base --db-dir={{ overpass_database_dir }} --meta --allow-duplicate-queries=yes
 ExecStop=/usr/local/bin/dispatcher --osm-base --terminate
 Restart=always
 RestartSec=30s

roles/overpass-api/templates/overpass-setup.py

@@ -16,15 +16,16 @@
 
 
 def parse_state_txt(url):
+    log.info(url)
     resp = requests.get(url)
 
     for line in resp.text.split('\n'):
         if line.startswith('sequenceNumber'):
             sequence_number = line.split('=')[1]
         elif line.startswith('timestamp'):
             timestamp = line.split('=')[1]
-
-    return (sequence_number, datetime.strptime(timestamp[:-5],
+    cleaned_timestamp = timestamp.replace("\:", ":") # Looks like the datetime is not escaped in the URL https://osm-planet-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com/planet/replication/minute/state.txt
+    return (sequence_number, datetime.strptime(cleaned_timestamp[:-5],
                                                '%Y-%m-%dT%H:%M'))
 
 
@@ -68,6 +69,9 @@ def md5(fname):
 obj_datetime = pbf_datetime - delta
 url = '{{ overpass_replication_url }}/state.txt'
 
+log.info(url)
+log.info(obj_datetime)
+
 repl_seq, repl_datetime = parse_state_txt(url)
 diff_datetime = obj_datetime - repl_datetime
 while diff_datetime > timedelta(
@@ -76,10 +80,12 @@ def md5(fname):
         int(repl_seq) + int(diff_datetime.total_seconds() // divider))
     url = '{{ overpass_replication_url }}/{}/{}/{}.state.txt'.format(
         repl_seq[:-6], repl_seq[-6:-3], repl_seq[-3:])
+
     repl_seq, repl_datetime = parse_state_txt(url)
     diff_datetime = obj_datetime - repl_datetime
 
 log.info('download pbf')
+log.info(url)
 pbf_url = '{{ overpass_pbf_root_url }}/{{ overpass_pbf_region }}-{}.osm.pbf'.format(
     pbf_datetime.strftime('%y%m%d'))
 pbf_dest = '{{ overpass_database_dir }}/{{ overpass_pbf_region }}-{}.osm.pbf'.format(