generate sitemap as part of deployment and automate IndexNow

This allows adding the lastmod field based on the last modification date
of the file along with automating submission to IndexNow.

Author: thestinger

deploy-static

@@ -16,10 +16,16 @@ [email protected]
 
 # use last modified timestamps from attestation.app
 rsync -rptcv --chmod=D755,F644 --delete --fsync --preallocate $remote:/srv/attestation.app/ static-production
+rsync -pcv --chmod=D755,F644 --fsync --preallocate static-production/sitemap.xml{,.gz,.br} static-tmp/
 rsync -rpcv --chmod=D755,F644 --delete --fsync --preallocate static-tmp/ static-production
 for f in static-production/**.*(br|gz); do
     touch -r "${f%.*}" "$f"
 done
+changed="$(./generate-sitemap)"
+xmllint --noblanks static-tmp/sitemap.xml --output static-tmp/sitemap.xml
+brotli -f static-tmp/sitemap.xml
+zopfli static-tmp/sitemap.xml
+rsync -pcv --chmod=D755,F644 --fsync --preallocate static-tmp/sitemap.xml{,.gz,.br} static-production/
 
 active=$(ssh $remote readlink /srv/attestation.app)
 
@@ -48,3 +54,7 @@ rsync -pcv --chmod=755 --fsync --preallocate remote-backup $remote:/usr/local/bi
 rsync -pcv --chmod=644 --fsync --preallocate systemd/system/remote-backup.timer $remote:/etc/systemd/system/remote-backup.timer
 rsync -pcv --chmod=644 --fsync --preallocate systemd/system/remote-backup.service $remote:/etc/systemd/system/remote-backup.service
 rsync -pcv --chmod=644 --chown attestation:attestation --fsync --preallocate backup-public-key.txt cloud-archive.sh $remote:/var/lib/attestation/
+
+if [[ -n "$changed" ]]; then
+    ./indexnow <<< "$changed"
+fi

generate-sitemap

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+from datetime import datetime, timezone
+from os.path import getmtime
+from pathlib import Path
+
+base = "https://attestation.app"
+
+pages = [
+    ["/", 1.0],
+    ["/.well-known/security.txt", 0.0],
+    ["/LICENSE.txt", 0.0],
+    ["/about", 1.0],
+    ["/donate", 0.5],
+    ["/contact", 0.1],
+    ["/humans.txt", 0.0],
+    ["/privacy-policy", 0.2],
+    ["/source", 0.1],
+    ["/tutorial", 1.0]
+]
+
+base_mtime = getmtime("static-tmp")
+entries = []
+
+for page in pages:
+    path = page[0]
+    loc = base + path
+    filepath = "static-production" + path
+    if path[-1] == '/':
+        filepath += "index.html"
+    elif "." not in path:
+        filepath += ".html"
+
+    mtime = getmtime(filepath)
+    if mtime > base_mtime:
+        print(loc)
+    lastmod = datetime.fromtimestamp(mtime, timezone.utc).strftime("%Y-%m-%dT%H:%M:%S%:z")
+    priority = page[1]
+    entries.append(f"""
+    <url>
+        <loc>{loc}</loc>
+        <lastmod>{lastmod}</lastmod>
+        <priority>{priority}</priority>
+    </url>""")
+
+sitemap = f"""<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9 http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">{"".join(entries)}
+</urlset>
+"""
+
+with open("static-tmp/sitemap.xml", "w") as f:
+    f.write(sitemap)