Merge branch 'tilserver-stats'

* tilserver-stats:
  Update tileserver stats calculation
  Added archival notice
  Add 'truncate' command
  Updated requests processing code
  Updated gitignore
  Updated procedures to have less nulls
  Relax python version constraint
  Does everything and takes two minutes for steno
  Basic module structure

Author: davenquinn

.idea/copilot.data.migration.agent.xml

@@ -3,13 +3,12 @@
 
 from dotenv import load_dotenv
 from dynaconf import Dynaconf, Validator
+from macrostrat.app_frame.control_command import BackendType
+from macrostrat.utils import get_logger
 from sqlalchemy.engine import make_url
 from sqlalchemy.engine.url import URL
 from toml import load as load_toml
 
-from macrostrat.app_frame.control_command import BackendType
-from macrostrat.utils import get_logger
-
 from .resolvers import cast_sources, setup_source_roots_environment
 from .utils import convert_to_string, find_macrostrat_config, path_list_resolver
 
@@ -82,6 +81,8 @@ def get(self, key, default=None):
     # Backend information. We could potentially infer this from other environment variables
     Validator("backend", default="kubernetes", cast=BackendType),
     Validator("sources", cast=cast_sources, default=None),
+    # Settings to control the location of arbitrary named databases
+    Validator("databases", default={}),
 )
 
 macrostrat_env = getattr(settings, "env", "default")
@@ -120,6 +121,9 @@ def get(self, key, default=None):
     docker_localhost = getattr(settings, "docker_localhost", "localhost")
     PG_DATABASE_DOCKER = PG_DATABASE.replace("localhost", docker_localhost)
 
+    # add this to the settings.databases mapping
+    settings.databases["macrostrat"] = PG_DATABASE
+
     # Set environment variables
     url = make_url(PG_DATABASE)
 

.idea/copilot.data.migration.edit.xml

@@ -0,0 +1,4 @@
+.env
+.venv
+__pycache__
+.idea

core/macrostrat/core/config.py

@@ -0,0 +1,5 @@
+# Tileserver stats
+
+Macrostrat library that subsets tileserver statistics.
+
+**Note**: This library has been archived and subsumed within the [Macrostrat](https://github.com/UW-Macrostrat/macrostrat) library.

py-modules/tileserver-stats/.gitignore

@@ -0,0 +1,2 @@
+[virtualenvs]
+in-project = true

py-modules/tileserver-stats/README.md

@@ -0,0 +1,26 @@
+[tool.poetry]
+authors = ["Daven Quinn <[email protected]>"]
+description = "Harvest tileserver statistics"
+license = "MIT"
+name = "tileserver_stats"
+packages = [{include = "tileserver_stats"}]
+readme = "README.md"
+version = "0.1.0"
+
+# Command-line scripts
+[tool.poetry.scripts]
+tileserver-stats = "tileserver_stats:app"
+
+[tool.poetry.dependencies]
+"macrostrat.database" = "^3.1.1"
+python = "^3.8"
+python-dotenv = "^0.21.0"
+rich = "^12.6.0"
+typer = "^0.12.5"
+
+[tool.poetry.group.dev.dependencies]
+black = "^24.8.0"
+
+[build-system]
+build-backend = "poetry.core.masonry.api"
+requires = ["poetry-core"]

py-modules/tileserver-stats/poetry.lock

@@ -0,0 +1,77 @@
+from datetime import datetime
+from os import environ
+from pathlib import Path
+
+from dotenv import load_dotenv
+from macrostrat.database import Database
+from macrostrat.utils import relative_path
+from rich import print
+from sqlalchemy.sql import text
+from typer import Typer
+
+load_dotenv()
+
+app = Typer(no_args_is_help=True)
+
+app.command()
+
+
+@app.command()
+def run(truncate: bool = False):
+    """Run the update procedure."""
+    tileserver_db = environ.get("TILESERVER_STATS_DATABASE")
+    db = Database(tileserver_db)
+
+    # Run update
+    fn = Path(relative_path(__file__, "procedures")) / "run-update.sql"
+    sql = text(fn.read_text().replace(":", "\:"))
+
+    # check query timing
+    conn = db.engine.connect()
+    n_results = 10000
+    start = datetime.now()
+    step = start
+    while n_results > 0:
+        res = conn.execute(sql, execution_options=dict(no_parameters=True)).first()
+        n_results = res.n_rows
+        conn.execute(text("COMMIT"))
+        next_step = datetime.now()
+        dt = (next_step - step).total_seconds()
+        print(f"{res.last_row_id} ({dt*1000:.0f} ms)")
+        step = next_step
+
+    if truncate and n_results == 0:
+        conn.execute(text("TRUNCATE TABLE requests"))
+
+    print(f"Total time: {datetime.now() - start}")
+
+
+@app.command()
+def reset(drop: bool = False):
+    """Empty the stats schema and re-run the schema creation scripts."""
+    tileserver_db = environ.get("TILESERVER_STATS_DATABASE")
+    db = Database(tileserver_db)
+
+    if drop:
+        db.engine.execute("DROP SCHEMA IF EXISTS stats CASCADE")
+
+    files = Path(relative_path(__file__, "schema")).glob("*.sql")
+    files = list(files)
+    files.sort()
+
+    for file in files:
+        list(db.run_sql(file))
+
+
+@app.command()
+def truncate():
+    """Create the stats schema."""
+    tileserver_db = environ.get("TILESERVER_STATS_DATABASE")
+    db = Database(tileserver_db)
+
+    files = Path(relative_path(__file__, "schema")).glob("*.sql")
+    files = list(files)
+    files.sort()
+
+    for file in files:
+        list(db.run_sql(file))

py-modules/tileserver-stats/poetry.toml

@@ -0,0 +1,69 @@
+WITH a AS (
+	SELECT
+    req_id,
+		layer,
+		ext,
+		x,
+		y,
+		z,
+		substring(nullif(referrer, '') from '(?:.*://)?(?:www\.)?([^(/:)?]*)') referrer,
+		app,
+		app_version,
+		date_trunc('day', time) date
+	FROM requests
+  WHERE req_id > (SELECT last_row_id FROM stats.processing_status ORDER BY last_row_time DESC LIMIT 1)
+	ORDER BY req_id
+	LIMIT 100000
+),
+b AS (
+  INSERT INTO stats.day_index (layer, ext, referrer, app, app_version, date, num_requests)
+	SELECT
+    layer,
+    ext,
+    coalesce(referrer, 'none'),
+    coalesce(app, 'none'),
+    coalesce(app_version, 'none'),
+    date,
+    count(*)
+  FROM a
+	GROUP BY layer, ext, app, referrer, app_version, date
+  ON CONFLICT (layer, ext, referrer, app, app_version, date)
+  DO UPDATE SET
+    num_requests = stats.day_index.num_requests + EXCLUDED.num_requests
+  RETURNING *
+),
+reduced_complexity_locations AS (
+	SELECT
+    layer,
+    ext,
+    CASE WHEN z > 8 THEN
+      x >> (z - 8) -- Bit shift right by the difference between the current zoom level and 8
+    ELSE x END x,
+    CASE WHEN z > 8 THEN
+      y >> (z - 8)
+    ELSE y END y,
+    CASE WHEN z > 8 THEN 8 ELSE z END z,
+    z orig_z
+	FROM a
+),
+c AS (
+  INSERT INTO stats.location_index (layer, ext, x, y, z, orig_z, num_requests)
+  SELECT
+    layer,
+    ext,
+    x,
+    y,
+    z,
+    orig_z,
+    count(*) num_requests
+  FROM reduced_complexity_locations
+  GROUP BY layer, ext, x, y, z, orig_z
+  ON CONFLICT (layer, ext, x, y, z, orig_z)
+  DO UPDATE SET
+    num_requests = stats.location_index.num_requests + EXCLUDED.num_requests
+),
+d AS (
+  INSERT INTO stats.processing_status (last_row_id, last_row_time)
+  SELECT req_id, time FROM requests WHERE req_id = (SELECT max(req_id) FROM a)
+)
+SELECT max(req_id) last_row_id, count(*) n_rows FROM a;