packaging, tests: reuse webutil.testutil

and other packaging tweaks

Author: snarfed

.readthedocs.yml

@@ -14,8 +14,8 @@ python:
         path: .
         extra_requirements:
           - datastore
+          - docs
           - flask
-      - requirements: docs/requirements.txt
 
 sphinx:
   configuration: docs/conf.py

README.md

@@ -373,12 +373,10 @@ Here's how to package, test, and ship a new release.
     ```
 1. Run the unit tests.
     ```sh
-    source local/bin/activate.csh
+    source .venv/bin/activate.csh
     python -m unittest discover
-    python -m unittest arroba.tests.mst_test_suite # more extensive, slower tests (deliberately excluded from autodiscovery)
     ```
 1. Bump the version number in `pyproject.toml` and `docs/conf.py`. `git grep` the old version number to make sure it only appears in the changelog. Change the current changelog entry in `README.md` for this new version from _unreleased_ to the current date.
-1. Bump the version specifiers in `pyproject.toml` for `pywebutil`, `lexrpc`, and any other relevant dependencies to their most recent versions.
 1. Build the docs. If you added any new modules, add them to the appropriate file(s) in `docs/source/`. Then run `./docs/build.sh`. Check that the generated HTML looks fine by opening `docs/_build/html/index.html` and looking around.
 1. ```sh
    setenv ver X.Y
@@ -387,16 +385,15 @@ Here's how to package, test, and ship a new release.
 1. Upload to [test.pypi.org](https://test.pypi.org/) for testing.
     ```sh
     uv build
-    twine upload -r pypitest dist/arroba-$ver*
+    twine upload -r pypitest dist/arroba-$ver.tar.gz dist/arroba-$ver-py3-none-any.whl
     ```
 1. Install from test.pypi.org.
     ```sh
     cd /tmp
-    python -m venv local
-    source local/bin/activate.csh
+    python -m venv .venv
+    source .venv/bin/activate.csh
     # make sure we force pip to use the uploaded version
     pip uninstall arroba
-    pip install --upgrade pip
     pip install -i https://test.pypi.org/simple --extra-index-url https://pypi.org/simple arroba==$ver
     ```
 1. Smoke test that the code trivially loads and runs.
@@ -414,7 +411,7 @@ Here's how to package, test, and ship a new release.
 1. [Click here to draft a new release on GitHub.](https://github.com/snarfed/arroba/releases/new) Enter `vX.Y` in the _Tag version_ box. Leave _Release title_ empty. Copy `### Notable changes` and the changelog contents into the description text box.
 1. Upload to [pypi.org](https://pypi.org/)!
     ```sh
-    twine upload dist/arroba-$ver*
+    twine upload dist/dist/arroba-$ver.tar.gz dist/arroba-$ver-py3-none-any.whl
     ```
 1. [Wait for the docs to build on Read the Docs](https://readthedocs.org/projects/arroba/builds/), then check that they look ok.
 1. On the [Versions page](https://readthedocs.org/projects/arroba/versions/), check that the new version is active, If it's not, activate it in the _Activate a Version_ section.

arroba/datastore_storage.py

@@ -27,6 +27,7 @@
 from multiformats import CID, multicodec, multihash
 from pymediainfo import MediaInfo
 from webutil.models import EncryptedProperty, WriteOnceBlobProperty
+import webutil.util
 
 from .mst import MST
 from .repo import Repo
@@ -433,7 +434,7 @@ class AtpRemoteBlob(ndb.Model):
 
 
     @classmethod
-    def get_or_create(cls, *, url=None, repo=None, get_fn=util.session.get,
+    def get_or_create(cls, *, url=None, repo=None, get_fn=webutil.util.session.get,
                       max_size=None, accept_types=None, name=''):
         """Returns a new or existing :class:`AtpRemoteBlob` for a given URL.
 
@@ -497,13 +498,14 @@ def maybe_fetch(self, get_fn=requests.get):
             and self.mime_type.split('/')[0] not in BLOB_REFETCH_TYPES):
             # already fetched, and we don't refetch this type
             return
-        elif self.last_fetched and self.last_fetched >= util.now() - BLOB_REFETCH_AGE:
+        elif (self.last_fetched
+              and self.last_fetched >= webutil.util.now() - BLOB_REFETCH_AGE):
             # we've (re)fetched this recently
             return
 
         url = self.url or self.key.id()
         logger.info(f'(Re)fetching blob URL {url}')
-        self.last_fetched = util.now()
+        self.last_fetched = webutil.util.now()
 
         try:
             resp = get_fn(url, stream=True)

arroba/did.py

@@ -26,6 +26,7 @@
 import dag_cbor
 from multiformats import multibase, multicodec
 import requests
+from webutil.util import session
 
 from . import util
 
@@ -71,7 +72,7 @@ def resolve(did, **kwargs):
 
 @cached(TTLCache(maxsize=CACHE_SIZE, ttl=CACHE_TTL.total_seconds()),
         lock=threading.Lock())
-def resolve_plc(did, get_fn=util.session.get):
+def resolve_plc(did, get_fn=session.get):
     """Resolves a ``did:plc`` by fetching its DID document from a PLC directory.
 
     The PLC directory hostname is specified in the ``PLC_HOST`` environment
@@ -111,7 +112,7 @@ def create_plc(handle, **kwargs):
     return write_plc(handle=handle, **kwargs)
 
 
-def update_plc(did, handle=None, get_fn=util.session.get, **kwargs):
+def update_plc(did, handle=None, get_fn=session.get, **kwargs):
     """Updates an existing ``did:plc`` in a PLC directory.
 
     Args are documented in :func:`write_plc`.
@@ -140,7 +141,7 @@ def update_plc(did, handle=None, get_fn=util.session.get, **kwargs):
 
 def write_plc(did=None, handle=None, signing_key=None, rotation_key=None,
               new_rotation_key=None, pds_url=None, also_known_as=None,
-              prev=None, get_fn=util.session.get, post_fn=util.session.post):
+              prev=None, get_fn=session.get, post_fn=session.post):
     """Writes a PLC operation to a PLC directory.
 
     Generally used to create a new ``did:plc`` or update an existing one.
@@ -245,7 +246,7 @@ def _encode_key(key):
     return did_plc._replace(signing_key=signing_key, rotation_key=rotation_key)
 
 
-def write_plc_operation(op, rotation_key, did=None, post_fn=util.session.post):
+def write_plc_operation(op, rotation_key, did=None, post_fn=session.post):
     """Signs and sends a PLC operation to the directory.
 
     Args:
@@ -282,7 +283,7 @@ def write_plc_operation(op, rotation_key, did=None, post_fn=util.session.post):
 
 
 def rollback_plc(did, rotation_key, num_operations=1,
-                 get_fn=util.session.get, post_fn=util.session.post):
+                 get_fn=session.get, post_fn=session.post):
     """Reverts a DID PLC document to its last version.
 
     Reads a did:plc's audit log from the directory, extracts its *previous*
@@ -451,7 +452,7 @@ def plc_operation_to_did_doc(op):
 
 @cached(TTLCache(maxsize=CACHE_SIZE, ttl=CACHE_TTL.total_seconds()),
         lock=threading.Lock())
-def resolve_web(did, get_fn=util.session.get):
+def resolve_web(did, get_fn=session.get):
     """Resolves a ``did:web`` by fetching its DID document.
 
     ``did:web`` spec: https://w3c-ccg.github.io/did-method-web/
@@ -483,7 +484,7 @@ def resolve_web(did, get_fn=util.session.get):
 
 @cached(TTLCache(maxsize=CACHE_SIZE, ttl=CACHE_TTL.total_seconds()),
         lock=threading.Lock())
-def resolve_handle(handle, get_fn=util.session.get):
+def resolve_handle(handle, get_fn=session.get):
     """Resolves an ATProto handle to a DID.
 
     Supports the DNS TXT record and HTTPS well-known methods.

arroba/firehose.py

@@ -18,6 +18,7 @@
 import dag_cbor
 from google.auth.credentials import AnonymousCredentials
 from multiformats import CID
+import webutil.util
 
 from .mst import MST
 from . import server
@@ -350,7 +351,7 @@ def _collect(self):
                 last_event = time.time()
                 header, payload = process_event(event)
                 did = payload.get('did') or payload.get('repo')
-                delay_s = int((util.now() - datetime.fromisoformat(payload['time']))\
+                delay_s = int((webutil.util.now() - datetime.fromisoformat(payload['time']))\
                               .total_seconds())
                 logger.info(f'Emitting live to {len(subscribers)} subscribers: {payload["seq"]} {did} {header.get("t")} ({delay_s} s behind)')
 

arroba/memcache.py

@@ -3,7 +3,7 @@
 import logging
 import time
 
-from . import util
+import webutil.util
 
 logger = logging.getLogger(__name__)
 
@@ -63,7 +63,7 @@ def acquire(self):
             # False if it already existed
             if self.client.add(self.key, 'locked', noreply=False,
                                expire=int(self.expiration.total_seconds())):
-                self.expires_at = util.now() + self.expiration
+                self.expires_at = webutil.util.now() + self.expiration
                 logger.debug(f'acquired memcache lease {self.key}')
                 return
 
@@ -79,7 +79,7 @@ def release(self):
         """Release the lease if we still hold it (hasn't expired)."""
         assert self.expires_at
 
-        if util.now() <= self.expires_at:
+        if webutil.util.now() <= self.expires_at:
             self.client.delete(self.key)
             logger.debug(f'released memcache lease {self.key}')
         else:

arroba/storage.py

@@ -12,6 +12,7 @@
 
 import dag_cbor
 from multiformats import CID, multicodec, multihash
+import webutil.util
 
 from . import mst as mst_mod
 from .repo import Write
@@ -108,7 +109,7 @@ def __init__(self, *, cid=None, decoded=None, encoded=None, seq=None,
         self._decoded = decoded
         self.seq = seq
         self.ops = ops
-        self.time = time or util.now()
+        self.time = time or webutil.util.now()
         self.repo = repo
 
     def __str__(self):
@@ -473,7 +474,7 @@ def write_event(self, repo, type, **kwargs):
             '$type': f'com.atproto.sync.subscribeRepos#{type}',
             'seq': seq,
             'did': repo.did,
-            'time': util.now().isoformat(),
+            'time': webutil.util.now().isoformat(),
             **kwargs,
         }, seq=seq)