xrpc_sync.list_repos: add minimal kwarg

skips keys and other setup. should speed up listRepos in prod by 90%+

Author: snarfed

arroba/datastore_storage.py

@@ -715,7 +715,7 @@ def load_repo(self, did_or_handle):
                          rotation_key=atp_repo.rotation_key)
 
     @ndb_context
-    def load_repos(self, after=None, limit=500):
+    def load_repos(self, after=None, limit=500, minimal=False):
         query = AtpRepo.query()
         if after:
             query = query.filter(AtpRepo.key > AtpRepo(id=after).key)
@@ -727,6 +727,10 @@ def load_repos(self, after=None, limit=500):
         blocks = self.read_many(cids)  # dict mapping CID to block
         heads = [blocks[cid] for cid in cids]
 
+        if minimal:
+            return [Repo(storage=self, head=head, status=atp_repo.status)
+                    for atp_repo, head in zip(atp_repos, heads)]
+
         # MST.load doesn't read from storage
         msts = [MST.load(storage=self, cid=block.decoded['data']) for block in heads]
         return [Repo(storage=self, mst=mst, head=head, status=atp_repo.status,

arroba/repo.py

@@ -76,7 +76,6 @@ def __init__(self, *, storage=None, mst=None, head=None, handle=None,
           rotation_key (ec.EllipticCurvePrivateKey)
         """
         assert storage
-        assert signing_key
 
         self.storage = storage
         self.mst = mst

arroba/storage.py

@@ -232,7 +232,7 @@ def store_repo(self, repo):
         """
         raise NotImplementedError()
 
-    def load_repos(self, after=None, limit=500):
+    def load_repos(self, after=None, limit=500, minimal=False):
         """Loads multiple repos from storage.
 
         Repos are returned in lexicographic order of their DIDs, ascending.
@@ -241,6 +241,10 @@ def load_repos(self, after=None, limit=500):
         Args:
           after (str): optional DID to start at, *exclusive*
           limit (int): maximum number of repos to return
+          minimal (bool): if True, returned :class:`Repo` s will only include
+            ``head`` and ``status``; other attributes like ``signing_key``,
+            ``rotation_key``, ``mst``, and ``handle`` may be None. Intended
+            for ``com.atproto.sync.listRepos``.
 
         Returns:
           sequence of Repo:
@@ -679,7 +683,7 @@ def load_repo(self, did_or_handle):
     def store_repo(self, repo):
         self.repos[repo.did] = repo
 
-    def load_repos(self, after=None, limit=500):
+    def load_repos(self, after=None, limit=500, minimal=False):
         it = iter(sorted(self.repos.values(), key=lambda repo: repo.did))
 
         if after:

arroba/tests/test_storage.py

@@ -228,6 +228,19 @@ def test_load_repos_limit(self):
         self.assertEqual(1, len(got))
         self.assertEqual('did:plc:bob', got[0].did)
 
+    def test_load_repos_minimal(self):
+        alice = Repo.create(self.storage, 'did:web:alice', signing_key=self.key)
+        bob = Repo.create(self.storage, 'did:plc:bob', signing_key=self.key)
+        self.storage.tombstone_repo(bob)
+
+        got_bob, got_alice = self.storage.load_repos(minimal=True)
+
+        self.assertEqual(alice.head, got_alice.head)
+        self.assertIsNone(got_alice.status)
+
+        self.assertEqual(bob.head, got_bob.head)
+        self.assertEqual('tombstoned', got_bob.status)
+
     def test_tombstone_repo(self):
         seen = []
         repo = Repo.create(self.storage, 'did:user', signing_key=self.key)
@@ -500,6 +513,15 @@ def test_commit_callback(self):
 
 class DatastoreStorageTest(StorageTest, DatastoreTest):
     """Run all of StorageTest's tests with DatastoreStorage."""
+    def test_load_repos_minimal_skips_keys_and_mst(self):
+        Repo.create(self.storage, 'did:web:alice', signing_key=self.key)
+
+        got = self.storage.load_repos(minimal=True)
+        self.assertEqual(1, len(got))
+        self.assertIsNone(got[0].signing_key)
+        self.assertIsNone(got[0].rotation_key)
+        self.assertIsNone(got[0].mst)
+
     def test_create_commit_datastore_transaction_retry(self):
         # fake what a Repo.create => Storage.commit retry due to datastore transaction
         # contention  would look like.

arroba/xrpc_sync.py

@@ -76,7 +76,7 @@ def list_repos(input, limit=500, cursor=None):
     STATUSES = {'tombstoned': 'deactivated'}
 
     repos = []
-    for repo in server.storage.load_repos(limit=limit, after=cursor):
+    for repo in server.storage.load_repos(limit=limit, after=cursor, minimal=True):
         repo_obj = {
             'did': repo.did,
             'head': repo.head.cid.encode('base32'),