ZMI polish: PG-backed Catalog, Advanced, and Indexes & Metadata tabs

Replace inherited ZCatalog BTree-based ZMI views with PG-aware versions:
- Catalog tab: summary (object count, backend status), paginated object
  table with path filter, object detail with idx JSONB and searchable text
- Advanced tab: simplified to Update Catalog and Clear and Rebuild
- Indexes & Metadata tab: merged view reflecting IndexRegistry (replaces
  two tabs that showed always-zero BTree counts)
- Lexicon cleanup: remove orphaned ZCTextIndex lexicons at install time

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: jensens

CHANGES.md

@@ -1,5 +1,39 @@
 # Changelog
 
+## 1.0.0b9
+
+### Changed
+
+- **ZMI polish**: All ZMI tabs now use Bootstrap 4 cards/tables matching
+  Zope 5's modern look (was old-style `<table>` layout with `section-bar`).
+
+- **Catalog tab** (`manage_catalogView`): Replaced inherited ZCatalog
+  BTree-based view with PG-backed version. Shows catalog summary (object
+  count, index/metadata count, search backend with BM25/Tsvector status),
+  path filter, and server-side paginated object table (20/page) with
+  Previous/Next navigation. Object detail shows full idx JSONB and
+  searchable text preview.
+
+- **Advanced tab** (`manage_catalogAdvanced`): Simplified to only show
+  Update Catalog and Clear and Rebuild actions. Removed ZCatalog-specific
+  features (subtransactions, progress logging, standalone Clear Catalog)
+  that don't apply to PostgreSQL.
+
+- **Indexes & Metadata tab** (`manage_catalogIndexesAndMetadata`): Merged
+  the separate Indexes and Metadata tabs into one read-only view showing
+  all registered indexes (name, type, PG storage location, source attrs)
+  and metadata columns.  Reflects the IndexRegistry rather than BTree
+  counts (which were always 0).
+
+- **Removed tabs**: Query Report, Query Plan (BTree timing), and the
+  separate Indexes / Metadata tabs are hidden — replaced by PG-aware
+  equivalents.
+
+- **Lexicon cleanup**: `setuphandlers.install()` now removes orphaned
+  ZCTextIndex lexicons (`htmltext_lexicon`, `plaintext_lexicon`,
+  `plone_lexicon`) created by Plone's `catalog.xml` — unused with
+  PG-backed text search.
+
 ## 1.0.0b8
 
 ### Changed

src/plone/pgcatalog/catalog.py

@@ -14,7 +14,9 @@
 """
 
 from AccessControl import ClassSecurityInfo
+from App.special_dtml import DTMLFile
 from contextlib import contextmanager
+from plone.pgcatalog.backends import BM25Backend
 from plone.pgcatalog.backends import get_backend
 from plone.pgcatalog.brain import CatalogSearchResults
 from plone.pgcatalog.brain import PGCatalogBrain
@@ -269,6 +271,37 @@ class PlonePGCatalogTool(CatalogTool):
     meta_type = "PG Catalog Tool"
     security = ClassSecurityInfo()
 
+    # Replace irrelevant ZCatalog tabs with PG-aware versions:
+    # - Remove: Query Report, Query Plan (BTree timing), Indexes, Metadata
+    # - Replace Indexes + Metadata with merged "Indexes & Metadata" tab
+    manage_options = (
+        *(
+            tab
+            for tab in CatalogTool.manage_options
+            if tab["action"]
+            not in (
+                "manage_catalogReport",
+                "manage_catalogPlan",
+                "manage_catalogIndexes",
+                "manage_catalogSchema",
+            )
+        ),
+        {"action": "manage_catalogIndexesAndMetadata", "label": "Indexes & Metadata"},
+    )
+
+    # Simplified Advanced tab: only Update Catalog and Clear and Rebuild.
+    # ZCatalog's subtransactions and progress logging don't apply to PG.
+    manage_catalogAdvanced = DTMLFile("www/catalogAdvanced", globals())
+
+    # PG-backed Catalog tab and object detail view.
+    manage_catalogView = DTMLFile("www/catalogView", globals())
+    manage_objectInformation = DTMLFile("www/catalogObjectInformation", globals())
+
+    # Merged Indexes & Metadata tab.
+    manage_catalogIndexesAndMetadata = DTMLFile(
+        "www/catalogIndexesAndMetadata", globals()
+    )
+
     # Override ZCatalog's Indexes container to wrap each index with
     # PGIndex — provides PG-backed _index and uniqueValues().
     Indexes = PGCatalogIndexes()
@@ -277,6 +310,15 @@ class PlonePGCatalogTool(CatalogTool):
     security.declareProtected("Manage ZCatalog Entries", "refreshCatalog")
     security.declareProtected("Manage ZCatalog Entries", "reindexIndex")
     security.declareProtected("Manage ZCatalog Entries", "clearFindAndRebuild")
+    security.declareProtected("Manage ZCatalog Entries", "manage_get_catalog_summary")
+    security.declareProtected("Manage ZCatalog Entries", "manage_get_catalog_objects")
+    security.declareProtected("Manage ZCatalog Entries", "manage_get_object_detail")
+    security.declareProtected(
+        "Manage ZCatalog Entries", "manage_get_indexes_and_metadata"
+    )
+    security.declareProtected(
+        "Manage ZCatalog Entries", "manage_catalogIndexesAndMetadata"
+    )
 
     @contextmanager
     def _pg_connection(self):
@@ -706,6 +748,139 @@ def getrid(self, path, default=None):
             row = cur.fetchone()
         return row["zoid"] if row else default
 
+    # -- ZMI helpers ---------------------------------------------------------
+
+    _ZMI_PAGE_SIZE = 20
+
+    def manage_get_catalog_summary(self):
+        """Return summary dict for the Catalog tab header."""
+        conn = self._get_pg_read_connection()
+        with conn.cursor() as cur:
+            cur.execute(
+                "SELECT COUNT(*) AS cnt FROM object_state WHERE idx IS NOT NULL"
+            )
+            row = cur.fetchone()
+        object_count = row["cnt"] if row else 0
+
+        registry = get_registry()
+        backend = get_backend()
+        has_bm25 = isinstance(backend, BM25Backend)
+        return {
+            "object_count": object_count,
+            "index_count": len(registry),
+            "metadata_count": len(registry.metadata),
+            "backend_name": "BM25" if has_bm25 else "Tsvector",
+            "has_bm25": has_bm25,
+            "bm25_languages": list(getattr(backend, "languages", [])),
+        }
+
+    def manage_get_catalog_objects(self, batch_start=0, filterpath=""):
+        """Return paginated list of cataloged objects for the Catalog tab."""
+        batch_start = int(batch_start)
+        conn = self._get_pg_read_connection()
+
+        params = {}
+        where = "idx IS NOT NULL"
+        if filterpath:
+            where += " AND path LIKE %(prefix)s"
+            # Escape LIKE wildcards in user input, then add trailing %
+            safe = (
+                filterpath.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+            )
+            params["prefix"] = safe + "%"
+
+        sql = (
+            "SELECT zoid, path, idx->>'portal_type' AS portal_type, "
+            "COUNT(*) OVER() AS _total "
+            f"FROM object_state WHERE {where} "
+            "ORDER BY path "
+            f"LIMIT {self._ZMI_PAGE_SIZE} OFFSET %(offset)s"
+        )
+        params["offset"] = batch_start
+
+        with conn.cursor() as cur:
+            cur.execute(sql, params)
+            rows = cur.fetchall()
+
+        total = rows[0]["_total"] if rows else 0
+        objects = [
+            {
+                "zoid": r["zoid"],
+                "path": r["path"],
+                "portal_type": r["portal_type"] or "",
+            }
+            for r in rows
+        ]
+        return {"objects": objects, "total": total, "batch_start": batch_start}
+
+    def manage_get_object_detail(self, zoid):
+        """Return detail dict for a single cataloged object.
+
+        Returns idx_items as a pre-sorted list of {"key", "value"} dicts
+        so the DTML template doesn't need isinstance/sorted (restricted).
+        """
+        zoid = int(zoid)
+        conn = self._get_pg_read_connection()
+        with conn.cursor() as cur:
+            cur.execute(
+                "SELECT path, idx, "
+                "searchable_text IS NOT NULL AS has_searchable_text, "
+                "left(searchable_text::text, 200) AS searchable_text_preview "
+                "FROM object_state WHERE zoid = %(zoid)s",
+                {"zoid": zoid},
+            )
+            row = cur.fetchone()
+        if row is None:
+            return None
+        idx = row["idx"] or {}
+        idx_items = []
+        for k in sorted(idx):
+            v = idx[k]
+            if isinstance(v, list):
+                display = ", ".join(str(i) for i in v)
+            elif v is None:
+                display = ""
+            elif isinstance(v, bool):
+                display = "True" if v else "False"
+            else:
+                display = str(v)
+            idx_items.append({"key": k, "value": display, "is_none": v is None})
+        return {
+            "path": row["path"],
+            "idx_items": idx_items,
+            "has_searchable_text": row["has_searchable_text"],
+            "searchable_text_preview": row["searchable_text_preview"] or "",
+        }
+
+    def manage_get_indexes_and_metadata(self):
+        """Return indexes and metadata from the IndexRegistry for ZMI display.
+
+        Returns a dict with:
+        - indexes: sorted list of dicts with name, index_type, storage info
+        - metadata: sorted list of metadata column names
+        - index_count / metadata_count: totals
+        """
+        registry = get_registry()
+        indexes = []
+        for name, (idx_type, idx_key, source_attrs) in sorted(registry.items()):
+            indexes.append(
+                {
+                    "name": name,
+                    "index_type": idx_type.value,
+                    "idx_key": idx_key or "",
+                    "is_special": idx_key is None,
+                    "storage": "dedicated column" if idx_key is None else "idx JSONB",
+                    "source_attrs": ", ".join(source_attrs),
+                }
+            )
+        metadata = sorted(registry.metadata)
+        return {
+            "indexes": indexes,
+            "metadata": metadata,
+            "index_count": len(indexes),
+            "metadata_count": len(metadata),
+        }
+
     # -- Helpers -------------------------------------------------------------
 
     def _wrap_object(self, obj):

src/plone/pgcatalog/setuphandlers.py

@@ -26,6 +26,7 @@ def install(context):
 
     site = context.getSite()
     _ensure_catalog_indexes(site)
+    _remove_lexicons(site)
 
 
 def _ensure_catalog_indexes(site):
@@ -74,3 +75,20 @@ def _ensure_catalog_indexes(site):
             log.info("Re-applied catalog config from %s", profile_id)
         except Exception:
             log.debug("Could not apply catalog from %s", profile_id, exc_info=True)
+
+
+def _remove_lexicons(site):
+    """Remove ZCTextIndex lexicons — unused with PG-backed text search.
+
+    Plone's catalog.xml creates htmltext_lexicon, plaintext_lexicon, and
+    plone_lexicon for ZCTextIndex stemming/splitting.  With pgcatalog,
+    full-text search uses PostgreSQL tsvector/BM25, so these are orphaned.
+    """
+    catalog = getattr(site, "portal_catalog", None)
+    if catalog is None:
+        return
+
+    for name in ("htmltext_lexicon", "plaintext_lexicon", "plone_lexicon"):
+        if name in catalog.objectIds():
+            catalog._delObject(name)
+            log.info("Removed orphaned lexicon: %s", name)

src/plone/pgcatalog/www/catalogAdvanced.dtml

@@ -0,0 +1,47 @@
+<dtml-var manage_page_header>
+<dtml-var manage_tabs>
+
+<main class="container-fluid">
+
+<div class="card mt-3 mb-3">
+  <div class="card-header">
+    <strong>Catalog Maintenance</strong>
+  </div>
+  <div class="card-body p-3">
+    <div class="d-flex justify-content-between align-items-center mb-3">
+      <div>
+        <strong>Update Catalog</strong>
+        <p class="text-muted small mb-0">
+          Re-catalog all currently indexed objects in PostgreSQL.
+        </p>
+      </div>
+      <form action="&dtml-URL1;" class="ml-3">
+        <button class="btn btn-sm btn-primary" type="submit"
+                name="manage_catalogReindex:method" value="Update">
+          Update Catalog
+        </button>
+      </form>
+    </div>
+    <hr class="my-2" />
+    <div class="d-flex justify-content-between align-items-center">
+      <div>
+        <strong>Clear and Rebuild</strong>
+        <p class="text-muted small mb-0">
+          Remove all catalog data from PostgreSQL, then walk the entire portal
+          and re-index every content object. Use this after schema changes or
+          data corruption.
+        </p>
+      </div>
+      <form action="&dtml-URL1;" class="ml-3">
+        <button class="btn btn-sm btn-warning" type="submit"
+                name="manage_catalogRebuild:method" value="Rebuild">
+          Clear and Rebuild
+        </button>
+      </form>
+    </div>
+  </div>
+</div>
+
+</main>
+
+<dtml-var manage_page_footer>