add evaluation

Author: cl117

evaluation/rankers.py

@@ -162,9 +162,60 @@ def ranker_weighted_sum(query, alpha=0.5, exact_boost=0.5, pr_scale=1e6):
     return [d for d, _ in scored]
 
 
+def ranker_es_equation(query, pool_sort='_text_match(buckets: 1):desc,pagerank:desc'):
+    """
+    FAITHFUL PORT of the old Elasticsearch ranking (master branch), for
+    calibration. master/search.py ranks with a function_score whose script is:
+
+        script:     _score * Math.log(pagerank + 1)               # ln, natural log
+
+    BUT master sets NO `boost_mode`, and ES defaults boost_mode to MULTIPLY, so
+    the effective score master actually computes is:
+
+        final = _score * (_score * ln(pagerank+1)) = _score^2 * ln(pagerank+1)
+
+    i.e. the text score is SQUARED. We reproduce that here: `text_match` is the
+    Typesense analog of ES's `_score`, squared, times ln(pagerank+1). NO
+    normalization, NO pr_scale knob (those belong to the new weighted_sum/
+    multiplicative rankers, which are left intact).
+
+    Caveat: ES `_score` (BM25) and Typesense `text_match` are different engines'
+    relevance scores, so this is the ES *equation* on Typesense's index, not
+    byte-identical ES output.
+
+    pool_sort controls how the candidate pool is pulled:
+      - buckets:1 (default) -- pagerank-aware pool; text score is flattened into
+        one bucket, so the re-score is pagerank-dominated (squaring text has no
+        effect). This is the faithful "old /search" pool.
+      - '_text_match:desc,pagerank:desc' -- RAW continuous text score pool; lets
+        text_match (and its square) actually reshuffle results, closer to how ES
+        ranks on continuous BM25. Risk: with >pool exact-name matches, a canonical
+        high-pagerank part can fall OUT of the pool entirely (the original bug).
+    """
+    hits = _raw_search(query, pool_sort, per_page=CANDIDATE_POOL)
+    if not hits:
+        return []
+    scored = []
+    seen = set()
+    for h in hits:
+        doc = h['document']
+        did = doc.get('displayId')
+        if not did or did in seen:
+            continue
+        seen.add(did)
+        pr = doc.get('pagerank', 0) or 0
+        tm = h.get('text_match', 0)
+        score = tm * tm * log1p(pr)   # _score^2 * ln(pagerank + 1)  (master's real boost_mode=multiply)
+        scored.append((did, score))
+    scored.sort(key=lambda x: x[1], reverse=True)
+    return [d for d, _ in scored]
+
+
 # Registry the runner iterates over. Add new strategies here.
 # Original strategies are kept untouched for experiment reproducibility.
 RANKERS = {
+    'es_equation(ln)': ranker_es_equation,       # faithful old-ES baseline: text^2 * ln(pr+1), buckets:1 pool
+    'es_eq(rawpool)': lambda q: ranker_es_equation(q, '_text_match:desc,pagerank:desc'),  # raw continuous text pool
     'current(buckets10)': ranker_current,
     'buckets1': ranker_buckets1,
     'multiplicative': ranker_multiplicative,

evaluation/run_eval.py

@@ -9,6 +9,7 @@
 Usage:
     python run_eval.py                 # summary table over all rankers
     python run_eval.py --per-query     # also show per-query NDCG + top-gold rank
+    python run_eval.py --raw-ranks     # grid of top-gold RAW RANK: query x ranker
     python run_eval.py --gold mygold.json
 
 Requires Typesense to be running (docker start typesense).
@@ -29,15 +30,45 @@ def load_gold(path):
         return json.load(f)
 
 
-def run(gold_path, per_query):
+def print_raw_rank_grid(gold_set, ranker_names, rank_matrix):
+    """Grid of the top-gold RAW RANK for every (query, ranker). '-' = not found.
+
+    rank_matrix[ranker_name][query] -> int rank or None. This is the position
+    view the summary metrics compress away -- it exposes catastrophic burials
+    (e.g. terminator -> rank 64) that a mean NDCG hides.
+    """
+    queries = [e['query'] for e in gold_set]
+    qw = max(len(q) for q in queries)
+    # shorten ranker headers so the grid stays narrow
+    def short(n):
+        return n if len(n) <= 11 else n[:10] + '.'
+    headers = [short(n) for n in ranker_names]
+    colw = [max(len(h), 5) for h in headers]
+
+    header = f'{"query":<{qw}}' + ''.join(f'  {h:>{w}}' for h, w in zip(headers, colw))
+    print('=== top-gold RAW RANK grid (lower is better; - = not retrieved) ===')
+    print(header)
+    print('-' * len(header))
+    for q in queries:
+        row = f'{q:<{qw}}'
+        for name, w in zip(ranker_names, colw):
+            r = rank_matrix[name].get(q)
+            row += f'  {(str(r) if r is not None else "-"):>{w}}'
+        print(row)
+    print()
+
+
+def run(gold_path, per_query, raw_ranks):
     gold_set = load_gold(gold_path)
     print(f'Gold set: {len(gold_set)} queries from {os.path.basename(gold_path)}\n')
 
     summary = {}
+    rank_matrix = {}  # ranker_name -> {query: top_gold_rank}
     for name, fn in rankers.RANKERS.items():
         agg = {'P@1': 0.0, 'P@5': 0.0, 'P@10': 0.0, 'MRR': 0.0, 'NDCG@10': 0.0}
         latencies = []
         per_query_rows = []
+        rank_matrix[name] = {}
 
         for entry in gold_set:
             q, gold = entry['query'], entry['gold']
@@ -49,6 +80,7 @@ def run(gold_path, per_query):
             for key in agg:
                 agg[key] += m[key]
             per_query_rows.append((q, m['NDCG@10'], m['top_gold_rank']))
+            rank_matrix[name][q] = m['top_gold_rank']
 
         n = len(gold_set)
         for key in agg:
@@ -63,6 +95,9 @@ def run(gold_path, per_query):
                 print(f'  {q:28} NDCG@10={nd:.3f}  top_gold_rank={rank_s}')
             print()
 
+    if raw_ranks:
+        print_raw_rank_grid(gold_set, list(rankers.RANKERS.keys()), rank_matrix)
+
     # Summary table
     cols = ['P@1', 'P@5', 'P@10', 'MRR', 'NDCG@10', 'latency_ms']
     header = f'{"ranker":22}' + ''.join(f'{c:>12}' for c in cols)
@@ -80,5 +115,7 @@ def run(gold_path, per_query):
     ap = argparse.ArgumentParser()
     ap.add_argument('--gold', default=GOLD_DEFAULT)
     ap.add_argument('--per-query', action='store_true')
+    ap.add_argument('--raw-ranks', action='store_true',
+                    help='print a query x ranker grid of the top-gold raw rank')
     args = ap.parse_args()
-    run(args.gold, args.per_query)
+    run(args.gold, args.per_query, args.raw_ranks)

flask/explorer.py

@@ -218,5 +218,30 @@ def search_by_string():
         log.error(f'Error during search by string: {e}')
         raise
 
+@app.route('/facets', methods=['GET'])
+def facets_by_string():
+    """
+    Return facet counts (part role, part type, ...) for a text query, used to
+    build the left-hand filter sidebar. Served entirely by Typesense -- NO SPARQL.
+
+    Example:  GET /facets?query=rbs&facet_by=role,type,sboltype
+    Response: {"found": 491,
+               "facets": {"role": [{"value": "...Composite", "count": 163}, ...],
+                          "type": [...], "sboltype": [...]}}
+    """
+    try:
+        client = typesense_manager.get_client()
+        collection_name = config_manager.get_typesense_collection_name()
+        if not _collection_exists(client, collection_name):
+            abort(503, 'Typesense is not working or the collection does not exist.')
+
+        query = request.args.get('query', '')
+        facet_by = request.args.get('facet_by', 'role,type,sboltype')
+        facet_fields = [f.strip() for f in facet_by.split(',') if f.strip()]
+        return jsonify(search.get_facets(query, facet_fields))
+    except Exception as e:
+        log.error(f'Error during facet search: {e}')
+        raise
+
 if __name__ == "__main__":
     app.run(debug=False, threaded=True) # threaded=True

flask/index.py

@@ -19,8 +19,8 @@
         {'name': 'name',        'type': 'string', 'optional': True},
         {'name': 'description', 'type': 'string', 'optional': True},
         {'name': 'type',        'type': 'string', 'optional': True, 'facet': True},
-        {'name': 'role',        'type': 'string', 'optional': True},
-        {'name': 'sboltype',    'type': 'string', 'optional': True},
+        {'name': 'role',        'type': 'string', 'optional': True, 'facet': True},
+        {'name': 'sboltype',    'type': 'string', 'optional': True, 'facet': True},
         {'name': 'keywords',    'type': 'string', 'optional': True},
         {'name': 'graph',       'type': 'string', 'facet': True},
         {'name': 'pagerank',    'type': 'float'},
@@ -51,6 +51,42 @@ def create_parts_collection(collection_name):
     logger_.log('Collection created', True)
 
 
+def upsert_synonyms(collection_name):
+    """
+    Registers curated search synonyms (domain abbreviations, e.g. lac<->lacI,
+    rfp<->mRFP1, "ribosome binding site"<->rbs) on the collection.
+
+    Synonyms are COLLECTION-SCOPED, so create_parts_collection() wipes them every
+    time it recreates the collection -- this MUST run on every reindex or the
+    recall fixes silently disappear. The list lives in synonyms.json (curated
+    separately from code); each entry is {"id", "synonyms": [...]} for a
+    multi-way set, optionally with "root" for a one-way expansion.
+
+    Missing/empty synonyms.json is non-fatal: indexing proceeds without synonyms.
+    """
+    try:
+        with open('synonyms.json', 'r') as f:
+            synonym_sets = json.load(f)
+    except FileNotFoundError:
+        logger_.log('No synonyms.json found -> skipping synonyms', True)
+        return
+
+    collection = typesense_manager.get_client().collections[collection_name]
+
+    # Clear existing synonyms first so the collection matches synonyms.json
+    # exactly -- keeps this idempotent even when called standalone (not via a
+    # full reindex, which would already recreate the collection from scratch).
+    for existing in collection.synonyms.retrieve().get('synonyms', []):
+        collection.synonyms[existing['id']].delete()
+
+    for entry in synonym_sets:
+        body = {'synonyms': entry['synonyms']}
+        if entry.get('root'):
+            body['root'] = entry['root']
+        collection.synonyms.upsert(entry['id'], body)
+    logger_.log(f'Registered {len(synonym_sets)} synonym set(s)', True)
+
+
 def add_pagerank(parts_response, uri2rank):
     """
     Adds the pagerank score for each part.
@@ -153,6 +189,7 @@ def update_index(uri2rank):
     add_sbol_type(parts_response)
     create_parts_collection(collection_name)
     bulk_index_parts(parts_response, collection_name)
+    upsert_synonyms(collection_name)
 
     logger_.log(f'******** Finished adding {len(parts_response)} parts to index ********', True)
     logger_.log('------------ Successfully updated index ------------\n', True)

flask/search.py

@@ -89,6 +89,50 @@ def search_es(es_query: str) -> Dict:
         logger_.log("search_es(es_query: str)")
         raise
 
+def get_facets(query_string: str, facet_fields: List[str]) -> Dict:
+    """
+    Return facet counts for a text query WITHOUT fetching the documents.
+
+    A "facet" is a category field (part role, part type, ...). For the current
+    query's matching set, Typesense counts how many parts fall under each value
+    -- e.g. for query "rbs": role "Composite" -> 163, "Coding" -> 9. Those are the
+    numbers shown in the left-hand filter dropdowns of the SynBioHub UI.
+
+    IMPORTANT: the counts are computed entirely by Typesense over the indexed
+    facet fields -- NO SPARQL / Virtuoso call happens here. A field can only be
+    faceted if it was declared with {'facet': True} in the collection schema
+    (index.py) at index-build time, which requires a reindex to take effect.
+
+    Args:
+        query_string: the user's search text (same string as /search); '' -> all.
+        facet_fields: field names to facet on, e.g. ['role', 'type', 'sboltype'].
+
+    Returns:
+        {'found': <total matches>,
+         'facets': {'role': [{'value': ..., 'count': ...}, ...], ...}}
+    """
+    collection_name = config_manager.get_typesense_collection_name()
+    collection = typesense_manager.get_client().collections[collection_name]
+
+    params = {
+        'q': query_string or '*',       # '*' = match everything (empty query)
+        'query_by': TEXT_QUERY_BY,
+        'num_typos': '2',
+        'facet_by': ','.join(facet_fields),
+        'max_facet_values': 100,        # distinct values returned per facet
+        'per_page': 1,                  # we only want the counts, not the docs
+    }
+    response = collection.documents.search(params)
+
+    facets = {}
+    for facet in response.get('facet_counts', []):
+        facets[facet['field_name']] = [
+            {'value': c['value'], 'count': c['count']}
+            for c in facet.get('counts', [])
+        ]
+    return {'found': response.get('found', 0), 'facets': facets}
+
+
 def _weighted_rerank(hits: List[Dict], query: str) -> List[Dict]:
     """
     Re-rank Typesense hits by  alpha*norm_text + (1-alpha)*norm_pr + boost*exact.