Clarify program flow for map sharpening and map to model

Author: terwill

libtbx/langchain/agent/best_files_tracker.py

@@ -1277,6 +1277,8 @@ def _is_intermediate_file(self, path):
             'superposed_predicted_untrimmed',  # Intermediate predictions
             '_ELBOW.',            # Elbow geometry files (not fitted ligands)
             'ELBOW.',             # Elbow geometry files
+            '/tnb/',              # trace_and_build intermediate directory
+            '/trace_and_build/',  # trace_and_build intermediate directory
         ]
 
         path_check = path.lower()
@@ -1287,6 +1289,11 @@ def _is_intermediate_file(self, path):
             if pattern_lower in path_check or pattern_lower in basename_check:
                 return True
 
+        # Basename-specific patterns (regex)
+        import re
+        if re.match(r'path_\d+\.pdb$', basename_check):
+            return True  # trace-and-build fragment files
+
         return False
 
 

libtbx/langchain/agent/directive_extractor.py

@@ -115,12 +115,15 @@
 - Resolution is the diffraction limit (d_min), typically 1.5-4.0 Å. Only extract as resolution if explicitly stated as resolution/d_min.
 - If the text says "Resolution limit: Not mentioned" or similar, do NOT extract a resolution value from anywhere else in the text.
 
-**CRITICAL: max_refine_cycles vs after_program**
+**CRITICAL: max_refine_cycles vs after_program vs after_cycle**
 - max_refine_cycles=N: Limits the NUMBER of refinement jobs to N. The workflow continues normally until refinement, then stops after N refinement jobs.
 - after_program="X": FORCES program X to be run IMMEDIATELY, bypassing normal workflow. Only use when user wants to skip directly to a specific program.
+- after_cycle=N: Stops after N AGENT CYCLES (each cycle = one program execution). ONLY use when user says "stop after N cycles" with an explicit number.
 - "maximum of one refinement" or "at most one refinement" → ONLY set max_refine_cycles=1, do NOT set after_program
 - "solve the structure with one refinement" → max_refine_cycles=1 (workflow proceeds normally: xtriage → model → refine)
+- "stop after refinement" or "stop after one refinement" → max_refine_cycles=1, skip_validation=true
 - "just run refinement" or "only refinement" → after_program="phenix.refine" (skip to refinement immediately)
+- Do NOT use after_cycle for "stop after refinement" — that would stop after the first agent cycle (e.g., xtriage), not after refinement.
 
 **CRITICAL: skip_validation RULE**
 If the user specifies ANY explicit stop condition (like "stop after X" or "Stop Condition: ..."),
@@ -851,6 +854,10 @@ def _log(msg):
     # e.g., after_program=phenix.map_symmetry but constraints say "build a model" → don't stop
     validated = _fix_multi_step_workflow_conflict(validated, _log)
 
+    # Fix after_cycle=1 when max_refine_cycles is set
+    # LLM often confuses "one refinement cycle" with "one agent cycle"
+    validated = _fix_after_cycle_refinement_conflict(validated, _log)
+
     return validated
 
 
@@ -1023,6 +1030,47 @@ def _fix_multi_step_workflow_conflict(directives, log):
     return directives
 
 
+def _fix_after_cycle_refinement_conflict(directives, log):
+    """
+    Fix conflict where LLM sets after_cycle=1 alongside max_refine_cycles.
+
+    When user says "stop after refinement" or "one refinement cycle", the LLM
+    sometimes produces both max_refine_cycles=1 AND after_cycle=1. The after_cycle=1
+    is wrong — it would stop after the first agent cycle (e.g., xtriage), not after
+    the refinement program completes.
+
+    Rules:
+    - after_cycle=1 + max_refine_cycles=N → remove after_cycle (keep max_refine_cycles)
+    - after_cycle=1 alone (no max_refine_cycles, no after_program) → suspicious,
+      only keep if no workflow programs would run before refinement
+    """
+    stop_conditions = directives.get("stop_conditions", {})
+    if not stop_conditions:
+        return directives
+
+    after_cycle = stop_conditions.get("after_cycle")
+    max_refine = stop_conditions.get("max_refine_cycles")
+    after_program = stop_conditions.get("after_program")
+
+    # Case 1: after_cycle + max_refine_cycles — the after_cycle is redundant/wrong
+    if after_cycle is not None and max_refine is not None:
+        log("DIRECTIVES: Removing after_cycle=%d (conflicts with max_refine_cycles=%d)" %
+            (after_cycle, max_refine))
+        log("DIRECTIVES: (LLM confused 'one refinement' with 'one agent cycle')")
+        del directives["stop_conditions"]["after_cycle"]
+
+    # Case 2: after_cycle=1 alone (no after_program, no max_refine)
+    # This almost always means the LLM misinterpreted "stop after refinement"
+    # as "stop after 1 cycle". Convert to max_refine_cycles=1.
+    elif after_cycle == 1 and after_program is None and max_refine is None:
+        log("DIRECTIVES: Converting after_cycle=1 → max_refine_cycles=1")
+        log("DIRECTIVES: (after_cycle=1 would stop after xtriage, not after refinement)")
+        del directives["stop_conditions"]["after_cycle"]
+        directives["stop_conditions"]["max_refine_cycles"] = 1
+
+    return directives
+
+
 def _fix_program_name(name):
     """
     Try to fix common variations in program names.

libtbx/langchain/agent/sanity_checker.py

@@ -211,6 +211,10 @@ def _check_model_for_refine(self, context: Dict) -> Optional[SanityIssue]:
 
         If user has only search_model but requests refinement, give a specific
         error explaining they need to run Phaser/docking first.
+
+        EXCEPTION: If Phaser/dock_in_map haven't run yet, this is a normal
+        workflow progression — the agent should be allowed to proceed so it
+        can choose the positioning program.
         """
         state = context.get("state", "")
 
@@ -234,9 +238,25 @@ def _check_model_for_refine(self, context: Dict) -> Optional[SanityIssue]:
 
         if not has_model:
             if has_search_model:
-                # User has templates but no positioned model - common mistake!
+                # Check if model-positioning programs haven't been attempted yet.
+                # If so, this is normal workflow progression — the agent should
+                # be allowed to proceed and choose Phaser/dock_in_map.
+                history = context.get("history", [])
+                programs_run = {h.get("program", "").lower() for h in history if isinstance(h, dict)}
                 exp_type = context.get("experiment_type", "unknown")
 
+                positioning_programs = {"phenix.phaser"} if exp_type == "xray" else {"phenix.dock_in_map"}
+                positioning_attempted = any(
+                    any(pos_prog in prog for pos_prog in positioning_programs)
+                    for prog in programs_run
+                )
+
+                if not positioning_attempted:
+                    # Model positioning hasn't been tried yet — let the agent proceed
+                    # so it can choose the appropriate positioning program
+                    return None
+
+                # Positioning was attempted but model still not placed
                 if exp_type == "xray":
                     suggestion = (
                         "You have a search model (predicted structure or template) but no positioned model. "

libtbx/langchain/agent/workflow_engine.py

@@ -337,10 +337,14 @@ def _has_placed_model(self, files, history_info, directives=None):
         return False
 
     def _has_refined_model(self, files, history_info):
-        """Check if model has been refined."""
+        """Check if model has been refined IN THIS SESSION.
+
+        IMPORTANT: Only trust history-based evidence, not file names.
+        User-provided input files may start with 'refine_' (e.g., refine_001_model.pdb)
+        without any actual refinement having been done in this session. Relying on
+        files.get('refined') would incorrectly skip to validation/STOP.
+        """
         return bool(
-            files.get("refined") or
-            files.get("rsr_output") or
             history_info.get("refine_count", 0) > 0 or
             history_info.get("rsr_count", 0) > 0
         )

libtbx/langchain/knowledge/file_categories.yaml

@@ -428,6 +428,9 @@ unclassified_pdb:
     - "*chainsaw*"  # Chainsaw output (processed search models)
     - "lig*"
     - "*ligand*"
+    - "path_[0-9]*.pdb"    # trace-and-build fragments (intermediate)
+    - "*TEMP*"              # Temporary files (intermediate)
+    - "*reference*"         # Reference models (intermediate)
   notes: "Default category for generic PDB files - assume positioned model ready for refinement"
 
 # =============================================================================
@@ -509,6 +512,14 @@ carryover_temp:
     - "*/CarryOn/*"
   notes: "NEVER use - predict_and_build intermediate files"
 
+trace_fragment:
+  description: "Trace-and-build fragment files from predict_and_build"
+  parent_category: intermediate
+  extensions: [".pdb"]
+  patterns:
+    - "path_[0-9]*.pdb"    # trace_and_build path fragments (path_1.pdb, path_54.pdb, etc.)
+  notes: "NEVER use - these are intermediate trace fragments, not complete models"
+
 # =============================================================================
 # MAP SUBCATEGORIES
 # =============================================================================

libtbx/langchain/knowledge/programs.yaml

@@ -876,7 +876,7 @@ phenix.process_predicted_model:
   outputs:
     files:
       - pattern: "*processed*.pdb"
-        type: model
+        type: processed_predicted  # NOT 'model' - this is still a search model
 
   command: "phenix.process_predicted_model {model}"
 
@@ -1243,7 +1243,7 @@ phenix.pdbtools:
 # -----------------------------------------------------------------------------
 
 phenix.resolve_cryo_em:
-  description: "Optimize cryo-EM map using density modification"
+  description: "Density modification of cryo-EM map (map optimization, NOT model building)"
   category: map_optimization
   experiment_types: [cryoem]
 

libtbx/langchain/knowledge/prompts_hybrid.py

@@ -659,6 +659,25 @@ def is_half_map(filepath):
                 "predict_and_build" in valid_list):
                 workflow_section += "NOTE: Use predict_and_build with strategy: {\"stop_after_predict\": true}\n"
 
+        # Add program descriptions so LLM understands each program's purpose
+        try:
+            from libtbx.langchain.knowledge.yaml_loader import get_program
+        except ImportError:
+            try:
+                from knowledge.yaml_loader import get_program
+            except ImportError:
+                get_program = None
+        if get_program:
+            desc_lines = []
+            for prog in workflow_state.get("valid_programs", []):
+                if prog == "STOP":
+                    continue
+                prog_def = get_program(prog)
+                if prog_def and prog_def.get("description"):
+                    desc_lines.append("  - %s: %s" % (prog, prog_def["description"]))
+            if desc_lines:
+                workflow_section += "\nProgram descriptions:\n" + "\n".join(desc_lines) + "\n"
+
         # Add recommendations section if available
         recommendations = format_recommendations_for_prompt(workflow_state)
         if recommendations:

libtbx/langchain/knowledge/workflows.yaml

@@ -325,7 +325,7 @@ cryoem:
             - has: half_map
             - not_has: full_map
             - not_done: resolve_cryo_em  # Map optimization should only run once
-          hint: "Create optimized map from half-maps"
+          hint: "Density modification to create optimized map from half-maps (NOT model building)"
 
         - program: phenix.map_sharpening
           conditions:
@@ -357,7 +357,7 @@ cryoem:
             - has: half_map
             - not_has: full_map
             - not_done: resolve_cryo_em
-          hint: "Create optimized full map from half-maps before docking"
+          hint: "Density modification: create optimized full map from half-maps before docking (NOT model building)"
 
         - program: phenix.map_sharpening
           conditions: