fix(thoughtspot): keep formulas that no column surfaces, and keep them hidden

A model's `formulas[]` may hold entries no `columns[]` entry references. Those
are HIDDEN in the ThoughtSpot UI, and the usual reason to write one is that
another formula uses it -- a date-parameter model is the common case, where
`_startDate`/`_endDate` compute a window the visible formulas then apply.

The converter walks `columns[]`, so a formula no column surfaces was never
visited and was dropped outright. Even the unattributed-formula stash could not
catch them: that only holds formulas that WERE visited and could not be
attributed to a dataset.

On the real model that surfaced this, 32 of 41 formulas were lost, and the 9
visible ones referencing them came back with dangling `[formula__startDate]`
references. Round-tripping it now:

    dangling references   4  ->  NONE
    reverse-leg ERRORs    4  ->  none      (the run exited 1 before)
    surfacing columns    45  ->  45        unchanged

They are stashed with their `id` -- that is what the surviving references name,
and re-minting an id from the display name is exactly how the references came
to point at nothing -- and restored as `formulas[]` entries with NO surfacing
column. Giving them one, as the unattributed-formula path deliberately does,
would make a helper the modeller kept private visible to users.

Mutation-checked. 936 tests pass.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Author: djwaldo

converters/thoughtspot/docs/vendor-payload.md

@@ -59,6 +59,7 @@ Every `custom_extensions` entry this converter writes uses `vendor_name` `THOUGH
 | `endpoints_swapped` | Relationship | shadows_derivable | Restored only if its witness companion key still matches the live document's current value; a mismatch means the document changed since the stash was written, so the value is re-derived instead. |
 | `referencing_join` | Relationship | shadows_derivable | Restored only if reconstructing it from the live document still agrees with the stashed value (self-verifying — no separate witness key); disagreement re-derives instead. |
 | `join_shape` | Relationship | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. |
+| `unsurfaced_formulas` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. |
 | `unattributed_formulas` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. |
 | `unrepresentable_joins` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. |
 | `model_properties` | Model | information_only | Restored as-is whenever present — nothing on the Ossie side could have diverged from it. |

converters/thoughtspot/src/ossie_thoughtspot/constants.py

@@ -99,6 +99,23 @@
 #: must agree on the exact spelling and nothing else enforces that.
 FIELD_STASH_DB_COLUMN_NAME = "db_column_name"
 
+#: Formulas present in `model.formulas[]` that NO `columns[]` entry surfaces --
+#: internal helpers, referenced by other formulas but not exposed to users. A
+#: date-parameter model is the common case: `_startDate`, `_endDate` and the
+#: like compute a window that the visible formulas then use.
+#:
+#: They were dropped outright: the converter walks `columns[]`, so a formula no
+#: column surfaces is never visited at all, and even the unattributed-formula
+#: stash only catches ones that WERE visited and could not be attributed. In one
+#: real model that was 32 of 41 formulas -- and the 9 visible ones that
+#: referenced them came back with dangling `[formula__startDate]` references, so
+#: the emitted document would not import.
+#:
+#: Stashed with their `id`, because that is what the surviving references name.
+#: Restored as `formulas[]` entries with NO surfacing column, which is what they
+#: were: giving them one would make an internal helper user-visible.
+MODEL_STASH_UNSURFACED_FORMULAS = "unsurfaced_formulas"
+
 #: A surfacing column's `aggregation` that is LOAD-BEARING and has no home in
 #: the Ossie metric's own expression, so it is preserved verbatim instead.
 #:
@@ -533,6 +550,7 @@ class StashKeyClass(Enum):
     RELATIONSHIP_STASH_JOIN_SHAPE: StashKeyClass.INFORMATION_ONLY,
 
     # -- Model scope --
+    MODEL_STASH_UNSURFACED_FORMULAS: StashKeyClass.INFORMATION_ONLY,
     MODEL_STASH_UNATTRIBUTED_FORMULAS: StashKeyClass.INFORMATION_ONLY,
     MODEL_STASH_UNREPRESENTABLE_JOINS: StashKeyClass.INFORMATION_ONLY,
     MODEL_STASH_MODEL_PROPERTIES: StashKeyClass.INFORMATION_ONLY,

converters/thoughtspot/src/ossie_thoughtspot/ossie_to_thoughtspot.py

@@ -104,6 +104,7 @@
     MODEL_STASH_MODEL_JOINS_WITH,
     MODEL_STASH_MODEL_PROPERTIES,
     MODEL_STASH_PARAMETERS,
+    MODEL_STASH_UNSURFACED_FORMULAS,
     MODEL_STASH_UNATTRIBUTED_FORMULAS,
     MODEL_STASH_UNREPRESENTABLE_JOINS,
     PORTABLE_DIALECT,
@@ -2083,6 +2084,23 @@ def build_model(semantic_model: dict, tables: Sequence[TmlDocument], log: IssueL
         formulas.append(formulas_entry)
         columns.append(columns_entry)
 
+    for entry in model_payload.get(MODEL_STASH_UNSURFACED_FORMULAS) or []:
+        # Re-emitted with NO surfacing columns[] entry, because that is what
+        # they were: internal helpers other formulas reference. Giving them one
+        # -- as the unattributed-formula path deliberately does -- would make a
+        # formula the source kept private visible to users.
+        #
+        # The id is preserved rather than re-minted: it is what the surviving
+        # references name, and re-minting it from the display name is exactly
+        # how `[formula__startDate]` came back pointing at nothing.
+        unsurfaced_entry = {
+            "id": entry.get("id") or _formula_id_from(entry.get("name") or "formula"),
+            "name": entry.get("name") or "",
+            "expr": entry.get("expr") or "",
+        }
+        _allocate_formula_id(unsurfaced_entry, None, taken_formula_ids, log, preserved=True)
+        formulas.append(unsurfaced_entry)
+
     for entry in model_payload.get(MODEL_STASH_UNATTRIBUTED_FORMULAS) or []:
         # A formula spanning two or more Ossie datasets has no single
         # dataset to belong to, which is exactly why the forward direction

converters/thoughtspot/src/ossie_thoughtspot/tml_to_ossie.py

@@ -118,6 +118,7 @@
     MODEL_STASH_MODEL_JOINS_WITH,
     MODEL_STASH_MODEL_PROPERTIES,
     MODEL_STASH_PARAMETERS,
+    MODEL_STASH_UNSURFACED_FORMULAS,
     MODEL_STASH_UNATTRIBUTED_FORMULAS,
     MODEL_STASH_UNREPRESENTABLE_JOINS,
     PORTABLE_DIALECT,
@@ -2329,6 +2330,22 @@ def resolve(table: str, column: str) -> str | None:
                     unattributed[FIELD_STASH_COLUMN_PROPERTIES] = properties
                 model_stash.setdefault(MODEL_STASH_UNATTRIBUTED_FORMULAS, []).append(unattributed)
 
+    # Formulas no columns[] entry surfaces. The loop above walks columns[], so
+    # these were never visited and were dropped outright -- and the references
+    # to them, from formulas that ARE surfaced, then dangled. Preserved with
+    # their ids, which is what those references name.
+    surfaced_formula_ids = {
+        column["formula_id"] for column in model_columns if column.get("formula_id")
+    }
+    for formula_id, formula_entry in formulas.items():
+        if formula_id in surfaced_formula_ids or "expr" not in formula_entry:
+            continue
+        model_stash.setdefault(MODEL_STASH_UNSURFACED_FORMULAS, []).append({
+            "id": formula_id,
+            "name": formula_entry.get("name") or formula_id,
+            "expr": formula_entry["expr"],
+        })
+
     # -- Phase 3.5: unsurfaced physical columns, and SQL View output aliases --
     # A Table/SQL-View column with no Ossie FIELD of its own is not part of
     # the semantic model *as a field*, but has to be preserved verbatim

converters/thoughtspot/tests/test_roundtrip.py

@@ -1009,3 +1009,64 @@ def test_a_wrapped_group_aggregate_is_unaffected(self):
 
     def test_a_plain_aggregate_is_unaffected(self):
         assert self._round_trip("sum ( [T::a] )", "SUM") == "SUM"
+
+
+class TestAFormulaNoColumnSurfaces:
+    """An internal helper formula must survive, and stay internal.
+
+    A model's `formulas[]` may hold entries that no `columns[]` entry surfaces --
+    helpers other formulas reference but users never see. A date-parameter model
+    is the common case: `_startDate` computes a window the visible formulas use.
+
+    The converter walks `columns[]`, so these were never visited and were
+    dropped outright. In one real model that was 32 of 41 formulas, and the
+    visible formulas referencing them came back with dangling
+    `[formula__startDate]` references -- an ERROR, a non-zero exit, and a
+    document ThoughtSpot would refuse.
+    """
+
+    @staticmethod
+    def _round_trip():
+        table = TmlDocument(kind="table", guid=None, body={
+            "name": "T", "db": "D", "schema": "S", "db_table": "T",
+            "connection": {"name": "Conn"},
+            "columns": [{"name": "d", "db_column_name": "D",
+                         "db_column_properties": {"data_type": "DATE"}}]})
+        model = TmlDocument(kind="model", guid=None, body={
+            "name": "M", "model_tables": [{"name": "T"}],
+            "columns": [
+                {"name": "d", "column_id": "T::d", "properties": {"column_type": "ATTRIBUTE"}},
+                {"name": "In Period", "formula_id": "f_vis",
+                 "properties": {"column_type": "ATTRIBUTE"}},
+            ],
+            "formulas": [
+                # Surfaced by nothing -- an internal helper.
+                {"id": "formula__startDate", "name": "_startDate",
+                 "expr": "start_of_year ( [T::d] )"},
+                {"id": "f_vis", "name": "In Period",
+                 "expr": "[T::d] >= [formula__startDate]"},
+            ]})
+        ossie = tml_to_ossie.convert(DocumentSet(model=model, tables=(table,)))
+        return ossie_to_thoughtspot.convert(ossie.model)
+
+    def test_the_helper_formula_survives(self):
+        formulas = {f["name"] for f in self._round_trip().documents.model.body["formulas"]}
+        assert "_startDate" in formulas
+
+    def test_it_keeps_its_id_so_references_still_resolve(self):
+        body = self._round_trip().documents.model.body
+        emitted = {f["id"] for f in body["formulas"]}
+        referenced = {
+            ref for f in body["formulas"]
+            for ref in re.findall(r"\[(formula_[A-Za-z0-9_]+)\]", f["expr"])
+        }
+        assert not (referenced - emitted), f"dangling: {sorted(referenced - emitted)}"
+
+    def test_it_is_not_given_a_surfacing_column(self):
+        # It was internal in the source; surfacing it would expose a helper
+        # the modeller deliberately kept private.
+        body = self._round_trip().documents.model.body
+        assert "_startDate" not in {c["name"] for c in body["columns"]}
+
+    def test_the_conversion_reports_no_error(self):
+        assert not self._round_trip().issues.has_errors()