diff --git a/.github/workflows/converter-databricks-ci.yml b/.github/workflows/converter-databricks-ci.yml index 54ec6f0f..b57d153d 100644 --- a/.github/workflows/converter-databricks-ci.yml +++ b/.github/workflows/converter-databricks-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/databricks/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-databricks-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/databricks/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-databricks-ci.yml' jobs: diff --git a/.github/workflows/converter-dbt-ci.yml b/.github/workflows/converter-dbt-ci.yml index 7d7e7a31..61b741d8 100644 --- a/.github/workflows/converter-dbt-ci.yml +++ b/.github/workflows/converter-dbt-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/dbt/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-dbt-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/dbt/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-dbt-ci.yml' jobs: diff --git a/.github/workflows/converter-gooddata-ci.yml b/.github/workflows/converter-gooddata-ci.yml index afa95bfd..30654ad9 100644 --- a/.github/workflows/converter-gooddata-ci.yml +++ b/.github/workflows/converter-gooddata-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/gooddata/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-gooddata-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/gooddata/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-gooddata-ci.yml' jobs: diff --git a/.github/workflows/converter-honeydew-ci.yml b/.github/workflows/converter-honeydew-ci.yml index 1e58149f..8d46abb7 100644 --- a/.github/workflows/converter-honeydew-ci.yml +++ b/.github/workflows/converter-honeydew-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/honeydew/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-honeydew-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/honeydew/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-honeydew-ci.yml' jobs: diff --git a/.github/workflows/converter-microsoft-ci.yml b/.github/workflows/converter-microsoft-ci.yml index fc177244..d1217277 100644 --- a/.github/workflows/converter-microsoft-ci.yml +++ b/.github/workflows/converter-microsoft-ci.yml @@ -24,13 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/microsoft/**' - - 'core-spec/ossie-schema.json' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-microsoft-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/microsoft/**' - - 'core-spec/ossie-schema.json' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-microsoft-ci.yml' jobs: diff --git a/.github/workflows/converter-nvidia-ci.yml b/.github/workflows/converter-nvidia-ci.yml index c86c873b..fb09cae9 100644 --- a/.github/workflows/converter-nvidia-ci.yml +++ b/.github/workflows/converter-nvidia-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/nvidia/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-nvidia-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/nvidia/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-nvidia-ci.yml' jobs: diff --git a/.github/workflows/converter-omni-ci.yml b/.github/workflows/converter-omni-ci.yml index 146b12a0..4f56ac53 100644 --- a/.github/workflows/converter-omni-ci.yml +++ b/.github/workflows/converter-omni-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/omni/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-omni-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/omni/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-omni-ci.yml' jobs: diff --git a/.github/workflows/converter-ontology-ci.yml b/.github/workflows/converter-ontology-ci.yml index 2ff88792..2abd3de4 100644 --- a/.github/workflows/converter-ontology-ci.yml +++ b/.github/workflows/converter-ontology-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/ontology/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-ontology-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/ontology/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-ontology-ci.yml' jobs: diff --git a/.github/workflows/converter-orionbelt-ci.yml b/.github/workflows/converter-orionbelt-ci.yml index 6ed92626..0b3aebf6 100644 --- a/.github/workflows/converter-orionbelt-ci.yml +++ b/.github/workflows/converter-orionbelt-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/orionbelt/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-orionbelt-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/orionbelt/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-orionbelt-ci.yml' jobs: diff --git a/.github/workflows/converter-polaris-ci.yml b/.github/workflows/converter-polaris-ci.yml index f01d55b1..c4572c52 100644 --- a/.github/workflows/converter-polaris-ci.yml +++ b/.github/workflows/converter-polaris-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/polaris/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-polaris-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/polaris/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-polaris-ci.yml' jobs: diff --git a/.github/workflows/converter-salesforce-ci.yml b/.github/workflows/converter-salesforce-ci.yml index 125b27c5..68e0677e 100644 --- a/.github/workflows/converter-salesforce-ci.yml +++ b/.github/workflows/converter-salesforce-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/salesforce/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-salesforce-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/salesforce/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-salesforce-ci.yml' jobs: diff --git a/.github/workflows/converter-sigma-ci.yml b/.github/workflows/converter-sigma-ci.yml index 7d3599bf..85d51176 100644 --- a/.github/workflows/converter-sigma-ci.yml +++ b/.github/workflows/converter-sigma-ci.yml @@ -24,12 +24,16 @@ on: branches: [ "main" ] paths: - 'converters/sigma/**' + - 'core-spec/**' + - 'examples/**' - 'python/**' - '.github/workflows/converter-sigma-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/sigma/**' + - 'core-spec/**' + - 'examples/**' - 'python/**' - '.github/workflows/converter-sigma-ci.yml' diff --git a/.github/workflows/converter-snowflake-ci.yml b/.github/workflows/converter-snowflake-ci.yml index 707cdb46..b0df8031 100644 --- a/.github/workflows/converter-snowflake-ci.yml +++ b/.github/workflows/converter-snowflake-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/snowflake/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-snowflake-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/snowflake/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-snowflake-ci.yml' jobs: diff --git a/.github/workflows/converter-wisdom-ci.yml b/.github/workflows/converter-wisdom-ci.yml index eae3f1cd..33b2f1f3 100644 --- a/.github/workflows/converter-wisdom-ci.yml +++ b/.github/workflows/converter-wisdom-ci.yml @@ -24,11 +24,15 @@ on: branches: [ "main" ] paths: - 'converters/wisdom/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-wisdom-ci.yml' pull_request: branches: [ "main" ] paths: - 'converters/wisdom/**' + - 'core-spec/**' + - 'examples/**' - '.github/workflows/converter-wisdom-ci.yml' permissions: diff --git a/.github/workflows/validation-ci.yml b/.github/workflows/validation-ci.yml index a5c670a3..360bf0ce 100644 --- a/.github/workflows/validation-ci.yml +++ b/.github/workflows/validation-ci.yml @@ -24,6 +24,7 @@ on: branches: ["main"] paths: - "validation/**" + - "python/**" - "core-spec/**" - "examples/tpcds_semantic_model.yaml" - ".github/workflows/validation-ci.yml" @@ -31,6 +32,7 @@ on: branches: ["main"] paths: - "validation/**" + - "python/**" - "core-spec/**" - "examples/tpcds_semantic_model.yaml" - ".github/workflows/validation-ci.yml" @@ -62,5 +64,8 @@ jobs: - name: Run validation pytest tests run: uv run --with pytest --with pyyaml --with jsonschema -m pytest validation/tests/ + - name: Run shared Python model tests + run: uv run --with pytest --with jsonschema --with ./python -m pytest python/tests/ + - name: Validate canonical example run: uv run validation/validate.py examples/tpcds_semantic_model.yaml diff --git a/converters/README.md b/converters/README.md index 59a50035..07b586c0 100644 --- a/converters/README.md +++ b/converters/README.md @@ -19,6 +19,16 @@ # Apache Ossie Converters +## Document format + +Converters use the flat `0.2.0.dev0` core format: one model per JSON/YAML document, +with `name`, `datasets`, `relationships`, and `metrics` at the root alongside +`version`. Legacy `semantic_model` arrays or object wrappers must be migrated +before conversion; see the [migration guidance](../core-spec/spec.md#migrating-earlier-document-shapes). +For catalog exports, write separate documents rather than wrapping multiple +models in one file. Ontology documents retain their embedded `semantic_model` +property as defined by the ontology schema. + ## Overview An Ossie Converter translates between the Ossie semantic model format and a specific vendor's semantic implementation. This enables teams to author a semantic model once in the Ossie standard and then generate the corresponding vendor-specific representation automatically. diff --git a/converters/databricks/README.md b/converters/databricks/README.md index ed9268b1..865b9638 100644 --- a/converters/databricks/README.md +++ b/converters/databricks/README.md @@ -38,6 +38,10 @@ Metric View only features (filter, window, format, rely, ...) are instead **pres [requirement](#requirements) **raises a `ConversionError`** -- the converter never silently drops a field or produces an invalid result. +Ossie documents contain one model directly at the root, with `version`, `name`, +`datasets`, and optional model properties. Legacy `semantic_model` wrappers +(arrays or objects) are rejected. + ## Installation ```bash @@ -77,7 +81,7 @@ Each row maps in both directions; the **Notes** flag where a behavior is specifi | Apache Ossie | Metric View (v1.1) | Notes | |---|---|---| -| `semantic_model.description` | `comment` | Model-level description only. | +| `description` | `comment` | Model-level description only. | | root dataset | `source` | The fact/grain. | | other `datasets` | nested `joins[]` | Export: the relationship graph is reassembled into the join tree; a dataset reached by two paths (a diamond) fans out into one aliased join per path. | | `relationship` `from_columns`/`to_columns` | join `on` (differing names) / `using` (shared names) | Decomposed into columns on import; rebuilt into `on`/`using` on export. | diff --git a/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py b/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py index 63be1bd6..4799416b 100644 --- a/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py +++ b/converters/databricks/src/ossie_databricks/metric_view_to_ossie.py @@ -91,7 +91,7 @@ def convert_metric_view_to_ossie(mv_yaml_str, model_name=None): ) model = _convert_view(view, model_name) - return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}) + return dump_yaml({"version": OSSIE_VERSION, **model}) def _convert_view(view, model_name): diff --git a/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py b/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py index f96d505c..de828bef 100644 --- a/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py +++ b/converters/databricks/src/ossie_databricks/ossie_to_metric_view.py @@ -77,13 +77,18 @@ def convert_ossie_to_metric_view(ossie_yaml_str, source=None): f"Unsupported Apache Ossie version '{version}'. Supported: {OSSIE_VERSION}" ) - models = root.get("semantic_model") - if not isinstance(models, list) or not models: - raise ConversionError("'semantic_model' must be a non-empty list") - if len(models) > 1: - _warn("model", "multiple semantic models found; converting only the first") + if "semantic_model" in root: + raise ConversionError( + "Legacy 'semantic_model' wrappers are not supported; " + "place the model properties directly at the document root" + ) + + if "dialects" in root or "vendors" in root: + raise ConversionError("Root dialects and vendors are not supported by the Ossie spec") + if not isinstance(root.get("name"), str): + raise ConversionError("Ossie model requires a string 'name' at the document root") - view = _convert_model(models[0], explicit_source=source) + view = _convert_model(root, explicit_source=source) return dump_yaml(view) diff --git a/converters/databricks/tests/_roundtrip_helpers.py b/converters/databricks/tests/_roundtrip_helpers.py index a74ebd71..b3dde449 100644 --- a/converters/databricks/tests/_roundtrip_helpers.py +++ b/converters/databricks/tests/_roundtrip_helpers.py @@ -251,7 +251,7 @@ def build_ossie(rnd): model["relationships"] = relationships if metrics: model["metrics"] = metrics - return {"version": OSSIE_VERSION, "semantic_model": [model]} + return {"version": OSSIE_VERSION, **model} def _three_part(rnd): @@ -348,7 +348,7 @@ def assert_ossie_roundtrip(ossie): mv_yaml = _convert(exporter.convert_ossie_to_metric_view, dump_yaml(ossie)) ossie2 = load_yaml(_convert(importer.convert_metric_view_to_ossie, mv_yaml)) - m1, m2 = ossie["semantic_model"][0], ossie2["semantic_model"][0] + m1, m2 = ossie, ossie2 assert ({d["name"]: (d["source"], _fields_map(d)) for d in m1["datasets"]} == {d["name"]: (d["source"], _fields_map(d)) for d in m2["datasets"]}), "datasets" assert _rel_set(m1) == _rel_set(m2), "relationships" diff --git a/converters/databricks/tests/_util.py b/converters/databricks/tests/_util.py index e82a1d0b..bce2e466 100644 --- a/converters/databricks/tests/_util.py +++ b/converters/databricks/tests/_util.py @@ -64,13 +64,13 @@ def strip_dropped(ossie): `rely.at_most_one_match` (MV) -> `unique_keys` + a relationship rely-stash. We drop both key forms and the relationship stash so the key info is compared as 'gone'.""" ossie = copy.deepcopy(ossie) - for model in ossie.get("semantic_model", []): - model.pop("name", None) # MV carries no model name - model.pop("description", None) # model + fact descriptions merge into one comment - for ds in model.get("datasets", []): - ds.pop("primary_key", None) - ds.pop("unique_keys", None) - ds.pop("description", None) # no per-source comment in single-source MV - for rel in model.get("relationships", []): - rel.pop("custom_extensions", None) # derived rely-stash from a declared key + model = ossie + model.pop("name", None) # MV carries no model name + model.pop("description", None) # model + fact descriptions merge into one comment + for ds in model.get("datasets", []): + ds.pop("primary_key", None) + ds.pop("unique_keys", None) + ds.pop("description", None) # no per-source comment in single-source MV + for rel in model.get("relationships", []): + rel.pop("custom_extensions", None) # derived rely-stash from a declared key return ossie diff --git a/converters/databricks/tests/fixtures/fixtureA_ossie.yaml b/converters/databricks/tests/fixtures/fixtureA_ossie.yaml index 49baf43b..8501da8b 100644 --- a/converters/databricks/tests/fixtures/fixtureA_ossie.yaml +++ b/converters/databricks/tests/fixtures/fixtureA_ossie.yaml @@ -23,57 +23,56 @@ version: "0.2.0.dev0" -semantic_model: - - name: sales - description: Sales orders with customer attributes - datasets: - - name: orders # fact: no incoming relationship -> becomes `source` - source: samples.tpch.orders - primary_key: [o_orderkey] # dropped on export (Apache Ossie-only) - description: One row per order - fields: - - name: o_orderkey - expression: - dialects: - - dialect: DATABRICKS - expression: o_orderkey - description: Order identifier - - name: o_orderdate - expression: - dialects: - - dialect: DATABRICKS - expression: o_orderdate - label: Order Date - ai_context: - synonyms: [order date, date] - - name: customer - source: samples.tpch.customer - primary_key: [c_custkey] - fields: - - name: c_name - expression: - dialects: - - dialect: DATABRICKS - expression: c_name - description: Customer name - relationships: - - name: orders_to_customer - from: orders - to: customer - from_columns: [o_custkey] - to_columns: [c_custkey] - metrics: - - name: total_revenue +name: sales +description: Sales orders with customer attributes +datasets: + - name: orders # fact: no incoming relationship -> becomes `source` + source: samples.tpch.orders + primary_key: [o_orderkey] # dropped on export (Apache Ossie-only) + description: One row per order + fields: + - name: o_orderkey expression: dialects: - dialect: DATABRICKS - expression: SUM(o_totalprice) # fact columns are bare in measures - description: Total order revenue + expression: o_orderkey + description: Order identifier + - name: o_orderdate + expression: + dialects: + - dialect: DATABRICKS + expression: o_orderdate + label: Order Date ai_context: - synonyms: [revenue, total revenue, sales] - - name: order_count + synonyms: [order date, date] + - name: customer + source: samples.tpch.customer + primary_key: [c_custkey] + fields: + - name: c_name expression: dialects: - dialect: DATABRICKS - expression: COUNT(*) - description: Number of orders + expression: c_name + description: Customer name +relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: [o_custkey] + to_columns: [c_custkey] +metrics: + - name: total_revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(o_totalprice) # fact columns are bare in measures + description: Total order revenue + ai_context: + synonyms: [revenue, total revenue, sales] + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(*) + description: Number of orders diff --git a/converters/databricks/tests/fixtures/fixtureB_ossie.yaml b/converters/databricks/tests/fixtures/fixtureB_ossie.yaml index 7fa53dfd..d9ac2e98 100644 --- a/converters/databricks/tests/fixtures/fixtureB_ossie.yaml +++ b/converters/databricks/tests/fixtures/fixtureB_ossie.yaml @@ -24,49 +24,48 @@ version: "0.2.0.dev0" -semantic_model: +name: lineitem +description: Line item shipping metrics +datasets: - name: lineitem - description: Line item shipping metrics - datasets: - - name: lineitem - source: samples.tpch.lineitem - fields: - - name: line_number - expression: - dialects: - - dialect: DATABRICKS - expression: l_linenumber - custom_extensions: - - vendor_name: DATABRICKS - data: '{"_v": 1, "format": {"type": "number", "decimal_places": {"type": "exact", "places": 0}}}' - - name: orders - source: samples.tpch.orders - unique_keys: - - [o_orderkey] - relationships: - - name: lineitem_to_orders - from: lineitem - to: orders - from_columns: [l_orderkey] - to_columns: [o_orderkey] - custom_extensions: - - vendor_name: DATABRICKS - data: '{"_v": 1, "rely": {"at_most_one_match": true}}' - metrics: - - name: revenue + source: samples.tpch.lineitem + fields: + - name: line_number expression: dialects: - dialect: DATABRICKS - expression: SUM(l_extendedprice * (1 - l_discount)) - description: Net revenue + expression: l_linenumber custom_extensions: - vendor_name: DATABRICKS - data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD", "decimal_places": {"type": "exact", "places": 2}}}' - - name: order_count - expression: - dialects: - - dialect: DATABRICKS - expression: COUNT(DISTINCT l_orderkey) + data: '{"_v": 1, "format": {"type": "number", "decimal_places": {"type": "exact", "places": 0}}}' + - name: orders + source: samples.tpch.orders + unique_keys: + - [o_orderkey] +relationships: + - name: lineitem_to_orders + from: lineitem + to: orders + from_columns: [l_orderkey] + to_columns: [o_orderkey] + custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "rely": {"at_most_one_match": true}}' +metrics: + - name: revenue + expression: + dialects: + - dialect: DATABRICKS + expression: SUM(l_extendedprice * (1 - l_discount)) + description: Net revenue custom_extensions: - vendor_name: DATABRICKS - data: '{"_v": 1, "filter": "l_returnflag = ''N''"}' + data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD", "decimal_places": {"type": "exact", "places": 2}}}' + - name: order_count + expression: + dialects: + - dialect: DATABRICKS + expression: COUNT(DISTINCT l_orderkey) +custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "filter": "l_returnflag = ''N''"}' diff --git a/converters/databricks/tests/fixtures/tpcds_ossie.yaml b/converters/databricks/tests/fixtures/tpcds_ossie.yaml index e055eab1..70dba8f7 100644 --- a/converters/databricks/tests/fixtures/tpcds_ossie.yaml +++ b/converters/databricks/tests/fixtures/tpcds_ossie.yaml @@ -16,74 +16,73 @@ # under the License. version: "0.2.0.dev0" -semantic_model: - - name: tpcds_store_sales - description: Store sales enriched with date, item, and customer dimensions - datasets: - - name: store_sales - source: tpcds.public.store_sales - fields: - - name: ticket_number - expression: - dialects: [{dialect: DATABRICKS, expression: ss_ticket_number}] - - name: date_dim - source: tpcds.public.date_dim - primary_key: [d_date_sk] - fields: - - name: sold_year - expression: - dialects: [{dialect: DATABRICKS, expression: d_year}] - label: Year - ai_context: {synonyms: [year, yr]} - - name: sold_date - expression: - dialects: [{dialect: DATABRICKS, expression: d_date}] - - name: item - source: tpcds.public.item - primary_key: [i_item_sk] - fields: - - name: item_category - expression: - dialects: [{dialect: DATABRICKS, expression: i_category}] - ai_context: {synonyms: [category, product type]} - - name: item_brand - expression: - dialects: [{dialect: DATABRICKS, expression: i_brand}] - - name: customer - source: tpcds.public.customer - primary_key: [c_customer_sk] - fields: - - name: birth_country - expression: - dialects: [{dialect: DATABRICKS, expression: c_birth_country}] - relationships: - - name: store_sales_to_date_dim - from: store_sales - to: date_dim - from_columns: [ss_sold_date_sk] - to_columns: [d_date_sk] - - name: store_sales_to_item - from: store_sales - to: item - from_columns: [ss_item_sk] - to_columns: [i_item_sk] - - name: store_sales_to_customer - from: store_sales - to: customer - from_columns: [ss_customer_sk] - to_columns: [c_customer_sk] - metrics: - - name: total_sales +name: tpcds_store_sales +description: Store sales enriched with date, item, and customer dimensions +datasets: + - name: store_sales + source: tpcds.public.store_sales + fields: + - name: ticket_number expression: - dialects: [{dialect: DATABRICKS, expression: SUM(ss_ext_sales_price)}] - description: Total sales revenue - custom_extensions: - - vendor_name: DATABRICKS - data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD"}}' - - name: total_quantity + dialects: [{dialect: DATABRICKS, expression: ss_ticket_number}] + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] + fields: + - name: sold_year expression: - dialects: [{dialect: DATABRICKS, expression: SUM(ss_quantity)}] - description: Total units sold + dialects: [{dialect: DATABRICKS, expression: d_year}] + label: Year + ai_context: {synonyms: [year, yr]} + - name: sold_date + expression: + dialects: [{dialect: DATABRICKS, expression: d_date}] + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] + fields: + - name: item_category + expression: + dialects: [{dialect: DATABRICKS, expression: i_category}] + ai_context: {synonyms: [category, product type]} + - name: item_brand + expression: + dialects: [{dialect: DATABRICKS, expression: i_brand}] + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] + fields: + - name: birth_country + expression: + dialects: [{dialect: DATABRICKS, expression: c_birth_country}] +relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] +metrics: + - name: total_sales + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_ext_sales_price)}] + description: Total sales revenue custom_extensions: - vendor_name: DATABRICKS - data: '{"_v": 1, "filter": "ss_net_profit > 0"}' + data: '{"_v": 1, "format": {"type": "currency", "currency_code": "USD"}}' + - name: total_quantity + expression: + dialects: [{dialect: DATABRICKS, expression: SUM(ss_quantity)}] + description: Total units sold +custom_extensions: + - vendor_name: DATABRICKS + data: '{"_v": 1, "filter": "ss_net_profit > 0"}' diff --git a/converters/databricks/tests/test_metric_view_to_ossie.py b/converters/databricks/tests/test_metric_view_to_ossie.py index 02890037..cf8d9c0d 100644 --- a/converters/databricks/tests/test_metric_view_to_ossie.py +++ b/converters/databricks/tests/test_metric_view_to_ossie.py @@ -37,7 +37,7 @@ def test_fields_is_accepted_as_alias_for_dimensions(): "fields:\n- {name: region, expr: region}\n" ) ossie = parse(importer.convert_metric_view_to_ossie(mv)) - fields = ossie["semantic_model"][0]["datasets"][0].get("fields", []) + fields = ossie["datasets"][0].get("fields", []) assert [f["name"] for f in fields] == ["region"] @@ -58,14 +58,14 @@ def test_both_dimensions_and_fields_present_warns_and_uses_dimensions(): with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") ossie = parse(importer.convert_metric_view_to_ossie(mv)) - names = [f["name"] for f in ossie["semantic_model"][0]["datasets"][0].get("fields", [])] + names = [f["name"] for f in ossie["datasets"][0].get("fields", [])] assert names == ["kept"] assert any("fields" in str(w.message) and "ignored" in str(w.message) for w in caught) def test_stash_written_at_each_level(): ossie = parse(importer.convert_metric_view_to_ossie(load_fixture("fixtureB_metric_view.yaml"))) - model = ossie["semantic_model"][0] + model = ossie # model-level filter assert any(e["vendor_name"] == "DATABRICKS" and "filter" in e["data"] @@ -81,7 +81,7 @@ def test_stash_written_at_each_level(): def test_name_override(): ossie = parse(importer.convert_metric_view_to_ossie( load_fixture("fixtureB_metric_view.yaml"), model_name="custom")) - assert ossie["semantic_model"][0]["name"] == "custom" + assert ossie["name"] == "custom" def test_cross_join_rejected(): @@ -104,7 +104,7 @@ def test_complex_joined_dimension_filed_under_join_dataset(): "dimensions:\n- name: full\n expr: cust.a || cust.b\n" ) ossie = parse(importer.convert_metric_view_to_ossie(mv)) - cust = next(d for d in ossie["semantic_model"][0]["datasets"] if d["name"] == "cust") + cust = next(d for d in ossie["datasets"] if d["name"] == "cust") assert any(f["name"] == "full" for f in cust.get("fields", [])) @@ -149,7 +149,7 @@ def test_one_to_many_join_flips_from_to_and_stashes_source(): "measures:\n- {name: order_count, expr: COUNT(*)}\n" ) ossie = parse(importer.convert_metric_view_to_ossie(mv)) - model = ossie["semantic_model"][0] + model = ossie rel = model["relationships"][0] assert rel["from"] == "line_items" # many side (holds the FK) assert rel["to"] == "orders" # one side (holds the PK) @@ -169,7 +169,7 @@ def test_at_most_one_match_recovers_unique_key(): " rely: {at_most_one_match: true}\n" ) ossie = parse(importer.convert_metric_view_to_ossie(mv)) - cust = next(d for d in ossie["semantic_model"][0]["datasets"] if d["name"] == "customer") + cust = next(d for d in ossie["datasets"] if d["name"] == "customer") assert cust.get("unique_keys") == [["id"]] @@ -187,7 +187,7 @@ def test_sql_source_name_defaults_to_metric_view(): `metric_view` (not a token sliced out of the SQL).""" mv = "version: '1.1'\nsource: SELECT a, b FROM main.sales.orders\n" ossie = parse(importer.convert_metric_view_to_ossie(mv)) - assert ossie["semantic_model"][0]["name"] == "metric_view" + assert ossie["name"] == "metric_view" def test_join_missing_source_raises(): @@ -202,7 +202,7 @@ def test_measure_rewrite_with_regex_special_name(): --name containing regex backreference syntax (e.g. \1) does not raise a re.error.""" mv = "version: '1.1'\nsource: c.s.fact\nmeasures:\n- {name: rev, expr: SUM(source.amount)}\n" ossie = parse(importer.convert_metric_view_to_ossie(mv, model_name=r"a\1b")) - expr = ossie["semantic_model"][0]["metrics"][0]["expression"]["dialects"][0]["expression"] + expr = ossie["metrics"][0]["expression"]["dialects"][0]["expression"] assert expr == r"SUM(a\1b.amount)" @@ -236,7 +236,7 @@ def test_boollike_string_values_stay_strings_for_a_yaml_1_1_reader(): mv = ("version: '1.1'\nsource: c.s.t\n" "dimensions:\n- {name: status, expr: status, synonyms: [on, off]}\n") ossie_out = importer.convert_metric_view_to_ossie(mv) - field = yaml.safe_load(ossie_out)["semantic_model"][0]["datasets"][0]["fields"][0] + field = yaml.safe_load(ossie_out)["datasets"][0]["fields"][0] assert field["ai_context"]["synonyms"] == ["on", "off"] @@ -253,7 +253,7 @@ def test_fact_qualifier_variants_in_on_decompose(): "customer.c_custkey = o_custkey", # reversed operand order, bare fact ): rel = parse(importer.convert_metric_view_to_ossie( - base.format(cond=cond)))["semantic_model"][0]["relationships"][0] + base.format(cond=cond)))["relationships"][0] assert rel["from"] == "orders" and rel["to"] == "customer" assert rel["from_columns"] == ["o_custkey"] assert rel["to_columns"] == ["c_custkey"] @@ -264,7 +264,7 @@ def test_multi_column_on_with_bare_and_tablename_fact(): mv = ("version: '1.1'\nsource: c.s.orders\n" "joins:\n- name: customer\n source: c.s.customer\n" " on: o_a = customer.c_a AND orders.o_b = customer.c_b\n") - rel = parse(importer.convert_metric_view_to_ossie(mv))["semantic_model"][0]["relationships"][0] + rel = parse(importer.convert_metric_view_to_ossie(mv))["relationships"][0] assert rel["from_columns"] == ["o_a", "o_b"] assert rel["to_columns"] == ["c_a", "c_b"] @@ -300,7 +300,7 @@ def test_with_paren_subquery_source_accepted(): not mistaken for a (non-3-part) identifier (review finding).""" mv = "version: '1.1'\nsource: WITH(t AS (SELECT 1 AS a)) SELECT a FROM t\n" ossie = parse(importer.convert_metric_view_to_ossie(mv)) - assert ossie["semantic_model"][0]["datasets"][0]["source"].startswith("WITH(") + assert ossie["datasets"][0]["source"].startswith("WITH(") def test_nested_join_bare_column_rejected(): diff --git a/converters/databricks/tests/test_ossie_to_metric_view.py b/converters/databricks/tests/test_ossie_to_metric_view.py index 8b8808b2..77fa8988 100644 --- a/converters/databricks/tests/test_ossie_to_metric_view.py +++ b/converters/databricks/tests/test_ossie_to_metric_view.py @@ -40,7 +40,7 @@ def test_tpcds_export_matches_expected(): def test_unsupported_version_rejected(): - ossie = "version: '9.9.9'\nsemantic_model:\n - name: m\n datasets:\n - {name: d, source: c.s.t}\n" + ossie = "version: '9.9.9'\nname: m\ndatasets:\n - {name: d, source: c.s.t}\n" with pytest.raises(ConversionError): exporter.convert_ossie_to_metric_view(ossie) @@ -48,17 +48,13 @@ def test_unsupported_version_rejected(): def _model(rels): return { "version": exporter.OSSIE_VERSION, - "semantic_model": [ - { - "name": "m", - "datasets": [ - {"name": "a", "source": "c.s.a"}, - {"name": "b", "source": "c.s.b"}, - {"name": "x", "source": "c.s.x"}, - ], - "relationships": rels, - } + "name": "m", + "datasets": [ + {"name": "a", "source": "c.s.a"}, + {"name": "b", "source": "c.s.b"}, + {"name": "x", "source": "c.s.x"}, ], + "relationships": rels, } @@ -92,8 +88,7 @@ def test_mto_diamond_fans_out(): """A shared dimension reached by two parents (orders->customers->regions and orders->suppliers->regions) is fanned out into two aliased joins, not rejected.""" import yaml - ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ - "name": "m", + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "name": "m", "datasets": [ {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amount")]}, {"name": "customers", "source": "c.s.customers"}, @@ -105,8 +100,7 @@ def test_mto_diamond_fans_out(): {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, - ], - }]}) + ]}) out = parse(exporter.convert_ossie_to_metric_view(ossie)) region_joins = [j for top in out["joins"] for j in top.get("joins", []) if j["source"] == "c.s.regions"] assert {j["name"] for j in region_joins} == {"customers_regions", "suppliers_regions"} @@ -120,8 +114,7 @@ def test_otm_diamond_fans_out(): """customers (fact) -> past_orders/future_orders -> line_items: the shared line_items is fanned out, and every join is one_to_many.""" import yaml - ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ - "name": "m", + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "name": "m", "datasets": [ {"name": "customers", "source": "c.s.customers"}, {"name": "past_orders", "source": "c.s.past_orders"}, @@ -134,8 +127,7 @@ def test_otm_diamond_fans_out(): {"name": "r3", "from": "line_items", "to": "past_orders", "from_columns": ["oid"], "to_columns": ["id"]}, {"name": "r4", "from": "line_items", "to": "future_orders", "from_columns": ["oid"], "to_columns": ["id"]}, ], - "metrics": [{"name": "cnt", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], - }]}) + "metrics": [{"name": "cnt", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}]}) out = parse(exporter.convert_ossie_to_metric_view(ossie, source="customers")) leaf_names = {j["name"] for top in out["joins"] for j in top.get("joins", [])} assert leaf_names == {"past_orders_line_items", "future_orders_line_items"} @@ -172,14 +164,12 @@ def _single_fact_model(metric_expr): import yaml return yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "m", - "datasets": [{"name": "orders", "source": "c.s.orders", - "fields": [{"name": "k", "expression": {"dialects": [ - {"dialect": "DATABRICKS", "expression": "k"}]}}]}], - "metrics": [{"name": "rev", "expression": {"dialects": [ - {"dialect": "DATABRICKS", "expression": metric_expr}]}}], - }], + "name": "m", + "datasets": [{"name": "orders", "source": "c.s.orders", + "fields": [{"name": "k", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "k"}]}}]}], + "metrics": [{"name": "rev", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": metric_expr}]}}], }) @@ -199,7 +189,7 @@ def test_invalid_source_rejected(): import yaml ossie = yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{"name": "m", "datasets": [{"name": "d", "source": "justatable"}]}], + "name": "m", "datasets": [{"name": "d", "source": "justatable"}], }) with pytest.raises(ConversionError, match="source"): exporter.convert_ossie_to_metric_view(ossie) @@ -212,17 +202,15 @@ def test_duplicate_dimension_name_rejected(): import yaml ossie = yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "m", - "datasets": [ - {"name": "orders", "source": "c.s.orders", - "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, - {"name": "customer", "source": "c.s.customer", - "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, - ], - "relationships": [{"name": "r", "from": "orders", "to": "customer", - "from_columns": ["cid"], "to_columns": ["id"]}], - }], + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", + "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, + {"name": "customer", "source": "c.s.customer", + "fields": [{"name": "id", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}]}, + ], + "relationships": [{"name": "r", "from": "orders", "to": "customer", + "from_columns": ["cid"], "to_columns": ["id"]}], }) with pytest.raises(ConversionError, match="collides"): exporter.convert_ossie_to_metric_view(ossie) @@ -234,16 +222,14 @@ def test_measure_name_collides_with_dimension_rejected(): import yaml ossie = yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "m", - "datasets": [ - {"name": "orders", "source": "c.s.orders", - "fields": [{"name": "total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "total"}]}}]}, - ], - "metrics": [ - {"name": "Total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "SUM(total)"}]}}, - ], - }], + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", + "fields": [{"name": "total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "total"}]}}]}, + ], + "metrics": [ + {"name": "Total", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "SUM(total)"}]}}, + ], }) with pytest.raises(ConversionError, match="collides"): exporter.convert_ossie_to_metric_view(ossie) @@ -255,16 +241,14 @@ def test_cascade_drop_downstream_measure_reference(): import yaml ossie = yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "m", - "datasets": [{"name": "f", "source": "c.s.f"}], - "metrics": [ - {"name": "base", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "SUM(x)"}]}}, # dropped (no DBX/ANSI) - {"name": "derived", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(base) * 2"}]}}, - {"name": "derived2", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(derived) + 1"}]}}, # transitive - {"name": "ok", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}, - ], - }], + "name": "m", + "datasets": [{"name": "f", "source": "c.s.f"}], + "metrics": [ + {"name": "base", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "SUM(x)"}]}}, # dropped (no DBX/ANSI) + {"name": "derived", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(base) * 2"}]}}, + {"name": "derived2", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "measure(derived) + 1"}]}}, # transitive + {"name": "ok", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}, + ], }) out = parse(exporter.convert_ossie_to_metric_view(ossie)) names = [m["name"] for m in out.get("measures", [])] @@ -276,14 +260,12 @@ def test_cascade_drop_downstream_dimension_reference(): import yaml ossie = yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "m", - "datasets": [{"name": "f", "source": "c.s.f", "fields": [ - {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, # dropped dim - {"name": "label", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "upper(region)"}]}}, # references region - {"name": "keep", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}, - ]}], - }], + "name": "m", + "datasets": [{"name": "f", "source": "c.s.f", "fields": [ + {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, # dropped dim + {"name": "label", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "upper(region)"}]}}, # references region + {"name": "keep", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "id"}]}}, + ]}], }) out = parse(exporter.convert_ossie_to_metric_view(ossie)) dims = [d["name"] for d in out.get("dimensions", [])] @@ -298,18 +280,16 @@ def test_orientation_unverifiable_when_to_side_has_no_key_warns(): import yaml ossie = yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "m", - "datasets": [ - {"name": "a", "source": "c.s.a", "primary_key": ["a_id"], "fields": [ - {"name": "a_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "a_name"}]}}]}, - {"name": "b", "source": "c.s.b", "fields": [ - {"name": "b_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "b_name"}]}}]}, - ], - # from columns cover a's PK, but b (the `to` side) declares no key - "relationships": [{"name": "a_to_b", "from": "a", "to": "b", - "from_columns": ["a_id"], "to_columns": ["b_x"]}], - }], + "name": "m", + "datasets": [ + {"name": "a", "source": "c.s.a", "primary_key": ["a_id"], "fields": [ + {"name": "a_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "a_name"}]}}]}, + {"name": "b", "source": "c.s.b", "fields": [ + {"name": "b_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "b_name"}]}}]}, + ], + # from columns cover a's PK, but b (the `to` side) declares no key + "relationships": [{"name": "a_to_b", "from": "a", "to": "b", + "from_columns": ["a_id"], "to_columns": ["b_x"]}], }) with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") @@ -325,22 +305,20 @@ def test_cascade_drop_skips_qualified_join_alias_collision(): import yaml ossie = yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "m", - "datasets": [ - {"name": "orders", "source": "c.s.orders", "fields": [ - # dropped (no DBX/ANSI dialect); its name collides with the `region` join - {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, - # references the join alias `region`, not the dropped field -> must survive - {"name": "summary", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "region.r_name"}]}}, - ]}, - {"name": "region", "source": "c.s.region", "primary_key": ["r_key"], "fields": [ - {"name": "r_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "r_name"}]}}, - ]}, - ], - "relationships": [{"name": "orr", "from": "orders", "to": "region", - "from_columns": ["o_rkey"], "to_columns": ["r_key"]}], - }], + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "fields": [ + # dropped (no DBX/ANSI dialect); its name collides with the `region` join + {"name": "region", "expression": {"dialects": [{"dialect": "SNOWFLAKE", "expression": "r"}]}}, + # references the join alias `region`, not the dropped field -> must survive + {"name": "summary", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "region.r_name"}]}}, + ]}, + {"name": "region", "source": "c.s.region", "primary_key": ["r_key"], "fields": [ + {"name": "r_name", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "r_name"}]}}, + ]}, + ], + "relationships": [{"name": "orr", "from": "orders", "to": "region", + "from_columns": ["o_rkey"], "to_columns": ["r_key"]}], }) out = parse(exporter.convert_ossie_to_metric_view(ossie)) dims = [d["name"] for d in out.get("dimensions", [])] @@ -355,21 +333,19 @@ def _orders_lineitems_ossie(): import yaml return yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "sales", - "datasets": [ - {"name": "orders", "source": "c.s.orders", "primary_key": ["order_id"], - "fields": [{"name": "order_date", "expression": {"dialects": [ - {"dialect": "DATABRICKS", "expression": "o_order_date"}]}}]}, - {"name": "line_items", "source": "c.s.line_items", - "fields": [{"name": "product_sk", "expression": {"dialects": [ - {"dialect": "DATABRICKS", "expression": "l_product_sk"}]}}]}, - ], - "relationships": [{"name": "li_to_order", "from": "line_items", "to": "orders", - "from_columns": ["l_order_id"], "to_columns": ["order_id"]}], - "metrics": [{"name": "order_count", "expression": {"dialects": [ - {"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], - }], + "name": "sales", + "datasets": [ + {"name": "orders", "source": "c.s.orders", "primary_key": ["order_id"], + "fields": [{"name": "order_date", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "o_order_date"}]}}]}, + {"name": "line_items", "source": "c.s.line_items", + "fields": [{"name": "product_sk", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "l_product_sk"}]}}]}, + ], + "relationships": [{"name": "li_to_order", "from": "line_items", "to": "orders", + "from_columns": ["l_order_id"], "to_columns": ["order_id"]}], + "metrics": [{"name": "order_count", "expression": {"dialects": [ + {"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], }) @@ -406,20 +382,18 @@ def test_one_to_many_subtree_must_stay_one_to_many(): import yaml ossie = yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "m", - "datasets": [ - {"name": "orders", "source": "c.s.orders"}, - {"name": "line_items", "source": "c.s.line_items"}, - {"name": "product", "source": "c.s.product"}, - ], - "relationships": [ - {"name": "li_to_order", "from": "line_items", "to": "orders", # orders->li : OTM - "from_columns": ["l_order_id"], "to_columns": ["order_id"]}, - {"name": "li_to_product", "from": "line_items", "to": "product", # li->product : MTO - "from_columns": ["l_product_sk"], "to_columns": ["p_sk"]}, - ], - }], + "name": "m", + "datasets": [ + {"name": "orders", "source": "c.s.orders"}, + {"name": "line_items", "source": "c.s.line_items"}, + {"name": "product", "source": "c.s.product"}, + ], + "relationships": [ + {"name": "li_to_order", "from": "line_items", "to": "orders", # orders->li : OTM + "from_columns": ["l_order_id"], "to_columns": ["order_id"]}, + {"name": "li_to_product", "from": "line_items", "to": "product", # li->product : MTO + "from_columns": ["l_product_sk"], "to_columns": ["p_sk"]}, + ], }) with pytest.raises(ConversionError, match="one-to-many"): exporter.convert_ossie_to_metric_view(ossie, source="orders") @@ -433,12 +407,10 @@ def test_primary_key_deduces_at_most_one_match(): def model(dim_extra): dim = {"name": "customer", "source": "c.s.customer"} dim.update(dim_extra) - return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ - "name": "m", + return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "name": "m", "datasets": [{"name": "orders", "source": "c.s.orders"}, dim], "relationships": [{"name": "r", "from": "orders", "to": "customer", - "from_columns": ["cid"], "to_columns": ["id"]}], - }]}) + "from_columns": ["cid"], "to_columns": ["id"]}]}) join = parse(exporter.convert_ossie_to_metric_view(model({"primary_key": ["id"]})))["joins"][0] assert join.get("rely") == {"at_most_one_match": True} @@ -454,8 +426,7 @@ def test_mislabeled_from_to_reoriented_by_key(): import yaml def model(frm, to, from_cols, to_cols): - return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ - "name": "m", + return yaml.safe_dump({"version": exporter.OSSIE_VERSION, "name": "m", "datasets": [ {"name": "orders", "source": "c.s.orders", "primary_key": ["order_id"], "fields": [_field("amt", "amount")]}, @@ -463,8 +434,7 @@ def model(frm, to, from_cols, to_cols): "fields": [_field("cname", "c_name")]}, ], "relationships": [{"name": "r", "from": frm, "to": to, - "from_columns": from_cols, "to_columns": to_cols}], - }]}) + "from_columns": from_cols, "to_columns": to_cols}]}) well = parse(exporter.convert_ossie_to_metric_view( model("orders", "customer", ["cust_id"], ["c_id"]))) @@ -481,15 +451,13 @@ def test_dataset_named_source_is_renamed(): """A dataset literally named `source` must not collide with the fact's reserved `source` alias (would otherwise emit an ambiguous join).""" import yaml - ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ - "name": "m", + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "name": "m", "datasets": [ {"name": "orders", "source": "c.s.orders"}, {"name": "source", "source": "c.s.dim", "fields": [_field("x", "xcol")]}, ], "relationships": [{"name": "r", "from": "orders", "to": "source", - "from_columns": ["sid"], "to_columns": ["id"]}], - }]}) + "from_columns": ["sid"], "to_columns": ["id"]}]}) out = parse(exporter.convert_ossie_to_metric_view(ossie)) join = out["joins"][0] assert join["name"] != "source" @@ -501,8 +469,7 @@ def test_fanout_alias_collision_deduped(): """A real dataset whose name equals a synthesized fan-out alias still gets a distinct alias -- no two joins share a name.""" import yaml - ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ - "name": "m", + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "name": "m", "datasets": [ {"name": "orders", "source": "c.s.orders"}, {"name": "customers", "source": "c.s.customers"}, @@ -516,8 +483,7 @@ def test_fanout_alias_collision_deduped(): {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, {"name": "r5", "from": "orders", "to": "customers_regions", "from_columns": ["xid"], "to_columns": ["id"]}, - ], - }]}) + ]}) out = parse(exporter.convert_ossie_to_metric_view(ossie)) names = [] @@ -534,7 +500,7 @@ def test_malformed_input_raises_conversion_error(): """Missing required keys surface as ConversionError, not a raw KeyError traceback.""" import yaml bad = yaml.safe_dump({"version": exporter.OSSIE_VERSION, - "semantic_model": [{"name": "m", "datasets": [{"source": "c.s.t"}]}]}) + "name": "m", "datasets": [{"source": "c.s.t"}]}) with pytest.raises(ConversionError, match="missing required 'name'"): exporter.convert_ossie_to_metric_view(bad) @@ -543,8 +509,7 @@ def test_nameless_relationship_with_ai_context_does_not_crash(): """A relationship may omit `name`; the dropped-ai_context warning must not raise a raw KeyError when it has ai_context but no name.""" import yaml - ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ - "name": "m", + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "name": "m", "datasets": [ {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amt")]}, {"name": "customers", "source": "c.s.customers"}, @@ -552,8 +517,7 @@ def test_nameless_relationship_with_ai_context_does_not_crash(): "relationships": [ {"from": "orders", "to": "customers", "from_columns": ["cid"], "to_columns": ["id"], "ai_context": "joins orders to customers"}, - ], - }]}) + ]}) out = parse(exporter.convert_ossie_to_metric_view(ossie)) # must not raise assert out["joins"][0]["name"] == "customers" @@ -567,8 +531,7 @@ def test_fanout_complex_expr_dropped_not_emitted_ambiguously(): dimension per instance, but a complex expression -- which cannot be attributed to a single instance -- is dropped rather than emitted ambiguously.""" import yaml - ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ - "name": "m", + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "name": "m", "datasets": [ {"name": "orders", "source": "c.s.orders"}, {"name": "customers", "source": "c.s.customers"}, @@ -581,8 +544,7 @@ def test_fanout_complex_expr_dropped_not_emitted_ambiguously(): {"name": "r2", "from": "orders", "to": "suppliers", "from_columns": ["sid"], "to_columns": ["id"]}, {"name": "r3", "from": "customers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, {"name": "r4", "from": "suppliers", "to": "regions", "from_columns": ["rid"], "to_columns": ["id"]}, - ], - }]}) + ]}) dims = parse(exporter.convert_ossie_to_metric_view(ossie)).get("dimensions", []) # the simple column fans out into two unambiguous, alias-qualified dimensions ... assert sum(1 for d in dims if d["name"].endswith("_r_name")) == 2 @@ -602,8 +564,7 @@ def test_nested_join_uses_full_path_qualification(): Databricks nested-join rule -- not the single-level `nation.n_name`. A depth-1 join stays single-name.""" import yaml - ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "semantic_model": [{ - "name": "m", + ossie = yaml.safe_dump({"version": exporter.OSSIE_VERSION, "name": "m", "datasets": [ {"name": "orders", "source": "c.s.orders", "fields": [_field("amt", "amount")]}, {"name": "customer", "source": "c.s.customer", "fields": [_field("cname", "c_name")]}, @@ -612,8 +573,7 @@ def test_nested_join_uses_full_path_qualification(): "relationships": [ {"name": "r1", "from": "orders", "to": "customer", "from_columns": ["ckey"], "to_columns": ["c_key"]}, {"name": "r2", "from": "customer", "to": "nation", "from_columns": ["nkey"], "to_columns": ["n_key"]}, - ], - }]}) + ]}) out = parse(exporter.convert_ossie_to_metric_view(ossie)) exprs = {d["name"]: d["expr"] for d in out["dimensions"]} assert exprs["cname"] == "customer.c_name" # depth-1: the join's own name @@ -626,8 +586,7 @@ def test_nested_join_uses_full_path_qualification(): def test_case_variant_dataset_name_rejected(): """DBR identifiers are case-insensitive, so two datasets differing only in case (`customer`/`Customer`) collide and are rejected (review finding).""" - ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" - " - {name: customer, source: c.s.c}\n - {name: Customer, source: c.s.c2}\n") + ossie = ('version: 0.2.0.dev0\nname: m\ndatasets:\n- {name: customer, source: c.s.c}\n- {name: Customer, source: c.s.c2}\n') with pytest.raises(ConversionError, match="duplicate"): exporter.convert_ossie_to_metric_view(ossie) @@ -635,9 +594,7 @@ def test_case_variant_dataset_name_rejected(): def test_non_string_field_expression_raises_clean_error(): """A non-string dialect expression raises a ConversionError, not a raw crash (review finding).""" - ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" - " - name: o\n source: c.s.o\n fields:\n - name: d\n expression:\n" - " dialects:\n - {dialect: DATABRICKS, expression: 123}\n") + ossie = ('version: 0.2.0.dev0\nname: m\ndatasets:\n- name: o\n source: c.s.o\n fields:\n - name: d\n expression:\n dialects:\n - {dialect: DATABRICKS, expression: 123}\n') with pytest.raises(ConversionError, match="must be a string"): exporter.convert_ossie_to_metric_view(ossie) @@ -645,9 +602,7 @@ def test_non_string_field_expression_raises_clean_error(): def test_scalar_join_columns_rejected(): """`from_columns`/`to_columns` given as a scalar string (not a list) raise a clear 'must be lists' error rather than a misleading character-count length error.""" - ossie = ("version: 0.2.0.dev0\nsemantic_model:\n- name: m\n datasets:\n" - " - {name: a, source: c.s.a}\n - {name: b, source: c.s.b}\n relationships:\n" - " - {name: ab, from: a, to: b, from_columns: cid, to_columns: id}\n") + ossie = ('version: 0.2.0.dev0\nname: m\ndatasets:\n- {name: a, source: c.s.a}\n- {name: b, source: c.s.b}\nrelationships:\n- {name: ab, from: a, to: b, from_columns: cid, to_columns: id}\n') with pytest.raises(ConversionError, match="must be lists"): exporter.convert_ossie_to_metric_view(ossie) @@ -658,13 +613,49 @@ def test_malformed_stash_json_raises_conversion_error(): import yaml ossie = yaml.safe_dump({ "version": exporter.OSSIE_VERSION, - "semantic_model": [{ - "name": "m", - "custom_extensions": [{"vendor_name": "DATABRICKS", "data": "{not valid json"}], - "datasets": [{"name": "f", "source": "c.s.f", - "fields": [{"name": "x", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "x"}]}}]}], - "metrics": [{"name": "n", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], - }], + "name": "m", + "custom_extensions": [{"vendor_name": "DATABRICKS", "data": "{not valid json"}], + "datasets": [{"name": "f", "source": "c.s.f", + "fields": [{"name": "x", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "x"}]}}]}], + "metrics": [{"name": "n", "expression": {"dialects": [{"dialect": "DATABRICKS", "expression": "COUNT(*)"}]}}], }) with pytest.raises(ConversionError, match="not valid JSON"): exporter.convert_ossie_to_metric_view(ossie) + + +@pytest.mark.parametrize( + "wrapper", + [[], [{"name": "first"}], [{"name": "first"}, {"name": "second"}], {"name": "first"}, None], +) +def test_legacy_model_wrappers_are_rejected(wrapper): + import yaml + + document = {"version": exporter.OSSIE_VERSION, "semantic_model": wrapper} + with pytest.raises(ConversionError, match="Legacy 'semantic_model'"): + exporter.convert_ossie_to_metric_view(yaml.safe_dump(document)) + + +@pytest.mark.parametrize("property_name", ["dialects", "vendors"]) +@pytest.mark.parametrize("value", [None, [], ["legacy"]]) +def test_removed_root_metadata_is_rejected(property_name, value): + import yaml + + document = parse(load_fixture("fixtureA_ossie.yaml")) + document[property_name] = value + with pytest.raises(ConversionError, match="Root dialects and vendors"): + exporter.convert_ossie_to_metric_view(yaml.safe_dump(document)) + + +@pytest.mark.parametrize( + "name_properties", + [{}, {"name": None}, {"name": 123}, {"name": True}, {"name": []}, {"name": {}}], + ids=["missing", "null", "number", "boolean", "list", "object"], +) +def test_root_name_must_be_a_string(name_properties): + import yaml + + document = parse(load_fixture("fixtureA_ossie.yaml")) + del document["name"] + document.update(name_properties) + with pytest.raises(ConversionError, match="string 'name' at the document root"): + exporter.convert_ossie_to_metric_view(yaml.safe_dump(document)) diff --git a/converters/dbt/README.md b/converters/dbt/README.md index 4efc6a7b..06385b84 100644 --- a/converters/dbt/README.md +++ b/converters/dbt/README.md @@ -66,6 +66,12 @@ By default the Ossie semantic model is named `semantic_model`. Override it with ossie-dbt msi-to-ossie -i target/semantic_manifest.json -o semantic_model.yaml --model-name my_project ``` +The Ossie output contains one model at the document root (`name`, `datasets`, +`relationships`, and `metrics`, alongside document metadata). Each dbt semantic +model becomes an Ossie dataset; dbt manifests can still contain multiple +`semantic_models`. Ossie input must use the flat root format without a +`semantic_model` wrapper. + Conversion issues (e.g. dropped CONVERSION or PRIVATE metrics) are printed as warnings to stderr. The output file is still written. ### Apache Ossie → dbt diff --git a/converters/dbt/src/ossie_dbt/msi_to_ossie.py b/converters/dbt/src/ossie_dbt/msi_to_ossie.py index 027e20ba..638be16f 100644 --- a/converters/dbt/src/ossie_dbt/msi_to_ossie.py +++ b/converters/dbt/src/ossie_dbt/msi_to_ossie.py @@ -31,7 +31,6 @@ OssieField, OssieMetric, OssieRelationship, - OssieSemanticModel, ) from ossie_dbt.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult from ossie_dbt.filter_utils import _collect_filter_sql, _merge_filter_sqls @@ -122,15 +121,10 @@ def convert( return ConverterResult( output=OssieDocument( version="0.2.0.dev0", - dialects=[self._dialect], - semantic_model=[ - OssieSemanticModel( - name=ossie_model_name, - datasets=datasets, - relationships=relationships if relationships else None, - metrics=ossie_metrics if ossie_metrics else None, - ) - ], + name=ossie_model_name, + datasets=datasets, + relationships=relationships if relationships else None, + metrics=ossie_metrics if ossie_metrics else None, ), issues=issues, ) diff --git a/converters/dbt/src/ossie_dbt/ossie_to_msi.py b/converters/dbt/src/ossie_dbt/ossie_to_msi.py index 0bd10b61..4374d61e 100644 --- a/converters/dbt/src/ossie_dbt/ossie_to_msi.py +++ b/converters/dbt/src/ossie_dbt/ossie_to_msi.py @@ -101,10 +101,9 @@ def convert(self, document: OssieDocument) -> ConverterResult[PydanticSemanticMa semantic_models: List[PydanticSemanticModel] = [] metrics: List[PydanticMetric] = [] - for ossie_sm in document.semantic_model: - for dataset in ossie_sm.datasets: - semantic_models.append(self._convert_dataset(dataset, ossie_sm)) - metrics.extend(self._convert_metrics(ossie_sm)) + for dataset in document.datasets: + semantic_models.append(self._convert_dataset(dataset, document)) + metrics.extend(self._convert_metrics(document)) return ConverterResult( output=PydanticSemanticManifest( diff --git a/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr b/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr index facd135e..88d1f9b8 100644 --- a/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr +++ b/converters/dbt/tests/__snapshots__/test_msi_to_ossie.ambr @@ -17,184 +17,172 @@ # under the License. # name: TestMetricConversion.test_derived_metric_nested ''' - version: 0.2.0.dev0 - dialects: - - ANSI_SQL - semantic_model: - - name: semantic_model - datasets: - - name: orders - source: schema.table - fields: - - name: revenue - expression: - dialects: - - dialect: ANSI_SQL - expression: amount - - name: cost - expression: - dialects: - - dialect: ANSI_SQL - expression: cost_amount - - name: expenses - expression: - dialects: - - dialect: ANSI_SQL - expression: expense_amount - metrics: + name: semantic_model + datasets: + - name: orders + source: schema.table + fields: - name: revenue expression: dialects: - dialect: ANSI_SQL - expression: SUM(orders.amount) + expression: amount - name: cost expression: dialects: - dialect: ANSI_SQL - expression: SUM(orders.cost_amount) + expression: cost_amount - name: expenses expression: dialects: - dialect: ANSI_SQL - expression: SUM(orders.expense_amount) - - name: gross_profit - expression: - dialects: - - dialect: ANSI_SQL - expression: SUM(orders.amount) - SUM(orders.cost_amount) - - name: net_profit - expression: - dialects: - - dialect: ANSI_SQL - expression: (SUM(orders.amount) - SUM(orders.cost_amount)) - SUM(orders.expense_amount) + expression: expense_amount + metrics: + - name: revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + - name: cost + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.cost_amount) + - name: expenses + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.expense_amount) + - name: gross_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) - SUM(orders.cost_amount) + - name: net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: (SUM(orders.amount) - SUM(orders.cost_amount)) - SUM(orders.expense_amount) + version: 0.2.0.dev0 ''' # --- # name: TestMetricConversion.test_ratio_metric_inlines_sub_expressions ''' - version: 0.2.0.dev0 - dialects: - - ANSI_SQL - semantic_model: - - name: semantic_model - datasets: - - name: orders - source: schema.table - fields: - - name: revenue - expression: - dialects: - - dialect: ANSI_SQL - expression: amount - - name: order_count - expression: - dialects: - - dialect: ANSI_SQL - expression: CASE WHEN order_id IS NOT NULL THEN 1 ELSE 0 END - metrics: + name: semantic_model + datasets: + - name: orders + source: schema.table + fields: - name: revenue expression: dialects: - dialect: ANSI_SQL - expression: SUM(orders.amount) + expression: amount - name: order_count expression: dialects: - dialect: ANSI_SQL - expression: SUM(CASE WHEN orders.order_id IS NOT NULL THEN 1 ELSE 0 END) - - name: arpu - expression: - dialects: - - dialect: ANSI_SQL - expression: (SUM(orders.amount)) / (SUM(CASE WHEN orders.order_id IS NOT NULL - THEN 1 ELSE 0 END)) + expression: CASE WHEN order_id IS NOT NULL THEN 1 ELSE 0 END + metrics: + - name: revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + - name: order_count + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(CASE WHEN orders.order_id IS NOT NULL THEN 1 ELSE 0 END) + - name: arpu + expression: + dialects: + - dialect: ANSI_SQL + expression: (SUM(orders.amount)) / (SUM(CASE WHEN orders.order_id IS NOT NULL + THEN 1 ELSE 0 END)) + version: 0.2.0.dev0 ''' # --- # name: TestMetricFilterFlattening.test_metric_and_measure_filters_combined_with_and ''' - version: 0.2.0.dev0 - dialects: - - ANSI_SQL - semantic_model: - - name: semantic_model - datasets: - - name: orders - source: schema.table - fields: - - name: revenue - expression: - dialects: - - dialect: ANSI_SQL - expression: amount - metrics: - - name: paid_intl_revenue + name: semantic_model + datasets: + - name: orders + source: schema.table + fields: + - name: revenue expression: dialects: - dialect: ANSI_SQL - expression: SUM(CASE WHEN (status = 'paid') AND (region = 'intl') THEN orders.amount - END) + expression: amount + metrics: + - name: paid_intl_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(CASE WHEN (status = 'paid') AND (region = 'intl') THEN orders.amount + END) + version: 0.2.0.dev0 ''' # --- # name: TestRelationshipConversion.test_three_datasets_produce_all_pairs ''' + name: semantic_model + datasets: + - name: users_a + source: schema.table + primary_key: + - user_id + fields: + - name: user + expression: + dialects: + - dialect: ANSI_SQL + expression: user_id + - name: users_b + source: schema.table + unique_keys: + - - user_id + fields: + - name: user + expression: + dialects: + - dialect: ANSI_SQL + expression: user_id + - name: orders + source: schema.table + fields: + - name: user + expression: + dialects: + - dialect: ANSI_SQL + expression: user_id + relationships: + - name: users_a__users_b__user + from: users_a + to: users_b + from_columns: + - user_id + to_columns: + - user_id + - name: orders__users_a__user + from: orders + to: users_a + from_columns: + - user_id + to_columns: + - user_id + - name: orders__users_b__user + from: orders + to: users_b + from_columns: + - user_id + to_columns: + - user_id version: 0.2.0.dev0 - dialects: - - ANSI_SQL - semantic_model: - - name: semantic_model - datasets: - - name: users_a - source: schema.table - primary_key: - - user_id - fields: - - name: user - expression: - dialects: - - dialect: ANSI_SQL - expression: user_id - - name: users_b - source: schema.table - unique_keys: - - - user_id - fields: - - name: user - expression: - dialects: - - dialect: ANSI_SQL - expression: user_id - - name: orders - source: schema.table - fields: - - name: user - expression: - dialects: - - dialect: ANSI_SQL - expression: user_id - relationships: - - name: users_a__users_b__user - from: users_a - to: users_b - from_columns: - - user_id - to_columns: - - user_id - - name: orders__users_a__user - from: orders - to: users_a - from_columns: - - user_id - to_columns: - - user_id - - name: orders__users_b__user - from: orders - to: users_b - from_columns: - - user_id - to_columns: - - user_id ''' # --- diff --git a/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr b/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr index 211037fa..7b89427a 100644 --- a/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr +++ b/converters/dbt/tests/__snapshots__/test_ossie_to_msi.ambr @@ -17,49 +17,46 @@ # under the License. # name: TestOssieToMSIRoundTrip.test_ossie_to_msi_to_ossie_preserves_structure ''' - version: 0.2.0.dev0 - dialects: - - ANSI_SQL - semantic_model: - - name: semantic_model - datasets: - - name: orders - source: analytics.orders - primary_key: - - order_id - fields: - - name: order_id - expression: - dialects: - - dialect: ANSI_SQL - expression: order_id - - name: status - expression: - dialects: - - dialect: ANSI_SQL - expression: status - dimension: - is_time: false - - name: created_at - expression: - dialects: - - dialect: ANSI_SQL - expression: created_at - dimension: - is_time: true - - name: amount - expression: - dialects: - - dialect: ANSI_SQL - expression: amount - dimension: - is_time: false - metrics: - - name: revenue + name: semantic_model + datasets: + - name: orders + source: analytics.orders + primary_key: + - order_id + fields: + - name: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id + - name: status expression: dialects: - dialect: ANSI_SQL - expression: SUM(orders.amount) + expression: status + dimension: + is_time: false + - name: created_at + expression: + dialects: + - dialect: ANSI_SQL + expression: created_at + dimension: + is_time: true + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + dimension: + is_time: false + metrics: + - name: revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + version: 0.2.0.dev0 ''' # --- diff --git a/converters/dbt/tests/helpers.py b/converters/dbt/tests/helpers.py index 644d53ba..85ce2964 100644 --- a/converters/dbt/tests/helpers.py +++ b/converters/dbt/tests/helpers.py @@ -27,7 +27,6 @@ OssieField, OssieMetric, OssieRelationship, - OssieSemanticModel, ) from metricflow_semantic_interfaces.implementations.elements.dimension import ( PydanticDimension, @@ -218,12 +217,8 @@ def _ossie_doc( model_name: str = "test", ) -> OssieDocument: return OssieDocument( - semantic_model=[ - OssieSemanticModel( - name=model_name, - datasets=datasets or [], - metrics=metrics if metrics else None, - relationships=relationships if relationships else None, - ) - ] + name=model_name, + datasets=datasets or [], + metrics=metrics if metrics else None, + relationships=relationships if relationships else None, ) diff --git a/converters/dbt/tests/test_msi_to_ossie.py b/converters/dbt/tests/test_msi_to_ossie.py index 1734d17c..7ed0aa89 100644 --- a/converters/dbt/tests/test_msi_to_ossie.py +++ b/converters/dbt/tests/test_msi_to_ossie.py @@ -64,7 +64,7 @@ def _fields(result: OssieDocument, dataset_idx: int = 0) -> list: """Return fields for a dataset, asserting they exist.""" - fields = result.semantic_model[0].datasets[dataset_idx].fields + fields = result.datasets[dataset_idx].fields assert fields is not None return fields @@ -76,7 +76,7 @@ def _field_expr(result: OssieDocument, field_idx: int = 0) -> str: def _ossie_metrics(result: OssieDocument) -> list: """Return Ossie metrics for the first semantic model, asserting they exist.""" - metrics = result.semantic_model[0].metrics + metrics = result.metrics assert metrics is not None return metrics @@ -91,11 +91,10 @@ def test_empty_manifest_produces_empty_datasets(self) -> None: result = MSIToOssieConverter().convert(_manifest(), ossie_model_name="test").output assert result.version == "0.2.0.dev0" - assert len(result.semantic_model) == 1 - assert result.semantic_model[0].name == "test" - assert result.semantic_model[0].datasets == [] - assert result.semantic_model[0].metrics is None - assert result.semantic_model[0].relationships is None + assert result.name == "test" + assert result.datasets == [] + assert result.metrics is None + assert result.relationships is None def test_semantic_model_becomes_dataset(self) -> None: sm = semantic_model_with_guaranteed_meta( @@ -105,7 +104,7 @@ def test_semantic_model_becomes_dataset(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output - dataset = result.semantic_model[0].datasets[0] + dataset = result.datasets[0] assert dataset.name == "orders" assert dataset.source == "analytics.orders_table" assert dataset.description == "Order data" @@ -117,14 +116,14 @@ def test_source_includes_database_when_present(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output - assert result.semantic_model[0].datasets[0].source == "prod.analytics.orders_table" + assert result.datasets[0].source == "prod.analytics.orders_table" def test_multiple_semantic_models_become_multiple_datasets(self) -> None: sm_a = semantic_model_with_guaranteed_meta(name="orders") sm_b = semantic_model_with_guaranteed_meta(name="users") result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm_a, sm_b])).output - names = [ds.name for ds in result.semantic_model[0].datasets] + names = [ds.name for ds in result.datasets] assert names == ["orders", "users"] @@ -274,7 +273,7 @@ def test_key_extraction( ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output - dataset = result.semantic_model[0].datasets[0] + dataset = result.datasets[0] assert dataset.primary_key == expected_pk assert dataset.unique_keys == expected_uk @@ -303,7 +302,7 @@ def test_default_dialect_is_ansi_sql(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output - assert result.dialects == [OssieDialect.ANSI_SQL] + assert "dialects" not in json.loads(result.to_ossie_json()) assert _fields(result)[0].expression.dialects[0].dialect == OssieDialect.ANSI_SQL def test_configurable_dialect(self) -> None: @@ -313,7 +312,7 @@ def test_configurable_dialect(self) -> None: ) result = MSIToOssieConverter(dialect=OssieDialect.SNOWFLAKE).convert(_manifest(semantic_models=[sm])).output - assert result.dialects == [OssieDialect.SNOWFLAKE] + assert "dialects" not in json.loads(result.to_ossie_json()) assert _fields(result)[0].expression.dialects[0].dialect == OssieDialect.SNOWFLAKE @@ -329,7 +328,7 @@ def test_shared_entity_name_produces_relationship(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[listings, bookings])).output - rels = result.semantic_model[0].relationships + rels = result.relationships assert rels is not None assert len(rels) == 1 rel = rels[0] @@ -349,7 +348,7 @@ def test_same_type_entities_produce_relationship(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[users_a, users_b])).output - rels = result.semantic_model[0].relationships + rels = result.relationships assert rels is not None assert len(rels) == 1 assert rels[0].from_columns == ["user_id"] @@ -362,7 +361,7 @@ def test_single_dataset_with_entity_produces_no_relationship(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[bookings])).output - assert result.semantic_model[0].relationships is None + assert result.relationships is None def test_same_dataset_entities_excluded(self) -> None: orders = semantic_model_with_guaranteed_meta( @@ -374,7 +373,7 @@ def test_same_dataset_entities_excluded(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[orders])).output - assert result.semantic_model[0].relationships is None + assert result.relationships is None def test_three_datasets_produce_all_pairs(self, snapshot: SnapshotAssertion) -> None: users_a = semantic_model_with_guaranteed_meta( @@ -391,7 +390,7 @@ def test_three_datasets_produce_all_pairs(self, snapshot: SnapshotAssertion) -> ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[users_a, users_b, orders])).output - rels = result.semantic_model[0].relationships + rels = result.relationships assert rels is not None assert len(rels) == 3 pairs = {(r.from_dataset, r.to) for r in rels} @@ -409,7 +408,7 @@ def test_columns_use_expr_when_present(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[listings, bookings])).output - rels = result.semantic_model[0].relationships + rels = result.relationships assert rels is not None rel = rels[0] assert rel.from_columns == ["fk_lid"] @@ -426,7 +425,7 @@ def test_columns_fall_back_to_name_without_expr(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[listings, bookings])).output - rels = result.semantic_model[0].relationships + rels = result.relationships assert rels is not None assert rels[0].from_columns == ["listing"] assert rels[0].to_columns == ["listing"] @@ -446,7 +445,7 @@ def test_primary_entity_shorthand_does_not_produce_relationship(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[bookings, orders])).output - assert result.semantic_model[0].relationships is None + assert result.relationships is None def test_relationship_name_format(self) -> None: listings = semantic_model_with_guaranteed_meta( @@ -459,7 +458,7 @@ def test_relationship_name_format(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[listings, bookings])).output - rels = result.semantic_model[0].relationships + rels = result.relationships assert rels is not None assert rels[0].name == "bookings__listings__listing" @@ -474,7 +473,7 @@ def test_natural_entity_excluded(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[users, orders])).output - assert result.semantic_model[0].relationships is None + assert result.relationships is None def test_direction_based_on_entity_type_not_manifest_order(self) -> None: beta = semantic_model_with_guaranteed_meta( @@ -487,7 +486,7 @@ def test_direction_based_on_entity_type_not_manifest_order(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[beta, alpha])).output - rels = result.semantic_model[0].relationships + rels = result.relationships assert rels is not None assert rels[0].from_dataset == "alpha" assert rels[0].to == "beta" @@ -514,7 +513,7 @@ def test_foreign_foreign_pair_produces_no_relationship(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[customers, orders, reviews])).output - rels = result.semantic_model[0].relationships + rels = result.relationships assert rels is not None pairs = {(r.from_dataset, r.to) for r in rels} # orders and reviews each join to customers; orders-reviews (both FOREIGN on `customer`) is excluded. @@ -532,7 +531,7 @@ def test_foreign_foreign_pair_excluded_even_when_only_pair(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[orders, reviews])).output - assert result.semantic_model[0].relationships is None + assert result.relationships is None class TestMetricConversion: @@ -848,7 +847,7 @@ def test_no_metrics_produces_no_ossie_metrics(self) -> None: sm = semantic_model_with_guaranteed_meta(name="orders") result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output - assert result.semantic_model[0].metrics is None + assert result.metrics is None def test_multiple_metrics_all_converted(self) -> None: sm = semantic_model_with_guaranteed_meta( @@ -901,7 +900,7 @@ def test_conversion_metric_skipped(self) -> None: ) result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm], metrics=[conversion])).output - assert result.semantic_model[0].metrics is None + assert result.metrics is None class TestConverterIssues: @@ -1210,15 +1209,15 @@ def test_to_ossie_json_produces_valid_json(self) -> None: parsed = json.loads(result.to_ossie_json()) assert parsed["version"] == "0.2.0.dev0" - assert len(parsed["semantic_model"]) == 1 - assert parsed["semantic_model"][0]["name"] == "my_project" + assert "semantic_model" not in parsed + assert parsed["name"] == "my_project" def test_to_ossie_json_excludes_none_fields(self) -> None: sm = semantic_model_with_guaranteed_meta(name="orders") result = MSIToOssieConverter().convert(_manifest(semantic_models=[sm])).output parsed = json.loads(result.to_ossie_json()) - dataset = parsed["semantic_model"][0]["datasets"][0] + dataset = parsed["datasets"][0] assert "primary_key" not in dataset assert "unique_keys" not in dataset assert "fields" not in dataset diff --git a/converters/dbt/tests/test_ossie_to_msi.py b/converters/dbt/tests/test_ossie_to_msi.py index dcc8304a..c68e547d 100644 --- a/converters/dbt/tests/test_ossie_to_msi.py +++ b/converters/dbt/tests/test_ossie_to_msi.py @@ -481,7 +481,7 @@ def test_ossie_to_msi_to_ossie_preserves_structure(self, snapshot: SnapshotAsser ossie_doc = MSIToOssieConverter().convert(msi).output - dataset = ossie_doc.semantic_model[0].datasets[0] + dataset = ossie_doc.datasets[0] assert dataset.name == "orders" field_names = {f.name for f in dataset.fields or []} @@ -490,7 +490,7 @@ def test_ossie_to_msi_to_ossie_preserves_structure(self, snapshot: SnapshotAsser assert "created_at" in field_names assert "amount" in field_names - metrics = ossie_doc.semantic_model[0].metrics or [] + metrics = ossie_doc.metrics or [] assert len(metrics) == 1 assert metrics[0].name == "revenue" assert metrics[0].expression.dialects[0].expression == "SUM(orders.amount)" @@ -528,7 +528,7 @@ def test_discrete_percentile_survives_round_trip(self) -> None: _manifest(semantic_models=[orders], metrics=[metric]) ).output - ossie_expr = ossie_doc.semantic_model[0].metrics[0].expression.dialects[0].expression + ossie_expr = ossie_doc.metrics[0].expression.dialects[0].expression assert ossie_expr == "PERCENTILE_DISC(0.95) WITHIN GROUP (ORDER BY orders.amount)" back = OssieToMSIConverter().convert(ossie_doc).output diff --git a/converters/gooddata/README.md b/converters/gooddata/README.md index 9bdfde7f..51e3f61f 100644 --- a/converters/gooddata/README.md +++ b/converters/gooddata/README.md @@ -23,6 +23,11 @@ Bidirectional converter between GoodData's declarative Logical Data Model (LDM) and the [Apache Ossie](https://github.com/apache/ossie) semantic model specification. +Ossie input and output use one model per document, with `name`, `datasets`, +`relationships`, and `metrics` directly at the root beside `version`. Migrate +legacy `semantic_model` wrappers before conversion; see the +[format migration guidance](../../core-spec/spec.md#migrating-earlier-document-shapes). + ## Features - **GoodData → Ossie**: Convert a GoodData declarative LDM JSON to Ossie semantic model YAML diff --git a/converters/gooddata/src/ossie_gooddata/gooddata_to_ossie.py b/converters/gooddata/src/ossie_gooddata/gooddata_to_ossie.py index 05a51981..a63ea4d9 100644 --- a/converters/gooddata/src/ossie_gooddata/gooddata_to_ossie.py +++ b/converters/gooddata/src/ossie_gooddata/gooddata_to_ossie.py @@ -95,7 +95,7 @@ def gooddata_to_ossie( return { "version": OSSIE_VERSION, - "semantic_model": [semantic_model], + **semantic_model, } diff --git a/converters/gooddata/src/ossie_gooddata/models.py b/converters/gooddata/src/ossie_gooddata/models.py index e96a8a60..c3daedbe 100644 --- a/converters/gooddata/src/ossie_gooddata/models.py +++ b/converters/gooddata/src/ossie_gooddata/models.py @@ -36,7 +36,7 @@ └── granularitiesFormatting {titleBase, titlePattern} Ossie semantic model structure: - version, semantic_model[] + version, name, datasets[], relationships[], metrics[] ├── name, description, ai_context, custom_extensions[] ├── datasets[] {name, source, primary_key[], fields[], custom_extensions[]} ├── relationships[] {name, from, to, from_columns[], to_columns[]} diff --git a/converters/gooddata/src/ossie_gooddata/ossie_to_gooddata.py b/converters/gooddata/src/ossie_gooddata/ossie_to_gooddata.py index 21a4a910..35c7be86 100644 --- a/converters/gooddata/src/ossie_gooddata/ossie_to_gooddata.py +++ b/converters/gooddata/src/ossie_gooddata/ossie_to_gooddata.py @@ -62,21 +62,30 @@ def ossie_to_gooddata( datasets: list[GdDataset] = [] date_instances: list[GdDateInstance] = [] - for sm in ossie_model.get("semantic_model", []): - relationship_map = _build_relationship_map(sm) - # Pre-pass: for each Ossie dataset, record whether it is a date instance - # and map its physical source columns to the attribute ids that will - # be generated. Reference target columns resolve via this map. - target_info = _build_target_info(sm) - - for ds in sm.get("datasets", []): - gd_ds, date_inst = _convert_ossie_dataset( - ds, relationship_map, target_info, data_source_id, - ) - if date_inst: - date_instances.append(date_inst) - else: - datasets.append(gd_ds) + if not isinstance(ossie_model, dict): + raise ValueError("Ossie input must be a mapping") + if "semantic_model" in ossie_model: + raise ValueError("Ossie model properties must be at the root; semantic_model wrappers are not supported") + if "dialects" in ossie_model or "vendors" in ossie_model: + raise ValueError("Root dialects and vendors are not supported by the Ossie spec") + if not isinstance(ossie_model.get("name"), str) or not isinstance(ossie_model.get("datasets"), list): + raise ValueError("Ossie input requires name and datasets at the root") + + sm = ossie_model + relationship_map = _build_relationship_map(sm) + # Pre-pass: for each Ossie dataset, record whether it is a date instance + # and map its physical source columns to the attribute ids that will + # be generated. Reference target columns resolve via this map. + target_info = _build_target_info(sm) + + for ds in sm.get("datasets", []): + gd_ds, date_inst = _convert_ossie_dataset( + ds, relationship_map, target_info, data_source_id, + ) + if date_inst: + date_instances.append(date_inst) + else: + datasets.append(gd_ds) return GdDeclarativeModel(ldm=GdLdm(datasets=datasets, date_instances=date_instances)) diff --git a/converters/gooddata/tests/fixtures/ossie_tpcds.yaml b/converters/gooddata/tests/fixtures/ossie_tpcds.yaml index af14a2fd..790bb950 100644 --- a/converters/gooddata/tests/fixtures/ossie_tpcds.yaml +++ b/converters/gooddata/tests/fixtures/ossie_tpcds.yaml @@ -17,215 +17,214 @@ version: "0.2.0.dev0" -semantic_model: - - name: tpcds_model - description: TPC-DS retail model +name: tpcds_model +description: TPC-DS retail model - datasets: - - name: store_sales - source: tpcds.public.store_sales - primary_key: [ss_item_sk, ss_ticket_number] - description: Fact table containing all store sales transactions - fields: - - name: ss_item_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_item_sk - - dialect: MAQL - expression: "{label/store_sales.attr.store_sales.ss_item_sk}" - dimension: - is_time: false - - name: ss_ticket_number - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_ticket_number - - dialect: MAQL - expression: "{label/store_sales.attr.store_sales.ss_ticket_number}" - dimension: - is_time: false - - name: ss_customer_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_customer_sk - - dialect: MAQL - expression: "{label/store_sales.attr.store_sales.ss_customer_sk}" - dimension: - is_time: false - - name: ss_store_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_store_sk - - dialect: MAQL - expression: "{label/store_sales.attr.store_sales.ss_store_sk}" - dimension: - is_time: false - - name: ss_quantity - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_quantity - - dialect: MAQL - expression: "{fact/store_sales.fact.store_sales.ss_quantity}" - description: Quantity of items sold - - name: ss_sales_price - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_sales_price - - dialect: MAQL - expression: "{fact/store_sales.fact.store_sales.ss_sales_price}" - description: Sales price per unit - - name: ss_ext_sales_price - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_ext_sales_price - - dialect: MAQL - expression: "{fact/store_sales.fact.store_sales.ss_ext_sales_price}" - description: Extended sales price (quantity * price) - - name: ss_net_profit - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_net_profit - - dialect: MAQL - expression: "{fact/store_sales.fact.store_sales.ss_net_profit}" - description: Net profit from the sale +datasets: + - name: store_sales + source: tpcds.public.store_sales + primary_key: [ss_item_sk, ss_ticket_number] + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_item_sk}" + dimension: + is_time: false + - name: ss_ticket_number + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ticket_number + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_ticket_number}" + dimension: + is_time: false + - name: ss_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_customer_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_customer_sk}" + dimension: + is_time: false + - name: ss_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_store_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_store_sk}" + dimension: + is_time: false + - name: ss_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_quantity + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_quantity}" + description: Quantity of items sold + - name: ss_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_sales_price}" + description: Sales price per unit + - name: ss_ext_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ext_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_ext_sales_price}" + description: Extended sales price (quantity * price) + - name: ss_net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_profit + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_net_profit}" + description: Net profit from the sale - - name: customer - source: tpcds.public.customer - primary_key: [c_customer_sk] - description: Customer dimension with demographic information - fields: - - name: c_customer_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: c_customer_sk - dimension: - is_time: false - - name: c_first_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_first_name - dimension: - is_time: false - description: Customer first name - - name: c_last_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_last_name - dimension: - is_time: false - description: Customer last name + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_sk + dimension: + is_time: false + - name: c_first_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name + dimension: + is_time: false + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_last_name + dimension: + is_time: false + description: Customer last name - - name: item - source: tpcds.public.item - primary_key: [i_item_sk] - description: Item/Product dimension - fields: - - name: i_item_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_sk - dimension: - is_time: false - - name: i_brand - expression: - dialects: - - dialect: ANSI_SQL - expression: i_brand - dimension: - is_time: false - - name: i_category - expression: - dialects: - - dialect: ANSI_SQL - expression: i_category - dimension: - is_time: false - - name: i_current_price - expression: - dialects: - - dialect: ANSI_SQL - expression: i_current_price - description: Current price of the item + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] + description: Item/Product dimension + fields: + - name: i_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_sk + dimension: + is_time: false + - name: i_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: i_brand + dimension: + is_time: false + - name: i_category + expression: + dialects: + - dialect: ANSI_SQL + expression: i_category + dimension: + is_time: false + - name: i_current_price + expression: + dialects: + - dialect: ANSI_SQL + expression: i_current_price + description: Current price of the item - - name: store - source: tpcds.public.store - primary_key: [s_store_sk] - description: Store dimension with location attributes - fields: - - name: s_store_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_sk - dimension: - is_time: false - - name: s_store_name - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_name - dimension: - is_time: false - - name: s_city - expression: - dialects: - - dialect: ANSI_SQL - expression: s_city - dimension: - is_time: false - - name: s_state - expression: - dialects: - - dialect: ANSI_SQL - expression: s_state - dimension: - is_time: false - - name: s_number_employees - expression: - dialects: - - dialect: ANSI_SQL - expression: s_number_employees - description: Number of employees at the store + - name: store + source: tpcds.public.store + primary_key: [s_store_sk] + description: Store dimension with location attributes + fields: + - name: s_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_sk + dimension: + is_time: false + - name: s_store_name + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_name + dimension: + is_time: false + - name: s_city + expression: + dialects: + - dialect: ANSI_SQL + expression: s_city + dimension: + is_time: false + - name: s_state + expression: + dialects: + - dialect: ANSI_SQL + expression: s_state + dimension: + is_time: false + - name: s_number_employees + expression: + dialects: + - dialect: ANSI_SQL + expression: s_number_employees + description: Number of employees at the store - - name: date_dim - source: date_dim - description: Date dimension with calendar attributes - custom_extensions: - - vendor_name: GOODDATA - data: '{"date_dimension": true, "granularities": ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]}' + - name: date_dim + source: date_dim + description: Date dimension with calendar attributes + custom_extensions: + - vendor_name: GOODDATA + data: '{"date_dimension": true, "granularities": ["DAY", "WEEK", "MONTH", "QUARTER", "YEAR"]}' - relationships: - - name: store_sales_to_date_dim - from: store_sales - to: date_dim - from_columns: [ss_sold_date_sk] - to_columns: [ss_sold_date_sk] +relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [ss_sold_date_sk] - - name: store_sales_to_customer - from: store_sales - to: customer - from_columns: [ss_customer_sk] - to_columns: [c_customer_sk] + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] - - name: store_sales_to_item - from: store_sales - to: item - from_columns: [ss_item_sk] - to_columns: [i_item_sk] + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] - - name: store_sales_to_store - from: store_sales - to: store - from_columns: [ss_store_sk] - to_columns: [s_store_sk] + - name: store_sales_to_store + from: store_sales + to: store + from_columns: [ss_store_sk] + to_columns: [s_store_sk] diff --git a/converters/gooddata/tests/test_gooddata_to_ossie.py b/converters/gooddata/tests/test_gooddata_to_ossie.py index a714713f..49d85675 100644 --- a/converters/gooddata/tests/test_gooddata_to_ossie.py +++ b/converters/gooddata/tests/test_gooddata_to_ossie.py @@ -119,16 +119,16 @@ def test_basic_conversion(gooddata_tpcds_model: GdDeclarativeModel): result = gooddata_to_ossie(gooddata_tpcds_model, model_name="tpcds_test") assert result["version"] == "0.2.0.dev0" - assert len(result["semantic_model"]) == 1 + assert "semantic_model" not in result - sm = result["semantic_model"][0] + sm = result assert sm["name"] == "tpcds_test" def test_datasets_converted(gooddata_tpcds_model: GdDeclarativeModel): """Verify all datasets (regular + date instances) are converted.""" result = gooddata_to_ossie(gooddata_tpcds_model) - sm = result["semantic_model"][0] + sm = result # 4 regular datasets + 1 date instance = 5 datasets assert len(sm["datasets"]) == 5 @@ -144,7 +144,7 @@ def test_datasets_converted(gooddata_tpcds_model: GdDeclarativeModel): def test_dataset_source(gooddata_tpcds_model: GdDeclarativeModel): """Verify source is built from dataSourceTableId.""" result = gooddata_to_ossie(gooddata_tpcds_model) - sm = result["semantic_model"][0] + sm = result store_sales = next(ds for ds in sm["datasets"] if ds["name"] == "store_sales") assert store_sales["source"] == "tpcds.public.store_sales" @@ -153,7 +153,7 @@ def test_dataset_source(gooddata_tpcds_model: GdDeclarativeModel): def test_primary_key_from_grain(gooddata_tpcds_model: GdDeclarativeModel): """Verify primary_key is derived from grain attributes' source columns.""" result = gooddata_to_ossie(gooddata_tpcds_model) - sm = result["semantic_model"][0] + sm = result store_sales = next(ds for ds in sm["datasets"] if ds["name"] == "store_sales") assert set(store_sales["primary_key"]) == {"ss_item_sk", "ss_ticket_number"} @@ -162,7 +162,7 @@ def test_primary_key_from_grain(gooddata_tpcds_model: GdDeclarativeModel): def test_attributes_become_dimension_fields(gooddata_tpcds_model: GdDeclarativeModel): """Verify GoodData attributes become Ossie fields with dimension metadata.""" result = gooddata_to_ossie(gooddata_tpcds_model) - sm = result["semantic_model"][0] + sm = result customer = next(ds for ds in sm["datasets"] if ds["name"] == "customer") fields = customer["fields"] @@ -179,7 +179,7 @@ def test_attributes_become_dimension_fields(gooddata_tpcds_model: GdDeclarativeM def test_facts_become_plain_fields(gooddata_tpcds_model: GdDeclarativeModel): """Verify GoodData facts become Ossie fields without dimension metadata.""" result = gooddata_to_ossie(gooddata_tpcds_model) - sm = result["semantic_model"][0] + sm = result store_sales = next(ds for ds in sm["datasets"] if ds["name"] == "store_sales") fields = store_sales["fields"] @@ -195,7 +195,7 @@ def test_omitted_fixture_source_types_remain_unspecified(gooddata_tpcds_model: G """Verify absent GoodData source types are not promoted from field roles.""" result = gooddata_to_ossie(gooddata_tpcds_model) store_sales = next( - ds for ds in result["semantic_model"][0]["datasets"] if ds["name"] == "store_sales" + ds for ds in result["datasets"] if ds["name"] == "store_sales" ) item_key = next(field for field in store_sales["fields"] if field["name"] == "ss_item_sk") @@ -207,7 +207,7 @@ def test_omitted_fixture_source_types_remain_unspecified(gooddata_tpcds_model: G def test_maql_expressions(gooddata_tpcds_model: GdDeclarativeModel): """Verify MAQL dialect expressions are generated for fields.""" result = gooddata_to_ossie(gooddata_tpcds_model) - sm = result["semantic_model"][0] + sm = result store_sales = next(ds for ds in sm["datasets"] if ds["name"] == "store_sales") quantity_field = next(f for f in store_sales["fields"] if f["name"] == "ss_quantity") @@ -225,7 +225,7 @@ def test_maql_expressions(gooddata_tpcds_model: GdDeclarativeModel): def test_references_become_relationships(gooddata_tpcds_model: GdDeclarativeModel): """Verify GoodData references become Ossie relationships.""" result = gooddata_to_ossie(gooddata_tpcds_model) - sm = result["semantic_model"][0] + sm = result rels = sm["relationships"] assert len(rels) == 4 @@ -238,7 +238,7 @@ def test_references_become_relationships(gooddata_tpcds_model: GdDeclarativeMode def test_date_instance_converted(gooddata_tpcds_model: GdDeclarativeModel): """Verify date instances become Ossie datasets with custom_extensions.""" result = gooddata_to_ossie(gooddata_tpcds_model) - sm = result["semantic_model"][0] + sm = result date_ds = next(ds for ds in sm["datasets"] if ds["name"] == "date_dim") assert "custom_extensions" in date_ds @@ -254,7 +254,7 @@ def test_date_instance_converted(gooddata_tpcds_model: GdDeclarativeModel): def test_labels_in_custom_extensions(gooddata_tpcds_model: GdDeclarativeModel): """Verify attribute labels are preserved in custom_extensions.""" result = gooddata_to_ossie(gooddata_tpcds_model) - sm = result["semantic_model"][0] + sm = result customer = next(ds for ds in sm["datasets"] if ds["name"] == "customer") # First attribute (c_customer_sk) has 2 labels @@ -273,7 +273,7 @@ def test_labels_in_custom_extensions(gooddata_tpcds_model: GdDeclarativeModel): def test_data_source_id_extension(gooddata_tpcds_model: GdDeclarativeModel): """Verify data_source_id is stored in model-level custom_extensions.""" result = gooddata_to_ossie(gooddata_tpcds_model, data_source_id="my_pg") - sm = result["semantic_model"][0] + sm = result assert "custom_extensions" in sm diff --git a/converters/gooddata/tests/test_ossie_to_gooddata.py b/converters/gooddata/tests/test_ossie_to_gooddata.py index 5f44be70..7422ffc0 100644 --- a/converters/gooddata/tests/test_ossie_to_gooddata.py +++ b/converters/gooddata/tests/test_ossie_to_gooddata.py @@ -195,39 +195,35 @@ def test_omitted_datatype_keeps_role_default_without_warning(converter): def test_temporal_datatype_does_not_turn_regular_dataset_into_date_instance(): """Verify a temporal role does not imply GoodData's special date dataset kind.""" model = { - "semantic_model": [ + "name": "events", + "datasets": [ { - "name": "events", - "datasets": [ - { - "name": "event_times", - "source": "analytics.event_times", - "fields": [ - _direct_field( - "occurred_at", - datatype="DateTimeTz", - dimension={}, - description="Event occurrence time", - ) - ], - }, - { - "name": "observations", - "source": "analytics.observations", - "fields": [_direct_field("event_time", dimension={})], - }, - ], - "relationships": [ - { - "name": "observation_event_time", - "from": "observations", - "to": "event_times", - "from_columns": ["event_time"], - "to_columns": ["occurred_at"], - } + "name": "event_times", + "source": "analytics.event_times", + "fields": [ + _direct_field( + "occurred_at", + datatype="DateTimeTz", + dimension={}, + description="Event occurrence time", + ) ], + }, + { + "name": "observations", + "source": "analytics.observations", + "fields": [_direct_field("event_time", dimension={})], + }, + ], + "relationships": [ + { + "name": "observation_event_time", + "from": "observations", + "to": "event_times", + "from_columns": ["event_time"], + "to_columns": ["occurred_at"], } - ] + ], } result = ossie_to_gooddata(model) @@ -246,20 +242,16 @@ def test_temporal_datatype_does_not_turn_regular_dataset_into_date_instance(): def test_legacy_all_explicit_time_fields_still_create_date_instance(): """Verify the converter's pre-datatype date-dataset heuristic remains compatible.""" model = { - "semantic_model": [ + "name": "calendar", + "datasets": [ { - "name": "calendar", - "datasets": [ - { - "name": "calendar_dates", - "source": "analytics.calendar_dates", - "fields": [ - _direct_field("date_label", dimension={"is_time": True}) - ], - } + "name": "calendar_dates", + "source": "analytics.calendar_dates", + "fields": [ + _direct_field("date_label", dimension={"is_time": True}) ], } - ] + ], } result = ossie_to_gooddata(model) @@ -370,3 +362,25 @@ def test_data_source_table_id(ossie_tpcds_dict: dict): assert store_sales.data_source_table_id is not None assert store_sales.data_source_table_id.data_source_id == "tpcds" assert "store_sales" in store_sales.data_source_table_id.path + + +@pytest.mark.parametrize("wrapper", [None, [], {}, [{"name": "old", "datasets": []}]]) +def test_rejects_legacy_wrapper_even_with_root_model(ossie_tpcds_dict: dict, wrapper): + document = {**ossie_tpcds_dict, "semantic_model": wrapper} + + with pytest.raises(ValueError, match="at the root"): + ossie_to_gooddata(document) + + +@pytest.mark.parametrize("document", [None, [], {}, {"name": "sales"}]) +def test_rejects_missing_root_model(document): + with pytest.raises(ValueError, match="mapping|name and datasets"): + ossie_to_gooddata(document) + + +@pytest.mark.parametrize("property_name", ["dialects", "vendors"]) +@pytest.mark.parametrize("value", [None, [], ["legacy"]]) +def test_removed_root_metadata_is_rejected(ossie_tpcds_dict: dict, property_name, value): + document = {**ossie_tpcds_dict, property_name: value} + with pytest.raises(ValueError, match="Root dialects and vendors"): + ossie_to_gooddata(document) diff --git a/converters/gooddata/tests/test_roundtrip.py b/converters/gooddata/tests/test_roundtrip.py index 3b36565d..7a101d56 100644 --- a/converters/gooddata/tests/test_roundtrip.py +++ b/converters/gooddata/tests/test_roundtrip.py @@ -70,7 +70,7 @@ def test_roundtrip_preserves_native_source_types(source_type: str): def test_roundtrip_preserves_unknown_source_type_through_opaque(): """Verify an unknown GoodData type round-trips through Opaque extension data.""" ossie = gooddata_to_ossie(_model_with_attribute("CUSTOM_TYPE")) - field = ossie["semantic_model"][0]["datasets"][0]["fields"][0] + field = ossie["datasets"][0]["fields"][0] assert field["datatype"] == "Opaque" assert json.loads(field["custom_extensions"][0]["data"])["source_column_data_type"] == "CUSTOM_TYPE" @@ -84,7 +84,7 @@ def test_roundtrip_keeps_missing_source_type_unasserted(): model = _model_with_attribute(None) ossie = gooddata_to_ossie(model) - field = ossie["semantic_model"][0]["datasets"][0]["fields"][0] + field = ossie["datasets"][0]["fields"][0] result = gd_model_to_dict(ossie_to_gooddata(ossie)) attribute = result["ldm"]["datasets"][0]["attributes"][0] diff --git a/converters/honeydew/README.md b/converters/honeydew/README.md index 4419862b..d1e58955 100644 --- a/converters/honeydew/README.md +++ b/converters/honeydew/README.md @@ -25,6 +25,10 @@ and [Honeydew](https://honeydew.ai/docs) [workspace YAML](https://honeydew.ai/do Honeydew documents this integration from its own side under [Apache Ossie (OSI)](https://honeydew.ai/docs/integration/apache-ossie-osi). +Ossie documents contain one model directly at the root, with `version`, `name`, +`datasets`, and optional model properties. Legacy `semantic_model` wrappers +(arrays or objects) are rejected. + ## Overview | Direction | Input | Output | @@ -36,7 +40,7 @@ Honeydew documents this integration from its own side under | Ossie concept | Honeydew concept | |-------------|-----------------| -| `semantic_model.name` | `workspace.yml name` | +| `name` | `workspace.yml name` | | `dataset` | Entity + dataset files under `schema//` | | `dataset.source` | `dataset.sql` | | `dataset.primary_key` | `entity.keys` | @@ -50,7 +54,7 @@ Honeydew documents this integration from its own side under | Honeydew concept | Ossie concept | |-----------------|-------------| -| `workspace.name` | `semantic_model.name` | +| `workspace.name` | `name` | | Entity + primary dataset | `dataset` | | `entity.keys` | `dataset.primary_key` (and `dataset.unique_keys`) | | `dataset.attributes` (columns) | `fields` with `ANSI_SQL` expression = column name | diff --git a/converters/honeydew/src/ossie_honeydew/converter.py b/converters/honeydew/src/ossie_honeydew/converter.py index 33793e74..3ead23d1 100644 --- a/converters/honeydew/src/ossie_honeydew/converter.py +++ b/converters/honeydew/src/ossie_honeydew/converter.py @@ -43,7 +43,7 @@ _OSSIE_METADATA_SECTION = "ossie" # Workspaces written before the Ossie rebrand named the section "osi". Still # read it, so exporting such a workspace does not silently drop the fields it -# preserves (ai_context, label, unique_keys, custom_extensions, vendors). +# preserves (ai_context, label, unique_keys, custom_extensions). _LEGACY_OSSIE_METADATA_SECTION = "osi" _HD_ATTR_KEYS = ("display_name", "hidden", "folder", "format_string", "timegrain") @@ -98,21 +98,18 @@ def convert_ossie_to_honeydew(ossie_yaml_str: str) -> dict[str, str]: f"Unsupported Ossie version '{version_str}'. Supported: {SUPPORTED_OSSIE_VERSION}" ) - semantic_models = root.get("semantic_model") - if not isinstance(semantic_models, list) or not semantic_models: - raise HoneydewConversionError("'semantic_model' must be a non-empty list") - - if len(semantic_models) > 1: - warnings.warn( - f"Ossie YAML contains {len(semantic_models)} semantic models; " - "only the first will be converted" + if "semantic_model" in root: + raise HoneydewConversionError( + "Legacy 'semantic_model' wrappers are not supported; " + "place the model properties directly at the document root" ) - vendors = [v for v in (root.get("vendors") or []) if v != HONEYDEW_VENDOR] - return _model_to_files(semantic_models[0], extra_vendors=vendors) + if "dialects" in root or "vendors" in root: + raise HoneydewConversionError("Root dialects and vendors are not supported by the Ossie spec") + return _model_to_files(root) -def _model_to_files(sm: dict[str, Any], *, extra_vendors: list[str] | None = None) -> dict[str, str]: +def _model_to_files(sm: dict[str, Any]) -> dict[str, str]: name = sm.get("name") if not name: raise HoneydewConversionError("Missing 'name' in semantic model") @@ -123,13 +120,12 @@ def _model_to_files(sm: dict[str, Any], *, extra_vendors: list[str] | None = Non if sm.get("description"): workspace["description"] = sm["description"] - # Preserve model-level ai_context, non-HONEYDEW custom_extensions, and extra vendors + # Preserve model-level ai_context and non-HONEYDEW custom_extensions. model_ai_ctx = sm.get("ai_context") model_ext = [e for e in (sm.get("custom_extensions") or []) if e.get("vendor_name") != HONEYDEW_VENDOR] ws_meta = _build_ossie_metadata( ai_context=model_ai_ctx, custom_extensions=model_ext or None, - extra_vendors=extra_vendors or None, ) if ws_meta: workspace["metadata"] = [ws_meta] @@ -606,12 +602,9 @@ def convert_honeydew_to_ossie(workspace_dir: str) -> str: if ossie_metrics: sm["metrics"] = ossie_metrics - extra_vendors = ws_ossie_meta.get("vendors") or [] - vendors = [HONEYDEW_VENDOR] + [v for v in extra_vendors if v != HONEYDEW_VENDOR] root: dict[str, Any] = { "version": SUPPORTED_OSSIE_VERSION, - "vendors": vendors, - "semantic_model": [sm], + **sm, } return _dump(root) @@ -934,7 +927,6 @@ def _build_ossie_metadata( label: str | None = None, unique_keys: Any = None, custom_extensions: list | None = None, - extra_vendors: list[str] | None = None, ) -> dict[str, Any] | None: """Build a Honeydew metadata entry that stores Ossie-only fields for round-tripping.""" items: list[dict[str, Any]] = [] @@ -948,8 +940,6 @@ def _build_ossie_metadata( items.append({"name": "unique_keys", "value": json.dumps(unique_keys)}) if custom_extensions: items.append({"name": "custom_extensions", "value": json.dumps(custom_extensions)}) - if extra_vendors: - items.append({"name": "vendors", "value": json.dumps(extra_vendors)}) if not items: return None @@ -975,7 +965,7 @@ def _read_ossie_metadata(obj: dict[str, Any]) -> dict[str, Any]: result[key] = raw elif key == "label": result[key] = raw - elif key in ("unique_keys", "custom_extensions", "vendors"): + elif key in ("unique_keys", "custom_extensions"): try: result[key] = json.loads(raw) except (json.JSONDecodeError, TypeError): diff --git a/converters/honeydew/tests/test_ossie_honeydew_converter.py b/converters/honeydew/tests/test_ossie_honeydew_converter.py index d83523b4..40a2d4ac 100644 --- a/converters/honeydew/tests/test_ossie_honeydew_converter.py +++ b/converters/honeydew/tests/test_ossie_honeydew_converter.py @@ -52,7 +52,7 @@ def _ossie(model_dict): return yaml.dump( - {"version": OSSIE_VERSION, "semantic_model": [model_dict]}, + {"version": OSSIE_VERSION, **model_dict}, default_flow_style=False, sort_keys=False, ) @@ -140,7 +140,7 @@ def _ossie_roundtrip(model_dict, tmp_path): p = tmp_path / rel_path p.parent.mkdir(parents=True, exist_ok=True) p.write_text(content) - return yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))["semantic_model"][0] + return yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path))) def _honeydew_roundtrip(entities, tmp_path): @@ -596,24 +596,23 @@ def test_ossie_to_honeydew_metric_entity_hint_overrides_expression(): def test_ossie_to_honeydew_invalid_version_raises(): with pytest.raises(HoneydewConversionError, match="Unsupported"): - convert_ossie_to_honeydew("version: '9.9.9'\nsemantic_model:\n - name: m\n") + convert_ossie_to_honeydew("version: '9.9.9'\nname: m\n") -def test_ossie_to_honeydew_missing_semantic_model_raises(): +def test_ossie_to_honeydew_missing_model_name_raises(): with pytest.raises(HoneydewConversionError): convert_ossie_to_honeydew(f"version: '{OSSIE_VERSION}'\n") -def test_ossie_to_honeydew_multiple_models_warns(): - doc = yaml.dump({"version": OSSIE_VERSION, "semantic_model": [ - {"name": "m1", "datasets": []}, - {"name": "m2", "datasets": []}, - ]}) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - files = convert_ossie_to_honeydew(doc) - assert any("only the first" in str(x.message) for x in w) - assert yaml.safe_load(files["workspace.yml"]) == {"type": "workspace", "name": "m1"} +@pytest.mark.parametrize( + "wrapper", + [[], [{"name": "m1"}], [{"name": "m1"}, {"name": "m2"}], {"name": "m1"}, None], +) +def test_ossie_to_honeydew_legacy_model_wrappers_are_rejected(wrapper): + doc = yaml.dump({"version": OSSIE_VERSION, "semantic_model": wrapper}) + with pytest.raises(HoneydewConversionError, match="Legacy 'semantic_model'"): + convert_ossie_to_honeydew(doc) + # ───────────────────────────────────────────────────────────────────────────── @@ -621,7 +620,7 @@ def test_ossie_to_honeydew_multiple_models_warns(): # ───────────────────────────────────────────────────────────────────────────── def _hd_root(sm): - return {"version": OSSIE_VERSION, "vendors": ["HONEYDEW"], "semantic_model": [sm]} + return {"version": OSSIE_VERSION, **sm} def _ansi(expr): @@ -813,8 +812,8 @@ def test_honeydew_to_ossie_missing_workspace_raises(tmp_path): def test_honeydew_to_ossie_missing_schema_dir_empty_model(tmp_path): (tmp_path / "workspace.yml").write_text(yaml.dump({"type": "workspace", "name": "ws"})) result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path))) - assert result == {"version": OSSIE_VERSION, "vendors": ["HONEYDEW"], - "semantic_model": [{"name": "ws", "datasets": []}]} + assert result == {"version": OSSIE_VERSION, + "name": "ws", "datasets": []} def test_honeydew_to_ossie_empty_metric_sql_skipped(tmp_path): @@ -824,7 +823,7 @@ def test_honeydew_to_ossie_empty_metric_sql_skipped(tmp_path): "datatype": "number", "sql": ""}]}]) with warnings.catch_warnings(record=True): result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path))) - assert "metrics" not in result["semantic_model"][0] + assert "metrics" not in result def test_honeydew_to_ossie_duplicate_relations_deduplicated(tmp_path): @@ -839,7 +838,7 @@ def test_honeydew_to_ossie_duplicate_relations_deduplicated(tmp_path): "dataset_attrs": []}, ]) result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path))) - assert len(result["semantic_model"][0].get("relationships", [])) == 1 + assert len(result.get("relationships", [])) == 1 def test_honeydew_to_ossie_relation_target_columns_are_unique_keys(tmp_path): @@ -855,7 +854,7 @@ def test_honeydew_to_ossie_relation_target_columns_are_unique_keys(tmp_path): {"name": "customers", "keys": ["id"], "key_dataset": "customers", "sql": "db.s.customers", "dataset_attrs": []}, ]) - sm = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path)))["semantic_model"][0] + sm = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path))) datasets = {ds["name"]: ds for ds in sm["datasets"]} rel = sm["relationships"][0] target_ds = datasets[rel["to"]] @@ -986,7 +985,9 @@ def test_honeydew_to_ossie_relation_target_columns_are_unique_keys(tmp_path): ), ]) def test_ossie_roundtrip_sm(tmp_path, model, expected_sm): - assert _ossie_roundtrip(model, tmp_path) == expected_sm + assert _ossie_roundtrip(model, tmp_path) == { + "version": OSSIE_VERSION, **expected_sm + } def test_ossie_roundtrip_tpcds_example(tmp_path): @@ -1003,7 +1004,7 @@ def test_ossie_roundtrip_tpcds_example(tmp_path): p.parent.mkdir(parents=True, exist_ok=True) p.write_text(content) result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path))) - sm = result["semantic_model"][0] + sm = result assert sm["name"] == "tpcds_retail_model" ds_names = {ds["name"] for ds in sm["datasets"]} assert "store_sales" in ds_names and "customer" in ds_names @@ -1331,6 +1332,7 @@ def test_metric_string_ai_context_preserved_in_roundtrip(tmp_path): "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": "SUM(orders.total)"}]}}]} sm = _ossie_roundtrip(model, tmp_path) assert sm == { + "version": OSSIE_VERSION, "name": "m", "datasets": [{"name": "orders", "source": "db.s.orders"}], "metrics": [{ @@ -1418,28 +1420,15 @@ def test_connectionless_relation_warns(): # ───────────────────────────────────────────────────────────────────────────── -# Vendors round-trip +# Removed document metadata # ───────────────────────────────────────────────────────────────────────────── -@pytest.mark.parametrize("input_vendors,expected_vendors", [ - (["SNOWFLAKE", "HONEYDEW"], ["HONEYDEW", "SNOWFLAKE"]), - (["SNOWFLAKE"], ["HONEYDEW", "SNOWFLAKE"]), - (["HONEYDEW"], ["HONEYDEW"]), -]) -def test_vendors_roundtrip(tmp_path, input_vendors, expected_vendors): - doc = yaml.dump({ - "version": OSSIE_VERSION, - "vendors": input_vendors, - "semantic_model": [{"name": "m", "datasets": []}], - }) - files = convert_ossie_to_honeydew(doc) - for rel_path, content in files.items(): - p = tmp_path / rel_path - p.parent.mkdir(parents=True, exist_ok=True) - p.write_text(content) - result = yaml.safe_load(convert_honeydew_to_ossie(str(tmp_path))) - assert result == {"version": OSSIE_VERSION, "vendors": expected_vendors, - "semantic_model": [{"name": "m", "datasets": []}]} +@pytest.mark.parametrize("property_name", ["dialects", "vendors"]) +def test_rejects_removed_root_metadata(property_name): + doc = {"version": OSSIE_VERSION, "name": "m", "datasets": []} + doc[property_name] = [] + with pytest.raises(HoneydewConversionError, match="Root dialects and vendors"): + convert_ossie_to_honeydew(yaml.safe_dump(doc)) # ───────────────────────────────────────────────────────────────────────────── @@ -1451,9 +1440,9 @@ def test_main_ossie_to_honeydew(tmp_path): input_file = tmp_path / "model.yaml" input_file.write_text(yaml.dump({ "version": OSSIE_VERSION, - "semantic_model": [{"name": "m", "datasets": [ + "name": "m", "datasets": [ {"name": "orders", "source": "db.s.orders", "fields": []} - ]}], + ], })) output_dir = tmp_path / "out" result = subprocess.run( @@ -1482,11 +1471,10 @@ def test_main_honeydew_to_ossie(tmp_path): assert result.returncode == 0 assert yaml.safe_load(output_file.read_text()) == { "version": OSSIE_VERSION, - "vendors": ["HONEYDEW"], - "semantic_model": [{"name": "ws", "datasets": [ + "name": "ws", "datasets": [ {"name": "orders", "source": "DB.S.ORDERS", "primary_key": ["id"], "unique_keys": [["id"]]}, - ]}], + ], } @@ -1494,9 +1482,8 @@ def test_main_path_traversal_rejected(tmp_path): import subprocess input_file = tmp_path / "model.yaml" input_file.write_text( - f"version: '{OSSIE_VERSION}'\nsemantic_model:\n" - " - name: m\n datasets:\n" - " - name: '../../evil'\n source: db.s.evil\n fields: []\n" + f"version: '{OSSIE_VERSION}'\nname: m\ndatasets:\n" + " - name: '../../evil'\n source: db.s.evil\n fields: []\n" ) output_dir = tmp_path / "out" result = subprocess.run( diff --git a/converters/microsoft/README.md b/converters/microsoft/README.md index e9abb765..be36ee6e 100644 --- a/converters/microsoft/README.md +++ b/converters/microsoft/README.md @@ -24,6 +24,11 @@ semantic model. Power BI output is available as either a TMSL `model.bim` mappin TMDL document. The conversion is offline and requires no Power BI or Fabric connection. +Each Ossie JSON/YAML document contains one model, with `name`, `datasets`, +`relationships`, and `metrics` at the root alongside `version`. Legacy +`semantic_model` wrappers must be unwrapped before conversion; split documents +containing multiple models into separate files. + ## Installation ```bash @@ -171,7 +176,7 @@ logging.getLogger("ossie_microsoft").addHandler(logging.StreamHandler()) | Power BI (TMSL) | Apache Ossie | |-----------------|--------------| -| `name` / `model.description` | `semantic_model.name` / `.description` | +| `name` / `model.description` | `name` / `description` | | `model.tables[]` | `datasets[]` | | table partition source (`entity`, `m`, `query`, `calculated`) | `dataset.source` | | `table.columns[]` | `dataset.fields[]` | diff --git a/converters/microsoft/src/ossie_microsoft/ossie_to_semantic_model.py b/converters/microsoft/src/ossie_microsoft/ossie_to_semantic_model.py index d80814b5..981ed12d 100644 --- a/converters/microsoft/src/ossie_microsoft/ossie_to_semantic_model.py +++ b/converters/microsoft/src/ossie_microsoft/ossie_to_semantic_model.py @@ -163,16 +163,16 @@ def convert_ossie_to_semantic_model( f"{OSSIE_VERSION}; conversion may be incomplete", ) - models = document.get("semantic_model") - if not isinstance(models, list) or not models or not isinstance(models[0], dict): - raise ValueError("document is missing a 'semantic_model' entry") - if len(models) > 1: - warn( - "document", - f"a model.bim holds a single model; converting the first of {len(models)} " - "and skipping the rest", + if "semantic_model" in document: + raise ValueError( + "Legacy 'semantic_model' wrappers are not supported; " + "place model properties at the document root" ) - semantic_model = models[0] + if "dialects" in document or "vendors" in document: + raise ValueError("Root dialects and vendors are not supported by the Ossie spec") + if not document.get("name") or "datasets" not in document: + raise ValueError("document requires 'name' and 'datasets' at the root") + semantic_model = document stash = read_stash(semantic_model) _warn_foreign_extensions("model", semantic_model) diff --git a/converters/microsoft/src/ossie_microsoft/semantic_model_to_ossie.py b/converters/microsoft/src/ossie_microsoft/semantic_model_to_ossie.py index a65cf12c..f413fa18 100644 --- a/converters/microsoft/src/ossie_microsoft/semantic_model_to_ossie.py +++ b/converters/microsoft/src/ossie_microsoft/semantic_model_to_ossie.py @@ -179,7 +179,7 @@ def build_ossie_document(bim_file): excluded_measures, ) - return {"version": OSSIE_VERSION, "semantic_model": [semantic_model]} + return {"version": OSSIE_VERSION, **semantic_model} # --------------------------------------------------------------------------- diff --git a/converters/microsoft/tests/conftest.py b/converters/microsoft/tests/conftest.py index bbf4a9cf..2a3d8b3b 100644 --- a/converters/microsoft/tests/conftest.py +++ b/converters/microsoft/tests/conftest.py @@ -46,4 +46,4 @@ def osi(bim): @pytest.fixture(scope="module") def model(osi): - return osi["semantic_model"][0] + return osi diff --git a/converters/microsoft/tests/test_edge_cases.py b/converters/microsoft/tests/test_edge_cases.py index 1e05accf..c8632434 100644 --- a/converters/microsoft/tests/test_edge_cases.py +++ b/converters/microsoft/tests/test_edge_cases.py @@ -26,6 +26,7 @@ import warnings import pytest +import yaml from ossie_microsoft import convert_ossie_to_semantic_model from ossie_microsoft._common import ( @@ -59,7 +60,7 @@ def _model(**overrides): ], } semantic_model.update(overrides) - return {"version": OSSIE_VERSION, "semantic_model": [semantic_model]} + return {"version": OSSIE_VERSION, **semantic_model} def _convert(document): @@ -121,7 +122,7 @@ def test_a_malformed_extension_entry_is_ignored(): def test_a_non_dict_table_is_skipped(): bim = {"name": "m", "model": {"tables": ["nonsense", {"name": None}]}} - assert build_ossie_document(bim)["semantic_model"][0].get("datasets") == [] + assert build_ossie_document(bim).get("datasets") == [] def test_a_non_dict_column_is_skipped(): @@ -129,7 +130,7 @@ def test_a_non_dict_column_is_skipped(): "name": "m", "model": {"tables": [{"name": "T", "columns": ["nonsense", {"noName": 1}]}]}, } - model = build_ossie_document(bim)["semantic_model"][0] + model = build_ossie_document(bim) assert model["datasets"][0].get("fields") is None @@ -138,13 +139,13 @@ def test_a_non_dict_measure_is_skipped(): "name": "m", "model": {"tables": [{"name": "T", "measures": ["nonsense", {"noName": 1}]}]}, } - model = build_ossie_document(bim)["semantic_model"][0] + model = build_ossie_document(bim) assert model.get("metrics") is None def test_a_non_dict_relationship_is_skipped(): bim = {"name": "m", "model": {"tables": [], "relationships": ["nonsense"]}} - assert build_ossie_document(bim)["semantic_model"][0].get("relationships") is None + assert build_ossie_document(bim).get("relationships") is None def test_a_non_dict_dataset_is_skipped(): @@ -154,7 +155,7 @@ def test_a_non_dict_dataset_is_skipped(): def test_a_non_dict_field_is_skipped(): document = _model() - document["semantic_model"][0]["datasets"][0]["fields"] = ["nonsense", {"noName": 1}] + document["datasets"][0]["fields"] = ["nonsense", {"noName": 1}] bim = _convert(document) assert bim["model"]["tables"][0]["columns"] == [] @@ -172,7 +173,7 @@ def test_a_non_dict_metric_is_skipped(): def test_a_composite_unique_key_is_reported(): """TMSL marks uniqueness per column; a composite constraint has no equivalent.""" document = _model() - document["semantic_model"][0]["datasets"][0]["unique_keys"] = [["C", "D"]] + document["datasets"][0]["unique_keys"] = [["C", "D"]] with pytest.warns(UserWarning, match="composite unique constraint"): bim = _convert(document) assert "isUnique" not in bim["model"]["tables"][0]["columns"][0] @@ -180,13 +181,13 @@ def test_a_composite_unique_key_is_reported(): def test_a_malformed_unique_key_is_ignored(): document = _model() - document["semantic_model"][0]["datasets"][0]["unique_keys"] = ["not a list"] + document["datasets"][0]["unique_keys"] = ["not a list"] _convert(document) def test_an_unrecognized_datatype_is_reported_and_left_unspecified(): document = _model() - document["semantic_model"][0]["datasets"][0]["fields"][0]["datatype"] = "Fictional" + document["datasets"][0]["fields"][0]["datatype"] = "Fictional" with pytest.warns(UserWarning, match="unrecognized Apache Ossie data type"): bim = _convert(document) assert "dataType" not in bim["model"]["tables"][0]["columns"][0] @@ -229,7 +230,7 @@ def test_a_duplicate_measure_name_is_qualified_by_its_table(): }, } with pytest.warns(UserWarning, match="duplicate measure name"): - model = build_ossie_document(bim)["semantic_model"][0] + model = build_ossie_document(bim) assert [m["name"] for m in model["metrics"]] == ["Total", "B.Total"] @@ -261,7 +262,7 @@ def test_a_relationship_missing_an_endpoint_is_reported_and_preserved(): }, } with pytest.warns(UserWarning, match="missing an endpoint"): - model = build_ossie_document(bim)["semantic_model"][0] + model = build_ossie_document(bim) # Preserved, so a round trip back to Power BI does not delete it. assert read_stash(model)["excludedRelationships"][0]["name"] == "broken" @@ -287,7 +288,7 @@ def test_a_duplicate_relationship_name_is_reported_and_preserved(): }, } with pytest.warns(UserWarning, match="duplicate relationship"): - model = build_ossie_document(bim)["semantic_model"][0] + model = build_ossie_document(bim) assert len(model["relationships"]) == 1 assert read_stash(model)["excludedRelationships"][0]["name"] == "dup" @@ -303,7 +304,7 @@ def test_the_cli_writes_to_stdout_without_an_output_path(tmp_path, capsys): src = tmp_path / "m.bim" src.write_text(json.dumps({"name": "m", "model": {"tables": []}}), encoding="utf-8") assert main(["import", "-i", str(src)]) == 0 - assert "semantic_model" in capsys.readouterr().out + assert yaml.safe_load(capsys.readouterr().out)["name"] == "m" def test_the_cli_reports_a_bad_file_without_a_traceback(tmp_path, capsys): diff --git a/converters/microsoft/tests/test_ossie_to_semantic_model.py b/converters/microsoft/tests/test_ossie_to_semantic_model.py index ea4684b0..951ed929 100644 --- a/converters/microsoft/tests/test_ossie_to_semantic_model.py +++ b/converters/microsoft/tests/test_ossie_to_semantic_model.py @@ -42,7 +42,7 @@ def bim_out(model): with warnings.catch_warnings(): warnings.simplefilter("ignore") return convert_ossie_to_semantic_model( - {"version": OSSIE_VERSION, "semantic_model": [model]} + {"version": OSSIE_VERSION, **model} ) @@ -60,7 +60,7 @@ def _annotation(target, name): def _convert(semantic_model): return convert_ossie_to_semantic_model( - {"version": OSSIE_VERSION, "semantic_model": [semantic_model]} + {"version": OSSIE_VERSION, **semantic_model} ) @@ -136,19 +136,29 @@ def test_a_document_without_a_model_is_rejected(): def test_a_foreign_spec_version_warns(): with pytest.warns(UserWarning, match="targets Apache Ossie spec"): convert_ossie_to_semantic_model( - {"version": "9.9.9", "semantic_model": [_minimal()]} + {"version": "9.9.9", **_minimal()} ) -def test_only_the_first_model_is_converted(): - document = {"version": OSSIE_VERSION, "semantic_model": [_minimal(), _minimal()]} - with pytest.warns(UserWarning, match="single model"): - bim = convert_ossie_to_semantic_model(document) - assert len(bim["model"]["tables"]) == 1 +@pytest.mark.parametrize("wrapper", [None, [], {}, [_minimal()], [_minimal(), _minimal()]]) +@pytest.mark.parametrize("include_root_model", [False, True]) +def test_legacy_wrappers_are_rejected(wrapper, include_root_model): + document = {"version": OSSIE_VERSION, "semantic_model": wrapper} + if include_root_model: + document.update(_minimal()) + with pytest.raises(ValueError, match="Legacy 'semantic_model'"): + convert_ossie_to_semantic_model(document) + + +@pytest.mark.parametrize("property_name", ["dialects", "vendors"]) +def test_removed_root_metadata_is_rejected(property_name): + document = {"version": OSSIE_VERSION, **_minimal(), property_name: []} + with pytest.raises(ValueError, match="Root dialects and vendors"): + convert_ossie_to_semantic_model(document) def test_tmsl_is_the_default_and_can_be_selected_explicitly(): - document = {"version": OSSIE_VERSION, "semantic_model": [_minimal()]} + document = {"version": OSSIE_VERSION, **_minimal()} assert convert_ossie_to_semantic_model(document) == convert_ossie_to_semantic_model( document, output_format="tmsl" @@ -156,7 +166,7 @@ def test_tmsl_is_the_default_and_can_be_selected_explicitly(): def test_tmdl_serializes_the_completed_tmsl_model(monkeypatch): - document = {"version": OSSIE_VERSION, "semantic_model": [_minimal()]} + document = {"version": OSSIE_VERSION, **_minimal()} expected = "database Model\n\n\tmodel Model\n" received = [] @@ -454,7 +464,7 @@ def test_a_preserved_partition_is_replayed(bim_out): def test_yaml_text_and_source_parameters_generate_a_direct_lake_partition(): - document = {"version": OSSIE_VERSION, "semantic_model": [_minimal()]} + document = {"version": OSSIE_VERSION, **_minimal()} bim = convert_ossie_to_semantic_model( yaml.safe_dump(document), source={"workspaceId": "workspace", "itemId": "item"}, @@ -531,7 +541,7 @@ def test_an_unqualified_source_names_the_entity_without_inventing_a_schema(): def test_a_missing_onelake_location_is_reported_rather_than_assumed(): - document = {"version": OSSIE_VERSION, "semantic_model": [_minimal()]} + document = {"version": OSSIE_VERSION, **_minimal()} with pytest.warns(UserWarning, match="placeholder ids"): bim = convert_ossie_to_semantic_model(document, source={"workspaceId": "w"}) @@ -539,7 +549,7 @@ def test_a_missing_onelake_location_is_reported_rather_than_assumed(): def test_a_non_mapping_onelake_location_is_rejected(): - document = {"version": OSSIE_VERSION, "semantic_model": [_minimal()]} + document = {"version": OSSIE_VERSION, **_minimal()} with pytest.raises(TypeError, match="workspaceId and itemId"): convert_ossie_to_semantic_model(document, source="workspace/item") @@ -570,7 +580,7 @@ def test_an_explicit_compatible_source_reuses_the_preserved_database_query(): {"name": "Other", "kind": "m", "expression": "42"}, ] semantic_model, existing_partition = _mixed_partition_model(expressions) - document = {"version": OSSIE_VERSION, "semantic_model": [semantic_model]} + document = {"version": OSSIE_VERSION, **semantic_model} bim = convert_ossie_to_semantic_model(document, source=source) @@ -589,7 +599,7 @@ def test_a_conflicting_database_query_gets_a_collision_free_name(): {"name": "Unrelated", "kind": "m", "expression": "let X = 1 in X"}, ] semantic_model, existing_partition = _mixed_partition_model(expressions) - document = {"version": OSSIE_VERSION, "semantic_model": [semantic_model]} + document = {"version": OSSIE_VERSION, **semantic_model} bim = convert_ossie_to_semantic_model( document, source={"workspaceId": "current-workspace", "itemId": "current-item"} @@ -615,7 +625,7 @@ def test_a_non_m_database_query_is_not_reused_for_new_partitions(): {"name": "DatabaseQuery", "kind": "parameter", "expression": '"old"'} ] semantic_model, _ = _mixed_partition_model(expressions) - document = {"version": OSSIE_VERSION, "semantic_model": [semantic_model]} + document = {"version": OSSIE_VERSION, **semantic_model} bim = convert_ossie_to_semantic_model( document, source={"workspaceId": "workspace", "itemId": "item"} @@ -635,7 +645,7 @@ def test_a_scalar_database_query_expression_can_be_reused(): generated = _database_query("workspace", "item") generated["expression"] = "\n".join(generated["expression"]) semantic_model, _ = _mixed_partition_model([generated]) - document = {"version": OSSIE_VERSION, "semantic_model": [semantic_model]} + document = {"version": OSSIE_VERSION, **semantic_model} bim = convert_ossie_to_semantic_model( document, source={"workspaceId": "workspace", "itemId": "item"} @@ -1060,7 +1070,7 @@ def test_a_row_number_column_is_restored(): warnings.simplefilter("ignore") osi = yaml.safe_load(convert_semantic_model_to_ossie(bim)) result = convert_ossie_to_semantic_model(osi) - assert [f["name"] for f in osi["semantic_model"][0]["datasets"][0]["fields"]] == ["C"] + assert [f["name"] for f in osi["datasets"][0]["fields"]] == ["C"] assert [c["name"] for c in _table(result, "T")["columns"]] == ["RowNumber", "C"] diff --git a/converters/microsoft/tests/test_semantic_model_to_ossie.py b/converters/microsoft/tests/test_semantic_model_to_ossie.py index 95f13a42..e044e9b7 100644 --- a/converters/microsoft/tests/test_semantic_model_to_ossie.py +++ b/converters/microsoft/tests/test_semantic_model_to_ossie.py @@ -82,7 +82,7 @@ def test_cli_writes_ossie_yaml(tmp_path): out = tmp_path / "model.yaml" assert main(["import", "-i", str(FIXTURES / "sales_model.bim"), "-o", str(out)]) == 0 document = yaml.safe_load(out.read_text(encoding="utf-8")) - assert document["semantic_model"][0]["name"] == "sales_model" + assert document["name"] == "sales_model" def test_cli_reports_errors_without_traceback(tmp_path, capsys): @@ -99,7 +99,8 @@ def test_cli_reports_errors_without_traceback(tmp_path, capsys): def test_document_header(osi): assert osi["version"] == "0.2.0.dev0" - assert len(osi["semantic_model"]) == 1 + assert "semantic_model" not in osi + assert osi["name"] == "sales_model" def test_model_name_and_description(model): @@ -145,7 +146,7 @@ def test_calculation_group_is_skipped_with_a_warning(): } with pytest.warns(UserWarning, match="calculation groups are not converted"): document = build_ossie_document(bim) - assert document["semantic_model"][0]["datasets"] == [] + assert document["datasets"] == [] def test_calculated_table_is_skipped_with_a_warning(): @@ -170,7 +171,7 @@ def test_calculated_table_is_skipped_with_a_warning(): } with pytest.warns(UserWarning, match="calculated tables are not converted"): document = build_ossie_document(bim) - assert document["semantic_model"][0]["datasets"] == [] + assert document["datasets"] == [] def test_row_number_column_is_skipped(model): @@ -304,7 +305,7 @@ def _flip_osi(): def test_a_one_to_many_relationship_is_flipped_to_many_to_one(): - model = _flip_osi()["semantic_model"][0] + model = _flip_osi() rel = model["relationships"][0] assert rel["from"] == "Sales" assert rel["from_columns"] == ["OrderDate"] @@ -313,7 +314,7 @@ def test_a_one_to_many_relationship_is_flipped_to_many_to_one(): def test_a_flipped_relationship_records_its_original_orientation(): - model = _flip_osi()["semantic_model"][0] + model = _flip_osi() stash = read_stash(model["relationships"][0]) assert stash["flipped"] is True assert stash["fromCardinality"] == "one" @@ -335,7 +336,7 @@ def test_a_flipped_relationship_is_exported_the_way_power_bi_wrote_it(): def test_an_unchanged_pre_snapshot_stash_still_restores_the_original_orientation(): osi = _flip_osi() - relationship = osi["semantic_model"][0]["relationships"][0] + relationship = osi["relationships"][0] stash = read_stash(relationship) stash.pop("normalizedEndpoints") write_stash(relationship, stash) @@ -350,7 +351,7 @@ def test_an_unchanged_pre_snapshot_stash_still_restores_the_original_orientation def test_reversed_ossie_endpoints_are_not_reversed_again_by_a_stale_flip_marker(): osi = _flip_osi() - relationship = osi["semantic_model"][0]["relationships"][0] + relationship = osi["relationships"][0] relationship["from"], relationship["to"] = relationship["to"], relationship["from"] relationship["from_columns"], relationship["to_columns"] = ( relationship["to_columns"], @@ -368,7 +369,7 @@ def test_reversed_ossie_endpoints_are_not_reversed_again_by_a_stale_flip_marker( def test_edited_ossie_endpoints_do_not_replay_stale_cardinalities(): osi = _flip_osi() - relationship = osi["semantic_model"][0]["relationships"][0] + relationship = osi["relationships"][0] relationship["from_columns"] = ["AlternateOrderDate"] relationship["to_columns"] = ["AlternateDate"] @@ -462,7 +463,7 @@ def test_a_model_without_power_bi_specifics_has_no_stash(): }, } osi = yaml.safe_load(convert_semantic_model_to_ossie(bim)) - dataset = osi["semantic_model"][0]["datasets"][0] + dataset = osi["datasets"][0] assert "custom_extensions" not in dataset assert "custom_extensions" not in dataset["fields"][0] @@ -545,7 +546,7 @@ def annotated(name, value): } document = build_ossie_document(bim) - model = document["semantic_model"][0] + model = document dataset = _dataset(model, "Orders") field = _field(dataset, "CustomerId") metric = _metric(model, "Order Count") @@ -652,7 +653,7 @@ def _single_field_datatype(tmsl_type, format_string=None): column["formatString"] = format_string bim = {"name": "m", "model": {"tables": [{"name": "T", "columns": [column]}]}} osi = yaml.safe_load(convert_semantic_model_to_ossie(bim)) - return osi["semantic_model"][0]["datasets"][0]["fields"][0].get("datatype") + return osi["datasets"][0]["fields"][0].get("datatype") # --- lossy steps are reported ---------------------------------------------- @@ -694,7 +695,7 @@ def test_a_measure_without_an_expression_is_preserved_exactly(): } with pytest.warns(UserWarning, match="no expression"): osi = build_ossie_document(bim) - model = osi["semantic_model"][0] + model = osi assert read_stash(model)["excludedMeasures"] == [ {"table": "T", "measure": measure, "index": 1} ] @@ -732,7 +733,7 @@ def test_an_excluded_measure_with_a_missing_home_table_warns(): } with pytest.warns(UserWarning, match="no expression"): osi = build_ossie_document(bim) - osi["semantic_model"][0]["datasets"] = [] + osi["datasets"] = [] with pytest.warns(UserWarning, match="home table 'Gone' is missing"): out = convert_ossie_to_semantic_model(osi) @@ -757,7 +758,7 @@ def test_an_authored_metric_wins_over_an_excluded_measure_collision(): osi = build_ossie_document(bim) metric = {"name": "M", "expression": make_expression("1", "DAX")} write_stash(metric, {"table": "T"}) - osi["semantic_model"][0]["metrics"] = [metric] + osi["metrics"] = [metric] out = convert_ossie_to_semantic_model(osi) assert out["model"]["tables"][0]["measures"] == [{"name": "M", "expression": "1"}] diff --git a/converters/microsoft/tests/test_tom_integration.py b/converters/microsoft/tests/test_tom_integration.py index 2e7c0018..c7f94f96 100644 --- a/converters/microsoft/tests/test_tom_integration.py +++ b/converters/microsoft/tests/test_tom_integration.py @@ -84,8 +84,8 @@ def test_a_tmdl_document_imports_back_to_an_equivalent_ossie_model(monkeypatch): ) from_tmdl = yaml.safe_load(convert_semantic_model_to_ossie(tmdl)) - expected = from_tmsl["semantic_model"][0] - received = from_tmdl["semantic_model"][0] + expected = from_tmsl + received = from_tmdl assert received["name"] == expected["name"] assert received["description"] == expected["description"] assert [d["name"] for d in received["datasets"]] == [ diff --git a/converters/nvidia/README.md b/converters/nvidia/README.md index 3c564d3b..1351c587 100644 --- a/converters/nvidia/README.md +++ b/converters/nvidia/README.md @@ -23,6 +23,11 @@ Offline conversion between Apache Ossie YAML and NVIDIA GSF's native `GsfModelDocument` YAML contract. Conversion itself does not require GSF, Neo4j, a database, or network access. +Ossie input and output use one model per document, with `name`, `datasets`, +`relationships`, and `metrics` directly at the root beside `version`. Migrate +legacy `semantic_model` wrappers before conversion; see the +[format migration guidance](../../core-spec/spec.md#migrating-earlier-document-shapes). + ## Mapping | Apache Ossie | Native GSF model document | diff --git a/converters/nvidia/src/ossie_nvidia_gsf/native_converter.py b/converters/nvidia/src/ossie_nvidia_gsf/native_converter.py index f2c4fd7e..3cc13992 100644 --- a/converters/nvidia/src/ossie_nvidia_gsf/native_converter.py +++ b/converters/nvidia/src/ossie_nvidia_gsf/native_converter.py @@ -239,7 +239,7 @@ def convert_ossie_to_gsf( database_name: str | None = None, ) -> str: """Convert one Apache Ossie model to a native ``GsfModelDocument``.""" - _, model = _parse_ossie(ossie_yaml) + model = _parse_ossie(ossie_yaml) source_datasets = model.get("datasets") or [] if not isinstance(source_datasets, list) or not source_datasets: raise GSFConversionError( @@ -773,7 +773,7 @@ def convert_gsf_to_ossie( semantic_model["relationships"] = relationships if metrics: semantic_model["metrics"] = metrics - return _dump_yaml({"version": OSSIE_VERSION, "semantic_model": [semantic_model]}) + return _dump_yaml({"version": OSSIE_VERSION, **semantic_model}) def _build_catalog( @@ -1366,21 +1366,24 @@ def _parse_gsf(value: str) -> dict[str, Any]: return root -def _parse_ossie(value: str) -> tuple[dict[str, Any], dict[str, Any]]: +def _parse_ossie(value: str) -> dict[str, Any]: root = _load_yaml(value, "Ossie") - unknown = sorted(set(root) - {"version", "semantic_model"}) + if "semantic_model" in root: + raise GSFConversionError( + "Ossie model properties must be at the root; semantic_model wrappers are not supported" + ) + unknown = sorted(set(root) - { + "version", "name", "description", "ai_context", + "datasets", "relationships", "metrics", "custom_extensions", + }) if unknown: raise GSFConversionError( "Unsupported Ossie root properties: " + ", ".join(unknown) ) _check_ossie_version(root.get("version")) - models = root.get("semantic_model") - if not isinstance(models, list) or len(models) != 1: - raise GSFConversionError("Ossie input must contain exactly one semantic model") - model = models[0] - if not isinstance(model, dict) or not model.get("name"): + if not isinstance(root.get("name"), str) or not root["name"]: raise GSFConversionError("Ossie semantic model requires a name") - return root, model + return root def _check_ossie_version(value: Any) -> None: diff --git a/converters/nvidia/tests/fixtures/sales.ossie.yaml b/converters/nvidia/tests/fixtures/sales.ossie.yaml index 4e0bd6f2..a5c245eb 100644 --- a/converters/nvidia/tests/fixtures/sales.ossie.yaml +++ b/converters/nvidia/tests/fixtures/sales.ossie.yaml @@ -1,80 +1,79 @@ version: 0.2.0.dev0 -semantic_model: -- name: sales - description: Sales model +name: sales +description: Sales model +ai_context: + instructions: Use approved metrics +datasets: +- name: orders + source: analytics.public.orders + primary_key: + - order_id + description: Orders ai_context: - instructions: Use approved metrics - datasets: - - name: orders - source: analytics.public.orders - primary_key: - - order_id - description: Orders - ai_context: - synonyms: - - purchases - fields: - - name: order_id - expression: - dialects: - - dialect: ANSI_SQL - expression: order_id - dimension: - is_time: false - - name: customer_id - expression: - dialects: - - dialect: ANSI_SQL - expression: customer_id - dimension: - is_time: false - - name: order_date - expression: - dialects: - - dialect: ANSI_SQL - expression: order_date - dimension: - is_time: true - - name: net_total - expression: - dialects: - - dialect: ANSI_SQL - expression: subtotal - discount - dimension: - is_time: false - - name: customers - source: analytics.public.customers - primary_key: - - customer_id - fields: - - name: customer_id - expression: - dialects: - - dialect: ANSI_SQL - expression: customer_id - dimension: - is_time: false - - name: customer_name - expression: - dialects: - - dialect: ANSI_SQL - expression: name - dimension: - is_time: false - relationships: - - name: orders_to_customers - from: orders - to: customers - from_columns: - - customer_id - to_columns: - - customer_id - metrics: - - name: revenue_per_customer - description: Revenue per customer + synonyms: + - purchases + fields: + - name: order_id expression: dialects: - dialect: ANSI_SQL - expression: SUM(orders.subtotal) / COUNT(DISTINCT customers.customer_id) - - dialect: SNOWFLAKE - expression: SUM(orders.subtotal)::NUMBER / COUNT(DISTINCT customers.customer_id) + expression: order_id + dimension: + is_time: false + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + dimension: + is_time: false + - name: order_date + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date + dimension: + is_time: true + - name: net_total + expression: + dialects: + - dialect: ANSI_SQL + expression: subtotal - discount + dimension: + is_time: false +- name: customers + source: analytics.public.customers + primary_key: + - customer_id + fields: + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + dimension: + is_time: false + - name: customer_name + expression: + dialects: + - dialect: ANSI_SQL + expression: name + dimension: + is_time: false +relationships: +- name: orders_to_customers + from: orders + to: customers + from_columns: + - customer_id + to_columns: + - customer_id +metrics: +- name: revenue_per_customer + description: Revenue per customer + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.subtotal) / COUNT(DISTINCT customers.customer_id) + - dialect: SNOWFLAKE + expression: SUM(orders.subtotal)::NUMBER / COUNT(DISTINCT customers.customer_id) diff --git a/converters/nvidia/tests/test_converter.py b/converters/nvidia/tests/test_converter.py index 70fd5e1c..21a8a2cf 100644 --- a/converters/nvidia/tests/test_converter.py +++ b/converters/nvidia/tests/test_converter.py @@ -144,7 +144,7 @@ def test_generated_ossie_passes_official_validation(tmp_path: Path) -> None: def test_round_trip_preserves_ossie_semantics_and_global_metrics() -> None: result = yaml.safe_load(convert_gsf_to_ossie(convert_ossie_to_gsf(_ossie_yaml()))) - model = result["semantic_model"][0] + model = result datasets = {dataset["name"]: dataset for dataset in model["datasets"]} order_fields = {field["name"]: field for field in datasets["orders"]["fields"]} @@ -167,7 +167,7 @@ def test_round_trip_preserves_ossie_semantics_and_global_metrics() -> None: def test_edited_ossie_expressions_replace_preserved_native_sql() -> None: ossie = yaml.safe_load(convert_gsf_to_ossie(_gsf_yaml())) - model = ossie["semantic_model"][0] + model = ossie orders = next( dataset for dataset in model["datasets"] if dataset["name"] == "orders" ) @@ -224,7 +224,7 @@ def test_native_round_trip_preserves_ids_catalog_sql_source_and_zones() -> None: native["semantic_layer"]["sql_attributes"]["manual"] = [] ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) - assert _native_extension(ossie["semantic_model"][0])["native_document"] == native + assert _native_extension(ossie)["native_document"] == native restored = yaml.safe_load( convert_ossie_to_gsf(yaml.safe_dump(ossie, sort_keys=False)) @@ -250,7 +250,7 @@ def test_native_round_trip_preserves_ids_catalog_sql_source_and_zones() -> None: def test_relationship_edits_replace_preserved_native_records() -> None: ossie = yaml.safe_load(convert_gsf_to_ossie(_gsf_yaml())) - relationship = ossie["semantic_model"][0]["relationships"][0] + relationship = ossie["relationships"][0] relationship["from_columns"] = ["order_id"] regenerated = yaml.safe_load( @@ -275,7 +275,7 @@ def test_relationship_edits_replace_preserved_native_records() -> None: def test_relationship_deletion_removes_preserved_native_records() -> None: ossie = yaml.safe_load(convert_gsf_to_ossie(_gsf_yaml())) - ossie["semantic_model"][0].pop("relationships") + ossie.pop("relationships") regenerated = yaml.safe_load( convert_ossie_to_gsf(yaml.safe_dump(ossie, sort_keys=False)) @@ -326,7 +326,7 @@ def test_relationship_reconciliation_preserves_catalog_only_records() -> None: native["data_layer"]["foreign_keys"].append(audit_fk) ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) - ossie["semantic_model"][0].pop("relationships") + ossie.pop("relationships") regenerated = yaml.safe_load( convert_ossie_to_gsf(yaml.safe_dump(ossie, sort_keys=False)) ) @@ -337,7 +337,7 @@ def test_relationship_reconciliation_preserves_catalog_only_records() -> None: def test_multiple_databases_are_supported_and_name_falls_back() -> None: ossie = yaml.safe_load(_ossie_yaml()) - model = ossie["semantic_model"][0] + model = ossie model["datasets"][1]["source"] = "crm.public.customers" model["relationships"] = [] model["metrics"] = [] @@ -353,14 +353,14 @@ def test_multiple_databases_are_supported_and_name_falls_back() -> None: assert database_names == {"analytics", "crm"} assert len(native["data_layer"]["databases"]) == 2 - assert restored["semantic_model"][0]["name"] == "gsf_model" + assert restored["name"] == "gsf_model" def test_shared_physical_source_uses_one_catalog_table_and_valid_ossie( tmp_path: Path, ) -> None: ossie = yaml.safe_load(_ossie_yaml()) - model = ossie["semantic_model"][0] + model = ossie model["datasets"].append( { "name": "order_amounts", @@ -400,7 +400,7 @@ def test_shared_physical_source_uses_one_catalog_table_and_valid_ossie( restored = yaml.safe_load(convert_gsf_to_ossie(native_yaml)) assert { - dataset["name"] for dataset in restored["semantic_model"][0]["datasets"] + dataset["name"] for dataset in restored["datasets"] } >= {"orders", "order_amounts"} output_path = tmp_path / "shared-source.ossie.yaml" output_path.write_text(yaml.safe_dump(restored), encoding="utf-8") @@ -415,7 +415,7 @@ def test_shared_physical_source_uses_one_catalog_table_and_valid_ossie( def test_cross_database_ossie_metric_is_rejected() -> None: ossie = yaml.safe_load(_ossie_yaml()) - ossie["semantic_model"][0]["datasets"][1]["source"] = "crm.public.customers" + ossie["datasets"][1]["source"] = "crm.public.customers" with pytest.raises(GSFConversionError, match="spans multiple databases"): convert_ossie_to_gsf(yaml.safe_dump(ossie)) @@ -423,7 +423,7 @@ def test_cross_database_ossie_metric_is_rejected() -> None: def test_cross_database_full_query_field_is_rejected() -> None: ossie = yaml.safe_load(_ossie_yaml()) - model = ossie["semantic_model"][0] + model = ossie model["datasets"][1]["source"] = "crm.public.customers" model["metrics"] = [] model["relationships"] = [] @@ -451,7 +451,7 @@ def test_cross_database_full_query_field_is_rejected() -> None: @pytest.mark.parametrize("kind", ["sql_attribute", "custom_analysis"]) def test_cross_database_gsf_sql_objects_are_rejected(kind: str) -> None: ossie = yaml.safe_load(_ossie_yaml()) - model = ossie["semantic_model"][0] + model = ossie model["datasets"][1]["source"] = "crm.public.customers" model["metrics"] = [] model["relationships"] = [] @@ -512,8 +512,8 @@ def test_relationships_emit_join_physical_fk_and_semantic_fk() -> None: native["data_layer"]["joins"] = [] restored = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) - assert restored["semantic_model"][0]["relationships"][0]["from"] == "orders" - assert restored["semantic_model"][0]["relationships"][0]["to"] == "customers" + assert restored["relationships"][0]["from"] == "orders" + assert restored["relationships"][0]["to"] == "customers" def test_gsf_requires_one_represented_table_per_term() -> None: @@ -565,7 +565,7 @@ def test_date_part_keywords_do_not_become_catalog_columns( unit: str, ) -> None: ossie = yaml.safe_load(_ossie_yaml()) - ossie["semantic_model"][0]["datasets"][0]["fields"].append( + ossie["datasets"][0]["fields"].append( { "name": "order_age", "expression": { @@ -684,7 +684,7 @@ def test_columns_named_like_units_survive_outside_the_unit_slot( ) -> None: """Only the unit argument itself is treated as a keyword.""" ossie = yaml.safe_load(_ossie_yaml()) - ossie["semantic_model"][0]["datasets"][0]["fields"].append( + ossie["datasets"][0]["fields"].append( { "name": "order_age", "expression": { @@ -732,7 +732,7 @@ def test_expression_dialect_follows_the_gsf_connection() -> None: ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) orders = next( dataset - for dataset in ossie["semantic_model"][0]["datasets"] + for dataset in ossie["datasets"] if dataset["name"] == "orders" ) dialects = { @@ -752,7 +752,7 @@ def test_dialects_ossie_cannot_name_stay_ansi() -> None: ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) orders = next( dataset - for dataset in ossie["semantic_model"][0]["datasets"] + for dataset in ossie["datasets"] if dataset["name"] == "orders" ) net_total = next( @@ -767,7 +767,7 @@ def test_every_mappable_datatype_survives_a_round_trip(datatype: str) -> None: ossie = yaml.safe_load(_ossie_yaml()) orders = next( dataset - for dataset in ossie["semantic_model"][0]["datasets"] + for dataset in ossie["datasets"] if dataset["name"] == "orders" ) next(field for field in orders["fields"] if field["name"] == "order_id")[ @@ -778,7 +778,7 @@ def test_every_mappable_datatype_survives_a_round_trip(datatype: str) -> None: restored = yaml.safe_load(convert_gsf_to_ossie(native)) field = next( item - for dataset in restored["semantic_model"][0]["datasets"] + for dataset in restored["datasets"] if dataset["name"] == "orders" for item in dataset["fields"] if item["name"] == "order_id" @@ -842,7 +842,7 @@ def test_gsf_column_types_reach_the_ossie_field() -> None: ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) fields = { field["name"]: field.get("datatype") - for dataset in ossie["semantic_model"][0]["datasets"] + for dataset in ossie["datasets"] if dataset["name"] == "orders" for field in dataset["fields"] } @@ -870,7 +870,7 @@ def test_a_live_gsf_column_type_outranks_an_ossie_datatype() -> None: ossie = yaml.safe_load(convert_gsf_to_ossie(yaml.safe_dump(native))) next( field - for dataset in ossie["semantic_model"][0]["datasets"] + for dataset in ossie["datasets"] if dataset["name"] == "orders" for field in dataset["fields"] if field["name"] == "order_id" @@ -902,7 +902,7 @@ def test_old_fictional_gsf_root_is_rejected() -> None: def test_model_name_override() -> None: result = yaml.safe_load(convert_gsf_to_ossie(_gsf_yaml(), model_name="sales")) - assert result["semantic_model"][0]["name"] == "sales" + assert result["name"] == "sales" @pytest.mark.parametrize( @@ -966,4 +966,22 @@ def test_cli_converts_native_files( main(["import", "-i", str(gsf_path), "--name", "sales"]) output = yaml.safe_load(capsys.readouterr().out) assert output["version"] == OSSIE_VERSION - assert output["semantic_model"][0]["name"] == "sales" + assert output["name"] == "sales" + + +@pytest.mark.parametrize("wrapper", [None, [], {}, [{"name": "old", "datasets": []}]]) +def test_rejects_legacy_wrapper_even_with_root_model(wrapper: Any) -> None: + document = yaml.safe_load(_ossie_yaml()) + document["semantic_model"] = wrapper + + with pytest.raises(GSFConversionError, match="at the root"): + convert_ossie_to_gsf(yaml.safe_dump(document)) + + +@pytest.mark.parametrize("property_name", ["dialects", "vendors"]) +def test_rejects_removed_root_metadata(property_name: str) -> None: + document = yaml.safe_load(_ossie_yaml()) + document[property_name] = [] + + with pytest.raises(GSFConversionError, match="Unsupported Ossie root properties"): + convert_ossie_to_gsf(yaml.safe_dump(document)) diff --git a/converters/omni/README.md b/converters/omni/README.md index a4f2a57d..bfbcf634 100644 --- a/converters/omni/README.md +++ b/converters/omni/README.md @@ -58,6 +58,10 @@ topic curation, the model file, ...) are instead **preserved** in [requirement](#requirements) **raises a `ConversionError`** -- the converter never silently drops a field or produces an invalid result. +Ossie documents contain one model directly at the root, with `version`, `name`, +`datasets`, and optional model properties. Legacy `semantic_model` wrappers +(arrays or objects) are rejected. + ## Installation ```bash @@ -100,7 +104,7 @@ specific to **export** (Ossie -> Omni) or **import** (Omni -> Ossie). | Ossie | Omni | Notes | |---|---|---| -| `semantic_model.name` | topic file name | Import: the mapped topic's name (override with `--name`). | +| `name` | topic file name | Import: the mapped topic's name (override with `--name`). | | `model.description` / `ai_context.instructions` | topic `description` / `ai_context` | Import: taken from the sole topic, or `--topic`. | | dataset | `views/.view.yaml` | Import: a stashed original path (`DELIGHTED/response.view`) is restored on export. | | `dataset.source` `catalog.schema.table` / `schema.table` | view `catalog` + `schema` + `table_name` | `table_name` left implicit when it matches the file name; a part that is not a plain identifier is double-quoted (`"Omni Views".upload`). | diff --git a/converters/omni/src/ossie_omni/omni_to_ossie.py b/converters/omni/src/ossie_omni/omni_to_ossie.py index 97b6ed62..7970675d 100644 --- a/converters/omni/src/ossie_omni/omni_to_ossie.py +++ b/converters/omni/src/ossie_omni/omni_to_ossie.py @@ -282,7 +282,7 @@ def convert_omni_to_ossie(files, model_name=None, topic=None): stash["extra_files"] = extra_files write_stash(model, stash) - return dump_yaml({"version": OSSIE_VERSION, "semantic_model": [model]}) + return dump_yaml({"version": OSSIE_VERSION, **model}) def _convert_view(vname, view, meta): diff --git a/converters/omni/src/ossie_omni/ossie_to_omni.py b/converters/omni/src/ossie_omni/ossie_to_omni.py index 43ca7e58..415d2eaa 100644 --- a/converters/omni/src/ossie_omni/ossie_to_omni.py +++ b/converters/omni/src/ossie_omni/ossie_to_omni.py @@ -103,13 +103,18 @@ def convert_ossie_to_omni(ossie_yaml_str, base_view=None, dialect=None): f"Unsupported Ossie version '{version}'. Supported: {OSSIE_VERSION}" ) - models = root.get("semantic_model") - if not isinstance(models, list) or not models: - raise ConversionError("'semantic_model' must be a non-empty list") - if len(models) > 1: - _warn("model", "multiple semantic models found; converting only the first") + if "semantic_model" in root: + raise ConversionError( + "Legacy 'semantic_model' wrappers are not supported; " + "place the model properties directly at the document root" + ) + + if "dialects" in root or "vendors" in root: + raise ConversionError("Root dialects and vendors are not supported by the Ossie spec") + if not isinstance(root.get("name"), str): + raise ConversionError("Ossie model requires a string 'name' at the document root") - return _convert_model(models[0], base_view, dialect) + return _convert_model(root, base_view, dialect) def _convert_model(model, explicit_base_view, dialect): diff --git a/converters/omni/tests/_roundtrip_helpers.py b/converters/omni/tests/_roundtrip_helpers.py index 674fe56c..977bf13d 100644 --- a/converters/omni/tests/_roundtrip_helpers.py +++ b/converters/omni/tests/_roundtrip_helpers.py @@ -349,7 +349,7 @@ def build_ossie(rnd): model["relationships"] = relationships if metrics: model["metrics"] = metrics - return {"version": OSSIE_VERSION, "semantic_model": [model]} + return {"version": OSSIE_VERSION, **model} # --- Round-trip assertions ------------------------------------------------------- diff --git a/converters/omni/tests/_util.py b/converters/omni/tests/_util.py index 2e1418a4..b9d808ea 100644 --- a/converters/omni/tests/_util.py +++ b/converters/omni/tests/_util.py @@ -88,58 +88,58 @@ def strip_normalized(ossie): dimension on export, so it comes back as an extra (stash-only) field. """ ossie = copy.deepcopy(ossie) - for model in ossie.get("semantic_model", []): - model.pop("custom_extensions", None) - ai = model.get("ai_context") + model = ossie + model.pop("custom_extensions", None) + ai = model.get("ai_context") + if isinstance(ai, dict): + ai.pop("synonyms", None) + ai.pop("examples", None) + if not ai: + model.pop("ai_context") + for ds in model.get("datasets", []): + ds.pop("custom_extensions", None) + ds.pop("unique_keys", None) + ai = ds.get("ai_context") if isinstance(ai, dict): ai.pop("synonyms", None) - ai.pop("examples", None) if not ai: - model.pop("ai_context") - for ds in model.get("datasets", []): - ds.pop("custom_extensions", None) - ds.pop("unique_keys", None) - ai = ds.get("ai_context") - if isinstance(ai, dict): - ai.pop("synonyms", None) - if not ai: - ds.pop("ai_context") - fields = ds.get("fields", []) or [] - for field in fields: - field.pop("custom_extensions", None) - dimension = field.get("dimension") - is_time = False - if dimension is not None: - is_time_explicit = dimension.get("is_time") - if is_time_explicit is not None: - is_time = bool(is_time_explicit) - else: - is_time = field.get("datatype") in ("Date", "Time", "DateTime", "DateTimez") - field.pop("datatype", None) - if is_time: - field["dimension"] = {"is_time": True} + ds.pop("ai_context") + fields = ds.get("fields", []) or [] + for field in fields: + field.pop("custom_extensions", None) + dimension = field.get("dimension") + is_time = False + if dimension is not None: + is_time_explicit = dimension.get("is_time") + if is_time_explicit is not None: + is_time = bool(is_time_explicit) else: - field.pop("dimension", None) - # Drop backfilled key fields: a bare-column field named after a - # primary_key column, carrying nothing but its expression. - pk = set(ds.get("primary_key") or []) - ds_fields = [ - f for f in fields - if not (f["name"] in pk and set(f) <= {"name", "expression"}) - ] - if ds_fields: - ds["fields"] = ds_fields + is_time = field.get("datatype") in ("Date", "Time", "DateTime", "DateTimez") + field.pop("datatype", None) + if is_time: + field["dimension"] = {"is_time": True} else: - ds.pop("fields", None) - for rel in model.get("relationships", []) or []: - rel["name"] = f"{rel['from']}_to_{rel['to']}" - rel.pop("ai_context", None) - rel.pop("custom_extensions", None) - for metric in model.get("metrics", []) or []: - metric.pop("custom_extensions", None) - metric.pop("datatype", None) - # Metric order is not semantic; the import regroups metrics by the view - # their measure lives on. - if model.get("metrics"): - model["metrics"].sort(key=lambda m: m["name"]) + field.pop("dimension", None) + # Drop backfilled key fields: a bare-column field named after a + # primary_key column, carrying nothing but its expression. + pk = set(ds.get("primary_key") or []) + ds_fields = [ + f for f in fields + if not (f["name"] in pk and set(f) <= {"name", "expression"}) + ] + if ds_fields: + ds["fields"] = ds_fields + else: + ds.pop("fields", None) + for rel in model.get("relationships", []) or []: + rel["name"] = f"{rel['from']}_to_{rel['to']}" + rel.pop("ai_context", None) + rel.pop("custom_extensions", None) + for metric in model.get("metrics", []) or []: + metric.pop("custom_extensions", None) + metric.pop("datatype", None) + # Metric order is not semantic; the import regroups metrics by the view + # their measure lives on. + if model.get("metrics"): + model["metrics"].sort(key=lambda m: m["name"]) return ossie diff --git a/converters/omni/tests/fixtures/fixtureA_ossie.yaml b/converters/omni/tests/fixtures/fixtureA_ossie.yaml index 5800db47..d9e7febe 100644 --- a/converters/omni/tests/fixtures/fixtureA_ossie.yaml +++ b/converters/omni/tests/fixtures/fixtureA_ossie.yaml @@ -25,76 +25,75 @@ version: "0.2.0.dev0" -semantic_model: - - name: sales - description: Sales orders with customer attributes - ai_context: - instructions: "Use this model for order analysis." - datasets: - - name: orders # fact: no incoming relationship -> topic base_view - source: samples.tpch.orders - primary_key: [o_orderkey] - description: One row per order - fields: - - name: o_orderkey - expression: - dialects: - - dialect: ANSI_SQL - expression: o_orderkey - description: Order identifier - - name: o_custkey - expression: - dialects: - - dialect: ANSI_SQL - expression: o_custkey - - name: o_orderdate - expression: - dialects: - - dialect: ANSI_SQL - expression: o_orderdate - label: Order Date - dimension: - is_time: true - ai_context: - synonyms: [order date, date] - - name: o_totalprice - expression: - dialects: - - dialect: ANSI_SQL - expression: o_totalprice - - name: customer - source: samples.tpch.customer - primary_key: [c_custkey] - fields: - - name: c_custkey - expression: - dialects: - - dialect: ANSI_SQL - expression: c_custkey - - name: c_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_name - description: Customer name - relationships: - - name: orders_to_customer - from: orders - to: customer - from_columns: [o_custkey] - to_columns: [c_custkey] - metrics: - - name: total_revenue +name: sales +description: Sales orders with customer attributes +ai_context: + instructions: "Use this model for order analysis." +datasets: + - name: orders # fact: no incoming relationship -> topic base_view + source: samples.tpch.orders + primary_key: [o_orderkey] + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: ANSI_SQL + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: ANSI_SQL + expression: o_custkey + - name: o_orderdate expression: dialects: - dialect: ANSI_SQL - expression: SUM(orders.o_totalprice) - description: Total order revenue + expression: o_orderdate + label: Order Date + dimension: + is_time: true ai_context: - synonyms: [revenue, sales] - - name: order_count + synonyms: [order date, date] + - name: o_totalprice expression: dialects: - dialect: ANSI_SQL - expression: COUNT(*) - description: Number of orders + expression: o_totalprice + - name: customer + source: samples.tpch.customer + primary_key: [c_custkey] + fields: + - name: c_custkey + expression: + dialects: + - dialect: ANSI_SQL + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_name + description: Customer name +relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: [o_custkey] + to_columns: [c_custkey] +metrics: + - name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.o_totalprice) + description: Total order revenue + ai_context: + synonyms: [revenue, sales] + - name: order_count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(*) + description: Number of orders diff --git a/converters/omni/tests/test_omni_to_ossie.py b/converters/omni/tests/test_omni_to_ossie.py index c2795246..7b522743 100644 --- a/converters/omni/tests/test_omni_to_ossie.py +++ b/converters/omni/tests/test_omni_to_ossie.py @@ -30,7 +30,7 @@ def imp(files, **kwargs): with warnings.catch_warnings(): warnings.simplefilter("ignore") - return parse(convert_omni_to_ossie(files, **kwargs))["semantic_model"][0] + return parse(convert_omni_to_ossie(files, **kwargs)) def minimal_files(**view_overrides): @@ -381,7 +381,7 @@ def test_query_view_preserved_as_extra_file(): files["views/facts.query.view.yaml"] = "schema: s\nsql: SELECT 1\n" with warnings.catch_warnings(record=True) as ws: warnings.simplefilter("always") - model = parse(convert_omni_to_ossie(files))["semantic_model"][0] + model = parse(convert_omni_to_ossie(files)) assert any("query views" in str(w.message) for w in ws) assert "views/facts.query.view.yaml" in stash_of(model)["extra_files"] diff --git a/converters/omni/tests/test_ossie_to_omni.py b/converters/omni/tests/test_ossie_to_omni.py index d6de14b1..a3c8a53c 100644 --- a/converters/omni/tests/test_ossie_to_omni.py +++ b/converters/omni/tests/test_ossie_to_omni.py @@ -39,7 +39,7 @@ def minimal(**model_overrides): "fields": [_field("amount")]}], } model.update(model_overrides) - return dump_yaml({"version": "0.2.0.dev0", "semantic_model": [model]}) + return dump_yaml({"version": "0.2.0.dev0", **model}) def _field(name, expr=None, dialect="ANSI_SQL", **extra): @@ -251,10 +251,9 @@ def test_dialect_preference(): def test_names_are_sanitized(): - files = export(dump_yaml({"version": "0.2.0.dev0", "semantic_model": [{ - "name": "My Model", + files = export(dump_yaml({"version": "0.2.0.dev0", "name": "My Model", "datasets": [{"name": "Order Items", "source": "db.sch.t", - "fields": [_field("Total Price", "p")]}]}]})) + "fields": [_field("Total Price", "p")]}]})) assert "views/order_items.view.yaml" in files assert "topics/my_model.topic.yaml" in files dims = parse(files["views/order_items.view.yaml"])["dimensions"] @@ -293,7 +292,7 @@ def test_unknown_relationship_dataset_rejected(): def test_unsupported_version_rejected(): with pytest.raises(ConversionError, match="Unsupported Ossie version"): - export(dump_yaml({"version": "9.9.9", "semantic_model": [{"name": "m"}]})) + export(dump_yaml({"version": "9.9.9", "name": "m"})) # --- warnings --------------------------------------------------------------- @@ -343,9 +342,31 @@ def test_foreign_vendor_extensions_warn(): assert any("foreign-vendor" in m for m in msgs) -def test_multiple_models_warn_and_first_converts(): - ossie = parse(minimal()) - ossie["semantic_model"].append({"name": "second", "datasets": [ - {"name": "x", "source": "db.s.x"}]}) - msgs = _warnings_of(dump_yaml(ossie)) - assert any("multiple semantic models" in m for m in msgs) +@pytest.mark.parametrize( + "wrapper", + [[], [{"name": "first"}], [{"name": "first"}, {"name": "second"}], {"name": "first"}, None], +) +def test_legacy_model_wrappers_are_rejected(wrapper): + document = {"version": "0.2.0.dev0", "semantic_model": wrapper} + with pytest.raises(ConversionError, match="Legacy 'semantic_model'"): + export(dump_yaml(document)) + + +@pytest.mark.parametrize("property_name", ["dialects", "vendors"]) +@pytest.mark.parametrize("value", [None, [], ["legacy"]]) +def test_removed_root_metadata_is_rejected(property_name, value): + with pytest.raises(ConversionError, match="Root dialects and vendors"): + export(minimal(**{property_name: value})) + + +@pytest.mark.parametrize( + "name_properties", + [{}, {"name": None}, {"name": 123}, {"name": True}, {"name": []}, {"name": {}}], + ids=["missing", "null", "number", "boolean", "list", "object"], +) +def test_root_name_must_be_a_string(name_properties): + document = parse(minimal()) + del document["name"] + document.update(name_properties) + with pytest.raises(ConversionError, match="string 'name' at the document root"): + export(dump_yaml(document)) diff --git a/converters/omni/tests/test_real_world_layout.py b/converters/omni/tests/test_real_world_layout.py index d170437a..fd1c03cc 100644 --- a/converters/omni/tests/test_real_world_layout.py +++ b/converters/omni/tests/test_real_world_layout.py @@ -144,7 +144,7 @@ def test_api_layout_roundtrip_is_lossless(): def test_qualified_view_names_come_from_reference_comment(): with warnings.catch_warnings(): warnings.simplefilter("ignore") - ossie = load_yaml(convert_omni_to_ossie(REAL_FILES))["semantic_model"][0] + ossie = load_yaml(convert_omni_to_ossie(REAL_FILES)) names = {d["name"] for d in ossie["datasets"]} assert "delighted__response" in names assert "upload" in names # comment name wins even with a folder @@ -164,7 +164,7 @@ def test_qualified_view_names_come_from_reference_comment(): def test_unmappable_omni_features_drop_out_of_ossie_but_survive(): with warnings.catch_warnings(): warnings.simplefilter("ignore") - ossie = load_yaml(convert_omni_to_ossie(REAL_FILES))["semantic_model"][0] + ossie = load_yaml(convert_omni_to_ossie(REAL_FILES)) # The extends view is not a dataset; the templated dimension not a field. assert "delighted__response_ext" not in {d["name"] for d in ossie["datasets"]} response = next(d for d in ossie["datasets"] diff --git a/converters/ontology/src/ossie_ontology/spec.py b/converters/ontology/src/ossie_ontology/spec.py index 377c2821..e61ce52a 100644 --- a/converters/ontology/src/ossie_ontology/spec.py +++ b/converters/ontology/src/ossie_ontology/spec.py @@ -175,9 +175,11 @@ class Metric(OssieObject): class SemanticModel(OssieObject): - """A complete logical/semantic model (the body that the core spec calls - `semantic_model`): datasets plus the join paths and metrics defined over - them. One or more SemanticModels can feed a single OntologyMapping.""" + """Model contents: datasets plus the join paths and metrics defined over them. + + OntologyMapping embeds these contents under ``semantic_model``; standalone + core documents place them at the root alongside document metadata. + """ name: str description: str | None = None ai_context: AiContext | None = None diff --git a/converters/orionbelt/README.md b/converters/orionbelt/README.md index 122f1f63..15e590b8 100644 --- a/converters/orionbelt/README.md +++ b/converters/orionbelt/README.md @@ -31,6 +31,10 @@ developed in the repository (under `packages/ossie-orionbelt`) and published to PyPI from there; file issues and contributions upstream. +Ossie documents contain one model directly at the root, with `version`, `name`, +`datasets`, and optional model properties. Legacy `semantic_model` wrappers +(arrays or objects) are rejected. + ## Requirements - Python 3.12+ diff --git a/converters/orionbelt/ossie_obml_mapping_analysis.md b/converters/orionbelt/ossie_obml_mapping_analysis.md index 957e9cf0..08df96ce 100644 --- a/converters/orionbelt/ossie_obml_mapping_analysis.md +++ b/converters/orionbelt/ossie_obml_mapping_analysis.md @@ -25,7 +25,7 @@ | Aspect | Ossie v0.2.0.dev0 | OBML v1.0 | |---|---|---| -| **Top-level** | `semantic_model[]` (array of models) | Single model with `dataObjects`, `dimensions`, `measures`, `metrics` sections | +| **Top-level** | Single model at the root (`version`, `name`, `datasets`, ...) | Single model with `dataObjects`, `dimensions`, `measures`, `metrics` sections | | **Tables / Entities** | `datasets[]` (flat array) | `dataObjects{}` (named dictionary) | | **Column identifiers** | `fields[].name` (snake_case code) | `columns{}.code` (with display name as dict key) | | **Expressions** | `expression.dialects[]` per field (multi-dialect) | Single SQL expression via `code` (single dialect) | diff --git a/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md b/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md index 42768b05..8e4ebd0e 100644 --- a/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md +++ b/converters/orionbelt/ossie_obml_ontology_mapping_analysis.md @@ -68,7 +68,7 @@ ontology_mappings: | `join.joinType` | `Relationship.multiplicity` | `many-to-one`→`ManyToOne`, `one-to-one`→`OneToOne` | | `column.primaryKey` | entity `object_mappings[].expression` | `.`; identifies the entity | | `join.columnsFrom` (FK) | `link_mappings[].object_mapping.expression` | `.`; binds the relationship to its far role | -| whole core model | `ontology_mappings[].semantic_model` | embedded verbatim from `OBMLtoOssie.convert()` | +| whole core model | `ontology_mappings[].semantic_model` | embedded from `OBMLtoOssie.convert()` without standalone document metadata (`version`, `dialects`, `vendors`) | `
` is the final identifier of the dataset `source` (e.g. `db.schema.t` → `t`), falling back to the dataset name when `source` has no dotted physical table. diff --git a/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py b/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py index d46687eb..179acd1f 100644 --- a/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py +++ b/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py @@ -140,15 +140,10 @@ def convert(self) -> dict: self.obml.get("customExtensions"), sem_model["custom_extensions"] ) - ossie["semantic_model"] = [sem_model] - - # The published Ossie core schema forbids root-level ``dialects`` / - # ``vendors`` (root is additionalProperties:false, only ``version`` + - # ``semantic_model``). Dialects live per-expression in - # ``expression.dialects[]`` and vendors per-entity in - # ``custom_extensions[].vendor_name`` — the schema-valid homes — so the - # document stays fully conformant without root advertisement arrays. - # See Ossie PR #148 (and the single-document-dialect direction in #52). + ossie.update(sem_model) + + # Dialects and vendors are represented on the expressions and extensions + # that use them; root-level advertisement arrays are not supported. return ossie def _emit_foreign_extensions(self, obml_exts: list[dict] | None, ossie_exts: list[dict]) -> None: diff --git a/converters/orionbelt/src/ossie_orionbelt/ontology.py b/converters/orionbelt/src/ossie_orionbelt/ontology.py index d175946b..0dfac93c 100644 --- a/converters/orionbelt/src/ossie_orionbelt/ontology.py +++ b/converters/orionbelt/src/ossie_orionbelt/ontology.py @@ -105,7 +105,8 @@ def convert(self) -> dict: model_description=self.model_description, ai_instructions=self.ai_instructions, ) - sem_model = core_conv.convert()["semantic_model"][0] + sem_model = core_conv.convert() + sem_model.pop("version", None) self.warnings.extend(core_conv.warnings) data_objects = self.obml.get("dataObjects", {}) or {} diff --git a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py index 600d38cb..72bf28be 100644 --- a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py +++ b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py @@ -76,27 +76,23 @@ def _normalize_legacy_v01(self) -> None: if version and not version.startswith(("0.1", "0.0")): return # already v0.2+ (or future) — nothing to do - models = self.ossie.get("semantic_model", []) - if not isinstance(models, list): - return - - for model in models: - for ds in model.get("datasets", []) or []: - # Promote legacy primary_key / unique_keys from OBSL extras - # only if the dataset doesn't already declare them. - legacy = self._extract_obml_extras(ds) - if not legacy: - continue - if "primary_key" not in ds and legacy.get("obml_primary_key"): - pk = legacy["obml_primary_key"] - if isinstance(pk, list) and all(isinstance(c, str) for c in pk): - ds["primary_key"] = list(pk) - if "unique_keys" not in ds and legacy.get("obml_unique_keys"): - uk = legacy["obml_unique_keys"] - if isinstance(uk, list) and all( - isinstance(g, list) and all(isinstance(c, str) for c in g) for g in uk - ): - ds["unique_keys"] = [list(g) for g in uk] + model = self.ossie + for ds in model.get("datasets", []) or []: + # Promote legacy primary_key / unique_keys from OBSL extras + # only if the dataset doesn't already declare them. + legacy = self._extract_obml_extras(ds) + if not legacy: + continue + if "primary_key" not in ds and legacy.get("obml_primary_key"): + pk = legacy["obml_primary_key"] + if isinstance(pk, list) and all(isinstance(c, str) for c in pk): + ds["primary_key"] = list(pk) + if "unique_keys" not in ds and legacy.get("obml_unique_keys"): + uk = legacy["obml_unique_keys"] + if isinstance(uk, list) and all( + isinstance(g, list) and all(isinstance(c, str) for c in g) for g in uk + ): + ds["unique_keys"] = [list(g) for g in uk] if version.startswith(("0.0", "0.1")): self.warnings.append( @@ -111,21 +107,22 @@ def convert(self) -> dict: self.warnings = [] self._unconverted_metrics = [] - # v0.1.x inputs need the legacy shim to promote pre-v0.2 - # custom_extensions into v0.2 first-class fields before we parse. - self._normalize_legacy_v01() + if not isinstance(self.ossie, dict): + raise ValueError("Ossie input must be a mapping") + if "semantic_model" in self.ossie: + raise ValueError( + "Legacy 'semantic_model' wrappers are not supported; " + "place the model properties directly at the document root" + ) - models = self.ossie.get("semantic_model", []) - if not models: - raise ValueError("No semantic_model found in Ossie input") + if "dialects" in self.ossie or "vendors" in self.ossie: + raise ValueError("Root dialects and vendors are not supported by the Ossie spec") - # Take the first semantic model (OBML is a single-model format) - model = models[0] - if len(models) > 1: - self.warnings.append( - f"Ossie contains {len(models)} semantic models; " - f"only the first ('{model.get('name')}') is converted." - ) + # Retain legacy key metadata normalization for already flattened inputs. + self._normalize_legacy_v01() + model = self.ossie + if not model.get("name"): + raise ValueError("Ossie model requires a name at the document root") obml: dict[str, Any] = {"version": 1.0} diff --git a/converters/orionbelt/src/ossie_orionbelt/validation.py b/converters/orionbelt/src/ossie_orionbelt/validation.py index 0a814f2f..b4dbe6f1 100644 --- a/converters/orionbelt/src/ossie_orionbelt/validation.py +++ b/converters/orionbelt/src/ossie_orionbelt/validation.py @@ -212,74 +212,75 @@ def validate_ossie(ossie_dict: dict[str, Any], schema_path: Path | None = None) def _as_dict_list(value: Any) -> list[dict[str, Any]]: return [item for item in value if isinstance(item, dict)] if isinstance(value, list) else [] - models = _as_dict_list(ossie_dict.get("semantic_model", [])) + # Legacy wrappers are schema errors, not model contents to traverse. + if not isinstance(ossie_dict, dict) or "semantic_model" in ossie_dict: + return result + model = ossie_dict # 2. Unique name checks - for model in models: - model_name = model.get("name", "") - datasets = _as_dict_list(model.get("datasets", [])) - - # Unique dataset names - dataset_names: list[str] = [] - for ds in datasets: - name = ds.get("name", "") - if name in dataset_names: + model_name = model.get("name", "") + datasets = _as_dict_list(model.get("datasets", [])) + + # Unique dataset names + dataset_names: list[str] = [] + for ds in datasets: + name = ds.get("name", "") + if name in dataset_names: + result.semantic_errors.append( + f"[DUPLICATE_DATASET] Duplicate dataset name '{name}' in model '{model_name}'" + ) + dataset_names.append(name) + + # Unique field names within each dataset + for ds in datasets: + ds_name = ds.get("name", "") + field_names: list[str] = [] + for field in _as_dict_list(ds.get("fields", [])): + fname = field.get("name", "") + if fname in field_names: result.semantic_errors.append( - f"[DUPLICATE_DATASET] Duplicate dataset name '{name}' in model '{model_name}'" + f"[DUPLICATE_FIELD] Duplicate field name '{fname}' in dataset '{ds_name}'" ) - dataset_names.append(name) - - # Unique field names within each dataset - for ds in datasets: - ds_name = ds.get("name", "") - field_names: list[str] = [] - for field in _as_dict_list(ds.get("fields", [])): - fname = field.get("name", "") - if fname in field_names: - result.semantic_errors.append( - f"[DUPLICATE_FIELD] Duplicate field name '{fname}' in dataset '{ds_name}'" - ) - field_names.append(fname) - - # Unique metric names - metric_names: list[str] = [] - for m in _as_dict_list(model.get("metrics", [])): - mname = m.get("name", "") - if mname in metric_names: - result.semantic_errors.append( - f"[DUPLICATE_METRIC] Duplicate metric name '{mname}' in model '{model_name}'" - ) - metric_names.append(mname) - - # Unique relationship names - rel_names: list[str] = [] - for r in _as_dict_list(model.get("relationships", [])): - rname = r.get("name", "") - if rname in rel_names: - result.semantic_errors.append( - f"[DUPLICATE_RELATIONSHIP] Duplicate relationship name " - f"'{rname}' in model '{model_name}'" - ) - rel_names.append(rname) + field_names.append(fname) + + # Unique metric names + metric_names: list[str] = [] + for m in _as_dict_list(model.get("metrics", [])): + mname = m.get("name", "") + if mname in metric_names: + result.semantic_errors.append( + f"[DUPLICATE_METRIC] Duplicate metric name '{mname}' in model '{model_name}'" + ) + metric_names.append(mname) + + # Unique relationship names + rel_names: list[str] = [] + for r in _as_dict_list(model.get("relationships", [])): + rname = r.get("name", "") + if rname in rel_names: + result.semantic_errors.append( + f"[DUPLICATE_RELATIONSHIP] Duplicate relationship name " + f"'{rname}' in model '{model_name}'" + ) + rel_names.append(rname) # 3. Reference checks — relationships reference existing datasets - for model in models: - datasets = _as_dict_list(model.get("datasets", [])) - ds_name_set = {ds.get("name") for ds in datasets if ds.get("name")} - for rel in _as_dict_list(model.get("relationships", [])): - rel_name = rel.get("name", "") - from_ds = rel.get("from") - to_ds = rel.get("to") - if from_ds and from_ds not in ds_name_set: - result.semantic_errors.append( - f"[UNKNOWN_DATASET_REF] Relationship '{rel_name}' " - f"references unknown dataset '{from_ds}'" - ) - if to_ds and to_ds not in ds_name_set: - result.semantic_errors.append( - f"[UNKNOWN_DATASET_REF] Relationship '{rel_name}' " - f"references unknown dataset '{to_ds}'" - ) + datasets = _as_dict_list(model.get("datasets", [])) + ds_name_set = {ds.get("name") for ds in datasets if ds.get("name")} + for rel in _as_dict_list(model.get("relationships", [])): + rel_name = rel.get("name", "") + from_ds = rel.get("from") + to_ds = rel.get("to") + if from_ds and from_ds not in ds_name_set: + result.semantic_errors.append( + f"[UNKNOWN_DATASET_REF] Relationship '{rel_name}' " + f"references unknown dataset '{from_ds}'" + ) + if to_ds and to_ds not in ds_name_set: + result.semantic_errors.append( + f"[UNKNOWN_DATASET_REF] Relationship '{rel_name}' " + f"references unknown dataset '{to_ds}'" + ) return result diff --git a/converters/orionbelt/tests/fixtures/obml_as_ossie.yaml b/converters/orionbelt/tests/fixtures/obml_as_ossie.yaml index 11c8ccab..fcc4da85 100644 --- a/converters/orionbelt/tests/fixtures/obml_as_ossie.yaml +++ b/converters/orionbelt/tests/fixtures/obml_as_ossie.yaml @@ -16,789 +16,788 @@ # under the License. version: '0.2.0.dev0' -semantic_model: -- name: sales_analytics - description: Sales, Returns, Shipments and Purchases Analytics - ai_context: - instructions: Use this model for retail sales analysis, returns tracking, and purchase management. - datasets: - - name: calendar - source: ANALYTICS.PUBLIC.calendar - fields: - - name: date - expression: - dialects: - - dialect: ANSI_SQL - expression: date - dimension: - is_time: true - description: Calendar Date - ai_context: - synonyms: - - Calendar Date - - name: ym - expression: - dialects: - - dialect: ANSI_SQL - expression: ym - dimension: - is_time: false - description: Year Month - ai_context: - synonyms: - - Year Month - - name: weekday - expression: - dialects: - - dialect: ANSI_SQL - expression: weekday - dimension: - is_time: false - description: Weekday - - name: public_holiday - expression: - dialects: - - dialect: ANSI_SQL - expression: public_holiday - description: Public Holiday - ai_context: - synonyms: - - Public Holiday - - name: countries - source: ANALYTICS.PUBLIC.countries - fields: - - name: country_id - expression: - dialects: - - dialect: ANSI_SQL - expression: country_id - description: Country ID - ai_context: - synonyms: - - Country ID - - name: country_name - expression: - dialects: - - dialect: ANSI_SQL - expression: country_name - dimension: - is_time: false - description: Country Name - ai_context: - synonyms: - - Country Name - - name: clients - source: ANALYTICS.PUBLIC.clients - fields: - - name: client_id - expression: - dialects: - - dialect: ANSI_SQL - expression: client_id - description: Client ID - ai_context: - synonyms: - - Client ID - - name: client_name - expression: - dialects: - - dialect: ANSI_SQL - expression: client_name - dimension: - is_time: false - description: Client Name - ai_context: - synonyms: - - Client Name - - name: client_gender - expression: - dialects: - - dialect: ANSI_SQL - expression: client_gender - dimension: - is_time: false - description: Client Gender - ai_context: - synonyms: - - Client Gender - - name: client_country_id - expression: - dialects: - - dialect: ANSI_SQL - expression: client_country_id - description: Client Country ID - ai_context: - synonyms: - - Client Country ID - - name: suppliers - source: ANALYTICS.PUBLIC.suppliers - fields: - - name: supplier_id - expression: - dialects: - - dialect: ANSI_SQL - expression: supplier_id - description: Supplier ID - ai_context: - synonyms: - - Supplier ID - - name: supplier_name - expression: - dialects: - - dialect: ANSI_SQL - expression: supplier_name - dimension: - is_time: false - description: Supplier Name - ai_context: - synonyms: - - Supplier Name - - name: supplier_country_id - expression: - dialects: - - dialect: ANSI_SQL - expression: supplier_country_id - description: Supplier Country ID - ai_context: - synonyms: - - Supplier Country ID - - name: products - source: ANALYTICS.PUBLIC.products - fields: - - name: product_id - expression: - dialects: - - dialect: ANSI_SQL - expression: product_id - description: Product ID - ai_context: - synonyms: - - Product ID - - name: product_name - expression: - dialects: - - dialect: ANSI_SQL - expression: product_name - dimension: - is_time: false - description: Product Name - ai_context: - synonyms: - - Product Name - - name: product_cat - expression: - dialects: - - dialect: ANSI_SQL - expression: product_cat - dimension: - is_time: false - description: Product Category - ai_context: - synonyms: - - Product Category - - name: product_suppl - expression: - dialects: - - dialect: ANSI_SQL - expression: product_suppl - description: Product Supplier - ai_context: - synonyms: - - Product Supplier - - name: unit_cost - expression: - dialects: - - dialect: ANSI_SQL - expression: unit_cost - description: Unit Cost - ai_context: - synonyms: - - Unit Cost - - name: unit_price - expression: - dialects: - - dialect: ANSI_SQL - expression: unit_price - description: Unit Price - ai_context: - synonyms: - - Unit Price - - name: employees - source: ANALYTICS.PUBLIC.employees - fields: - - name: employee_id - expression: - dialects: - - dialect: ANSI_SQL - expression: employee_id - description: Employee ID - ai_context: - synonyms: - - Employee ID - - name: employee_name - expression: - dialects: - - dialect: ANSI_SQL - expression: employee_name - dimension: - is_time: false - description: Employee Name - ai_context: - synonyms: - - Employee Name - - name: department_name - expression: - dialects: - - dialect: ANSI_SQL - expression: department_name - dimension: - is_time: false - description: Department Name - ai_context: - synonyms: - - Department Name - - Department - - name: acct_bal - source: ANALYTICS.PUBLIC.acct_bal - fields: - - name: account_id - expression: - dialects: - - dialect: ANSI_SQL - expression: account_id - description: Account ID - ai_context: - synonyms: - - Account ID - - name: iban - expression: - dialects: - - dialect: ANSI_SQL - expression: iban - description: IBAN - - name: balance - expression: - dialects: - - dialect: ANSI_SQL - expression: balance - description: Balance - - name: sales - source: ANALYTICS.PUBLIC.sales - fields: - - name: sales_id - expression: - dialects: - - dialect: ANSI_SQL - expression: sales_id - description: Sales ID - ai_context: - synonyms: - - Sales ID - - name: sales_date - expression: - dialects: - - dialect: ANSI_SQL - expression: sales_date - dimension: - is_time: true - description: Sales Date - ai_context: - synonyms: - - Sales Date - - name: sales_ym - expression: - dialects: - - dialect: ANSI_SQL - expression: sales_ym - description: Sales YM - ai_context: - synonyms: - - Sales YM - - name: sales_client - expression: - dialects: - - dialect: ANSI_SQL - expression: sales_client - description: Sales Client - ai_context: - synonyms: - - Sales Client - - name: product - expression: - dialects: - - dialect: ANSI_SQL - expression: product - description: Sales Product - ai_context: - synonyms: - - Sales Product - - name: sales_emp_id - expression: - dialects: - - dialect: ANSI_SQL - expression: sales_emp_id - description: Sales Employee - ai_context: - synonyms: - - Sales Employee - - name: sales_payment_type - expression: - dialects: - - dialect: ANSI_SQL - expression: sales_payment_type - dimension: - is_time: false - description: Sales Payment Type - ai_context: - synonyms: - - Sales Payment Type - - Payment Type - - name: sales_quantity - expression: - dialects: - - dialect: ANSI_SQL - expression: sales_quantity - description: Sales Quantity - ai_context: - synonyms: - - Sales Quantity - - name: sales_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: sales_amount - description: Sales Amount - ai_context: - synonyms: - - Sales Amount - - name: notes - expression: - dialects: - - dialect: ANSI_SQL - expression: notes - description: Sales Notes - ai_context: - synonyms: - - Sales Notes - - name: returns - source: ANALYTICS.PUBLIC.returns - fields: - - name: return_id - expression: - dialects: - - dialect: ANSI_SQL - expression: return_id - description: Return ID - ai_context: - synonyms: - - Return ID - - name: return_sales_id - expression: - dialects: - - dialect: ANSI_SQL - expression: return_sales_id - description: Return Sales ID - ai_context: - synonyms: - - Return Sales ID - - name: return_date - expression: - dialects: - - dialect: ANSI_SQL - expression: return_date - dimension: - is_time: true - description: Return Date - ai_context: - synonyms: - - Return Date - - name: return_ym - expression: - dialects: - - dialect: ANSI_SQL - expression: return_ym - description: Return YM - ai_context: - synonyms: - - Return YM - - name: return_quantity - expression: - dialects: - - dialect: ANSI_SQL - expression: return_quantity - description: Return Quantity - ai_context: - synonyms: - - Return Quantity - - name: return_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: return_amount - description: Return Amount - ai_context: - synonyms: - - Return Amount - - name: notes - expression: - dialects: - - dialect: ANSI_SQL - expression: notes - description: Return Notes - ai_context: - synonyms: - - Return Notes - - name: shipments - source: ANALYTICS.PUBLIC.shipments - fields: - - name: shipment_id - expression: - dialects: - - dialect: ANSI_SQL - expression: shipment_id - description: Shipment ID - ai_context: - synonyms: - - Shipment ID - - name: shipment_sales_id - expression: - dialects: - - dialect: ANSI_SQL - expression: shipment_sales_id - description: Shipment Sales ID - ai_context: - synonyms: - - Shipment Sales ID - - name: shipment_date - expression: - dialects: - - dialect: ANSI_SQL - expression: shipment_date - dimension: - is_time: true - description: Shipment Date - ai_context: - synonyms: - - Shipment Date - - name: shipment_ym - expression: - dialects: - - dialect: ANSI_SQL - expression: shipment_ym - description: Shipment YM - ai_context: - synonyms: - - Shipment YM - - name: shipment_product - expression: - dialects: - - dialect: ANSI_SQL - expression: shipment_product - description: Shipment Product - ai_context: - synonyms: - - Shipment Product - - name: shipment_quantity - expression: - dialects: - - dialect: ANSI_SQL - expression: shipment_quantity - description: Shipment Quantity - ai_context: - synonyms: - - Shipment Quantity - - name: shipment_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: shipment_amount - description: Shipment Amount - ai_context: - synonyms: - - Shipment Amount - - name: notes - expression: - dialects: - - dialect: ANSI_SQL - expression: notes - description: Shipment Notes - ai_context: - synonyms: - - Shipment Notes - - name: purchases - source: ANALYTICS.PUBLIC.purchases - fields: - - name: purchase_id - expression: - dialects: - - dialect: ANSI_SQL - expression: purchase_id - description: Purchase ID - ai_context: - synonyms: - - Purchase ID - - name: purchase_date - expression: - dialects: - - dialect: ANSI_SQL - expression: purchase_date - dimension: - is_time: true - description: Purchase Date - ai_context: - synonyms: - - Purchase Date - - name: purchase_ym - expression: - dialects: - - dialect: ANSI_SQL - expression: purchase_ym - description: Purchase YM - ai_context: - synonyms: - - Purchase YM - - name: purchase_product_id - expression: - dialects: - - dialect: ANSI_SQL - expression: purchase_product_id - description: Purchase Product ID - ai_context: - synonyms: - - Purchase Product ID - - name: purchase_emp_id - expression: - dialects: - - dialect: ANSI_SQL - expression: purchase_emp_id - description: Purchase Employee - ai_context: - synonyms: - - Purchase Employee - - name: purchase_supplier - expression: - dialects: - - dialect: ANSI_SQL - expression: purchase_supplier - description: Purchase Supplier - ai_context: - synonyms: - - Purchase Supplier - - name: purchase_quantity - expression: - dialects: - - dialect: ANSI_SQL - expression: purchase_quantity - description: Purchase Quantity - ai_context: - synonyms: - - Purchase Quantity - - name: purchase_price - expression: - dialects: - - dialect: ANSI_SQL - expression: purchase_price - description: Purchase Price - ai_context: - synonyms: - - Purchase Price - relationships: - - name: clients_to_countries - from: clients - to: countries - from_columns: - - client_country_id - to_columns: - - country_id - - name: suppliers_to_countries_supplier_country - from: suppliers - to: countries - from_columns: - - supplier_country_id - to_columns: - - country_id - ai_context: - instructions: 'Secondary/alternative join path named: supplier_country. Use only when explicitly needed.' - - name: products_to_suppliers - from: products - to: suppliers - from_columns: - - product_suppl - to_columns: - - supplier_id - - name: sales_to_clients - from: sales - to: clients - from_columns: - - sales_client - to_columns: - - client_id - - name: sales_to_products_1 - from: sales - to: products - from_columns: - - product - to_columns: - - product_id - - name: sales_to_employees_2 - from: sales - to: employees - from_columns: - - sales_emp_id - to_columns: - - employee_id - - name: sales_to_calendar_3 - from: sales - to: calendar - from_columns: - - sales_ym - to_columns: - - ym - - name: sales_to_calendar_sales_date - from: sales - to: calendar - from_columns: - - sales_date - to_columns: - - date - ai_context: - instructions: 'Secondary/alternative join path named: sales_date. Use only when explicitly needed.' - - name: returns_to_sales - from: returns - to: sales - from_columns: - - return_sales_id - to_columns: - - sales_id - - name: shipments_to_sales - from: shipments - to: sales - from_columns: - - shipment_sales_id - to_columns: - - sales_id - - name: shipments_to_products_1 - from: shipments - to: products - from_columns: - - shipment_product - to_columns: - - product_id - - name: purchases_to_products - from: purchases - to: products - from_columns: - - purchase_product_id - to_columns: - - product_id - - name: purchases_to_employees_1 - from: purchases - to: employees - from_columns: - - purchase_emp_id - to_columns: - - employee_id - - name: purchases_to_suppliers_2 - from: purchases - to: suppliers - from_columns: - - purchase_supplier - to_columns: - - supplier_id - metrics: - - name: total_sales +name: sales_analytics +description: Sales, Returns, Shipments and Purchases Analytics +ai_context: + instructions: Use this model for retail sales analysis, returns tracking, and purchase management. +datasets: +- name: calendar + source: ANALYTICS.PUBLIC.calendar + fields: + - name: date + expression: + dialects: + - dialect: ANSI_SQL + expression: date + dimension: + is_time: true + description: Calendar Date + ai_context: + synonyms: + - Calendar Date + - name: ym + expression: + dialects: + - dialect: ANSI_SQL + expression: ym + dimension: + is_time: false + description: Year Month + ai_context: + synonyms: + - Year Month + - name: weekday + expression: + dialects: + - dialect: ANSI_SQL + expression: weekday + dimension: + is_time: false + description: Weekday + - name: public_holiday + expression: + dialects: + - dialect: ANSI_SQL + expression: public_holiday + description: Public Holiday + ai_context: + synonyms: + - Public Holiday +- name: countries + source: ANALYTICS.PUBLIC.countries + fields: + - name: country_id + expression: + dialects: + - dialect: ANSI_SQL + expression: country_id + description: Country ID + ai_context: + synonyms: + - Country ID + - name: country_name + expression: + dialects: + - dialect: ANSI_SQL + expression: country_name + dimension: + is_time: false + description: Country Name + ai_context: + synonyms: + - Country Name +- name: clients + source: ANALYTICS.PUBLIC.clients + fields: + - name: client_id + expression: + dialects: + - dialect: ANSI_SQL + expression: client_id + description: Client ID + ai_context: + synonyms: + - Client ID + - name: client_name + expression: + dialects: + - dialect: ANSI_SQL + expression: client_name + dimension: + is_time: false + description: Client Name + ai_context: + synonyms: + - Client Name + - name: client_gender expression: dialects: - dialect: ANSI_SQL - expression: SUM(sales.sales_amount) - description: Total Sales + expression: client_gender + dimension: + is_time: false + description: Client Gender ai_context: synonyms: - - Total Sales - - name: sales_count + - Client Gender + - name: client_country_id expression: dialects: - dialect: ANSI_SQL - expression: COUNT(DISTINCT sales.sales_id) - description: Sales Count + expression: client_country_id + description: Client Country ID ai_context: synonyms: - - Sales Count - - name: total_sales_qty + - Client Country ID +- name: suppliers + source: ANALYTICS.PUBLIC.suppliers + fields: + - name: supplier_id expression: dialects: - dialect: ANSI_SQL - expression: SUM(sales.sales_quantity) - description: Total Sales Qty + expression: supplier_id + description: Supplier ID ai_context: synonyms: - - Total Sales Qty - - name: total_returns + - Supplier ID + - name: supplier_name expression: dialects: - dialect: ANSI_SQL - expression: SUM(returns.return_amount) - description: Total Returns + expression: supplier_name + dimension: + is_time: false + description: Supplier Name ai_context: synonyms: - - Total Returns - - name: return_count + - Supplier Name + - name: supplier_country_id expression: dialects: - dialect: ANSI_SQL - expression: COUNT(DISTINCT sales.sales_id) - description: Return Count + expression: supplier_country_id + description: Supplier Country ID ai_context: synonyms: - - Return Count - - name: total_shipments + - Supplier Country ID +- name: products + source: ANALYTICS.PUBLIC.products + fields: + - name: product_id expression: dialects: - dialect: ANSI_SQL - expression: SUM(shipments.shipment_amount) - description: Total Shipments + expression: product_id + description: Product ID ai_context: synonyms: - - Total Shipments - - name: total_shipment_qty + - Product ID + - name: product_name expression: dialects: - dialect: ANSI_SQL - expression: SUM(shipments.shipment_quantity) - description: Total Shipment Qty + expression: product_name + dimension: + is_time: false + description: Product Name ai_context: synonyms: - - Total Shipment Qty - - name: total_purchases + - Product Name + - name: product_cat expression: dialects: - dialect: ANSI_SQL - expression: SUM(purchases.purchase_price) - description: Total Purchases + expression: product_cat + dimension: + is_time: false + description: Product Category ai_context: synonyms: - - Total Purchases - - name: total_purchase_qty + - Product Category + - name: product_suppl expression: dialects: - dialect: ANSI_SQL - expression: SUM(purchases.purchase_quantity) - description: Total Purchase Qty + expression: product_suppl + description: Product Supplier ai_context: synonyms: - - Total Purchase Qty - - name: return_rate + - Product Supplier + - name: unit_cost expression: dialects: - dialect: ANSI_SQL - expression: SUM(returns.return_amount) / SUM(sales.sales_amount) - description: Return Rate - - name: average_sale + expression: unit_cost + description: Unit Cost + ai_context: + synonyms: + - Unit Cost + - name: unit_price + expression: + dialects: + - dialect: ANSI_SQL + expression: unit_price + description: Unit Price + ai_context: + synonyms: + - Unit Price +- name: employees + source: ANALYTICS.PUBLIC.employees + fields: + - name: employee_id expression: dialects: - dialect: ANSI_SQL - expression: SUM(sales.sales_amount) / COUNT(DISTINCT sales.sales_id) - description: Average Sale - custom_extensions: - - vendor_name: COMMON - data: '{"source_format": "OBML", "source_version": "1.0", "converter": "ossie-orionbelt"}' + expression: employee_id + description: Employee ID + ai_context: + synonyms: + - Employee ID + - name: employee_name + expression: + dialects: + - dialect: ANSI_SQL + expression: employee_name + dimension: + is_time: false + description: Employee Name + ai_context: + synonyms: + - Employee Name + - name: department_name + expression: + dialects: + - dialect: ANSI_SQL + expression: department_name + dimension: + is_time: false + description: Department Name + ai_context: + synonyms: + - Department Name + - Department +- name: acct_bal + source: ANALYTICS.PUBLIC.acct_bal + fields: + - name: account_id + expression: + dialects: + - dialect: ANSI_SQL + expression: account_id + description: Account ID + ai_context: + synonyms: + - Account ID + - name: iban + expression: + dialects: + - dialect: ANSI_SQL + expression: iban + description: IBAN + - name: balance + expression: + dialects: + - dialect: ANSI_SQL + expression: balance + description: Balance +- name: sales + source: ANALYTICS.PUBLIC.sales + fields: + - name: sales_id + expression: + dialects: + - dialect: ANSI_SQL + expression: sales_id + description: Sales ID + ai_context: + synonyms: + - Sales ID + - name: sales_date + expression: + dialects: + - dialect: ANSI_SQL + expression: sales_date + dimension: + is_time: true + description: Sales Date + ai_context: + synonyms: + - Sales Date + - name: sales_ym + expression: + dialects: + - dialect: ANSI_SQL + expression: sales_ym + description: Sales YM + ai_context: + synonyms: + - Sales YM + - name: sales_client + expression: + dialects: + - dialect: ANSI_SQL + expression: sales_client + description: Sales Client + ai_context: + synonyms: + - Sales Client + - name: product + expression: + dialects: + - dialect: ANSI_SQL + expression: product + description: Sales Product + ai_context: + synonyms: + - Sales Product + - name: sales_emp_id + expression: + dialects: + - dialect: ANSI_SQL + expression: sales_emp_id + description: Sales Employee + ai_context: + synonyms: + - Sales Employee + - name: sales_payment_type + expression: + dialects: + - dialect: ANSI_SQL + expression: sales_payment_type + dimension: + is_time: false + description: Sales Payment Type + ai_context: + synonyms: + - Sales Payment Type + - Payment Type + - name: sales_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: sales_quantity + description: Sales Quantity + ai_context: + synonyms: + - Sales Quantity + - name: sales_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: sales_amount + description: Sales Amount + ai_context: + synonyms: + - Sales Amount + - name: notes + expression: + dialects: + - dialect: ANSI_SQL + expression: notes + description: Sales Notes + ai_context: + synonyms: + - Sales Notes +- name: returns + source: ANALYTICS.PUBLIC.returns + fields: + - name: return_id + expression: + dialects: + - dialect: ANSI_SQL + expression: return_id + description: Return ID + ai_context: + synonyms: + - Return ID + - name: return_sales_id + expression: + dialects: + - dialect: ANSI_SQL + expression: return_sales_id + description: Return Sales ID + ai_context: + synonyms: + - Return Sales ID + - name: return_date + expression: + dialects: + - dialect: ANSI_SQL + expression: return_date + dimension: + is_time: true + description: Return Date + ai_context: + synonyms: + - Return Date + - name: return_ym + expression: + dialects: + - dialect: ANSI_SQL + expression: return_ym + description: Return YM + ai_context: + synonyms: + - Return YM + - name: return_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: return_quantity + description: Return Quantity + ai_context: + synonyms: + - Return Quantity + - name: return_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: return_amount + description: Return Amount + ai_context: + synonyms: + - Return Amount + - name: notes + expression: + dialects: + - dialect: ANSI_SQL + expression: notes + description: Return Notes + ai_context: + synonyms: + - Return Notes +- name: shipments + source: ANALYTICS.PUBLIC.shipments + fields: + - name: shipment_id + expression: + dialects: + - dialect: ANSI_SQL + expression: shipment_id + description: Shipment ID + ai_context: + synonyms: + - Shipment ID + - name: shipment_sales_id + expression: + dialects: + - dialect: ANSI_SQL + expression: shipment_sales_id + description: Shipment Sales ID + ai_context: + synonyms: + - Shipment Sales ID + - name: shipment_date + expression: + dialects: + - dialect: ANSI_SQL + expression: shipment_date + dimension: + is_time: true + description: Shipment Date + ai_context: + synonyms: + - Shipment Date + - name: shipment_ym + expression: + dialects: + - dialect: ANSI_SQL + expression: shipment_ym + description: Shipment YM + ai_context: + synonyms: + - Shipment YM + - name: shipment_product + expression: + dialects: + - dialect: ANSI_SQL + expression: shipment_product + description: Shipment Product + ai_context: + synonyms: + - Shipment Product + - name: shipment_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: shipment_quantity + description: Shipment Quantity + ai_context: + synonyms: + - Shipment Quantity + - name: shipment_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: shipment_amount + description: Shipment Amount + ai_context: + synonyms: + - Shipment Amount + - name: notes + expression: + dialects: + - dialect: ANSI_SQL + expression: notes + description: Shipment Notes + ai_context: + synonyms: + - Shipment Notes +- name: purchases + source: ANALYTICS.PUBLIC.purchases + fields: + - name: purchase_id + expression: + dialects: + - dialect: ANSI_SQL + expression: purchase_id + description: Purchase ID + ai_context: + synonyms: + - Purchase ID + - name: purchase_date + expression: + dialects: + - dialect: ANSI_SQL + expression: purchase_date + dimension: + is_time: true + description: Purchase Date + ai_context: + synonyms: + - Purchase Date + - name: purchase_ym + expression: + dialects: + - dialect: ANSI_SQL + expression: purchase_ym + description: Purchase YM + ai_context: + synonyms: + - Purchase YM + - name: purchase_product_id + expression: + dialects: + - dialect: ANSI_SQL + expression: purchase_product_id + description: Purchase Product ID + ai_context: + synonyms: + - Purchase Product ID + - name: purchase_emp_id + expression: + dialects: + - dialect: ANSI_SQL + expression: purchase_emp_id + description: Purchase Employee + ai_context: + synonyms: + - Purchase Employee + - name: purchase_supplier + expression: + dialects: + - dialect: ANSI_SQL + expression: purchase_supplier + description: Purchase Supplier + ai_context: + synonyms: + - Purchase Supplier + - name: purchase_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: purchase_quantity + description: Purchase Quantity + ai_context: + synonyms: + - Purchase Quantity + - name: purchase_price + expression: + dialects: + - dialect: ANSI_SQL + expression: purchase_price + description: Purchase Price + ai_context: + synonyms: + - Purchase Price +relationships: +- name: clients_to_countries + from: clients + to: countries + from_columns: + - client_country_id + to_columns: + - country_id +- name: suppliers_to_countries_supplier_country + from: suppliers + to: countries + from_columns: + - supplier_country_id + to_columns: + - country_id + ai_context: + instructions: 'Secondary/alternative join path named: supplier_country. Use only when explicitly needed.' +- name: products_to_suppliers + from: products + to: suppliers + from_columns: + - product_suppl + to_columns: + - supplier_id +- name: sales_to_clients + from: sales + to: clients + from_columns: + - sales_client + to_columns: + - client_id +- name: sales_to_products_1 + from: sales + to: products + from_columns: + - product + to_columns: + - product_id +- name: sales_to_employees_2 + from: sales + to: employees + from_columns: + - sales_emp_id + to_columns: + - employee_id +- name: sales_to_calendar_3 + from: sales + to: calendar + from_columns: + - sales_ym + to_columns: + - ym +- name: sales_to_calendar_sales_date + from: sales + to: calendar + from_columns: + - sales_date + to_columns: + - date + ai_context: + instructions: 'Secondary/alternative join path named: sales_date. Use only when explicitly needed.' +- name: returns_to_sales + from: returns + to: sales + from_columns: + - return_sales_id + to_columns: + - sales_id +- name: shipments_to_sales + from: shipments + to: sales + from_columns: + - shipment_sales_id + to_columns: + - sales_id +- name: shipments_to_products_1 + from: shipments + to: products + from_columns: + - shipment_product + to_columns: + - product_id +- name: purchases_to_products + from: purchases + to: products + from_columns: + - purchase_product_id + to_columns: + - product_id +- name: purchases_to_employees_1 + from: purchases + to: employees + from_columns: + - purchase_emp_id + to_columns: + - employee_id +- name: purchases_to_suppliers_2 + from: purchases + to: suppliers + from_columns: + - purchase_supplier + to_columns: + - supplier_id +metrics: +- name: total_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(sales.sales_amount) + description: Total Sales + ai_context: + synonyms: + - Total Sales +- name: sales_count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT sales.sales_id) + description: Sales Count + ai_context: + synonyms: + - Sales Count +- name: total_sales_qty + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(sales.sales_quantity) + description: Total Sales Qty + ai_context: + synonyms: + - Total Sales Qty +- name: total_returns + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(returns.return_amount) + description: Total Returns + ai_context: + synonyms: + - Total Returns +- name: return_count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT sales.sales_id) + description: Return Count + ai_context: + synonyms: + - Return Count +- name: total_shipments + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(shipments.shipment_amount) + description: Total Shipments + ai_context: + synonyms: + - Total Shipments +- name: total_shipment_qty + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(shipments.shipment_quantity) + description: Total Shipment Qty + ai_context: + synonyms: + - Total Shipment Qty +- name: total_purchases + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(purchases.purchase_price) + description: Total Purchases + ai_context: + synonyms: + - Total Purchases +- name: total_purchase_qty + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(purchases.purchase_quantity) + description: Total Purchase Qty + ai_context: + synonyms: + - Total Purchase Qty +- name: return_rate + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(returns.return_amount) / SUM(sales.sales_amount) + description: Return Rate +- name: average_sale + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(sales.sales_amount) / COUNT(DISTINCT sales.sales_id) + description: Average Sale +custom_extensions: +- vendor_name: COMMON + data: '{"source_format": "OBML", "source_version": "1.0", "converter": "ossie-orionbelt"}' diff --git a/converters/orionbelt/tests/fixtures/tpcds_ossie.yaml b/converters/orionbelt/tests/fixtures/tpcds_ossie.yaml index 21ec6483..979a6778 100644 --- a/converters/orionbelt/tests/fixtures/tpcds_ossie.yaml +++ b/converters/orionbelt/tests/fixtures/tpcds_ossie.yaml @@ -23,573 +23,572 @@ version: "0.2.0.dev0" -semantic_model: - - name: tpcds_retail_model - description: TPC-DS retail semantic model for sales and customer analytics +name: tpcds_retail_model +description: TPC-DS retail semantic model for sales and customer analytics +ai_context: + instructions: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." + +datasets: + # Fact table: Store sales transactions + - name: store_sales + source: tpcds.public.store_sales + primary_key: [ss_item_sk, ss_ticket_number] # Composite primary key + unique_keys: + - [ss_item_sk, ss_ticket_number] # Composite key: item + ticket number uniquely identifies a line item + description: Fact table containing all store sales transactions ai_context: - instructions: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." - - datasets: - # Fact table: Store sales transactions - - name: store_sales - source: tpcds.public.store_sales - primary_key: [ss_item_sk, ss_ticket_number] # Composite primary key - unique_keys: - - [ss_item_sk, ss_ticket_number] # Composite key: item + ticket number uniquely identifies a line item - description: Fact table containing all store sales transactions + synonyms: + - "sales transactions" + - "store purchases" + - "retail sales" + - "POS data" + + fields: + - name: ss_sold_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sold_date_sk + description: Foreign key to date dimension + dimension: + is_time: false ai_context: synonyms: - - "sales transactions" - - "store purchases" - - "retail sales" - - "POS data" - - fields: - - name: ss_sold_date_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_sold_date_sk - description: Foreign key to date dimension - dimension: - is_time: false - ai_context: - synonyms: - - "sale date" - - "transaction date" - - - name: ss_item_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_item_sk - description: Foreign key to item dimension - dimension: - is_time: false - ai_context: - synonyms: - - "product" - - "item" - - - name: ss_customer_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_customer_sk - description: Foreign key to customer dimension - dimension: - is_time: false - ai_context: - synonyms: - - "customer" - - "buyer" - - - name: ss_store_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_store_sk - description: Foreign key to store dimension - dimension: - is_time: false - ai_context: - synonyms: - - "store" - - "location" - - - name: ss_quantity - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_quantity - description: Quantity of items sold - ai_context: - synonyms: - - "units sold" - - "quantity" - - - name: ss_sales_price - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_sales_price - description: Sales price per unit - ai_context: - synonyms: - - "unit price" - - "price" - - - name: ss_ext_sales_price - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_ext_sales_price - description: Extended sales price (quantity * price) - ai_context: - synonyms: - - "total price" - - "line total" - - - name: ss_net_profit - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_net_profit - description: Net profit from the sale - ai_context: - synonyms: - - "profit" - - "margin" - - # Dimension table: Date - - name: date_dim - source: tpcds.public.date_dim - primary_key: [d_date_sk] # Simple primary key - unique_keys: - - [d_date_sk] # Simple key: single column - description: Date dimension with calendar attributes + - "sale date" + - "transaction date" + + - name: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk + description: Foreign key to item dimension + dimension: + is_time: false ai_context: synonyms: - - "calendar" - - "dates" - - "time periods" - - fields: - - name: d_date_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: d_date_sk - description: Surrogate key for date - dimension: - is_time: false - - - name: d_date - expression: - dialects: - - dialect: ANSI_SQL - expression: d_date - description: Actual date value - dimension: - is_time: true - ai_context: - synonyms: - - "date" - - "calendar date" - - - name: d_year - expression: - dialects: - - dialect: ANSI_SQL - expression: d_year - description: Year - dimension: - is_time: true - ai_context: - synonyms: - - "year" - - - name: d_quarter_name - expression: - dialects: - - dialect: ANSI_SQL - expression: d_quarter_name - description: Quarter name (e.g., 2024Q1) - dimension: - is_time: true - ai_context: - synonyms: - - "quarter" - - "fiscal quarter" - - - name: d_moy - expression: - dialects: - - dialect: ANSI_SQL - expression: d_moy - description: Month of year (1-12) - dimension: - is_time: true - ai_context: - synonyms: - - "month" - - # Dimension table: Customer - - name: customer - source: tpcds.public.customer - primary_key: [c_customer_sk] # Simple primary key - unique_keys: - - [c_customer_sk] # Simple key: single column - description: Customer dimension with demographic information + - "product" + - "item" + + - name: ss_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_customer_sk + description: Foreign key to customer dimension + dimension: + is_time: false ai_context: synonyms: - - "customers" - - "shoppers" - - "buyers" - - fields: - - name: c_customer_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: c_customer_sk - description: Surrogate key for customer - dimension: - is_time: false - - - name: c_customer_id - expression: - dialects: - - dialect: ANSI_SQL - expression: c_customer_id - description: Business key for customer - dimension: - is_time: false - ai_context: - synonyms: - - "customer ID" - - "customer number" - - - name: c_first_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_first_name - description: Customer first name - dimension: - is_time: false - - - name: c_last_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_last_name - description: Customer last name - dimension: - is_time: false - - - name: customer_full_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_first_name || ' ' || c_last_name - description: Customer full name (computed field) - dimension: - is_time: false - ai_context: - synonyms: - - "full name" - - "customer name" - - - name: c_email_address - expression: - dialects: - - dialect: ANSI_SQL - expression: c_email_address - description: Customer email address - dimension: - is_time: false - ai_context: - synonyms: - - "email" - - "contact" - - # Dimension table: Item (Product) - - name: item - source: tpcds.public.item - primary_key: [i_item_sk] # Simple primary key - unique_keys: - - [i_item_sk] # Simple key: single column - description: Item/Product dimension with product attributes + - "customer" + - "buyer" + + - name: ss_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_store_sk + description: Foreign key to store dimension + dimension: + is_time: false ai_context: synonyms: - - "products" - - "items" - - "merchandise" - - fields: - - name: i_item_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_sk - description: Surrogate key for item - dimension: - is_time: false - - - name: i_item_id - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_id - description: Business key for item - dimension: - is_time: false - ai_context: - synonyms: - - "item ID" - - "product ID" - - "SKU" - - - name: i_item_desc - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_desc - description: Item description - dimension: - is_time: false - ai_context: - synonyms: - - "product description" - - "item name" - - - name: i_brand - expression: - dialects: - - dialect: ANSI_SQL - expression: i_brand - description: Brand name - dimension: - is_time: false - ai_context: - synonyms: - - "brand" - - "manufacturer" - - - name: i_category - expression: - dialects: - - dialect: ANSI_SQL - expression: i_category - description: Item category - dimension: - is_time: false - ai_context: - synonyms: - - "product category" - - "department" - - - name: i_current_price - expression: - dialects: - - dialect: ANSI_SQL - expression: i_current_price - description: Current price of the item - dimension: - is_time: false - ai_context: - synonyms: - - "price" - - "list price" - - # Dimension table: Store - - name: store - source: tpcds.public.store - primary_key: [s_store_sk] # Simple primary key - unique_keys: - - [s_store_id] # Simple key: single column - description: Store dimension with location and store attributes + - "store" + - "location" + + - name: ss_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_quantity + description: Quantity of items sold ai_context: synonyms: - - "stores" - - "retail locations" - - "branches" - - fields: - - name: s_store_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_sk - description: Surrogate key for store - dimension: - is_time: false - - - name: s_store_id - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_id - description: Business key for store - dimension: - is_time: false - ai_context: - synonyms: - - "store ID" - - "store number" - - - name: s_store_name - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_name - description: Store name - dimension: - is_time: false - ai_context: - synonyms: - - "store name" - - "location name" - - - name: s_city - expression: - dialects: - - dialect: ANSI_SQL - expression: s_city - description: City where store is located - dimension: - is_time: false - ai_context: - synonyms: - - "city" - - "location" - - - name: s_state - expression: - dialects: - - dialect: ANSI_SQL - expression: s_state - description: State where store is located - dimension: - is_time: false - ai_context: - synonyms: - - "state" - - "region" - - - name: s_number_employees - expression: - dialects: - - dialect: ANSI_SQL - expression: s_number_employees - description: Number of employees at the store - ai_context: - synonyms: - - "employee count" - - "staff size" - - # Relationships between datasets - relationships: - - name: store_sales_to_date - from: store_sales - to: date_dim - from_columns: [ss_sold_date_sk] - to_columns: [d_date_sk] + - "units sold" + - "quantity" + + - name: ss_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sales_price + description: Sales price per unit + ai_context: + synonyms: + - "unit price" + - "price" + + - name: ss_ext_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + ai_context: + synonyms: + - "total price" + - "line total" + + - name: ss_net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_profit + description: Net profit from the sale ai_context: synonyms: - - "sales date relationship" - - "when sale occurred" - - - name: store_sales_to_customer - from: store_sales - to: customer - from_columns: [ss_customer_sk] - to_columns: [c_customer_sk] + - "profit" + - "margin" + + # Dimension table: Date + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] # Simple primary key + unique_keys: + - [d_date_sk] # Simple key: single column + description: Date dimension with calendar attributes + ai_context: + synonyms: + - "calendar" + - "dates" + - "time periods" + + fields: + - name: d_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date_sk + description: Surrogate key for date + dimension: + is_time: false + + - name: d_date + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date + description: Actual date value + dimension: + is_time: true ai_context: synonyms: - - "customer purchase relationship" - - "who bought" - - - name: store_sales_to_item - from: store_sales - to: item - from_columns: [ss_item_sk] - to_columns: [i_item_sk] + - "date" + - "calendar date" + + - name: d_year + expression: + dialects: + - dialect: ANSI_SQL + expression: d_year + description: Year + dimension: + is_time: true ai_context: synonyms: - - "product sold relationship" - - "what was sold" - - - name: store_sales_to_store - from: store_sales - to: store - from_columns: [ss_store_sk] - to_columns: [s_store_sk] + - "year" + + - name: d_quarter_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_quarter_name + description: Quarter name (e.g., 2024Q1) + dimension: + is_time: true ai_context: synonyms: - - "store location relationship" - - "where sale occurred" + - "quarter" + - "fiscal quarter" - # Semantic model-level metrics spanning multiple datasets - metrics: - - name: total_sales + - name: d_moy expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) - description: Total sales revenue across all transactions + expression: d_moy + description: Month of year (1-12) + dimension: + is_time: true ai_context: synonyms: - - "total revenue" - - "gross sales" - - "sales amount" + - "month" + + # Dimension table: Customer + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] # Simple primary key + unique_keys: + - [c_customer_sk] # Simple key: single column + description: Customer dimension with demographic information + ai_context: + synonyms: + - "customers" + - "shoppers" + - "buyers" + + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_sk + description: Surrogate key for customer + dimension: + is_time: false - - name: total_profit + - name: c_customer_id expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_net_profit) - description: Total net profit from store sales + expression: c_customer_id + description: Business key for customer + dimension: + is_time: false ai_context: synonyms: - - "net profit" - - "total earnings" - - "profit" + - "customer ID" + - "customer number" + + - name: c_first_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name + description: Customer first name + dimension: + is_time: false + + - name: c_last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_last_name + description: Customer last name + dimension: + is_time: false + + - name: customer_full_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name || ' ' || c_last_name + description: Customer full name (computed field) + dimension: + is_time: false + ai_context: + synonyms: + - "full name" + - "customer name" + + - name: c_email_address + expression: + dialects: + - dialect: ANSI_SQL + expression: c_email_address + description: Customer email address + dimension: + is_time: false + ai_context: + synonyms: + - "email" + - "contact" + + # Dimension table: Item (Product) + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] # Simple primary key + unique_keys: + - [i_item_sk] # Simple key: single column + description: Item/Product dimension with product attributes + ai_context: + synonyms: + - "products" + - "items" + - "merchandise" + + fields: + - name: i_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_sk + description: Surrogate key for item + dimension: + is_time: false + + - name: i_item_id + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_id + description: Business key for item + dimension: + is_time: false + ai_context: + synonyms: + - "item ID" + - "product ID" + - "SKU" + + - name: i_item_desc + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_desc + description: Item description + dimension: + is_time: false + ai_context: + synonyms: + - "product description" + - "item name" + + - name: i_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: i_brand + description: Brand name + dimension: + is_time: false + ai_context: + synonyms: + - "brand" + - "manufacturer" + + - name: i_category + expression: + dialects: + - dialect: ANSI_SQL + expression: i_category + description: Item category + dimension: + is_time: false + ai_context: + synonyms: + - "product category" + - "department" + + - name: i_current_price + expression: + dialects: + - dialect: ANSI_SQL + expression: i_current_price + description: Current price of the item + dimension: + is_time: false + ai_context: + synonyms: + - "price" + - "list price" + + # Dimension table: Store + - name: store + source: tpcds.public.store + primary_key: [s_store_sk] # Simple primary key + unique_keys: + - [s_store_id] # Simple key: single column + description: Store dimension with location and store attributes + ai_context: + synonyms: + - "stores" + - "retail locations" + - "branches" + + fields: + - name: s_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_sk + description: Surrogate key for store + dimension: + is_time: false - - name: customer_lifetime_value + - name: s_store_id expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) - description: Average lifetime sales value per customer + expression: s_store_id + description: Business key for store + dimension: + is_time: false ai_context: synonyms: - - "CLV" - - "LTV" - - "customer value" - - "lifetime revenue" + - "store ID" + - "store number" - - name: sales_by_brand + - name: s_store_name expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) - description: Total sales by brand (requires grouping by item.i_brand) + expression: s_store_name + description: Store name + dimension: + is_time: false ai_context: synonyms: - - "brand sales" - - "brand performance" - - "brand revenue" + - "store name" + - "location name" - - name: store_productivity + - name: s_city expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) - description: Sales per employee across stores + expression: s_city + description: City where store is located + dimension: + is_time: false ai_context: synonyms: - - "sales per employee" - - "employee productivity" - - "revenue per employee" - - custom_extensions: - - vendor_name: SALESFORCE - data: | - { - "tableau_workbook_id": "tpcds_retail_dashboard", - "einstein_enabled": true, - "crm_sync": { - "enabled": true, - "sync_frequency": "daily", - "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" - }, - "tableau_semantics": { - "published": true, - "version": "1.0" - } - } - - - vendor_name: DBT - data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' + - "city" + - "location" + + - name: s_state + expression: + dialects: + - dialect: ANSI_SQL + expression: s_state + description: State where store is located + dimension: + is_time: false + ai_context: + synonyms: + - "state" + - "region" + + - name: s_number_employees + expression: + dialects: + - dialect: ANSI_SQL + expression: s_number_employees + description: Number of employees at the store + ai_context: + synonyms: + - "employee count" + - "staff size" + +# Relationships between datasets +relationships: + - name: store_sales_to_date + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + ai_context: + synonyms: + - "sales date relationship" + - "when sale occurred" + + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] + ai_context: + synonyms: + - "customer purchase relationship" + - "who bought" + + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + ai_context: + synonyms: + - "product sold relationship" + - "what was sold" + + - name: store_sales_to_store + from: store_sales + to: store + from_columns: [ss_store_sk] + to_columns: [s_store_sk] + ai_context: + synonyms: + - "store location relationship" + - "where sale occurred" + +# Semantic model-level metrics spanning multiple datasets +metrics: + - name: total_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales revenue across all transactions + ai_context: + synonyms: + - "total revenue" + - "gross sales" + - "sales amount" + + - name: total_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_net_profit) + description: Total net profit from store sales + ai_context: + synonyms: + - "net profit" + - "total earnings" + - "profit" + + - name: customer_lifetime_value + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + description: Average lifetime sales value per customer + ai_context: + synonyms: + - "CLV" + - "LTV" + - "customer value" + - "lifetime revenue" + + - name: sales_by_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales by brand (requires grouping by item.i_brand) + ai_context: + synonyms: + - "brand sales" + - "brand performance" + - "brand revenue" + + - name: store_productivity + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) + description: Sales per employee across stores + ai_context: + synonyms: + - "sales per employee" + - "employee productivity" + - "revenue per employee" + +custom_extensions: + - vendor_name: SALESFORCE + data: | + { + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "1.0" + } + } + + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' diff --git a/converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml b/converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml index df1f42c0..934644ce 100644 --- a/converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml +++ b/converters/orionbelt/tests/fixtures/tpcds_semantic_model.yaml @@ -23,573 +23,572 @@ version: "0.2.0.dev0" -semantic_model: - - name: tpcds_retail_model - description: TPC-DS retail semantic model for sales and customer analytics +name: tpcds_retail_model +description: TPC-DS retail semantic model for sales and customer analytics +ai_context: + instructions: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." + +datasets: + # Fact table: Store sales transactions + - name: store_sales + source: tpcds.public.store_sales + primary_key: [ss_item_sk, ss_ticket_number] # Composite primary key + unique_keys: + - [ss_item_sk, ss_ticket_number] # Composite key: item + ticket number uniquely identifies a line item + description: Fact table containing all store sales transactions ai_context: - instructions: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." - - datasets: - # Fact table: Store sales transactions - - name: store_sales - source: tpcds.public.store_sales - primary_key: [ss_item_sk, ss_ticket_number] # Composite primary key - unique_keys: - - [ss_item_sk, ss_ticket_number] # Composite key: item + ticket number uniquely identifies a line item - description: Fact table containing all store sales transactions + synonyms: + - "sales transactions" + - "store purchases" + - "retail sales" + - "POS data" + + fields: + - name: ss_sold_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sold_date_sk + description: Foreign key to date dimension + dimension: + is_time: false ai_context: synonyms: - - "sales transactions" - - "store purchases" - - "retail sales" - - "POS data" - - fields: - - name: ss_sold_date_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_sold_date_sk - description: Foreign key to date dimension - dimension: - is_time: false - ai_context: - synonyms: - - "sale date" - - "transaction date" - - - name: ss_item_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_item_sk - description: Foreign key to item dimension - dimension: - is_time: false - ai_context: - synonyms: - - "product" - - "item" - - - name: ss_customer_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_customer_sk - description: Foreign key to customer dimension - dimension: - is_time: false - ai_context: - synonyms: - - "customer" - - "buyer" - - - name: ss_store_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_store_sk - description: Foreign key to store dimension - dimension: - is_time: false - ai_context: - synonyms: - - "store" - - "location" - - - name: ss_quantity - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_quantity - description: Quantity of items sold - ai_context: - synonyms: - - "units sold" - - "quantity" - - - name: ss_sales_price - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_sales_price - description: Sales price per unit - ai_context: - synonyms: - - "unit price" - - "price" - - - name: ss_ext_sales_price - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_ext_sales_price - description: Extended sales price (quantity * price) - ai_context: - synonyms: - - "total price" - - "line total" - - - name: ss_net_profit - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_net_profit - description: Net profit from the sale - ai_context: - synonyms: - - "profit" - - "margin" - - # Dimension table: Date - - name: date_dim - source: tpcds.public.date_dim - primary_key: [d_date_sk] # Simple primary key - unique_keys: - - [d_date_sk] # Simple key: single column - description: Date dimension with calendar attributes + - "sale date" + - "transaction date" + + - name: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk + description: Foreign key to item dimension + dimension: + is_time: false ai_context: synonyms: - - "calendar" - - "dates" - - "time periods" - - fields: - - name: d_date_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: d_date_sk - description: Surrogate key for date - dimension: - is_time: false - - - name: d_date - expression: - dialects: - - dialect: ANSI_SQL - expression: d_date - description: Actual date value - dimension: - is_time: true - ai_context: - synonyms: - - "date" - - "calendar date" - - - name: d_year - expression: - dialects: - - dialect: ANSI_SQL - expression: d_year - description: Year - dimension: - is_time: true - ai_context: - synonyms: - - "year" - - - name: d_quarter_name - expression: - dialects: - - dialect: ANSI_SQL - expression: d_quarter_name - description: Quarter name (e.g., 2024Q1) - dimension: - is_time: true - ai_context: - synonyms: - - "quarter" - - "fiscal quarter" - - - name: d_moy - expression: - dialects: - - dialect: ANSI_SQL - expression: d_moy - description: Month of year (1-12) - dimension: - is_time: true - ai_context: - synonyms: - - "month" - - # Dimension table: Customer - - name: customer - source: tpcds.public.customer - primary_key: [c_customer_sk] # Simple primary key - unique_keys: - - [c_customer_sk] # Simple key: single column - description: Customer dimension with demographic information + - "product" + - "item" + + - name: ss_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_customer_sk + description: Foreign key to customer dimension + dimension: + is_time: false ai_context: synonyms: - - "customers" - - "shoppers" - - "buyers" - - fields: - - name: c_customer_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: c_customer_sk - description: Surrogate key for customer - dimension: - is_time: false - - - name: c_customer_id - expression: - dialects: - - dialect: ANSI_SQL - expression: c_customer_id - description: Business key for customer - dimension: - is_time: false - ai_context: - synonyms: - - "customer ID" - - "customer number" - - - name: c_first_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_first_name - description: Customer first name - dimension: - is_time: false - - - name: c_last_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_last_name - description: Customer last name - dimension: - is_time: false - - - name: customer_full_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_first_name || ' ' || c_last_name - description: Customer full name (computed field) - dimension: - is_time: false - ai_context: - synonyms: - - "full name" - - "customer name" - - - name: c_email_address - expression: - dialects: - - dialect: ANSI_SQL - expression: c_email_address - description: Customer email address - dimension: - is_time: false - ai_context: - synonyms: - - "email" - - "contact" - - # Dimension table: Item (Product) - - name: item - source: tpcds.public.item - primary_key: [i_item_sk] # Simple primary key - unique_keys: - - [i_item_sk] # Simple key: single column - description: Item/Product dimension with product attributes + - "customer" + - "buyer" + + - name: ss_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_store_sk + description: Foreign key to store dimension + dimension: + is_time: false ai_context: synonyms: - - "products" - - "items" - - "merchandise" - - fields: - - name: i_item_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_sk - description: Surrogate key for item - dimension: - is_time: false - - - name: i_item_id - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_id - description: Business key for item - dimension: - is_time: false - ai_context: - synonyms: - - "item ID" - - "product ID" - - "SKU" - - - name: i_item_desc - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_desc - description: Item description - dimension: - is_time: false - ai_context: - synonyms: - - "product description" - - "item name" - - - name: i_brand - expression: - dialects: - - dialect: ANSI_SQL - expression: i_brand - description: Brand name - dimension: - is_time: false - ai_context: - synonyms: - - "brand" - - "manufacturer" - - - name: i_category - expression: - dialects: - - dialect: ANSI_SQL - expression: i_category - description: Item category - dimension: - is_time: false - ai_context: - synonyms: - - "product category" - - "department" - - - name: i_current_price - expression: - dialects: - - dialect: ANSI_SQL - expression: i_current_price - description: Current price of the item - dimension: - is_time: false - ai_context: - synonyms: - - "price" - - "list price" - - # Dimension table: Store - - name: store - source: tpcds.public.store - primary_key: [s_store_sk] # Simple primary key - unique_keys: - - [s_store_id] # Simple key: single column - description: Store dimension with location and store attributes + - "store" + - "location" + + - name: ss_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_quantity + description: Quantity of items sold ai_context: synonyms: - - "stores" - - "retail locations" - - "branches" - - fields: - - name: s_store_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_sk - description: Surrogate key for store - dimension: - is_time: false - - - name: s_store_id - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_id - description: Business key for store - dimension: - is_time: false - ai_context: - synonyms: - - "store ID" - - "store number" - - - name: s_store_name - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_name - description: Store name - dimension: - is_time: false - ai_context: - synonyms: - - "store name" - - "location name" - - - name: s_city - expression: - dialects: - - dialect: ANSI_SQL - expression: s_city - description: City where store is located - dimension: - is_time: false - ai_context: - synonyms: - - "city" - - "location" - - - name: s_state - expression: - dialects: - - dialect: ANSI_SQL - expression: s_state - description: State where store is located - dimension: - is_time: false - ai_context: - synonyms: - - "state" - - "region" - - - name: s_number_employees - expression: - dialects: - - dialect: ANSI_SQL - expression: s_number_employees - description: Number of employees at the store - ai_context: - synonyms: - - "employee count" - - "staff size" - - # Relationships between datasets - relationships: - - name: store_sales_to_date - from: store_sales - to: date_dim - from_columns: [ss_sold_date_sk] - to_columns: [d_date_sk] + - "units sold" + - "quantity" + + - name: ss_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sales_price + description: Sales price per unit + ai_context: + synonyms: + - "unit price" + - "price" + + - name: ss_ext_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + ai_context: + synonyms: + - "total price" + - "line total" + + - name: ss_net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_profit + description: Net profit from the sale ai_context: synonyms: - - "sales date relationship" - - "when sale occurred" - - - name: store_sales_to_customer - from: store_sales - to: customer - from_columns: [ss_customer_sk] - to_columns: [c_customer_sk] + - "profit" + - "margin" + + # Dimension table: Date + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] # Simple primary key + unique_keys: + - [d_date_sk] # Simple key: single column + description: Date dimension with calendar attributes + ai_context: + synonyms: + - "calendar" + - "dates" + - "time periods" + + fields: + - name: d_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date_sk + description: Surrogate key for date + dimension: + is_time: false + + - name: d_date + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date + description: Actual date value + dimension: + is_time: true ai_context: synonyms: - - "customer purchase relationship" - - "who bought" - - - name: store_sales_to_item - from: store_sales - to: item - from_columns: [ss_item_sk] - to_columns: [i_item_sk] + - "date" + - "calendar date" + + - name: d_year + expression: + dialects: + - dialect: ANSI_SQL + expression: d_year + description: Year + dimension: + is_time: true ai_context: synonyms: - - "product sold relationship" - - "what was sold" - - - name: store_sales_to_store - from: store_sales - to: store - from_columns: [ss_store_sk] - to_columns: [s_store_sk] + - "year" + + - name: d_quarter_name + expression: + dialects: + - dialect: ANSI_SQL + expression: d_quarter_name + description: Quarter name (e.g., 2024Q1) + dimension: + is_time: true ai_context: synonyms: - - "store location relationship" - - "where sale occurred" + - "quarter" + - "fiscal quarter" - # Semantic model-level metrics spanning multiple datasets - metrics: - - name: total_sales + - name: d_moy expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) - description: Total sales revenue across all transactions + expression: d_moy + description: Month of year (1-12) + dimension: + is_time: true ai_context: synonyms: - - "total revenue" - - "gross sales" - - "sales amount" + - "month" + + # Dimension table: Customer + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] # Simple primary key + unique_keys: + - [c_customer_sk] # Simple key: single column + description: Customer dimension with demographic information + ai_context: + synonyms: + - "customers" + - "shoppers" + - "buyers" + + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_sk + description: Surrogate key for customer + dimension: + is_time: false - - name: total_profit + - name: c_customer_id expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_net_profit) - description: Total net profit from store sales + expression: c_customer_id + description: Business key for customer + dimension: + is_time: false ai_context: synonyms: - - "net profit" - - "total earnings" - - "profit" + - "customer ID" + - "customer number" + + - name: c_first_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name + description: Customer first name + dimension: + is_time: false + + - name: c_last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_last_name + description: Customer last name + dimension: + is_time: false + + - name: customer_full_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name || ' ' || c_last_name + description: Customer full name (computed field) + dimension: + is_time: false + ai_context: + synonyms: + - "full name" + - "customer name" + + - name: c_email_address + expression: + dialects: + - dialect: ANSI_SQL + expression: c_email_address + description: Customer email address + dimension: + is_time: false + ai_context: + synonyms: + - "email" + - "contact" + + # Dimension table: Item (Product) + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] # Simple primary key + unique_keys: + - [i_item_sk] # Simple key: single column + description: Item/Product dimension with product attributes + ai_context: + synonyms: + - "products" + - "items" + - "merchandise" + + fields: + - name: i_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_sk + description: Surrogate key for item + dimension: + is_time: false + + - name: i_item_id + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_id + description: Business key for item + dimension: + is_time: false + ai_context: + synonyms: + - "item ID" + - "product ID" + - "SKU" + + - name: i_item_desc + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_desc + description: Item description + dimension: + is_time: false + ai_context: + synonyms: + - "product description" + - "item name" + + - name: i_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: i_brand + description: Brand name + dimension: + is_time: false + ai_context: + synonyms: + - "brand" + - "manufacturer" + + - name: i_category + expression: + dialects: + - dialect: ANSI_SQL + expression: i_category + description: Item category + dimension: + is_time: false + ai_context: + synonyms: + - "product category" + - "department" + + - name: i_current_price + expression: + dialects: + - dialect: ANSI_SQL + expression: i_current_price + description: Current price of the item + dimension: + is_time: false + ai_context: + synonyms: + - "price" + - "list price" + + # Dimension table: Store + - name: store + source: tpcds.public.store + primary_key: [s_store_sk] # Simple primary key + unique_keys: + - [s_store_id] # Simple key: single column + description: Store dimension with location and store attributes + ai_context: + synonyms: + - "stores" + - "retail locations" + - "branches" + + fields: + - name: s_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_sk + description: Surrogate key for store + dimension: + is_time: false - - name: customer_lifetime_value + - name: s_store_id expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) - description: Average lifetime sales value per customer + expression: s_store_id + description: Business key for store + dimension: + is_time: false ai_context: synonyms: - - "CLV" - - "LTV" - - "customer value" - - "lifetime revenue" + - "store ID" + - "store number" - - name: sales_by_brand + - name: s_store_name expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) - description: Total sales by brand (requires grouping by item.i_brand) + expression: s_store_name + description: Store name + dimension: + is_time: false ai_context: synonyms: - - "brand sales" - - "brand performance" - - "brand revenue" + - "store name" + - "location name" - - name: store_productivity + - name: s_city expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) - description: Sales per employee across stores + expression: s_city + description: City where store is located + dimension: + is_time: false ai_context: synonyms: - - "sales per employee" - - "employee productivity" - - "revenue per employee" - - custom_extensions: - - vendor_name: SALESFORCE - data: | - { - "tableau_workbook_id": "tpcds_retail_dashboard", - "einstein_enabled": true, - "crm_sync": { - "enabled": true, - "sync_frequency": "daily", - "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" - }, - "tableau_semantics": { - "published": true, - "version": "0.1.1" - } - } - - - vendor_name: DBT - data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' + - "city" + - "location" + + - name: s_state + expression: + dialects: + - dialect: ANSI_SQL + expression: s_state + description: State where store is located + dimension: + is_time: false + ai_context: + synonyms: + - "state" + - "region" + + - name: s_number_employees + expression: + dialects: + - dialect: ANSI_SQL + expression: s_number_employees + description: Number of employees at the store + ai_context: + synonyms: + - "employee count" + - "staff size" + +# Relationships between datasets +relationships: + - name: store_sales_to_date + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + ai_context: + synonyms: + - "sales date relationship" + - "when sale occurred" + + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] + ai_context: + synonyms: + - "customer purchase relationship" + - "who bought" + + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + ai_context: + synonyms: + - "product sold relationship" + - "what was sold" + + - name: store_sales_to_store + from: store_sales + to: store + from_columns: [ss_store_sk] + to_columns: [s_store_sk] + ai_context: + synonyms: + - "store location relationship" + - "where sale occurred" + +# Semantic model-level metrics spanning multiple datasets +metrics: + - name: total_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales revenue across all transactions + ai_context: + synonyms: + - "total revenue" + - "gross sales" + - "sales amount" + + - name: total_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_net_profit) + description: Total net profit from store sales + ai_context: + synonyms: + - "net profit" + - "total earnings" + - "profit" + + - name: customer_lifetime_value + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + description: Average lifetime sales value per customer + ai_context: + synonyms: + - "CLV" + - "LTV" + - "customer value" + - "lifetime revenue" + + - name: sales_by_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales by brand (requires grouping by item.i_brand) + ai_context: + synonyms: + - "brand sales" + - "brand performance" + - "brand revenue" + + - name: store_productivity + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) + description: Sales per employee across stores + ai_context: + synonyms: + - "sales per employee" + - "employee productivity" + - "revenue per employee" + +custom_extensions: + - vendor_name: SALESFORCE + data: | + { + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "0.1.1" + } + } + + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' diff --git a/converters/orionbelt/tests/test_ossie_converter_cumulative.py b/converters/orionbelt/tests/test_ossie_converter_cumulative.py index 2dddf059..12a7905e 100644 --- a/converters/orionbelt/tests/test_ossie_converter_cumulative.py +++ b/converters/orionbelt/tests/test_ossie_converter_cumulative.py @@ -106,7 +106,7 @@ def _convert(self) -> tuple[dict, list[str]]: return result, converter.warnings def _find_metric(self, ossie: dict, name: str) -> dict | None: - for m in ossie["semantic_model"][0].get("metrics", []): + for m in ossie.get("metrics", []): if m["name"] == name: return m return None @@ -276,7 +276,7 @@ def test_cumulative_without_measure_skipped(self) -> None: converter = conv.OBMLtoOssie(obml) ossie = converter.convert() # Should be skipped with a warning - metrics = ossie["semantic_model"][0].get("metrics", []) + metrics = ossie.get("metrics", []) assert all(m["name"] != "Bad Cumulative" for m in metrics) assert any("Bad Cumulative" in w for w in converter.warnings) @@ -315,7 +315,7 @@ def test_cumulative_with_synonyms_roundtrip(self) -> None: ossie = converter1.convert() # Check synonyms in Ossie ai_context - ossie_metric = ossie["semantic_model"][0]["metrics"][-1] + ossie_metric = ossie["metrics"][-1] assert "cumulative revenue" in ossie_metric.get("ai_context", {}).get("synonyms", []) # Roundtrip back @@ -329,7 +329,7 @@ def test_mixed_model_all_metric_types(self) -> None: """Model with measures, derived metrics, and cumulative metrics all convert.""" converter = conv.OBMLtoOssie(_OBML_WITH_CUMULATIVES) ossie = converter.convert() - metric_names = [m["name"] for m in ossie["semantic_model"][0]["metrics"]] + metric_names = [m["name"] for m in ossie["metrics"]] # All four should be present: Revenue (measure), Running Revenue, # Rolling 7d Revenue, MTD Revenue, Derived Metric diff --git a/converters/orionbelt/tests/test_ossie_converter_filters.py b/converters/orionbelt/tests/test_ossie_converter_filters.py index 1e419bcb..931cb238 100644 --- a/converters/orionbelt/tests/test_ossie_converter_filters.py +++ b/converters/orionbelt/tests/test_ossie_converter_filters.py @@ -97,7 +97,7 @@ class TestOBMLtoOssieFilters: def test_filters_in_custom_extensions(self): converter = conv.OBMLtoOssie(_OBML_WITH_FILTERS) ossie = converter.convert() - sem = ossie["semantic_model"][0] + sem = ossie exts = sem.get("custom_extensions", []) assert len(exts) >= 1 common = next(e for e in exts if e["vendor_name"] == "ORIONBELT") @@ -111,7 +111,7 @@ def test_no_filters_no_key(self): obml = {**_OBML_WITH_FILTERS, "filters": []} converter = conv.OBMLtoOssie(obml) ossie = converter.convert() - sem = ossie["semantic_model"][0] + sem = ossie common = next(e for e in sem["custom_extensions"] if e["vendor_name"] == "ORIONBELT") data = json.loads(common["data"]) assert "obml_filters" not in data diff --git a/converters/orionbelt/tests/test_ossie_converter_measure_overrides.py b/converters/orionbelt/tests/test_ossie_converter_measure_overrides.py index 6c9e47f8..863450aa 100644 --- a/converters/orionbelt/tests/test_ossie_converter_measure_overrides.py +++ b/converters/orionbelt/tests/test_ossie_converter_measure_overrides.py @@ -274,7 +274,7 @@ def _delegated_obml(self) -> dict[str, Any]: @staticmethod def _ossie_metrics(ossie: dict[str, Any]) -> list[dict[str, Any]]: - return ossie["semantic_model"][0].get("metrics", []) + return ossie.get("metrics", []) def test_obml_to_ossie_emits_measure_placeholder(self): obml = self._delegated_obml() diff --git a/converters/orionbelt/tests/test_ossie_converter_ontology.py b/converters/orionbelt/tests/test_ossie_converter_ontology.py index b9698f3f..64933665 100644 --- a/converters/orionbelt/tests/test_ossie_converter_ontology.py +++ b/converters/orionbelt/tests/test_ossie_converter_ontology.py @@ -101,6 +101,7 @@ def test_embeds_core_semantic_model(self) -> None: omap = doc["ontology_mappings"][0] assert omap["name"] == "sales_map" assert "semantic_model" in omap + assert not {"version", "dialects", "vendors"} & set(omap["semantic_model"]) assert omap["semantic_model"]["name"] == "sales" assert {d["name"] for d in omap["semantic_model"]["datasets"]} == { "Customers", diff --git a/converters/orionbelt/tests/test_ossie_converter_pop.py b/converters/orionbelt/tests/test_ossie_converter_pop.py index 47b021a0..2dba14e6 100644 --- a/converters/orionbelt/tests/test_ossie_converter_pop.py +++ b/converters/orionbelt/tests/test_ossie_converter_pop.py @@ -152,7 +152,7 @@ def _convert(self) -> tuple[dict, list[str]]: return result, converter.warnings def _find_metric(self, ossie: dict, name: str) -> dict | None: - for m in ossie["semantic_model"][0].get("metrics", []): + for m in ossie.get("metrics", []): if m["name"] == name: return m return None @@ -342,7 +342,7 @@ def test_pop_without_config_skipped(self) -> None: } converter = conv.OBMLtoOssie(obml) ossie = converter.convert() - metrics = ossie["semantic_model"][0].get("metrics", []) + metrics = ossie.get("metrics", []) assert all(m["name"] != "Bad PoP" for m in metrics) assert any("Bad PoP" in w for w in converter.warnings) @@ -371,7 +371,7 @@ def test_pop_without_expression_skipped(self) -> None: } converter = conv.OBMLtoOssie(obml) ossie = converter.convert() - metrics = ossie["semantic_model"][0].get("metrics", []) + metrics = ossie.get("metrics", []) assert all(m["name"] != "Bad PoP" for m in metrics) assert any("Bad PoP" in w for w in converter.warnings) @@ -415,7 +415,7 @@ def test_pop_with_synonyms_roundtrip(self) -> None: # Check synonyms in Ossie ai_context ossie_metric = next( - m for m in ossie["semantic_model"][0]["metrics"] if m["name"] == "YoY Growth" + m for m in ossie["metrics"] if m["name"] == "YoY Growth" ) assert "year-over-year" in ossie_metric.get("ai_context", {}).get("synonyms", []) @@ -430,7 +430,7 @@ def test_mixed_model_all_metric_types(self) -> None: """Model with measures, derived, cumulative, and PoP metrics all convert.""" converter = conv.OBMLtoOssie(_OBML_WITH_POP) ossie = converter.convert() - metric_names = [m["name"] for m in ossie["semantic_model"][0]["metrics"]] + metric_names = [m["name"] for m in ossie["metrics"]] # Revenue (measure → Ossie metric) plus all 4 PoP metrics + derived assert "Revenue" in metric_names diff --git a/converters/orionbelt/tests/test_ossie_converter_roundtrip_robustness.py b/converters/orionbelt/tests/test_ossie_converter_roundtrip_robustness.py index d94f70f9..2958d862 100644 --- a/converters/orionbelt/tests/test_ossie_converter_roundtrip_robustness.py +++ b/converters/orionbelt/tests/test_ossie_converter_roundtrip_robustness.py @@ -63,7 +63,7 @@ class TestMetricRoundTripCodeVsName: def test_emitter_uses_physical_code(self) -> None: ossie = conv.OBMLtoOssie(self._OBML).convert() - sql = ossie["semantic_model"][0]["metrics"][0]["expression"]["dialects"][0]["expression"] + sql = ossie["metrics"][0]["expression"]["dialects"][0]["expression"] assert "fact_orders" in sql # confirms the emit side uses the code def test_measure_survives_round_trip(self) -> None: @@ -86,38 +86,34 @@ def test_third_party_field_name_differs_from_code(self) -> None: # the metric references the physical code. Must resolve, not drop. ossie = { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "sales", + "datasets": [ { - "name": "sales", - "datasets": [ + "name": "Orders", + "source": "WH.PUBLIC.fact_orders", + "fields": [ { - "name": "Orders", - "source": "WH.PUBLIC.fact_orders", - "fields": [ - { - "name": "Amount", - "expression": { - "dialects": [ - {"dialect": "ANSI_SQL", "expression": "amount"} - ] - }, - "data_type": "number", - } - ], - } - ], - "metrics": [ - { - "name": "Revenue", + "name": "Amount", "expression": { "dialects": [ - {"dialect": "ANSI_SQL", "expression": "SUM(fact_orders.amount)"} + {"dialect": "ANSI_SQL", "expression": "amount"} ] }, + "data_type": "number", } ], } ], + "metrics": [ + { + "name": "Revenue", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "SUM(fact_orders.amount)"} + ] + }, + } + ], } obml = conv.OssietoOBML(ossie).convert() assert "Revenue" in obml.get("measures", {}), obml.get("measures") @@ -132,41 +128,37 @@ def test_physical_code_maps_to_field_name_with_space(self) -> None: # fails query resolution. ossie = { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "sales", + "datasets": [ { - "name": "sales", - "datasets": [ + "name": "Orders", + "source": "WH.PUBLIC.fact_orders", + "fields": [ { - "name": "Orders", - "source": "WH.PUBLIC.fact_orders", - "fields": [ - { - "name": "Net Amount", - "expression": { - "dialects": [ - {"dialect": "ANSI_SQL", "expression": "net_amount"} - ] - }, - "data_type": "number", - } - ], - } - ], - "metrics": [ - { - "name": "Net Revenue", + "name": "Net Amount", "expression": { "dialects": [ - { - "dialect": "ANSI_SQL", - "expression": "SUM(fact_orders.net_amount)", - } + {"dialect": "ANSI_SQL", "expression": "net_amount"} ] }, + "data_type": "number", } ], } ], + "metrics": [ + { + "name": "Net Revenue", + "expression": { + "dialects": [ + { + "dialect": "ANSI_SQL", + "expression": "SUM(fact_orders.net_amount)", + } + ] + }, + } + ], } obml = conv.OssietoOBML(ossie).convert() assert "Net Revenue" in obml.get("measures", {}), obml.get("measures") @@ -182,41 +174,37 @@ def test_quoted_physical_identifiers_resolve(self) -> None: # must still resolve to a queryable measure, not fall through to LOSSY. ossie = { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "sales", + "datasets": [ { - "name": "sales", - "datasets": [ + "name": "Orders", + "source": 'WH.PUBLIC."fact_orders"', + "fields": [ { - "name": "Orders", - "source": 'WH.PUBLIC."fact_orders"', - "fields": [ - { - "name": "Amount", - "expression": { - "dialects": [ - {"dialect": "ANSI_SQL", "expression": '"net_amount"'} - ] - }, - "data_type": "number", - } - ], - } - ], - "metrics": [ - { - "name": "Revenue", + "name": "Amount", "expression": { "dialects": [ - { - "dialect": "ANSI_SQL", - "expression": "SUM(fact_orders.net_amount)", - } + {"dialect": "ANSI_SQL", "expression": '"net_amount"'} ] }, + "data_type": "number", } ], } ], + "metrics": [ + { + "name": "Revenue", + "expression": { + "dialects": [ + { + "dialect": "ANSI_SQL", + "expression": "SUM(fact_orders.net_amount)", + } + ] + }, + } + ], } obml = conv.OssietoOBML(ossie).convert() assert "Revenue" in obml.get("measures", {}), obml.get("measures") @@ -249,14 +237,10 @@ class TestDimensionNameCollision: def test_both_dimensions_survive(self) -> None: ossie = { "version": "0.2.0.dev0", - "semantic_model": [ - { - "name": "sales", - "datasets": [ - _dim_dataset("Orders", "WH.PUBLIC.orders"), - _dim_dataset("Invoices", "WH.PUBLIC.invoices"), - ], - } + "name": "sales", + "datasets": [ + _dim_dataset("Orders", "WH.PUBLIC.orders"), + _dim_dataset("Invoices", "WH.PUBLIC.invoices"), ], } c = conv.OssietoOBML(ossie) @@ -279,7 +263,7 @@ class TestValidateOssieRobustness: def test_malformed_datasets_does_not_raise(self) -> None: # datasets is a string, not a list — must return a result, not raise. r = conv.validate_ossie( - {"version": "0.1.1", "semantic_model": [{"name": "x", "datasets": "not-an-array"}]} + {"version": "0.1.1", "name": "x", "datasets": "not-an-array"} ) assert r is not None # No garbage semantic errors from iterating a string char-by-char. @@ -289,7 +273,7 @@ def test_malformed_fields_does_not_raise(self) -> None: r = conv.validate_ossie( { "version": "0.2.0.dev0", - "semantic_model": [{"name": "x", "datasets": [{"name": "D", "fields": "nope"}]}], + "name": "x", "datasets": [{"name": "D", "fields": "nope"}], } ) assert r is not None @@ -299,14 +283,10 @@ def test_still_flags_duplicate_datasets(self) -> None: r = conv.validate_ossie( { "version": "0.2.0.dev0", - "semantic_model": [ - { - "name": "m", - "datasets": [ - {"name": "D", "source": "a.b.d", "fields": []}, - {"name": "D", "source": "a.b.d2", "fields": []}, - ], - } + "name": "m", + "datasets": [ + {"name": "D", "source": "a.b.d", "fields": []}, + {"name": "D", "source": "a.b.d2", "fields": []}, ], } ) diff --git a/converters/orionbelt/tests/test_ossie_converter_trend_v26.py b/converters/orionbelt/tests/test_ossie_converter_trend_v26.py index e8775c62..e2c55fd2 100644 --- a/converters/orionbelt/tests/test_ossie_converter_trend_v26.py +++ b/converters/orionbelt/tests/test_ossie_converter_trend_v26.py @@ -121,7 +121,7 @@ def _ossie_metric(ossie: dict, name: str) -> dict: - for m in ossie["semantic_model"][0].get("metrics", []): + for m in ossie.get("metrics", []): if m["name"] == name: return m raise AssertionError(f"missing Ossie metric {name!r}") diff --git a/converters/orionbelt/tests/test_ossie_converter_vendors.py b/converters/orionbelt/tests/test_ossie_converter_vendors.py index 35c7d22a..536b9fbe 100644 --- a/converters/orionbelt/tests/test_ossie_converter_vendors.py +++ b/converters/orionbelt/tests/test_ossie_converter_vendors.py @@ -57,23 +57,19 @@ def test_obml_to_ossie_uses_orionbelt(self) -> None: }, } ossie = conv.OBMLtoOssie(obml).convert() - ce = ossie["semantic_model"][0]["custom_extensions"] + ce = ossie["custom_extensions"] assert all(e["vendor_name"] == "ORIONBELT" for e in ce) def test_ossie_to_obml_native_stash_uses_ossie_vendor(self) -> None: ossie = { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "m", + "datasets": [ { - "name": "m", - "datasets": [ - { - "name": "Customers", - "source": "WH.PUB.customers", - "unique_keys": [["customer_id"]], - "fields": [_ossie_field("customer_id")], - } - ], + "name": "Customers", + "source": "WH.PUB.customers", + "unique_keys": [["customer_id"]], + "fields": [_ossie_field("customer_id")], } ], } @@ -88,34 +84,30 @@ class TestForeignVendorRoundtrip: def _ossie_with_foreign(self) -> dict[str, Any]: return { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "demo", + "custom_extensions": [ + {"vendor_name": "DBT", "data": json.dumps({"model": "mart_x"})} + ], + "datasets": [ { - "name": "demo", + "name": "Customers", + "source": "WH.PUB.customers", "custom_extensions": [ - {"vendor_name": "DBT", "data": json.dumps({"model": "mart_x"})} - ], - "datasets": [ { - "name": "Customers", - "source": "WH.PUB.customers", - "custom_extensions": [ + "vendor_name": "SALESFORCE", + "data": json.dumps({"object": "Account"}), + } + ], + "fields": [ + _ossie_field( + "customer_id", + custom_extensions=[ { - "vendor_name": "SALESFORCE", - "data": json.dumps({"object": "Account"}), + "vendor_name": "GOODDATA", + "data": json.dumps({"ldm": "a"}), } ], - "fields": [ - _ossie_field( - "customer_id", - custom_extensions=[ - { - "vendor_name": "GOODDATA", - "data": json.dumps({"ldm": "a"}), - } - ], - ) - ], - } + ) ], } ], @@ -136,7 +128,7 @@ def test_foreign_carried_into_obml(self) -> None: def test_foreign_reemitted_to_ossie(self) -> None: obml = conv.OssietoOBML(self._ossie_with_foreign()).convert() ossie = conv.OBMLtoOssie(obml, "demo").convert() - sm = ossie["semantic_model"][0] + sm = ossie model_vendors = {e["vendor_name"] for e in sm["custom_extensions"]} ds_vendors = {e["vendor_name"] for e in sm["datasets"][0]["custom_extensions"]} field_vendors = { @@ -149,31 +141,27 @@ def test_foreign_reemitted_to_ossie(self) -> None: def test_foreign_metric_roundtrip(self) -> None: ossie_in = { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "demo", + "datasets": [ { - "name": "demo", - "datasets": [ - { - "name": "Sales", - "source": "WH.PUB.sales", - "fields": [_ossie_field("amount")], - } - ], - "metrics": [ - { - "name": "Total", - "data_type": "number", - "description": "d", - "custom_extensions": [ - {"vendor_name": "LOOKER", "data": json.dumps({"view": "sales"})} - ], - "expression": { - "dialects": [ - {"dialect": "ANSI_SQL", "expression": "SUM(sales.amount)"} - ] - }, - } + "name": "Sales", + "source": "WH.PUB.sales", + "fields": [_ossie_field("amount")], + } + ], + "metrics": [ + { + "name": "Total", + "data_type": "number", + "description": "d", + "custom_extensions": [ + {"vendor_name": "LOOKER", "data": json.dumps({"view": "sales"})} ], + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "SUM(sales.amount)"} + ] + }, } ], } @@ -185,7 +173,7 @@ def test_foreign_metric_roundtrip(self) -> None: "customExtensions" ] ossie_out = conv.OBMLtoOssie(obml, "demo").convert() - metric = ossie_out["semantic_model"][0]["metrics"][0] + metric = ossie_out["metrics"][0] assert any(e["vendor_name"] == "LOOKER" for e in metric["custom_extensions"]) def test_foreign_dimension_emitted_to_field(self) -> None: @@ -213,7 +201,7 @@ def test_foreign_dimension_emitted_to_field(self) -> None: } ossie = conv.OBMLtoOssie(obml).convert() field = next( - f for f in ossie["semantic_model"][0]["datasets"][0]["fields"] if f["name"] == "status" + f for f in ossie["datasets"][0]["fields"] if f["name"] == "status" ) assert any(e["vendor_name"] == "TABLEAU" for e in field["custom_extensions"]) @@ -248,7 +236,7 @@ def test_legacy_common_and_obsl_still_read(self) -> None: }, } ossie = conv.OBMLtoOssie(obml).convert() - ds = ossie["semantic_model"][0]["datasets"][0] + ds = ossie["datasets"][0] assert ds.get("unique_keys") == [["order_id"]] order_id = next(f for f in ds["fields"] if f["name"] == "order_id") assert order_id.get("label") == "filter" diff --git a/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py b/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py index d3fb61d0..e46ad84c 100644 --- a/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py +++ b/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py @@ -44,22 +44,18 @@ def _ossie_model(metrics: list[dict[str, Any]]) -> dict[str, Any]: """Minimal single-dataset Ossie v0.2 model carrying the given metrics.""" return { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "sales", + "datasets": [ { - "name": "sales", - "datasets": [ - { - "name": "Orders", - "source": "ANALYTICS.PUBLIC.ORDERS", - "fields": [ - {"name": "amount", "data_type": "number"}, - {"name": "id", "data_type": "integer"}, - ], - } + "name": "Orders", + "source": "ANALYTICS.PUBLIC.ORDERS", + "fields": [ + {"name": "amount", "data_type": "number"}, + {"name": "id", "data_type": "integer"}, ], - "metrics": metrics, } ], + "metrics": metrics, } @@ -230,7 +226,7 @@ def test_roundtrip_restores_preserved_metric(self) -> None: obml = conv.OssietoOBML(ossie).convert() ossie_again = conv.OBMLtoOssie(obml, "sales").convert() - metrics = ossie_again["semantic_model"][0].get("metrics", []) + metrics = ossie_again.get("metrics", []) restored = next((m for m in metrics if m["name"] == "Mdx Thing"), None) assert restored is not None # Verbatim: expression dialect + description survive the round trip. @@ -269,7 +265,7 @@ def test_preserved_metric_keeps_dialect_and_vendor_on_the_metric(self) -> None: ossie_again = conv.OBMLtoOssie(obml, "sales").convert() restored = next( - m for m in ossie_again["semantic_model"][0]["metrics"] if m["name"] == "Cube Metric" + m for m in ossie_again["metrics"] if m["name"] == "Cube Metric" ) assert [d["dialect"] for d in restored["expression"]["dialects"]] == ["MDX"] assert any(e["vendor_name"] == "GOODDATA" for e in restored["custom_extensions"]) @@ -310,7 +306,7 @@ def test_collision_drops_preserved_copy_not_duplicate(self) -> None: converter = conv.OBMLtoOssie(obml, "sales") ossie_again = converter.convert() - metrics = ossie_again["semantic_model"][0].get("metrics", []) + metrics = ossie_again.get("metrics", []) revenue = [m for m in metrics if m["name"] == "Revenue"] # Exactly one "Revenue" — no duplicate that would fail validation. assert len(revenue) == 1 @@ -365,7 +361,7 @@ def test_obml_to_ossie_convert_is_idempotent(self) -> None: warnings_after_first = list(converter.warnings) ossie_again = converter.convert() - metrics = ossie_again["semantic_model"][0].get("metrics", []) + metrics = ossie_again.get("metrics", []) assert [m["name"] for m in metrics].count("Mdx Thing") == 1 assert converter.warnings == warnings_after_first diff --git a/converters/orionbelt/tests/test_ossie_v02_compat.py b/converters/orionbelt/tests/test_ossie_v02_compat.py index b097f341..6b0d6ee1 100644 --- a/converters/orionbelt/tests/test_ossie_v02_compat.py +++ b/converters/orionbelt/tests/test_ossie_v02_compat.py @@ -20,7 +20,7 @@ Covers the v2.6 spec bump from Ossie v0.1.1 → v0.2.0.dev0: - Emitted ``version`` is the v0.2 constant -- Top-level ``dialects`` / ``vendors`` informational arrays are present +- Dialects and vendors are represented per expression and extension, not at the root - Dataset ``primary_key`` is promoted from per-column ``primaryKey: true`` - Dataset ``unique_keys`` round-trips lossly via OBSL custom_extensions - Field ``label`` round-trips via OBSL custom_extensions @@ -120,43 +120,39 @@ def walk(node: Any) -> None: _OSSIE_V01_INPUT: dict[str, Any] = { "version": "0.1.1", - "semantic_model": [ + "name": "ecommerce", + "datasets": [ { - "name": "ecommerce", - "datasets": [ + "name": "Orders", + "source": "WAREHOUSE.PUBLIC.orders", + # Legacy: PK stashed in custom_extensions (pre-v0.2 shape) + "custom_extensions": [ { - "name": "Orders", - "source": "WAREHOUSE.PUBLIC.orders", - # Legacy: PK stashed in custom_extensions (pre-v0.2 shape) - "custom_extensions": [ + "vendor_name": "COMMON", + "data": json.dumps( { - "vendor_name": "COMMON", - "data": json.dumps( - { - "obml_primary_key": ["order_id"], - "obml_unique_keys": [["order_id"], ["order_number"]], - } - ), + "obml_primary_key": ["order_id"], + "obml_unique_keys": [["order_id"], ["order_number"]], } - ], - "fields": [ - { - "name": "order_id", - "expression": { - "dialects": [{"dialect": "ANSI_SQL", "expression": "order_id"}] - }, - "data_type": "string", - }, - { - "name": "amount", - "expression": { - "dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}] - }, - "data_type": "number", - }, - ], + ), } ], + "fields": [ + { + "name": "order_id", + "expression": { + "dialects": [{"dialect": "ANSI_SQL", "expression": "order_id"}] + }, + "data_type": "string", + }, + { + "name": "amount", + "expression": { + "dialects": [{"dialect": "ANSI_SQL", "expression": "amount"}] + }, + "data_type": "number", + }, + ], } ], } @@ -174,17 +170,18 @@ def test_top_level_version_is_v02(self) -> None: assert ossie["version"].startswith("0.2") def test_no_root_dialects_or_vendors(self) -> None: - # The published Ossie core schema forbids root-level dialects/vendors - # (root is additionalProperties:false). See Ossie PR #148. + # Root advertisements are optional; the converter emits dialects and + # vendors at each expression/extension that uses them. ossie = conv.OBMLtoOssie(_OBML_WITH_PK_AND_LABEL).convert() assert "dialects" not in ossie assert "vendors" not in ossie - assert set(ossie.keys()) <= {"version", "semantic_model"} + assert "semantic_model" not in ossie + assert {"version", "name", "datasets"} <= set(ossie) def test_dialect_tagged_per_expression(self) -> None: # Dialects live on each expression, the schema-valid home. ossie = conv.OBMLtoOssie(_OBML_WITH_PK_AND_LABEL).convert() - for metric in ossie["semantic_model"][0].get("metrics", []): + for metric in ossie.get("metrics", []): tags = [d["dialect"] for d in metric["expression"]["dialects"]] assert "ANSI_SQL" in tags @@ -194,7 +191,7 @@ def test_vendor_tagged_per_entity_not_root(self) -> None: # field label (no vendor tag needed). ossie = conv.OBMLtoOssie(_OBML_WITH_PK_AND_LABEL).convert() assert "ORIONBELT" in _collect_vendor_names(ossie) - fields = ossie["semantic_model"][0]["datasets"][0]["fields"] + fields = ossie["datasets"][0]["fields"] assert any(f.get("label") for f in fields), "expected a native Ossie field label" @@ -206,7 +203,7 @@ def test_vendor_tagged_per_entity_not_root(self) -> None: class TestPrimaryKey: def test_composite_pk_emitted_in_declaration_order(self) -> None: ossie = conv.OBMLtoOssie(_OBML_WITH_PK_AND_LABEL).convert() - ds = ossie["semantic_model"][0]["datasets"][0] + ds = ossie["datasets"][0] # Two columns flagged primaryKey: emit composite in declaration order assert ds["primary_key"] == ["order_id", "line_no"] @@ -223,26 +220,22 @@ def test_pk_roundtrip_restores_per_column_flag(self) -> None: def test_unknown_pk_column_emits_warning(self) -> None: bad = { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "x", + "datasets": [ { - "name": "x", - "datasets": [ + "name": "Orders", + "source": "a.b.c", + "primary_key": ["no_such_column"], + "fields": [ { - "name": "Orders", - "source": "a.b.c", - "primary_key": ["no_such_column"], - "fields": [ - { - "name": "amount", - "expression": { - "dialects": [ - {"dialect": "ANSI_SQL", "expression": "amount"} - ] - }, - "data_type": "number", - }, - ], - } + "name": "amount", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "amount"} + ] + }, + "data_type": "number", + }, ], } ], @@ -256,7 +249,7 @@ def test_unknown_pk_column_emits_warning(self) -> None: class TestUniqueKeys: def test_unique_keys_roundtrip(self) -> None: ossie = conv.OBMLtoOssie(_OBML_WITH_PK_AND_LABEL).convert() - ds = ossie["semantic_model"][0]["datasets"][0] + ds = ossie["datasets"][0] assert ds.get("unique_keys") == [["order_id"], ["order_id", "line_no"]] @@ -268,7 +261,7 @@ def test_unique_keys_roundtrip(self) -> None: class TestFieldLabel: def test_label_emitted_from_custom_extensions(self) -> None: ossie = conv.OBMLtoOssie(_OBML_WITH_PK_AND_LABEL).convert() - fields = ossie["semantic_model"][0]["datasets"][0]["fields"] + fields = ossie["datasets"][0]["fields"] order_id_field = next(f for f in fields if f["name"] == "order_id") assert order_id_field["label"] == "filter" @@ -299,7 +292,7 @@ def test_v01_input_promotes_primary_key(self) -> None: # Manually invoke the shim converter.ossie = json.loads(json.dumps(_OSSIE_V01_INPUT)) # deep copy converter._normalize_legacy_v01() - ds = converter.ossie["semantic_model"][0]["datasets"][0] + ds = converter.ossie["datasets"][0] assert ds["primary_key"] == ["order_id"] assert ds["unique_keys"] == [["order_id"], ["order_number"]] @@ -313,25 +306,21 @@ def test_v02_input_not_normalized(self) -> None: # v0.2 input declaring primary_key directly — shim must be a no-op v02 = { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "x", + "datasets": [ { - "name": "x", - "datasets": [ + "name": "Orders", + "source": "a.b.c", + "primary_key": ["order_id"], + "fields": [ { - "name": "Orders", - "source": "a.b.c", - "primary_key": ["order_id"], - "fields": [ - { - "name": "order_id", - "expression": { - "dialects": [ - {"dialect": "ANSI_SQL", "expression": "order_id"} - ] - }, - "data_type": "string", - } - ], + "name": "order_id", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "order_id"} + ] + }, + "data_type": "string", } ], } @@ -372,13 +361,12 @@ def test_tpcds_fixture_passes_v02_validation(self, schema_validator: Any) -> Non assert errors == [], [e.message for e in errors[:5]] def test_schema_rejects_root_dialects_and_vendors(self, schema_validator: Any) -> None: - """Guard Ossie PR #148: root-level dialects/vendors are non-conformant.""" + """Dialect and vendor metadata belongs to expressions and extensions.""" ossie = conv.OBMLtoOssie(_OBML_WITH_PK_AND_LABEL).convert() ossie["dialects"] = ["ANSI_SQL"] ossie["vendors"] = ["ORIONBELT"] messages = [e.message for e in schema_validator.iter_errors(ossie)] - assert any("dialects" in m for m in messages), messages - assert any("vendors" in m for m in messages), messages + assert any("Additional properties" in message for message in messages) # --------------------------------------------------------------------------- @@ -390,38 +378,34 @@ class TestMAQLDialect: def test_maql_only_metric_does_not_raise(self) -> None: ossie_in: dict[str, Any] = { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "gd", + "datasets": [ { - "name": "gd", - "datasets": [ - { - "name": "Sales", - "source": "a.b.sales", - "fields": [ - { - "name": "amount", - "expression": { - "dialects": [ - {"dialect": "ANSI_SQL", "expression": "amount"} - ] - }, - "data_type": "number", - } - ], - } - ], - "metrics": [ + "name": "Sales", + "source": "a.b.sales", + "fields": [ { - "name": "total_revenue_maql", + "name": "amount", "expression": { "dialects": [ - {"dialect": "MAQL", "expression": "SELECT SUM(amount)"}, + {"dialect": "ANSI_SQL", "expression": "amount"} ] }, + "data_type": "number", } ], } ], + "metrics": [ + { + "name": "total_revenue_maql", + "expression": { + "dialects": [ + {"dialect": "MAQL", "expression": "SELECT SUM(amount)"}, + ] + }, + } + ], } # The MAQL dialect is OBSL-unparseable but must not raise; it # surfaces as a warning that the MAQL expression couldn't be @@ -429,3 +413,23 @@ def test_maql_only_metric_does_not_raise(self) -> None: converter = conv.OssietoOBML(ossie_in) converter.convert() # No crash is the contract; warnings are acceptable. + + +@pytest.mark.parametrize( + "wrapper", + [[], [{"name": "first"}], [{"name": "first"}, {"name": "second"}], {"name": "first"}, None], +) +def test_legacy_model_wrappers_are_rejected(wrapper): + document = {"version": "0.2.0.dev0", "semantic_model": wrapper} + assert not conv.validate_ossie(document).valid + with pytest.raises(ValueError, match="Legacy 'semantic_model'"): + conv.OssietoOBML(document).convert() + + +@pytest.mark.parametrize("property_name", ["dialects", "vendors"]) +@pytest.mark.parametrize("value", [None, [], ["legacy"]]) +def test_removed_root_metadata_is_rejected(property_name, value): + document = conv.OBMLtoOssie(_OBML_WITH_PK_AND_LABEL).convert() + document[property_name] = value + with pytest.raises(ValueError, match="Root dialects and vendors"): + conv.OssietoOBML(document).convert() diff --git a/converters/polaris/README.md b/converters/polaris/README.md index 353be7df..5d4d730e 100644 --- a/converters/polaris/README.md +++ b/converters/polaris/README.md @@ -35,7 +35,7 @@ Requires Java 21+. ### Import (Polaris → Apache Ossie) -Reads all namespaces and tables from a Polaris catalog and generates an Ossie YAML file. +Reads all namespaces and tables from a Polaris catalog and generates one Ossie YAML file per nonempty namespace. ```bash java -jar target/ossie-polaris-converter-0.1.0-SNAPSHOT.jar import \ @@ -43,14 +43,26 @@ java -jar target/ossie-polaris-converter-0.1.0-SNAPSHOT.jar import \ --catalog my_catalog \ --client-id \ --client-secret \ - -o output.yaml + --output-dir models ``` -Each Polaris namespace becomes a separate Ossie semantic model containing datasets for every table in that namespace. +Each nonempty Polaris namespace becomes a separate Ossie document containing +datasets for every table in that namespace. Files have the flat Ossie structure: +`version`, `name`, `datasets`, `relationships`, and `metrics` are at the root. + +`--output-dir` creates the directory if needed. Namespaces are sorted, and filenames +combine a numbered prefix with a sanitized namespace name (for example, +`0001-sales.yaml`). This keeps filenames safe and distinct even when namespace +names would otherwise collide. Existing files in the output directory are never overwritten. + +For a catalog with exactly one nonempty namespace, use `-o model.yaml` to write a +single file, or omit both output options to print it to stdout. Multiple nonempty +namespaces require `--output-dir`; they are never combined or silently dropped. +An empty catalog produces a warning and no document. ### Export (Apache Ossie → Polaris) -Reads an Ossie YAML file and creates namespaces and Iceberg tables in a Polaris catalog. +Reads one flat Ossie YAML file and creates a namespace and Iceberg tables in a Polaris catalog. ```bash java -jar target/ossie-polaris-converter-0.1.0-SNAPSHOT.jar export \ @@ -61,7 +73,9 @@ java -jar target/ossie-polaris-converter-0.1.0-SNAPSHOT.jar export \ model.yaml ``` -Each Ossie semantic model becomes a Polaris namespace, and each dataset becomes an Iceberg table. +The Ossie model becomes a Polaris namespace, and each dataset becomes an Iceberg +table. Legacy `semantic_model` wrappers are rejected; split older multi-model +files and flatten each model before exporting. ### Options @@ -72,7 +86,8 @@ Each Ossie semantic model becomes a Polaris namespace, and each dataset becomes | `--client-id ID` | OAuth2 client ID | | `--client-secret SECRET` | OAuth2 client secret | | `--token TOKEN` | Pre-existing bearer token (alternative to client credentials) | -| `-o FILE` | Output file for import mode (default: stdout) | +| `-o FILE` | Output file for one nonempty namespace (default: stdout); replaces an existing file | +| `--output-dir DIR` | One file per nonempty namespace; cannot be combined with `-o` | ## Mapping Reference @@ -80,7 +95,7 @@ Each Ossie semantic model becomes a Polaris namespace, and each dataset becomes | Polaris / Iceberg | Ossie | |-------------------|-----| -| Namespace | `semantic_model` (name, description) | +| Namespace | Document root (`name`, `description`) | | Table | `dataset` (name) | | Table location (`catalog.namespace.table`) | `dataset.source` | | Schema fields | `field` with `ANSI_SQL` dialect expression and logical `datatype` | @@ -93,7 +108,7 @@ Each Ossie semantic model becomes a Polaris namespace, and each dataset becomes | Ossie | Polaris / Iceberg | |-----|-------------------| -| `semantic_model` | Namespace | +| Document root (`name`, `description`) | Namespace | | `dataset` | Table | | `dataset.source` | Stored in table property `ossie.source` | | `dataset.primary_key` | `identifier-field-ids` | diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieModelParser.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieModelParser.java index d8378d44..c91ac20a 100644 --- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieModelParser.java +++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieModelParser.java @@ -51,21 +51,27 @@ public OssieModel parse(Path yamlPath) throws IOException { @SuppressWarnings("unchecked") public OssieModel parse(InputStream is) { Yaml yaml = new Yaml(); - Map root = yaml.load(is); + Object document = yaml.load(is); + if (!(document instanceof Map)) { + throw new IllegalArgumentException("An Ossie document must be an object"); + } + Map root = (Map) document; + if (root.containsKey("semantic_model")) { + throw new IllegalArgumentException( + "Legacy semantic_model wrappers are not supported; use version, name, and datasets at the root"); + } + if (root.containsKey("dialects") || root.containsKey("vendors")) { + throw new IllegalArgumentException("Root dialects and vendors are not supported by the Ossie spec"); + } + if (!(root.get("version") instanceof String) + || !(root.get("name") instanceof String) + || !(root.get("datasets") instanceof List)) { + throw new IllegalArgumentException("An Ossie document requires version, name, and datasets at the root"); + } OssieModel model = new OssieModel(); model.setVersion((String) root.get("version")); - - List> smList = (List>) root.get("semantic_model"); - if (smList == null) { - return model; - } - - List semanticModels = new ArrayList<>(); - for (Map smMap : smList) { - semanticModels.add(parseSemanticModel(smMap)); - } - model.setSemanticModels(semanticModels); + model.setSemanticModel(parseSemanticModel(root)); return model; } diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssiePolarisConverter.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssiePolarisConverter.java index 265ccfce..f16a8957 100644 --- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssiePolarisConverter.java +++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssiePolarisConverter.java @@ -24,20 +24,26 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; /** * CLI entry point for the Ossie Polaris converter. *

* Supports two modes: *

    - *
  • import: Reads from a Polaris catalog and generates an Ossie YAML file
    • + *
  • import: Reads from a Polaris catalog and generates one Ossie YAML file per nonempty namespace
    • *
  • export: Reads an Ossie YAML file and creates tables in a Polaris catalog
    • *
* * 
  * Usage:
- *   ossie-polaris-converter import --url URL --catalog CATALOG [--client-id ID --client-secret SECRET] [-o output.yaml]
+ *   ossie-polaris-converter import --url URL --catalog CATALOG [options] [-o output.yaml | --output-dir DIR]
  *   ossie-polaris-converter export --url URL --catalog CATALOG [--client-id ID --client-secret SECRET] <ossie_model.yaml>
  *
*/ @@ -56,6 +62,7 @@ public static void main(String[] args) throws Exception { String clientSecret = null; String token = null; String outputFile = null; + String outputDirectory = null; String inputFile = null; for (int i = 1; i < args.length; i++) { @@ -78,6 +85,12 @@ public static void main(String[] args) throws Exception { case "-o": if (i + 1 < args.length) outputFile = args[++i]; break; + case "--output-dir": + if (i + 1 >= args.length || args[i + 1].startsWith("-")) { + throw new IllegalArgumentException("--output-dir requires a directory"); + } + outputDirectory = args[++i]; + break; default: if (!args[i].startsWith("-")) { inputFile = args[i]; @@ -92,6 +105,10 @@ public static void main(String[] args) throws Exception { System.exit(1); } + if (outputDirectory != null && (!"import".equals(mode) || outputFile != null)) { + throw new IllegalArgumentException("--output-dir is only for import and cannot be combined with -o"); + } + PolarisClient client = new PolarisClient(url, catalog); // Authenticate @@ -103,7 +120,7 @@ public static void main(String[] args) throws Exception { switch (mode) { case "import": - doImport(client, outputFile); + doImport(client, outputFile, outputDirectory); break; case "export": doExport(client, inputFile); @@ -115,19 +132,47 @@ public static void main(String[] args) throws Exception { } } - private static void doImport(PolarisClient client, String outputFile) throws Exception { + static void doImport(PolarisClient client, String outputFile, String outputDirectory) throws Exception { + if (outputFile != null && outputDirectory != null) { + throw new IllegalArgumentException("Use either -o or --output-dir"); + } PolarisImporter importer = new PolarisImporter(client); - OssieModel model = importer.importCatalog(); + List models = importer.importCatalog(); - if (model.getSemanticModels().isEmpty()) { + if (models.isEmpty()) { System.err.println("Warning: no tables found in catalog."); + return; } OssieYamlGenerator generator = new OssieYamlGenerator(); - String yaml = generator.generate(model); + if (outputDirectory != null) { + Path directory = Paths.get(outputDirectory); + Files.createDirectories(directory); + List outputs = new ArrayList<>(); + for (int i = 0; i < models.size(); i++) { + // The ordinal avoids collisions between sanitized or flattened namespace names. + String name = models.get(i).getSemanticModel().getName().replaceAll("[^A-Za-z0-9_-]", "_"); + name = name.substring(0, Math.min(name.length(), 80)); + Path output = directory.resolve(String.format(Locale.ROOT, "%04d-%s.yaml", i + 1, name)); + if (Files.exists(output, LinkOption.NOFOLLOW_LINKS)) { + throw new IOException("Refusing to overwrite existing file: " + output); + } + outputs.add(output); + } + for (int i = 0; i < models.size(); i++) { + Files.writeString(outputs.get(i), generator.generate(models.get(i)), + StandardCharsets.UTF_8, StandardOpenOption.CREATE_NEW); + System.out.println("Ossie model written to " + outputs.get(i)); + } + return; + } + if (models.size() != 1) { + throw new IllegalArgumentException("Catalog contains multiple nonempty namespaces; use --output-dir"); + } + String yaml = generator.generate(models.get(0)); if (outputFile != null) { - Files.write(Paths.get(outputFile), yaml.getBytes(StandardCharsets.UTF_8)); + Files.writeString(Paths.get(outputFile), yaml, StandardCharsets.UTF_8); System.out.println("Ossie model written to " + outputFile); } else { System.out.println(yaml); @@ -143,21 +188,15 @@ private static void doExport(PolarisClient client, String inputFile) throws Exce OssieModelParser parser = new OssieModelParser(); OssieModel model = parser.parse(Paths.get(inputFile)); - if (model.getSemanticModels().isEmpty()) { - System.err.println("Error: no semantic_model found in " + inputFile); - System.exit(1); - } - PolarisExporter exporter = new PolarisExporter(client); exporter.exportModel(model); - System.out.println("Exported " + model.getSemanticModels().size() - + " semantic model(s) to Polaris catalog."); + System.out.println("Exported one semantic model to Polaris catalog."); } private static void printUsage() { System.err.println("Usage:"); - System.err.println(" ossie-polaris-converter import --url URL --catalog CATALOG [options] [-o output.yaml]"); + System.err.println(" ossie-polaris-converter import --url URL --catalog CATALOG [options] [-o output.yaml | --output-dir DIR]"); System.err.println(" ossie-polaris-converter export --url URL --catalog CATALOG [options] "); System.err.println(); System.err.println("Options:"); @@ -166,6 +205,7 @@ private static void printUsage() { System.err.println(" --client-id ID OAuth2 client ID for authentication"); System.err.println(" --client-secret SECRET OAuth2 client secret for authentication"); System.err.println(" --token TOKEN Pre-existing bearer token"); - System.err.println(" -o FILE Output file (import mode, default: stdout)"); + System.err.println(" -o FILE Output file for a single nonempty namespace (default: stdout)"); + System.err.println(" --output-dir DIR One YAML file per nonempty namespace (import mode)"); } } diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieYamlGenerator.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieYamlGenerator.java index a3f5b2b1..6b6e49c0 100644 --- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieYamlGenerator.java +++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieYamlGenerator.java @@ -35,26 +35,27 @@ public class OssieYamlGenerator { * Generate Ossie YAML string from a model. */ public String generate(OssieModel model) { + if (model.getSemanticModel() == null) { + throw new IllegalArgumentException("The document requires a semantic model"); + } StringBuilder sb = new StringBuilder(); sb.append("version: \"").append(model.getVersion()).append("\"\n\n"); - sb.append("semantic_model:\n"); - - for (SemanticModel sm : model.getSemanticModels()) { - generateSemanticModel(sb, sm); - } + generateSemanticModel(sb, model.getSemanticModel()); return sb.toString(); } private void generateSemanticModel(StringBuilder sb, SemanticModel sm) { - sb.append(" - name: ").append(sm.getName()).append("\n"); + sb.append("name: ").append(sm.getName()).append("\n"); if (sm.getDescription() != null) { - sb.append(" description: \"").append(escapeYaml(sm.getDescription())).append("\"\n"); + sb.append("description: \"").append(escapeYaml(sm.getDescription())).append("\"\n"); } // Datasets - if (!sm.getDatasets().isEmpty()) { - sb.append(" datasets:\n"); + if (sm.getDatasets().isEmpty()) { + sb.append("datasets: []\n"); + } else { + sb.append("datasets:\n"); for (Dataset ds : sm.getDatasets()) { generateDataset(sb, ds); } @@ -62,7 +63,7 @@ private void generateSemanticModel(StringBuilder sb, SemanticModel sm) { // Relationships if (!sm.getRelationships().isEmpty()) { - sb.append(" relationships:\n"); + sb.append("relationships:\n"); for (Relationship rel : sm.getRelationships()) { generateRelationship(sb, rel); } @@ -70,7 +71,7 @@ private void generateSemanticModel(StringBuilder sb, SemanticModel sm) { // Metrics if (!sm.getMetrics().isEmpty()) { - sb.append(" metrics:\n"); + sb.append("metrics:\n"); for (Metric metric : sm.getMetrics()) { generateMetric(sb, metric); } @@ -78,49 +79,49 @@ private void generateSemanticModel(StringBuilder sb, SemanticModel sm) { } private void generateDataset(StringBuilder sb, Dataset ds) { - sb.append(" - name: ").append(ds.getName()).append("\n"); - sb.append(" source: ").append(ds.getSource()).append("\n"); + sb.append(" - name: ").append(ds.getName()).append("\n"); + sb.append(" source: ").append(ds.getSource()).append("\n"); if (!ds.getPrimaryKey().isEmpty()) { - sb.append(" primary_key: [").append(String.join(", ", ds.getPrimaryKey())).append("]\n"); + sb.append(" primary_key: [").append(String.join(", ", ds.getPrimaryKey())).append("]\n"); } if (!ds.getUniqueKeys().isEmpty()) { - sb.append(" unique_keys:\n"); + sb.append(" unique_keys:\n"); for (List uk : ds.getUniqueKeys()) { - sb.append(" - [").append(String.join(", ", uk)).append("]\n"); + sb.append(" - [").append(String.join(", ", uk)).append("]\n"); } } if (ds.getDescription() != null) { - sb.append(" description: \"").append(escapeYaml(ds.getDescription())).append("\"\n"); + sb.append(" description: \"").append(escapeYaml(ds.getDescription())).append("\"\n"); } if (!ds.getFields().isEmpty()) { - sb.append(" fields:\n"); + sb.append(" fields:\n"); for (Field field : ds.getFields()) { generateField(sb, field); } } if (!ds.getCustomExtensions().isEmpty()) { - generateCustomExtensions(sb, ds.getCustomExtensions(), " "); + generateCustomExtensions(sb, ds.getCustomExtensions(), " "); } } private void generateField(StringBuilder sb, Field field) { - sb.append(" - name: ").append(field.getName()).append("\n"); + sb.append(" - name: ").append(field.getName()).append("\n"); if (field.getDatatype() != null) { - sb.append(" datatype: ").append(field.getDatatype()).append("\n"); + sb.append(" datatype: ").append(field.getDatatype()).append("\n"); } if (!field.getExpressions().isEmpty()) { - sb.append(" expression:\n"); - sb.append(" dialects:\n"); + sb.append(" expression:\n"); + sb.append(" dialects:\n"); for (DialectExpression de : field.getExpressions()) { - sb.append(" - dialect: ").append(de.getDialect()).append("\n"); - sb.append(" expression: "); + sb.append(" - dialect: ").append(de.getDialect()).append("\n"); + sb.append(" expression: "); String expr = de.getExpression(); if (needsQuoting(expr)) { sb.append("\"").append(escapeYaml(expr)).append("\""); @@ -132,16 +133,16 @@ private void generateField(StringBuilder sb, Field field) { } if (field.isTime()) { - sb.append(" dimension:\n"); - sb.append(" is_time: true\n"); + sb.append(" dimension:\n"); + sb.append(" is_time: true\n"); } if (field.getDescription() != null) { - sb.append(" description: \"").append(escapeYaml(field.getDescription())).append("\"\n"); + sb.append(" description: \"").append(escapeYaml(field.getDescription())).append("\"\n"); } if (!field.getCustomExtensions().isEmpty()) { - generateCustomExtensions(sb, field.getCustomExtensions(), " "); + generateCustomExtensions(sb, field.getCustomExtensions(), " "); } } @@ -156,22 +157,22 @@ private void generateCustomExtensions( } private void generateRelationship(StringBuilder sb, Relationship rel) { - sb.append(" - name: ").append(rel.getName()).append("\n"); - sb.append(" from: ").append(rel.getFrom()).append("\n"); - sb.append(" to: ").append(rel.getTo()).append("\n"); - sb.append(" from_columns: [").append(String.join(", ", rel.getFromColumns())).append("]\n"); - sb.append(" to_columns: [").append(String.join(", ", rel.getToColumns())).append("]\n"); + sb.append(" - name: ").append(rel.getName()).append("\n"); + sb.append(" from: ").append(rel.getFrom()).append("\n"); + sb.append(" to: ").append(rel.getTo()).append("\n"); + sb.append(" from_columns: [").append(String.join(", ", rel.getFromColumns())).append("]\n"); + sb.append(" to_columns: [").append(String.join(", ", rel.getToColumns())).append("]\n"); } private void generateMetric(StringBuilder sb, Metric metric) { - sb.append(" - name: ").append(metric.getName()).append("\n"); + sb.append(" - name: ").append(metric.getName()).append("\n"); if (!metric.getExpressions().isEmpty()) { - sb.append(" expression:\n"); - sb.append(" dialects:\n"); + sb.append(" expression:\n"); + sb.append(" dialects:\n"); for (DialectExpression de : metric.getExpressions()) { - sb.append(" - dialect: ").append(de.getDialect()).append("\n"); - sb.append(" expression: "); + sb.append(" - dialect: ").append(de.getDialect()).append("\n"); + sb.append(" expression: "); String expr = de.getExpression(); if (needsQuoting(expr)) { sb.append("\"").append(escapeYaml(expr)).append("\""); @@ -183,7 +184,7 @@ private void generateMetric(StringBuilder sb, Metric metric) { } if (metric.getDescription() != null) { - sb.append(" description: \"").append(escapeYaml(metric.getDescription())).append("\"\n"); + sb.append(" description: \"").append(escapeYaml(metric.getDescription())).append("\"\n"); } } diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java index 5b96b10f..9df0a81a 100644 --- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java +++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisExporter.java @@ -54,12 +54,10 @@ public PolarisExporter(PolarisClient client) { /** * Export the Ossie model to the Polaris catalog. - * Each semantic model becomes a namespace, and each dataset becomes a table. + * The semantic model becomes a namespace, and each dataset becomes a table. */ public void exportModel(OssieModel model) throws IOException, InterruptedException { - for (SemanticModel sm : model.getSemanticModels()) { - exportSemanticModel(sm); - } + exportSemanticModel(Objects.requireNonNull(model.getSemanticModel(), "The document requires a semantic model")); } /** diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java index 3d767b40..cb71254d 100644 --- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java +++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/PolarisImporter.java @@ -45,23 +45,33 @@ public PolarisImporter(PolarisClient client) { } /** - * Import all tables from all namespaces in the catalog into an Ossie model. - * Each namespace becomes a separate semantic model. + * Import all tables from all namespaces in the catalog. + * Each nonempty namespace becomes a separate Ossie document, in namespace order. */ - public OssieModel importCatalog() throws IOException, InterruptedException { - OssieModel model = new OssieModel(); - model.setVersion("0.2.0.dev0"); + public List importCatalog() throws IOException, InterruptedException { + List models = new ArrayList<>(); - List> namespaces = client.listNamespaces(); + List> namespaces = new ArrayList<>(client.listNamespaces()); + + namespaces.sort((left, right) -> { + for (int i = 0; i < Math.min(left.size(), right.size()); i++) { + int comparison = left.get(i).compareTo(right.get(i)); + if (comparison != 0) return comparison; + } + return Integer.compare(left.size(), right.size()); + }); for (List namespace : namespaces) { SemanticModel sm = importNamespace(namespace); if (sm != null && !sm.getDatasets().isEmpty()) { - model.getSemanticModels().add(sm); + OssieModel model = new OssieModel(); + model.setVersion("0.2.0.dev0"); + model.setSemanticModel(sm); + models.add(model); } } - return model; + return models; } /** diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/model/OssieModel.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/model/OssieModel.java index b888fa1a..c647fff3 100644 --- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/model/OssieModel.java +++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/model/OssieModel.java @@ -28,7 +28,7 @@ public class OssieModel { private String version; - private List semanticModels = new ArrayList<>(); + private SemanticModel semanticModel; public String getVersion() { return version; @@ -38,12 +38,12 @@ public void setVersion(String version) { this.version = version; } - public List getSemanticModels() { - return semanticModels; + public SemanticModel getSemanticModel() { + return semanticModel; } - public void setSemanticModels(List semanticModels) { - this.semanticModels = semanticModels; + public void setSemanticModel(SemanticModel semanticModel) { + this.semanticModel = semanticModel; } // ----------------------------------------------------------------------- diff --git a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OssiePolarisConverterTest.java b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OssiePolarisConverterTest.java index 62ba18d2..515fee38 100644 --- a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OssiePolarisConverterTest.java +++ b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OssiePolarisConverterTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.apache.ossie.converter.polaris.model.OssieModel; import org.apache.ossie.converter.polaris.model.OssieModel.*; @@ -29,6 +30,8 @@ import java.io.ByteArrayOutputStream; import java.io.PrintStream; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collections; import java.util.List; @@ -36,71 +39,73 @@ class OssiePolarisConverterTest { + @TempDir + Path tempDirectory; + private static final String MINIMAL_MODEL = "version: \"0.2.0.dev0\"\n" + "\n" - + "semantic_model:\n" - + " - name: test_model\n" - + " description: A test model\n" - + " datasets:\n" - + " - name: orders\n" - + " source: catalog.ns.orders\n" - + " primary_key: [order_id]\n" - + " description: Order fact table\n" - + " fields:\n" - + " - name: order_id\n" - + " datatype: Integer\n" - + " expression:\n" - + " dialects:\n" - + " - dialect: ANSI_SQL\n" - + " expression: order_id\n" - + " - name: total_amount\n" - + " datatype: Decimal\n" - + " expression:\n" - + " dialects:\n" - + " - dialect: ANSI_SQL\n" - + " expression: \"quantity * unit_price\"\n" - + " description: Computed total\n" - + " custom_extensions:\n" - + " - vendor_name: POLARIS\n" - + " data: '{\"iceberg_type\":\"decimal(18,2)\"}'\n" - + " - name: order_date\n" - + " datatype: Date\n" - + " expression:\n" - + " dialects:\n" - + " - dialect: ANSI_SQL\n" - + " expression: order_date\n" - + " dimension:\n" - + " is_time: true\n" - + " - name: customer\n" - + " source: catalog.ns.customer\n" - + " primary_key: [customer_id]\n" - + " fields:\n" - + " - name: customer_id\n" - + " datatype: String\n" - + " expression:\n" - + " dialects:\n" - + " - dialect: ANSI_SQL\n" - + " expression: customer_id\n" - + " - name: full_name\n" - + " datatype: String\n" - + " expression:\n" - + " dialects:\n" - + " - dialect: ANSI_SQL\n" - + " expression: \"first_name || ' ' || last_name\"\n" - + " relationships:\n" - + " - name: orders_to_customer\n" - + " from: orders\n" - + " to: customer\n" - + " from_columns: [customer_id]\n" - + " to_columns: [customer_id]\n" - + " metrics:\n" - + " - name: total_revenue\n" + + "name: test_model\n" + + "description: A test model\n" + + "datasets:\n" + + " - name: orders\n" + + " source: catalog.ns.orders\n" + + " primary_key: [order_id]\n" + + " description: Order fact table\n" + + " fields:\n" + + " - name: order_id\n" + + " datatype: Integer\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: ANSI_SQL\n" + + " expression: order_id\n" + + " - name: total_amount\n" + + " datatype: Decimal\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: ANSI_SQL\n" + + " expression: \"quantity * unit_price\"\n" + + " description: Computed total\n" + + " custom_extensions:\n" + + " - vendor_name: POLARIS\n" + + " data: '{\"iceberg_type\":\"decimal(18,2)\"}'\n" + + " - name: order_date\n" + + " datatype: Date\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: ANSI_SQL\n" + + " expression: order_date\n" + + " dimension:\n" + + " is_time: true\n" + + " - name: customer\n" + + " source: catalog.ns.customer\n" + + " primary_key: [customer_id]\n" + + " fields:\n" + + " - name: customer_id\n" + + " datatype: String\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: ANSI_SQL\n" + + " expression: customer_id\n" + + " - name: full_name\n" + + " datatype: String\n" + " expression:\n" + " dialects:\n" + " - dialect: ANSI_SQL\n" - + " expression: SUM(orders.total_amount)\n" - + " description: Total revenue across all orders\n"; + + " expression: \"first_name || ' ' || last_name\"\n" + + "relationships:\n" + + " - name: orders_to_customer\n" + + " from: orders\n" + + " to: customer\n" + + " from_columns: [customer_id]\n" + + " to_columns: [customer_id]\n" + + "metrics:\n" + + " - name: total_revenue\n" + + " expression:\n" + + " dialects:\n" + + " - dialect: ANSI_SQL\n" + + " expression: SUM(orders.total_amount)\n" + + " description: Total revenue across all orders\n"; // -- Parser tests ------------------------------------------------------- @@ -111,9 +116,9 @@ void testParseMinimalModel() { new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8))); assertEquals("0.2.0.dev0", model.getVersion()); - assertEquals(1, model.getSemanticModels().size()); + assertNotNull(model.getSemanticModel()); - SemanticModel sm = model.getSemanticModels().get(0); + SemanticModel sm = model.getSemanticModel(); assertEquals("test_model", sm.getName()); assertEquals(2, sm.getDatasets().size()); assertEquals(1, sm.getRelationships().size()); @@ -126,7 +131,7 @@ void testParseDatasetFields() { OssieModel model = parser.parse( new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8))); - Dataset orders = model.getSemanticModels().get(0).getDatasets().get(0); + Dataset orders = model.getSemanticModel().getDatasets().get(0); assertEquals("orders", orders.getName()); assertEquals("catalog.ns.orders", orders.getSource()); assertEquals(3, orders.getFields().size()); @@ -144,7 +149,7 @@ void testParseTimeDimension() { OssieModel model = parser.parse( new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8))); - Dataset orders = model.getSemanticModels().get(0).getDatasets().get(0); + Dataset orders = model.getSemanticModel().getDatasets().get(0); Field orderDate = orders.getFields().get(2); assertEquals("order_date", orderDate.getName()); assertEquals("Date", orderDate.getDatatype()); @@ -157,7 +162,7 @@ void testParseRelationship() { OssieModel model = parser.parse( new ByteArrayInputStream(MINIMAL_MODEL.getBytes(StandardCharsets.UTF_8))); - Relationship rel = model.getSemanticModels().get(0).getRelationships().get(0); + Relationship rel = model.getSemanticModel().getRelationships().get(0); assertEquals("orders_to_customer", rel.getName()); assertEquals("orders", rel.getFrom()); assertEquals("customer", rel.getTo()); @@ -194,9 +199,9 @@ void testYamlGenerationRoundTrip() { // Re-parse the generated YAML to verify it's valid OssieModel reparsed = parser.parse( new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8))); - assertEquals(1, reparsed.getSemanticModels().size()); - assertEquals("test_model", reparsed.getSemanticModels().get(0).getName()); - assertEquals(2, reparsed.getSemanticModels().get(0).getDatasets().size()); + assertNotNull(reparsed.getSemanticModel()); + assertEquals("test_model", reparsed.getSemanticModel().getName()); + assertEquals(2, reparsed.getSemanticModel().getDatasets().size()); } // -- Exporter tests (Iceberg schema generation) ------------------------- @@ -210,7 +215,7 @@ void testExporterBuildCreateTableRequest() throws Exception { PolarisClient client = new PolarisClient("http://localhost:8181", "test_catalog"); PolarisExporter exporter = new PolarisExporter(client); - Dataset orders = model.getSemanticModels().get(0).getDatasets().get(0); + Dataset orders = model.getSemanticModel().getDatasets().get(0); String json = exporter.buildCreateTableRequest(orders); ObjectMapper mapper = new ObjectMapper(); @@ -290,9 +295,11 @@ void testImporterMapTableToDataset() throws Exception { PolarisClient client = new FakePolarisClient(tableMetadata); PolarisImporter importer = new PolarisImporter(client); - OssieModel model = importer.importCatalog(); + List models = importer.importCatalog(); + assertEquals(1, models.size()); + OssieModel model = models.get(0); - Dataset ds = model.getSemanticModels().get(0).getDatasets().get(0); + Dataset ds = model.getSemanticModel().getDatasets().get(0); assertEquals("test_table", ds.getName()); assertEquals("test_catalog.test_ns.test_table", ds.getSource()); assertEquals(Collections.singletonList("id"), ds.getPrimaryKey()); @@ -336,7 +343,7 @@ void testImporterMapTableToDataset() throws Exception { // distinctions survive, while nested IDs are regenerated for the new schema. OssieModel reparsed = new OssieModelParser().parse( new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8))); - Dataset reparsedDataset = reparsed.getSemanticModels().get(0).getDatasets().get(0); + Dataset reparsedDataset = reparsed.getSemanticModel().getDatasets().get(0); JsonNode exportedSchema = mapper.readTree( new PolarisExporter(client).buildCreateTableRequest(reparsedDataset)).path("schema"); JsonNode exportedFields = exportedSchema.path("fields"); @@ -433,7 +440,7 @@ void testTypeInference() throws Exception { ds.setFields(fields); sm.setDatasets(Collections.singletonList(ds)); - model.setSemanticModels(Collections.singletonList(sm)); + model.setSemanticModel(sm); PolarisClient client = new PolarisClient("http://localhost:8181", "cat"); PolarisExporter exporter = new PolarisExporter(client); @@ -457,15 +464,145 @@ void testTypeInference() throws Exception { void testEmptyModel() { String emptyYaml = "version: \"0.2.0.dev0\"\n"; OssieModelParser parser = new OssieModelParser(); - OssieModel model = parser.parse( - new ByteArrayInputStream(emptyYaml.getBytes(StandardCharsets.UTF_8))); + assertThrows(IllegalArgumentException.class, () -> parser.parse( + new ByteArrayInputStream(emptyYaml.getBytes(StandardCharsets.UTF_8)))); + } - assertEquals("0.2.0.dev0", model.getVersion()); - assertTrue(model.getSemanticModels().isEmpty()); + @Test + void testRejectsLegacyWrappedDocuments() { + for (String value : List.of("[]", "[{name: legacy, datasets: []}]", "{name: legacy, datasets: []}")) { + String yaml = "version: 0.2.0.dev0\nsemantic_model: " + value + "\n"; + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> new OssieModelParser().parse( + new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)))); + assertTrue(exception.getMessage().contains("Legacy semantic_model wrappers")); + } + } + + @Test + void testRejectsRemovedRootMetadata() { + for (String property : List.of("dialects", "vendors")) { + for (String value : List.of("null", "[]", "[legacy]")) { + String yaml = MINIMAL_MODEL + property + ": " + value + "\n"; + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> new OssieModelParser().parse( + new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8)))); + assertTrue(exception.getMessage().contains("Root dialects and vendors")); + } + } + } + + @Test + void testEmptyDatasetListRoundTrips() { + String yaml = "version: 0.2.0.dev0\nname: empty\ndatasets: []\n"; + OssieModelParser parser = new OssieModelParser(); + OssieModel model = parser.parse(new ByteArrayInputStream(yaml.getBytes(StandardCharsets.UTF_8))); + String generated = new OssieYamlGenerator().generate(model); + assertFalse(generated.contains("semantic_model:")); + OssieModel reparsed = parser.parse(new ByteArrayInputStream(generated.getBytes(StandardCharsets.UTF_8))); + assertEquals("empty", reparsed.getSemanticModel().getName()); + assertTrue(reparsed.getSemanticModel().getDatasets().isEmpty()); + } + + @Test + void testBulkImportWritesSeparateDocumentsWithSafeDistinctNames() throws Exception { + PolarisClient client = namespaceClient(List.of( + List.of("a_b"), List.of("a", "b"), List.of("../outside"))); + Path directory = tempDirectory.resolve("models"); + OssiePolarisConverter.doImport(client, null, directory.toString()); + + List files; + try (var paths = Files.list(directory)) { + files = paths.sorted().toList(); + } + assertEquals(List.of("0001-___outside.yaml", "0002-a_b.yaml", "0003-a_b.yaml"), + files.stream().map(path -> path.getFileName().toString()).toList()); + assertEquals(List.of("../outside", "a_b", "a_b"), files.stream().map(path -> { + try { + return new OssieModelParser().parse(path).getSemanticModel().getName(); + } catch (Exception exception) { + throw new AssertionError(exception); + } + }).toList()); + assertEquals("test_catalog.a.b.test_table", + new OssieModelParser().parse(files.get(1)).getSemanticModel().getDatasets().get(0).getSource()); + assertEquals("test_catalog.a_b.test_table", + new OssieModelParser().parse(files.get(2)).getSemanticModel().getDatasets().get(0).getSource()); + + // API enumeration order does not change the output filenames or contents. + Path reordered = tempDirectory.resolve("reordered"); + OssiePolarisConverter.doImport(namespaceClient(List.of( + List.of("../outside"), List.of("a", "b"), List.of("a_b"))), null, reordered.toString()); + for (Path file : files) { + assertEquals(Files.readString(file), Files.readString(reordered.resolve(file.getFileName()))); + } + } + + @Test + void testMultipleNamespacesRequireOutputDirectory() { + PolarisClient client = namespaceClient(List.of(List.of("sales"), List.of("support"))); + Path output = tempDirectory.resolve("model.yaml"); + IllegalArgumentException exception = assertThrows(IllegalArgumentException.class, + () -> OssiePolarisConverter.doImport(client, output.toString(), null)); + assertTrue(exception.getMessage().contains("--output-dir")); + assertFalse(Files.exists(output)); + assertThrows(IllegalArgumentException.class, () -> OssiePolarisConverter.doImport(client, null, null)); + } + + @Test + void testSingleNamespaceFileAndStdout() throws Exception { + PolarisClient client = namespaceClient(List.of(List.of("sales"))); + Path output = tempDirectory.resolve("model.yaml"); + OssiePolarisConverter.doImport(client, output.toString(), null); + assertEquals("sales", new OssieModelParser().parse(output).getSemanticModel().getName()); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + PrintStream originalOutput = System.out; + try { + System.setOut(new PrintStream(bytes, true, StandardCharsets.UTF_8)); + OssiePolarisConverter.doImport(client, null, null); + } finally { + System.setOut(originalOutput); + } + assertEquals("sales", new OssieModelParser().parse( + new ByteArrayInputStream(bytes.toByteArray())).getSemanticModel().getName()); + } + + @Test + void testBulkImportRefusesExistingFilesBeforeWriting() throws Exception { + PolarisClient client = namespaceClient(List.of(List.of("sales"), List.of("support"))); + Path existing = tempDirectory.resolve("0002-support.yaml"); + Files.writeString(existing, "keep this file"); + assertThrows(java.io.IOException.class, + () -> OssiePolarisConverter.doImport(client, null, tempDirectory.toString())); + assertFalse(Files.exists(tempDirectory.resolve("0001-sales.yaml"))); + assertEquals("keep this file", Files.readString(existing)); + } + + @Test + void testEmptyCatalogDoesNotWriteInvalidDocument() throws Exception { + Path output = tempDirectory.resolve("empty.yaml"); + OssiePolarisConverter.doImport(namespaceClient(List.of()), output.toString(), null); + assertFalse(Files.exists(output)); + } + + @Test + void testOutputOptionsAreMutuallyExclusive() { + assertThrows(IllegalArgumentException.class, () -> OssiePolarisConverter.doImport( + namespaceClient(List.of()), "model.yaml", tempDirectory.toString())); } // -- Helpers ------------------------------------------------------------ + private PolarisClient namespaceClient(List> namespaces) { + return new FakePolarisClient(new ObjectMapper().createObjectNode()) { + @Override + public List> listNamespaces() { + return namespaces; + } + }; + } + private Field makeField(String name, boolean isTime) { Field f = new Field(); f.setName(name); diff --git a/converters/salesforce/README.md b/converters/salesforce/README.md index 3b11559c..823c26f0 100644 --- a/converters/salesforce/README.md +++ b/converters/salesforce/README.md @@ -25,6 +25,11 @@ This converter supports conversion in both directions between Ossie YAML and Salesforce Semantic Model JSON. Unmapped Salesforce properties are preserved in `custom_extensions`; see the mapping reference for direction-specific limits. +Each Ossie file contains one model, with `version`, `name`, `datasets`, +`relationships`, and `metrics` at the document root. Legacy `semantic_model` +wrappers are rejected; split older multi-model files and flatten each model +before conversion. The string API returns a singleton list in both directions. + ## Requirements - **Java 21+** @@ -42,7 +47,7 @@ This produces a self-contained executable jar at `target/ossie-salesforce-conver ## Setup -Both schemas must be obtained and placed under `src/main/resources/schemas/` before building, so they get bundled into the jar. +The canonical Ossie schema is bundled automatically during the build. Obtain the Salesforce schema below and place it under `src/main/resources/schemas/` before building. ### Salesforce Semantic Model Schema @@ -52,9 +57,8 @@ Both schemas must be obtained and placed under `src/main/resources/schemas/` bef ### Apache Ossie Schema -1. Visit the [Ossie schema on GitHub](https://github.com/apache/ossie/blob/main/core-spec/ossie-schema.json) -2. Copy the raw JSON contents -3. Save it to `src/main/resources/schemas/ossie-schema.json` +The build copies [the canonical Ossie schema](../../core-spec/ossie-schema.json) +into the jar. No separate download or duplicate schema file is needed. ## Usage diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/Converter.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/Converter.java index 6554643c..be20ddb2 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/Converter.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/Converter.java @@ -34,8 +34,8 @@ public interface Converter { /** * Converts the input file and writes results to the specified output directory. * - *

Each semantic model is written to a separate file named after its apiName. - * Example: "Sales_Model.json", "Marketing_Model.json" + *

The semantic model is written to a file named after its apiName or name. + * Example: "Sales_Model.json" or "Sales_Model.yaml" * * @param inputPath the path to the input file * @param outputDir the directory where output files will be written @@ -45,11 +45,11 @@ public interface Converter { /** * Converts string content from the source format to the target format. * - *

For Ossie to Salesforce: returns one Salesforce model per Ossie semantic_model entry. - *

For Salesforce to Ossie: returns one Ossie document with one semantic_model entry. + *

For Ossie to Salesforce: returns one Salesforce model for the Ossie document. + *

For Salesforce to Ossie: returns one Ossie document with model fields at the root. * * @param content the content to convert - * @return list of converted content strings (one per semantic model) + * @return a singleton list containing the converted document */ List convert(String content); } diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java index 044dfe4c..8e25e65e 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java @@ -40,7 +40,6 @@ public enum Level { // Ossie root structure public static final String VERSION = "version"; public static final String OSSIE_VERSION = "0.2.0.dev0"; - public static final String SEMANTIC_MODEL = "semantic_model"; // Ossie semantic model structure public static final String CUSTOM_EXTENSIONS = "custom_extensions"; diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java index c3735a4d..00226186 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterImpl.java @@ -20,17 +20,14 @@ package org.apache.ossie.converter; import static org.apache.ossie.converter.ConverterConstants.*; -import static org.apache.ossie.util.DataStructureUtils.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.ossie.converter.pipeline.*; -import org.apache.ossie.converter.pipeline.*; import org.apache.ossie.exception.ConversionException; import org.apache.ossie.validator.SchemaValidator; -import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -97,29 +94,21 @@ public List convert(String content) { } private List convertOssieToSalesforce(Map ossieRoot) { - List semanticModels = getList(ossieRoot, SEMANTIC_MODEL); - List results = new ArrayList<>(); - - for (Object modelObj : semanticModels) { - Map sourceData = asMap(modelObj); - String result = executePipeline(sourceData); - results.add(result); - } - return results; + return List.of(executePipeline(ossieRoot)); } private List convertSalesforceToOssie(Map sourceData) { String result = executePipeline(sourceData); - // Wrap output in Ossie root structure + // Add document metadata alongside the semantic model fields. try { Map outputData = yamlMapper.readValue(result, new TypeReference<>() {}); Map ossieRoot = new LinkedHashMap<>(); ossieRoot.put(VERSION, OSSIE_VERSION); - ossieRoot.put(SEMANTIC_MODEL, List.of(outputData)); + ossieRoot.putAll(outputData); return List.of(toYaml(ossieRoot)); } catch (JsonProcessingException e) { - throw new ConversionException("Failed to wrap output in Ossie root", e); + throw new ConversionException("Failed to create Ossie document", e); } } @@ -156,15 +145,6 @@ protected String extractModelName(String result) { String field = directionConfig.getExtractModelNameFrom(); - // Handle Ossie format (wrapped in semantic_model array) - if (direction == ConversionDirection.SALESFORCE_TO_OSSIE) { - List models = getList(data, SEMANTIC_MODEL); - if (models != null && !models.isEmpty()) { - Map firstModel = asMap(models.get(0)); - return firstModel.get(field).toString(); - } - } - return data.get(field).toString(); } catch (JsonProcessingException e) { throw new ConversionException("Failed to extract model name", e); diff --git a/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java index ffc77647..4e65039f 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/SalesforceToOssieConverterTest.java @@ -100,17 +100,13 @@ void testCompleteConversion() throws Exception { String ossieYaml = results.get(0); assertNotNull(ossieYaml); assertTrue(ossieYaml.contains("version: 0.2.0.dev0")); - assertTrue(ossieYaml.contains("semantic_model:")); + assertFalse(ossieYaml.contains("semantic_model:")); Map ossieRoot = yamlMapper.readValue(ossieYaml, Map.class); assertNotNull(ossieRoot); assertEquals("0.2.0.dev0", ossieRoot.get("version")); - List> semanticModels = (List>) ossieRoot.get("semantic_model"); - assertNotNull(semanticModels); - assertEquals(1, semanticModels.size()); - - Map model = semanticModels.get(0); + Map model = ossieRoot; assertEquals("Customer_Orders_Model", model.get("name")); assertNotNull(model.get("description")); } @@ -119,7 +115,7 @@ void testCompleteConversion() throws Exception { void testDatasetMapping() throws Exception { List results = converter.convert(salesforceJson); Map ossieRoot = yamlMapper.readValue(results.get(0), Map.class); - Map model = ((List>) ossieRoot.get("semantic_model")).get(0); + Map model = ossieRoot; List> datasets = (List>) model.get("datasets"); assertNotNull(datasets); @@ -140,7 +136,7 @@ void testDatasetMapping() throws Exception { void testFieldMapping() throws Exception { List results = converter.convert(salesforceJson); Map ossieRoot = yamlMapper.readValue(results.get(0), Map.class); - Map model = ((List>) ossieRoot.get("semantic_model")).get(0); + Map model = ossieRoot; List> datasets = (List>) model.get("datasets"); Map customersDataset = datasets.get(0); @@ -186,7 +182,7 @@ void testFieldMapping() throws Exception { void testCalculatedDimensionConversion() throws Exception { List results = converter.convert(salesforceJson); Map ossieRoot = yamlMapper.readValue(results.get(0), Map.class); - Map model = ((List>) ossieRoot.get("semantic_model")).get(0); + Map model = ossieRoot; List> datasets = (List>) model.get("datasets"); // customer_email_domain should be converted to a field in Customers dataset (single dependency) @@ -224,7 +220,7 @@ void testCalculatedDimensionConversion() throws Exception { void testRelationshipMapping() throws Exception { List results = converter.convert(salesforceJson); Map ossieRoot = yamlMapper.readValue(results.get(0), Map.class); - Map model = ((List>) ossieRoot.get("semantic_model")).get(0); + Map model = ossieRoot; List> relationships = (List>) model.get("relationships"); assertNotNull(relationships); @@ -250,7 +246,7 @@ void testRelationshipMapping() throws Exception { void testUnsupportedRelationshipsInCustomExtensions() throws Exception { List results = converter.convert(salesforceJson); Map ossieRoot = yamlMapper.readValue(results.get(0), Map.class); - Map model = ((List>) ossieRoot.get("semantic_model")).get(0); + Map model = ossieRoot; // Unsupported relationships (Formula/SemanticField) should be in custom_extensions List> customExtensions = (List>) model.get("custom_extensions"); @@ -271,7 +267,7 @@ void testUnsupportedRelationshipsInCustomExtensions() throws Exception { void testMetricMapping() throws Exception { List results = converter.convert(salesforceJson); Map ossieRoot = yamlMapper.readValue(results.get(0), Map.class); - Map model = ((List>) ossieRoot.get("semantic_model")).get(0); + Map model = ossieRoot; List> metrics = (List>) model.get("metrics"); assertNotNull(metrics); @@ -294,7 +290,7 @@ void testMetricMapping() throws Exception { void testCustomExtensionsPreservation() throws Exception { List results = converter.convert(salesforceJson); Map ossieRoot = yamlMapper.readValue(results.get(0), Map.class); - Map model = ((List>) ossieRoot.get("semantic_model")).get(0); + Map model = ossieRoot; // Check model-level custom_extensions List> customExtensions = (List>) model.get("custom_extensions"); @@ -318,7 +314,7 @@ void testCustomExtensionsPreservation() throws Exception { void testTimeDimensionMapping() throws Exception { List results = converter.convert(salesforceJson); Map ossieRoot = yamlMapper.readValue(results.get(0), Map.class); - Map model = ((List>) ossieRoot.get("semantic_model")).get(0); + Map model = ossieRoot; List> datasets = (List>) model.get("datasets"); Map ordersDataset = datasets.get(1); @@ -409,18 +405,14 @@ void testHandlerFactoryForSalesforceToOssie() { @Test void testConverterImplExtractModelNameFromOssieFormat() throws Exception { - // Test extractModelName specifically handles Ossie wrapped format + // The generated document keeps the model name at the root. List results = converter.convert(salesforceJson); String ossieYaml = results.get(0); - // The result is wrapped Ossie format - extractModelName should handle this + // The same name is used for file output. Map ossieRoot = yamlMapper.readValue(ossieYaml, Map.class); - assertTrue(ossieRoot.containsKey("semantic_model")); + assertFalse(ossieRoot.containsKey("semantic_model")); - // Verify it's properly wrapped - List> models = (List>) ossieRoot.get("semantic_model"); - assertNotNull(models); - assertEquals(1, models.size()); - assertEquals("Customer_Orders_Model", models.get(0).get("name")); + assertEquals("Customer_Orders_Model", ossieRoot.get("name")); } } diff --git a/converters/salesforce/src/test/java/org/apache/ossie/converter/FlatDocumentConversionTest.java b/converters/salesforce/src/test/java/org/apache/ossie/converter/FlatDocumentConversionTest.java new file mode 100644 index 00000000..8d8a21c6 --- /dev/null +++ b/converters/salesforce/src/test/java/org/apache/ossie/converter/FlatDocumentConversionTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.ossie.converter; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; +import org.apache.ossie.converter.pipeline.PipelineConfig; +import org.apache.ossie.converter.pipeline.PipelineConfigLoader; +import org.apache.ossie.exception.ValidationException; +import org.apache.ossie.validator.SchemaValidator; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class FlatDocumentConversionTest { + private final ObjectMapper yamlMapper = new ObjectMapper(new YAMLFactory()); + + @TempDir + Path tempDirectory; + + @Test + void convertsFlatOssieDocumentToOneSalesforceModel() throws Exception { + ConverterImpl converter = new ConverterImpl(ConversionDirection.OSSIE_TO_SALESFORCE); + String input = Files.readString(Path.of("src/test/resources/examples/ossieToSalesforce.yaml")); + List results = converter.convert(input); + assertEquals(1, results.size()); + var result = new ObjectMapper().readTree(results.get(0)); + assertEquals("Customer_Orders_Model", result.path("apiName").asText()); + assertEquals(3, result.path("semanticDataObjects").size()); + assertFalse(result.has("version")); + assertFalse(result.has("semantic_model")); + } + + @Test + void rejectsArrayAndObjectWrappers() throws Exception { + Converter converter = new ConverterImpl(ConversionDirection.OSSIE_TO_SALESFORCE); + Map model = Map.of("name", "legacy", "datasets", List.of()); + for (Object wrapper : List.of(model, List.of(model), List.of())) { + String input = yamlMapper.writeValueAsString(Map.of( + "version", "0.2.0.dev0", "semantic_model", wrapper)); + assertThrows(ValidationException.class, () -> converter.convert(input)); + } + } + + @Test + void convertsSalesforceToFlatOssieAndExtractsName() throws Exception { + ConverterImpl converter = salesforceImportConverter(); + String input = Files.readString(Path.of("src/test/resources/examples/salesforceToOssie.json")); + List results = converter.convert(input); + assertEquals(1, results.size()); + Map document = yamlMapper.readValue(results.get(0), new TypeReference<>() {}); + assertEquals("0.2.0.dev0", document.get("version")); + assertEquals("Customer_Orders_Model", document.get("name")); + assertEquals(3, ((List) document.get("datasets")).size()); + assertFalse(document.containsKey("semantic_model")); + assertEquals("Customer_Orders_Model", converter.extractModelName(results.get(0))); + new SchemaValidator(yamlMapper, SchemaValidator.OSSIE_SCHEMA_PATH).validate(document); + } + + @Test + void fileConversionUsesRootModelName() throws Exception { + ConverterImpl converter = salesforceImportConverter(); + converter.convert(Path.of("src/test/resources/examples/salesforceToOssie.json"), tempDirectory); + Path output = tempDirectory.resolve("Customer_Orders_Model.yaml"); + assertTrue(Files.isRegularFile(output)); + Map document = yamlMapper.readValue(Files.readString(output), new TypeReference<>() {}); + assertEquals("Customer_Orders_Model", document.get("name")); + assertFalse(document.containsKey("semantic_model")); + } + + private ConverterImpl salesforceImportConverter() { + PipelineConfig config = PipelineConfigLoader.loadFromResource(); + // Test the complete conversion pipeline without requiring a manually downloaded + // Salesforce schema. This fixture schema covers the example's input envelope; + // the generated Ossie document is validated against the canonical core schema. + config.getDirectionConfigs().get(ConversionDirection.SALESFORCE_TO_OSSIE.toPipelineKey()) + .setSchemaPath("/schemas/salesforce-input-fixture-schema.json"); + return new ConverterImpl(ConversionDirection.SALESFORCE_TO_OSSIE, config); + } +} diff --git a/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml b/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml index 101588f0..742eedd1 100644 --- a/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml +++ b/converters/salesforce/src/test/resources/examples/ossieToSalesforce.yaml @@ -16,388 +16,387 @@ # under the License. version: 0.2.0.dev0 -semantic_model: - - name: Customer_Orders_Model - description: Example model demonstrating calculated dimensions with single and multiple - data object dependencies +name: Customer_Orders_Model +description: Example model demonstrating calculated dimensions with single and multiple + data object dependencies +custom_extensions: +- vendor_name: SALESFORCE + data: |- + { + "dataspace" : "default" + } +datasets: +- description: Customer master data + name: Customers + source: Customers__dll + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Customers", + "dataObjectType" : "Dlo" + } + fields: + - name: customer_id + datatype: String + label: Customer ID + description: Unique customer identifier + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id__c custom_extensions: - vendor_name: SALESFORCE data: |- { - "dataspace" : "default" + "dataType" : "Text", + "displayCategory" : "Discrete" } - datasets: - - description: Customer master data - name: Customers - source: Customers__dll - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "label" : "Customers", - "dataObjectType" : "Dlo" - } - fields: - - name: customer_id - datatype: String - label: Customer ID - description: Unique customer identifier - dimension: - is_time: false - expression: - dialects: - - dialect: ANSI_SQL - expression: customer_id__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Text", - "displayCategory" : "Discrete" - } - - name: email - datatype: String - label: Email - description: Customer email address - dimension: - is_time: false - expression: - dialects: - - dialect: ANSI_SQL - expression: email__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Email" - } - - name: total_purchases - datatype: Decimal - label: Total Purchases - description: Lifetime purchase count - expression: - dialects: - - dialect: ANSI_SQL - expression: total_purchases__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Number", - "aggregationType" : "Sum", - "displayCategory" : "Continuous" - } - - name: lifetime_value - datatype: Decimal - label: Lifetime Value - description: Total customer value - expression: - dialects: - - dialect: ANSI_SQL - expression: lifetime_value__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Currency", - "aggregationType" : "Sum" - } - - name: customer_email_domain - datatype: String - label: Customer Email Domain - description: Customer Email Domain - dimension: - is_time: false - expression: - dialects: - - dialect: ANSI_SQL - expression: SUBSTRING([Customers].[email], POSITION('@' IN [Customers].[email]) - + 1, LENGTH([Customers].[email])) - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Text" - } - - description: Order transaction data - name: Orders - source: Orders__dll - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "label" : "Orders", - "dataObjectType" : "Dlo" - } - fields: - - name: order_id - datatype: String - label: Order ID - description: Unique order identifier - dimension: - is_time: false - expression: - dialects: - - dialect: ANSI_SQL - expression: order_id__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Text" - } - - name: customer_id - datatype: String - label: Customer ID - description: Foreign key to Customers - dimension: - is_time: false - expression: - dialects: - - dialect: ANSI_SQL - expression: customer_id__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Text", - "displayCategory" : "Discrete", - "isVisible" : true, - "sortOrder" : "Ascending" - } - - name: product_id - datatype: String - label: Product ID - description: Foreign key to Products - dimension: - is_time: false - expression: - dialects: - - dialect: ANSI_SQL - expression: product_id__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Text", - "displayCategory" : "Discrete", - "isVisible" : true, - "sortOrder" : "Ascending" - } - - name: order_date - datatype: Date - label: Order Date - description: Date when order was placed - dimension: - is_time: true - expression: - dialects: - - dialect: ANSI_SQL - expression: order_date__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Date", - "displayCategory" : "Discrete", - "isVisible" : true, - "sortOrder" : "Descending" - } - - name: amount - datatype: Decimal - label: Amount - description: Order amount in currency - expression: - dialects: - - dialect: ANSI_SQL - expression: amount__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Currency", - "aggregationType" : "Sum" - } - - name: quantity - datatype: Decimal - label: Quantity - description: Number of items ordered - expression: - dialects: - - dialect: ANSI_SQL - expression: quantity__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Number", - "aggregationType" : "Sum" - } - - name: order_year - datatype: Decimal - label: Order Year - description: Order Year - dimension: - is_time: false - expression: - dialects: - - dialect: ANSI_SQL - expression: YEAR([Orders].[order_date]) - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Number", - "displayCategory" : "Discrete" - } - - description: Product catalog data - name: Products - source: Products__dll - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "label" : "Products", - "dataObjectType" : "Dlo" - } - fields: - - name: product_id - datatype: String - label: Product ID - description: Unique product identifier - dimension: - is_time: false - expression: - dialects: - - dialect: ANSI_SQL - expression: product_id__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Text", - "displayCategory" : "Discrete", - "isVisible" : true, - "sortOrder" : "Ascending" - } - - name: product_name - datatype: String - label: Product Name - description: Product display name - dimension: - is_time: false - expression: - dialects: - - dialect: ANSI_SQL - expression: product_name__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Text", - "displayCategory" : "Discrete", - "isVisible" : true, - "sortOrder" : "Ascending" - } - - name: unit_price - datatype: Decimal - label: Unit Price - description: Price per unit - expression: - dialects: - - dialect: ANSI_SQL - expression: unit_price__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Currency" - } - - name: stock_level - datatype: Decimal - label: Stock Level - description: Current inventory count - expression: - dialects: - - dialect: ANSI_SQL - expression: stock_level__c - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "dataType" : "Number", - "aggregationType" : "Sum", - "displayCategory" : "Continuous", - "isVisible" : true, - "isAggregatable" : true, - "shouldTreatNullsAsZeros" : false, - "decimalPlace" : 0 - } - relationships: - - name: Customers_Orders - from: Customers - to: Orders - from_columns: - - customer_id - to_columns: - - customer_id - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "label" : "Customers to Orders", - "cardinality" : "OneToMany", - "joinType" : "Auto", - "isEnabled" : true - } - - name: Orders_Products - from: Orders - to: Products - from_columns: - - product_id - to_columns: - - product_id - - name: Customers_ByDomain - from: Customers - to: Orders - from_columns: - - customer_email_domain - to_columns: - - order_id - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "label" : "Invalid Relationship - Uses Calculated Field" - } - - name: Orders_ByYear - from: Orders - to: Products - from_columns: - - order_year - to_columns: - - product_id - custom_extensions: - - vendor_name: SALESFORCE - data: |- - { - "label" : "Invalid Relationship - Uses Calculated Field", - "cardinality" : "ManyToMany", - "joinType" : "Auto", - "isEnabled" : true - } - metrics: - - description: Sum of all order amounts - name: total_revenue - datatype: Decimal - expression: - dialects: - - dialect: ANSI_SQL - expression: SUM([Orders].[amount]) - - description: Average amount per order - name: avg_order_value - datatype: Decimal - expression: - dialects: - - dialect: ANSI_SQL - expression: AVG([Orders].[amount]) + - name: email + datatype: String + label: Email + description: Customer email address + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: email__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Email" + } + - name: total_purchases + datatype: Decimal + label: Total Purchases + description: Lifetime purchase count + expression: + dialects: + - dialect: ANSI_SQL + expression: total_purchases__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Number", + "aggregationType" : "Sum", + "displayCategory" : "Continuous" + } + - name: lifetime_value + datatype: Decimal + label: Lifetime Value + description: Total customer value + expression: + dialects: + - dialect: ANSI_SQL + expression: lifetime_value__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Currency", + "aggregationType" : "Sum" + } + - name: customer_email_domain + datatype: String + label: Customer Email Domain + description: Customer Email Domain + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: SUBSTRING([Customers].[email], POSITION('@' IN [Customers].[email]) + + 1, LENGTH([Customers].[email])) + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text" + } +- description: Order transaction data + name: Orders + source: Orders__dll + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Orders", + "dataObjectType" : "Dlo" + } + fields: + - name: order_id + datatype: String + label: Order ID + description: Unique order identifier + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text" + } + - name: customer_id + datatype: String + label: Customer ID + description: Foreign key to Customers + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Ascending" + } + - name: product_id + datatype: String + label: Product ID + description: Foreign key to Products + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: product_id__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Ascending" + } + - name: order_date + datatype: Date + label: Order Date + description: Date when order was placed + dimension: + is_time: true + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Date", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Descending" + } + - name: amount + datatype: Decimal + label: Amount + description: Order amount in currency + expression: + dialects: + - dialect: ANSI_SQL + expression: amount__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Currency", + "aggregationType" : "Sum" + } + - name: quantity + datatype: Decimal + label: Quantity + description: Number of items ordered + expression: + dialects: + - dialect: ANSI_SQL + expression: quantity__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Number", + "aggregationType" : "Sum" + } + - name: order_year + datatype: Decimal + label: Order Year + description: Order Year + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: YEAR([Orders].[order_date]) + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Number", + "displayCategory" : "Discrete" + } +- description: Product catalog data + name: Products + source: Products__dll + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Products", + "dataObjectType" : "Dlo" + } + fields: + - name: product_id + datatype: String + label: Product ID + description: Unique product identifier + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: product_id__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Ascending" + } + - name: product_name + datatype: String + label: Product Name + description: Product display name + dimension: + is_time: false + expression: + dialects: + - dialect: ANSI_SQL + expression: product_name__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Text", + "displayCategory" : "Discrete", + "isVisible" : true, + "sortOrder" : "Ascending" + } + - name: unit_price + datatype: Decimal + label: Unit Price + description: Price per unit + expression: + dialects: + - dialect: ANSI_SQL + expression: unit_price__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Currency" + } + - name: stock_level + datatype: Decimal + label: Stock Level + description: Current inventory count + expression: + dialects: + - dialect: ANSI_SQL + expression: stock_level__c + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "dataType" : "Number", + "aggregationType" : "Sum", + "displayCategory" : "Continuous", + "isVisible" : true, + "isAggregatable" : true, + "shouldTreatNullsAsZeros" : false, + "decimalPlace" : 0 + } +relationships: +- name: Customers_Orders + from: Customers + to: Orders + from_columns: + - customer_id + to_columns: + - customer_id + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Customers to Orders", + "cardinality" : "OneToMany", + "joinType" : "Auto", + "isEnabled" : true + } +- name: Orders_Products + from: Orders + to: Products + from_columns: + - product_id + to_columns: + - product_id +- name: Customers_ByDomain + from: Customers + to: Orders + from_columns: + - customer_email_domain + to_columns: + - order_id + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Invalid Relationship - Uses Calculated Field" + } +- name: Orders_ByYear + from: Orders + to: Products + from_columns: + - order_year + to_columns: + - product_id + custom_extensions: + - vendor_name: SALESFORCE + data: |- + { + "label" : "Invalid Relationship - Uses Calculated Field", + "cardinality" : "ManyToMany", + "joinType" : "Auto", + "isEnabled" : true + } +metrics: +- description: Sum of all order amounts + name: total_revenue + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM([Orders].[amount]) +- description: Average amount per order + name: avg_order_value + datatype: Decimal + expression: + dialects: + - dialect: ANSI_SQL + expression: AVG([Orders].[amount]) diff --git a/converters/salesforce/src/test/resources/schemas/salesforce-input-fixture-schema.json b/converters/salesforce/src/test/resources/schemas/salesforce-input-fixture-schema.json new file mode 100644 index 00000000..3f6492ed --- /dev/null +++ b/converters/salesforce/src/test/resources/schemas/salesforce-input-fixture-schema.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Salesforce input envelope for flat document conversion tests", + "type": "object", + "properties": { + "apiName": {"type": "string"}, + "semanticDataObjects": {"type": "array"} + }, + "required": ["apiName", "semanticDataObjects"] +} diff --git a/converters/sigma/LIMITATIONS.md b/converters/sigma/LIMITATIONS.md index 4a9790a3..3270be40 100644 --- a/converters/sigma/LIMITATIONS.md +++ b/converters/sigma/LIMITATIONS.md @@ -121,7 +121,5 @@ Sigma's backend. ## One semantic model per document -Sigma data models are single models; `OssieDocument.semantic_model` is a list. Only -`semantic_model[0]` is converted, with `EXTRA_MODEL_DROPPED` naming how many were -dropped. An empty `semantic_model` list has no model to convert at all; rather than -raising an opaque `IndexError`, `OssieToSigmaConverter` raises a `ConverterError`. +Each Ossie document contains one model at the root, which maps to one Sigma data +model. Legacy `semantic_model` wrappers are rejected when parsing an Ossie document. diff --git a/converters/sigma/README.md b/converters/sigma/README.md index f923bf41..d4fb117c 100644 --- a/converters/sigma/README.md +++ b/converters/sigma/README.md @@ -67,7 +67,7 @@ spec = json.loads(Path("data_model.json").read_text()) result = SigmaToOssieConverter().convert(spec) for issue in result.issues: print(f"[warning] {issue.issue_type.value}: {issue.element_name}") -Path("semantic_model.yaml").write_text(result.output.to_osi_yaml()) +Path("semantic_model.yaml").write_text(result.output.to_ossie_yaml()) # Ossie -> Sigma from ossie import OssieDocument @@ -82,7 +82,7 @@ Path("data_model.json").write_text(json.dumps(result.output, indent=2)) | Sigma concept | Ossie concept | Notes | |---|---|---| -| Data model (`name`, `description`) | `OssieSemanticModel` | `dataModelId`, `folderId`, `documentVersion`, `schemaVersion`, `createdAt`, `createdBy`, `updatedAt`, `updatedBy`, `ownerId`, `url` preserved in `custom_extensions` and written back on export | +| Data model (`name`, `description`) | `OssieDocument` root | `dataModelId`, `folderId`, `documentVersion`, `schemaVersion`, `createdAt`, `createdBy`, `updatedAt`, `updatedBy`, `ownerId`, `url` preserved in `custom_extensions` and written back on export | | Page | *(none)* | Ossie has no page/folder-of-elements concept, but `pages` is a required part of the spec, so page membership is preserved per-dataset in `custom_extensions` and rebuilt on export | | Element (`kind: table`) | `OssieDataset` | `source` = warehouse path joined with `.` for `warehouse-table`; the other five source kinds get a marker plus the native `source` block in `custom_extensions` | | Element (any other `kind`) | *not modeled* | Preserved verbatim in a model-level `custom_extensions` entry so `ossie-to-sigma` restores it unchanged | diff --git a/converters/sigma/src/ossie_sigma/converter_issues.py b/converters/sigma/src/ossie_sigma/converter_issues.py index 2a04005e..ea70f453 100644 --- a/converters/sigma/src/ossie_sigma/converter_issues.py +++ b/converters/sigma/src/ossie_sigma/converter_issues.py @@ -14,7 +14,6 @@ class ConverterIssueType(Enum): FILTER_NOT_MODELED = "FILTER_NOT_MODELED" CROSS_DATASET_METRIC_DROPPED = "CROSS_DATASET_METRIC_DROPPED" OPAQUE_DATATYPE = "OPAQUE_DATATYPE" - EXTRA_MODEL_DROPPED = "EXTRA_MODEL_DROPPED" MISSING_ID = "MISSING_ID" diff --git a/converters/sigma/src/ossie_sigma/ossie_to_sigma.py b/converters/sigma/src/ossie_sigma/ossie_to_sigma.py index f3532617..3effd900 100644 --- a/converters/sigma/src/ossie_sigma/ossie_to_sigma.py +++ b/converters/sigma/src/ossie_sigma/ossie_to_sigma.py @@ -17,7 +17,7 @@ OssieVendor, ) -from ossie_sigma.converter_issues import ConverterError, ConverterIssue, ConverterIssueType, ConverterResult +from ossie_sigma.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult from ossie_sigma.expression_utils import ansi_sql_text, infer_single_dataset_qualifier, sigma_dialect_text from ossie_sigma.sigma_formula import sql_to_sigma_formula from ossie_sigma.spec_keys import MODEL_LEVEL_SPEC_KEYS @@ -126,22 +126,7 @@ class OssieToSigmaConverter: def convert(self, document: OssieDocument) -> ConverterResult[dict[str, Any]]: issues: list[ConverterIssue] = [] - if not document.semantic_model: - raise ConverterError( - "OssieDocument.semantic_model is empty; there is no semantic model to convert " - "into a Sigma data model spec." - ) - - if len(document.semantic_model) > 1: - issues.append( - ConverterIssue( - ConverterIssueType.EXTRA_MODEL_DROPPED, - "document", - "Sigma data models are single semantic models; only semantic_model[0] " - f"was converted, {len(document.semantic_model) - 1} additional model(s) were dropped.", - ) - ) - model = document.semantic_model[0] + model = document model_ext = _sigma_ext(model) or {} spec: dict[str, Any] = {"kind": "data-model", "name": model.name} diff --git a/converters/sigma/src/ossie_sigma/sigma_to_ossie.py b/converters/sigma/src/ossie_sigma/sigma_to_ossie.py index b2fff4bf..9518741c 100644 --- a/converters/sigma/src/ossie_sigma/sigma_to_ossie.py +++ b/converters/sigma/src/ossie_sigma/sigma_to_ossie.py @@ -15,7 +15,6 @@ OssieField, OssieMetric, OssieRelationship, - OssieSemanticModel, OssieVendor, ) @@ -449,7 +448,7 @@ def convert(self, spec: dict[str, Any]) -> ConverterResult[OssieDocument]: ) ) - semantic_model = OssieSemanticModel( + document = OssieDocument( name=spec.get("name", "sigma_data_model"), description=spec.get("description"), datasets=datasets, @@ -458,5 +457,4 @@ def convert(self, spec: dict[str, Any]) -> ConverterResult[OssieDocument]: custom_extensions=[_vendor_ext(model_ext)] if model_ext else None, ) - document = OssieDocument(semantic_model=[semantic_model]) return ConverterResult(output=document, issues=issues) diff --git a/converters/sigma/tests/test_ossie_to_sigma.py b/converters/sigma/tests/test_ossie_to_sigma.py index b9204045..fd03011f 100644 --- a/converters/sigma/tests/test_ossie_to_sigma.py +++ b/converters/sigma/tests/test_ossie_to_sigma.py @@ -9,13 +9,13 @@ OssieExpression, OssieField, OssieMetric, - OssieSemanticModel, ) import pytest from ossie import OssieRelationship +from pydantic import ValidationError -from ossie_sigma.converter_issues import ConverterError, ConverterIssueType +from ossie_sigma.converter_issues import ConverterIssueType from ossie_sigma.ossie_to_sigma import OssieToSigmaConverter, _stable_id from ossie_sigma.sigma_to_ossie import SigmaToOssieConverter @@ -90,24 +90,20 @@ def test_relationship_ids_are_scoped_by_owning_dataset(): """Two unrelated relationships sharing a name, on different table pairs, must not collide onto the same synthesized Sigma relationship id.""" document = OssieDocument( - semantic_model=[ - OssieSemanticModel( - name="m", - datasets=[ - OssieDataset(name="orders", source="db.public.orders"), - OssieDataset(name="shipments", source="db.public.shipments"), - OssieDataset(name="customers", source="db.public.customers"), - OssieDataset(name="carriers", source="db.public.carriers"), - ], - relationships=[ - OssieRelationship( - name="Parent", **{"from": "orders"}, to="customers", from_columns=["x"], to_columns=["y"] - ), - OssieRelationship( - name="Parent", **{"from": "shipments"}, to="carriers", from_columns=["x"], to_columns=["y"] - ), - ], - ) + name="m", + datasets=[ + OssieDataset(name="orders", source="db.public.orders"), + OssieDataset(name="shipments", source="db.public.shipments"), + OssieDataset(name="customers", source="db.public.customers"), + OssieDataset(name="carriers", source="db.public.carriers"), + ], + relationships=[ + OssieRelationship( + name="Parent", **{"from": "orders"}, to="customers", from_columns=["x"], to_columns=["y"] + ), + OssieRelationship( + name="Parent", **{"from": "shipments"}, to="carriers", from_columns=["x"], to_columns=["y"] + ), ] ) @@ -122,10 +118,9 @@ def test_relationship_ids_are_scoped_by_owning_dataset(): assert len(set(rel_ids)) == 2, "relationships with the same name on different dataset pairs must not collide" -def test_empty_semantic_model_raises_a_clear_error(): - document = OssieDocument(semantic_model=[]) - with pytest.raises(ConverterError): - OssieToSigmaConverter().convert(document) +def test_legacy_semantic_model_wrapper_is_rejected(): + with pytest.raises(ValidationError, match="semantic_model"): + OssieDocument(semantic_model=[]) def test_model_level_metadata_round_trips_through_ossie_and_back(): @@ -149,54 +144,50 @@ def test_untranslatable_expression_omits_the_column_instead_of_faking_a_formula( whole document before applying any of it, so a placeholder would fail the entire upload rather than degrade one column.""" document = OssieDocument( - semantic_model=[ - OssieSemanticModel( - name="m", - datasets=[ - OssieDataset( - name="orders", - source="db.public.orders", - fields=[ - OssieField( - name="ok", - expression=OssieExpression( - dialects=[OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression="amount")] - ), - ), - OssieField( - name="untranslatable", - expression=OssieExpression( - dialects=[ - OssieDialectExpression( - dialect=OssieDialect.ANSI_SQL, - expression="SUM(amount) OVER (PARTITION BY region)", - ) - ] - ), - ), - OssieField( - name="no_usable_dialect", - expression=OssieExpression( - dialects=[OssieDialectExpression(dialect=OssieDialect.MDX, expression="[Measures].[X]")] - ), - ), - ], - ) - ], - metrics=[ - OssieMetric( - name="untranslatable_metric", + name="m", + datasets=[ + OssieDataset( + name="orders", + source="db.public.orders", + fields=[ + OssieField( + name="ok", + expression=OssieExpression( + dialects=[OssieDialectExpression(dialect=OssieDialect.ANSI_SQL, expression="amount")] + ), + ), + OssieField( + name="untranslatable", expression=OssieExpression( dialects=[ OssieDialectExpression( dialect=OssieDialect.ANSI_SQL, - expression="SUM(orders.amount) OVER (PARTITION BY orders.region)", + expression="SUM(amount) OVER (PARTITION BY region)", ) ] ), - ) + ), + OssieField( + name="no_usable_dialect", + expression=OssieExpression( + dialects=[OssieDialectExpression(dialect=OssieDialect.MDX, expression="[Measures].[X]")] + ), + ), ], ) + ], + metrics=[ + OssieMetric( + name="untranslatable_metric", + expression=OssieExpression( + dialects=[ + OssieDialectExpression( + dialect=OssieDialect.ANSI_SQL, + expression="SUM(orders.amount) OVER (PARTITION BY orders.region)", + ) + ] + ), + ) ] ) @@ -221,37 +212,33 @@ def test_synthesized_spec_carries_a_schema_version(): def test_datatypes_only_ever_emit_the_two_documented_format_kinds(): document = OssieDocument( - semantic_model=[ - OssieSemanticModel( - name="m", - datasets=[ - OssieDataset( - name="t", - source="db.public.t", - fields=[ - OssieField( - name=datatype.lower(), - datatype=datatype, - expression=OssieExpression( - dialects=[ - OssieDialectExpression( - dialect=OssieDialect.ANSI_SQL, expression=datatype.lower() - ) - ] - ), - ) - for datatype in ( - "String", - "Integer", - "Decimal", - "Float", - "Boolean", - "Date", - "Time", - "DateTime", - "DateTimeTz", - ) - ], + name="m", + datasets=[ + OssieDataset( + name="t", + source="db.public.t", + fields=[ + OssieField( + name=datatype.lower(), + datatype=datatype, + expression=OssieExpression( + dialects=[ + OssieDialectExpression( + dialect=OssieDialect.ANSI_SQL, expression=datatype.lower() + ) + ] + ), + ) + for datatype in ( + "String", + "Integer", + "Decimal", + "Float", + "Boolean", + "Date", + "Time", + "DateTime", + "DateTimeTz", ) ], ) diff --git a/converters/sigma/tests/test_roundtrip.py b/converters/sigma/tests/test_roundtrip.py index 0887a64c..01569965 100644 --- a/converters/sigma/tests/test_roundtrip.py +++ b/converters/sigma/tests/test_roundtrip.py @@ -21,7 +21,12 @@ def test_sigma_osi_sigma_roundtrip_through_yaml_serialization(fixture_name): document = SigmaToOssieConverter().convert(spec).output yaml_text = document.to_ossie_yaml() - reparsed_document = OssieDocument.model_validate(yaml.safe_load(yaml_text)) + serialized = yaml.safe_load(yaml_text) + assert serialized["name"] == spec["name"] + assert "semantic_model" not in serialized + assert "dialects" not in serialized + assert "vendors" not in serialized + reparsed_document = OssieDocument.model_validate(serialized) reconstructed_spec = OssieToSigmaConverter().convert(reparsed_document).output assert normalize(reconstructed_spec) == normalize(spec) @@ -39,7 +44,7 @@ def test_osi_sigma_osi_roundtrip_preserves_portable_fields(fixture_name): document_2 = SigmaToOssieConverter().convert(spec_2).output def portable(document): - model = document.semantic_model[0] + model = document return { "datasets": [(d.name, d.source, [(f.name, f.datatype) for f in d.fields or []]) for d in model.datasets], "relationships": [(r.name, r.from_dataset, r.to, r.from_columns, r.to_columns) for r in model.relationships or []], diff --git a/converters/sigma/tests/test_sigma_to_ossie.py b/converters/sigma/tests/test_sigma_to_ossie.py index 9e848f8a..1c6d4c0d 100644 --- a/converters/sigma/tests/test_sigma_to_ossie.py +++ b/converters/sigma/tests/test_sigma_to_ossie.py @@ -10,7 +10,7 @@ def test_basic_datasets_fields_relationships_metrics(): spec = load_fixture("fixtureA_sigma.json") result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output assert model.name == "Sales" assert {d.name for d in model.datasets} == {"Orders", "Customers"} @@ -40,7 +40,7 @@ def test_basic_datasets_fields_relationships_metrics(): def test_unique_keys_map_to_the_portable_primary_key(): spec = load_fixture("fixtureA_sigma.json") - model = SigmaToOssieConverter().convert(spec).output.semantic_model[0] + model = SigmaToOssieConverter().convert(spec).output orders = next(d for d in model.datasets if d.name == "Orders") assert orders.primary_key == ["Order ID"] @@ -49,7 +49,7 @@ def test_unique_keys_map_to_the_portable_primary_key(): def test_non_table_element_kinds_are_preserved_but_not_modeled(): spec = load_fixture("fixtureC_sigma.json") result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output assert {d.name for d in model.datasets} == {"Basic"} # never modeled as a dataset @@ -63,7 +63,7 @@ def test_unmapped_spec_keys_survive_as_native_residue(): import json spec = load_fixture("fixtureC_sigma.json") - model = SigmaToOssieConverter().convert(spec).output.semantic_model[0] + model = SigmaToOssieConverter().convert(spec).output basic = next(d for d in model.datasets if d.name == "Basic") dataset_ext = json.loads(basic.custom_extensions[0].data) @@ -79,7 +79,7 @@ def test_unmapped_spec_keys_survive_as_native_residue(): def test_relationship_resolves_inode_style_physical_column_refs(): spec = load_fixture("fixtureB_sigma.json") result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output rel = next(r for r in model.relationships if r.name == "relEventsToOrgUser") assert rel.from_columns == ["Org ID", "User ID"] @@ -99,7 +99,7 @@ def test_relationship_resolves_inode_style_physical_column_refs(): def test_every_non_warehouse_source_kind_gets_a_marker_and_an_issue(element_name, expected_source): spec = load_fixture("fixtureB_sigma.json") result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output dataset = next(d for d in model.datasets if d.name == element_name) assert dataset.source == expected_source @@ -114,7 +114,7 @@ def test_all_filter_kinds_are_preserved_with_an_issue(): spec = load_fixture("fixtureB_sigma.json") result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output events = next(d for d in model.datasets if d.name == "Events") filters = json.loads(events.custom_extensions[0].data)["native"]["filters"] @@ -134,7 +134,7 @@ def test_all_filter_kinds_are_preserved_with_an_issue(): def test_opaque_datatype_for_unrecognized_format(): spec = load_fixture("fixtureB_sigma.json") result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output events = next(d for d in model.datasets if d.name == "Events") payload = next(f for f in events.fields if f.name == "Payload") @@ -147,7 +147,7 @@ def test_opaque_datatype_for_unrecognized_format(): def test_untranslatable_formula_keeps_sigma_dialect_only(): spec = load_fixture("fixtureB_sigma.json") result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output events = next(d for d in model.datasets if d.name == "Events") running_total = next(f for f in events.fields if f.name == "Running Total") @@ -161,7 +161,7 @@ def test_untranslatable_formula_keeps_sigma_dialect_only(): def test_derived_element_preserved_with_issue(): spec = load_fixture("fixtureB_sigma.json") result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output active_events = next(d for d in model.datasets if d.name == "Active Events") assert active_events.description == "Derived view layered on Events, not a direct warehouse table" @@ -175,7 +175,7 @@ def test_native_ids_and_page_metadata_preserved_in_custom_extensions(): spec = load_fixture("fixtureA_sigma.json") result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output orders = next(d for d in model.datasets if d.name == "Orders") ext = json.loads(orders.custom_extensions[0].data) @@ -196,7 +196,7 @@ def test_model_level_metadata_is_captured_into_custom_extensions(): url="https://app.sigmacomputing.com/data-model/11111111", ) result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output model_ext = json.loads(model.custom_extensions[0].data) for key in ("createdAt", "createdBy", "updatedAt", "updatedBy", "ownerId", "url"): @@ -209,7 +209,7 @@ def test_element_with_no_id_is_dropped_with_a_converter_issue(): del element["id"] result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output assert element["name"] not in {d.name for d in model.datasets} assert any(i.issue_type is ConverterIssueType.MISSING_ID for i in result.issues) @@ -223,7 +223,7 @@ def test_column_with_no_id_is_dropped_with_a_converter_issue(): del column["id"] result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output dataset = next(d for d in model.datasets if d.name == element["name"]) assert dropped_name not in {f.name for f in dataset.fields} @@ -240,7 +240,7 @@ def test_metric_with_no_id_is_dropped_with_a_converter_issue(): del metric["id"] result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output assert dropped_name not in {m.name for m in (model.metrics or [])} assert any(i.issue_type is ConverterIssueType.MISSING_ID for i in result.issues) @@ -254,7 +254,7 @@ def test_relationship_with_no_id_is_dropped_with_a_converter_issue(): del element["relationships"][0]["id"] result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output assert not (model.relationships or []) assert any(i.issue_type is ConverterIssueType.MISSING_ID for i in result.issues) @@ -268,7 +268,7 @@ def test_unresolvable_unique_key_is_dropped_but_preserved_as_native_residue(): element["uniqueKeys"] = ["inode-nope/DOES_NOT_EXIST"] result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output dataset = next(d for d in model.datasets if d.name == element["name"]) assert dataset.primary_key is None @@ -289,7 +289,7 @@ def test_cross_table_qualified_column_ref_is_not_indexed_as_a_physical_column(): ) result = SigmaToOssieConverter().convert(spec) - rel = next(r for r in result.output.semantic_model[0].relationships if r.name == "relEventsToOrgUser") + rel = next(r for r in result.output.relationships if r.name == "relEventsToOrgUser") assert "AmountFromOtherTable" in rel.from_columns assert any(i.issue_type is ConverterIssueType.RELATIONSHIP_COLUMN_UNRESOLVED for i in result.issues) @@ -302,7 +302,7 @@ def test_case_insensitive_physical_column_collision_is_treated_as_unresolved(): element["uniqueKeys"] = ["inode-x/AMOUNT"] result = SigmaToOssieConverter().convert(spec) - model = result.output.semantic_model[0] + model = result.output dataset = next(d for d in model.datasets if d.name == element["name"]) assert dataset.primary_key is None diff --git a/converters/snowflake/README.md b/converters/snowflake/README.md index 656954c5..a7741afb 100644 --- a/converters/snowflake/README.md +++ b/converters/snowflake/README.md @@ -23,6 +23,10 @@ Converts Ossie YAML semantic models to [Snowflake Cortex Analyst](https://docs.s > **Note:** This converter is under active development. It handles common cases but has not been thoroughly tested against all edge cases — use with caution in production. +Ossie documents contain one model directly at the root, with `version`, `name`, +`datasets`, and optional model properties. Legacy `semantic_model` wrappers +(arrays or objects) are rejected. + ## Setup ```bash diff --git a/converters/snowflake/src/ossie_snowflake/converter.py b/converters/snowflake/src/ossie_snowflake/converter.py index d8588826..8e17b649 100644 --- a/converters/snowflake/src/ossie_snowflake/converter.py +++ b/converters/snowflake/src/ossie_snowflake/converter.py @@ -81,11 +81,11 @@ def convert_ossie_to_snowflake(ossie_yaml_str): """Top-level entry point. Parses Ossie YAML, validates, converts, returns Snowflake YAML string. - Expects the standard Ossie wrapped format:: + Expects the standard Ossie document format:: version: "0.2.0.dev0" - semantic_model: - - name: ... + name: ... + datasets: [...] Args: ossie_yaml_str: Ossie YAML as a string. @@ -107,25 +107,21 @@ def convert_ossie_to_snowflake(ossie_yaml_str): f"Supported: {SUPPORTED_VERSION}" ) - semantic_model = root.get("semantic_model") - if not isinstance(semantic_model, list) or len(semantic_model) == 0: + if "semantic_model" in root: raise OssieConversionError( - "Invalid Ossie YAML: 'semantic_model' must be a non-empty list" + "Legacy 'semantic_model' wrappers are not supported; " + "place the model properties directly at the document root" ) - if len(semantic_model) > 1: - warnings.warn( - f"Ossie YAML contains {len(semantic_model)} semantic models; " - f"only the first will be converted" - ) - - ossie = semantic_model[0] - if not isinstance(ossie, dict): - raise OssieConversionError( - "Invalid Ossie YAML: 'semantic_model' entries must be mappings" - ) + if "dialects" in root or "vendors" in root: + raise OssieConversionError("Root dialects and vendors are not supported by the Ossie spec") - snowflake_model = _convert_model(ossie) + # Document metadata is consumed here; it is not a dropped model property. + model = { + key: value for key, value in root.items() + if key != "version" + } + snowflake_model = _convert_model(model) return yaml.dump( snowflake_model, diff --git a/converters/snowflake/tests/test_ossie_to_snowflake_yaml_converter.py b/converters/snowflake/tests/test_ossie_to_snowflake_yaml_converter.py index bcc59fd4..b0b5d509 100644 --- a/converters/snowflake/tests/test_ossie_to_snowflake_yaml_converter.py +++ b/converters/snowflake/tests/test_ossie_to_snowflake_yaml_converter.py @@ -45,7 +45,7 @@ def _wrap_ossie(model_dict): """Wrap a model dict in the standard Ossie envelope.""" return yaml.dump( - {"version": "0.2.0.dev0", "semantic_model": [model_dict]}, + {"version": "0.2.0.dev0", **model_dict}, default_flow_style=False, ) @@ -606,45 +606,31 @@ def test_invalid_yaml_root_raises(self): convert_ossie_to_snowflake("- a list") def test_wrong_version_raises(self): - bad = yaml.dump({"version": "9.9.9", "semantic_model": [{"name": "m"}]}) + bad = yaml.dump({"version": "9.9.9", "name": "m"}) with pytest.raises(OssieConversionError, match="Unsupported Ossie specification"): convert_ossie_to_snowflake(bad) - def test_missing_semantic_model_raises(self): - bad = yaml.dump({"version": "0.2.0.dev0"}) - with pytest.raises(OssieConversionError, match="non-empty list"): - convert_ossie_to_snowflake(bad) - - def test_empty_semantic_model_raises(self): - bad = yaml.dump({"version": "0.2.0.dev0", "semantic_model": []}) - with pytest.raises(OssieConversionError, match="non-empty list"): - convert_ossie_to_snowflake(bad) - - def test_non_dict_model_entry_raises(self): - bad = yaml.dump({"version": "0.2.0.dev0", "semantic_model": ["not a dict"]}) - with pytest.raises(OssieConversionError, match="must be mappings"): - convert_ossie_to_snowflake(bad) - def test_missing_model_name_raises(self): - bad = yaml.dump({"version": "0.2.0.dev0", "semantic_model": [{"description": "x"}]}) + bad = yaml.dump({"version": "0.2.0.dev0", "description": "x"}) with pytest.raises(OssieConversionError, match="Missing required 'name'"): convert_ossie_to_snowflake(bad) - def test_multiple_models_warns(self): - multi = yaml.dump( - { - "version": "0.2.0.dev0", - "semantic_model": [ - _minimal_model(name="first"), - _minimal_model(name="second"), - ], - } - ) - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - result = yaml.safe_load(convert_ossie_to_snowflake(multi)) - assert result["name"] == "first" - assert any("only the first" in str(warning.message) for warning in w) + @pytest.mark.parametrize( + "wrapper", + [[], [{"name": "first"}], [{"name": "first"}, {"name": "second"}], {"name": "first"}, None], + ) + def test_legacy_model_wrappers_are_rejected(self, wrapper): + bad = yaml.dump({"version": "0.2.0.dev0", "semantic_model": wrapper}) + with pytest.raises(OssieConversionError, match="Legacy 'semantic_model'"): + convert_ossie_to_snowflake(bad) + + @pytest.mark.parametrize("property_name", ["dialects", "vendors"]) + @pytest.mark.parametrize("value", [None, [], ["legacy"]]) + def test_removed_root_metadata_is_rejected(self, property_name, value): + document = yaml.safe_load(_wrap_ossie(_minimal_model())) + document[property_name] = value + with pytest.raises(OssieConversionError, match="Root dialects and vendors"): + convert_ossie_to_snowflake(yaml.safe_dump(document)) def test_snowflake_dialect_preferred(self): model = { diff --git a/converters/wisdom/README.md b/converters/wisdom/README.md index 1e1aba13..4b40a0ce 100644 --- a/converters/wisdom/README.md +++ b/converters/wisdom/README.md @@ -42,9 +42,9 @@ python ../../validation/validate.py semantic_model.yaml --schema ../../core-spec | Ossie | Wisdom | |-------|--------| -| `semantic_model[].name` | domain `ref.name` | -| `semantic_model[].description` | domain `description` | -| `semantic_model[].ai_context` | `domainSystemInstructions` + each domain `knowledge[].content` as a bulleted list | +| `name` | domain `ref.name` | +| `description` | domain `description` | +| `ai_context` | `domainSystemInstructions` + each domain `knowledge[].content` as a bulleted list | | `datasets[].name` | table `ref.name` | | `datasets[].source` | table `location.database.schema.dbTable` | | `datasets[].description` | table `description` | diff --git a/converters/wisdom/src/ossie_wisdom/cli.py b/converters/wisdom/src/ossie_wisdom/cli.py index 0f15423b..72342e4e 100644 --- a/converters/wisdom/src/ossie_wisdom/cli.py +++ b/converters/wisdom/src/ossie_wisdom/cli.py @@ -41,7 +41,6 @@ ConverterIssueType.METRIC_NAME_COLLISION: "another table defines a measure with the same name; this one was prefixed with its table name", ConverterIssueType.STALE_MEASURE: "wisdom marked this measure stale; it was converted anyway", ConverterIssueType.DUPLICATE_FIELD_DROPPED: "the dataset already has a field with this name", - ConverterIssueType.EXTRA_MODEL_DROPPED: "a wisdom domain export holds a single domain; only the first semantic model was converted", ConverterIssueType.AI_CONTEXT_DROPPED: "wisdom has no equivalent for ai_context at this level (or for synonyms/examples)", ConverterIssueType.METRIC_TABLE_UNRESOLVED: "the metric expression references no known dataset; it was attached to the first dataset", ConverterIssueType.MISSING_DIALECT_EXPRESSION: "no expression was available in the dataset's dialect or ANSI_SQL; the first available dialect was used", @@ -52,7 +51,6 @@ _DROPPED_ISSUE_TYPES = { ConverterIssueType.RELATIONSHIP_DROPPED, ConverterIssueType.DUPLICATE_FIELD_DROPPED, - ConverterIssueType.EXTRA_MODEL_DROPPED, ConverterIssueType.AI_CONTEXT_DROPPED, ConverterIssueType.UNIQUE_KEYS_DROPPED, ConverterIssueType.CUSTOM_EXTENSION_DROPPED, diff --git a/converters/wisdom/src/ossie_wisdom/converter_issues.py b/converters/wisdom/src/ossie_wisdom/converter_issues.py index fcef92f0..9e499156 100644 --- a/converters/wisdom/src/ossie_wisdom/converter_issues.py +++ b/converters/wisdom/src/ossie_wisdom/converter_issues.py @@ -29,7 +29,6 @@ class ConverterIssueType(Enum): METRIC_NAME_COLLISION = "METRIC_NAME_COLLISION" STALE_MEASURE = "STALE_MEASURE" DUPLICATE_FIELD_DROPPED = "DUPLICATE_FIELD_DROPPED" - EXTRA_MODEL_DROPPED = "EXTRA_MODEL_DROPPED" AI_CONTEXT_DROPPED = "AI_CONTEXT_DROPPED" METRIC_TABLE_UNRESOLVED = "METRIC_TABLE_UNRESOLVED" MISSING_DIALECT_EXPRESSION = "MISSING_DIALECT_EXPRESSION" diff --git a/converters/wisdom/src/ossie_wisdom/ossie_to_wisdom.py b/converters/wisdom/src/ossie_wisdom/ossie_to_wisdom.py index 55524cda..b7be440e 100644 --- a/converters/wisdom/src/ossie_wisdom/ossie_to_wisdom.py +++ b/converters/wisdom/src/ossie_wisdom/ossie_to_wisdom.py @@ -62,9 +62,7 @@ class OssieToWisdomConverter: def convert(self, document: OssieDocument, exported_at: Optional[str] = None) -> ConverterResult[dict]: issues: List[ConverterIssue] = [] - model = document.semantic_model[0] - for extra in document.semantic_model[1:]: - issues.append(ConverterIssue(issue_type=ConverterIssueType.EXTRA_MODEL_DROPPED, element_name=extra.name)) + model = document self._report_custom_extensions(model, issues) domain_uuid = _stable_id("ET_DOMAIN", model.name) diff --git a/converters/wisdom/src/ossie_wisdom/wisdom_to_ossie.py b/converters/wisdom/src/ossie_wisdom/wisdom_to_ossie.py index 6ab3347e..67cf677e 100644 --- a/converters/wisdom/src/ossie_wisdom/wisdom_to_ossie.py +++ b/converters/wisdom/src/ossie_wisdom/wisdom_to_ossie.py @@ -35,7 +35,6 @@ OssieField, OssieMetric, OssieRelationship, - OssieSemanticModel, ) from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult @@ -77,7 +76,7 @@ def convert(self, export: dict) -> ConverterResult[OssieDocument]: dataset_names = {d.name for d in datasets} relationships = self._convert_relationships(domain, dataset_names, issues) - model = OssieSemanticModel( + document = OssieDocument( name=domain.get("ref", {}).get("name") or export.get("export_metadata", {}).get("domain_name", "domain"), description=domain.get("description") or None, ai_context=self._build_ai_context(domain), @@ -85,7 +84,7 @@ def convert(self, export: dict) -> ConverterResult[OssieDocument]: relationships=relationships or None, metrics=[metric for _, metric in metrics] or None, ) - return ConverterResult(output=OssieDocument(semantic_model=[model]), issues=issues) + return ConverterResult(output=document, issues=issues) def _build_dialect_index(self, export: dict, issues: List[ConverterIssue]) -> Dict[str, OssieDialect]: index: Dict[str, OssieDialect] = {} diff --git a/converters/wisdom/tests/test_ossie_to_wisdom.py b/converters/wisdom/tests/test_ossie_to_wisdom.py index e74947bc..f89f5413 100644 --- a/converters/wisdom/tests/test_ossie_to_wisdom.py +++ b/converters/wisdom/tests/test_ossie_to_wisdom.py @@ -28,7 +28,6 @@ OssieExpression, OssieField, OssieRelationship, - OssieSemanticModel, ) from ossie_wisdom import ConverterIssueType, OssieToWisdomConverter, WisdomToOssieConverter @@ -142,7 +141,7 @@ def test_deterministic_output(ossie_document, export): assert again == export -def test_extra_models_and_unrepresentable_elements_are_reported(): +def test_unrepresentable_elements_are_reported(): dataset = OssieDataset( name="orders", source="analytics.sales.orders", @@ -151,27 +150,20 @@ def test_extra_models_and_unrepresentable_elements_are_reported(): OssieField(name="order_id", expression=_snowflake("order_id"), ai_context="the identifier"), ], ) - second = OssieSemanticModel(name="second", datasets=[OssieDataset(name="d", source="a.b.c")]) document = OssieDocument( - semantic_model=[ - OssieSemanticModel( - name="first", - datasets=[dataset], - relationships=[ - OssieRelationship( - name="orders_to_missing", - from_dataset="orders", - to="missing", - from_columns=["x"], - to_columns=["y"], - ) - ], - ), - second, - ] + name="first", + datasets=[dataset], + relationships=[ + OssieRelationship( + name="orders_to_missing", + from_dataset="orders", + to="missing", + from_columns=["x"], + to_columns=["y"], + ) + ], ) result = OssieToWisdomConverter().convert(document, exported_at="2026-07-10T00:00:00+00:00") - assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.EXTRA_MODEL_DROPPED)] == ["second"] assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.UNIQUE_KEYS_DROPPED)] == ["orders"] assert [issue.element_name for issue in _issues_of(result, ConverterIssueType.AI_CONTEXT_DROPPED)] == [ "orders.order_id" @@ -184,23 +176,19 @@ def test_extra_models_and_unrepresentable_elements_are_reported(): def test_one_to_one_note_restores_relationship_type(): document = OssieDocument( - semantic_model=[ - OssieSemanticModel( - name="m", - datasets=[ - OssieDataset(name="a", source="db.s.a"), - OssieDataset(name="b", source="db.s.b"), - ], - relationships=[ - OssieRelationship( - name="a_to_b", - from_dataset="a", - to="b", - from_columns=["id"], - to_columns=["id"], - ai_context="one-to-one relationship", - ) - ], + name="m", + datasets=[ + OssieDataset(name="a", source="db.s.a"), + OssieDataset(name="b", source="db.s.b"), + ], + relationships=[ + OssieRelationship( + name="a_to_b", + from_dataset="a", + to="b", + from_columns=["id"], + to_columns=["id"], + ai_context="one-to-one relationship", ) ] ) @@ -213,13 +201,9 @@ def test_unresolved_metric_attaches_to_first_dataset(): from ossie import OssieMetric document = OssieDocument( - semantic_model=[ - OssieSemanticModel( - name="m", - datasets=[OssieDataset(name="a", source="db.s.a"), OssieDataset(name="b", source="db.s.b")], - metrics=[OssieMetric(name="row_count", expression=_snowflake("COUNT(*)"))], - ) - ] + name="m", + datasets=[OssieDataset(name="a", source="db.s.a"), OssieDataset(name="b", source="db.s.b")], + metrics=[OssieMetric(name="row_count", expression=_snowflake("COUNT(*)"))] ) result = OssieToWisdomConverter().convert(document, exported_at="2026-07-10T00:00:00+00:00") export = result.output diff --git a/converters/wisdom/tests/test_wisdom_to_ossie.py b/converters/wisdom/tests/test_wisdom_to_ossie.py index 00839579..e08c4d9f 100644 --- a/converters/wisdom/tests/test_wisdom_to_ossie.py +++ b/converters/wisdom/tests/test_wisdom_to_ossie.py @@ -35,8 +35,7 @@ def result(): @pytest.fixture(scope="module") def model(result): - assert len(result.output.semantic_model) == 1 - return result.output.semantic_model[0] + return result.output def _issues_of(result, issue_type): @@ -158,7 +157,10 @@ def test_stale_measure_is_kept_with_warning(result, model): def test_output_round_trips_through_ossie_yaml(result): - document = OssieDocument.model_validate(yaml.safe_load(result.output.to_ossie_yaml())) + serialized = yaml.safe_load(result.output.to_ossie_yaml()) + assert serialized["name"] == "Sample Sales" + assert "semantic_model" not in serialized + document = OssieDocument.model_validate(serialized) assert document == result.output diff --git a/docs/index.md b/docs/index.md index fb93fb14..5860ac68 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,7 +47,7 @@ Ossie addresses semantic fragmentation by providing: ### Specification at a Glance -The Ossie core specification (current version: **0.2.0.dev0**, latest released: **0.1.1**) defines a JSON/YAML format with one semantic model per document. Model properties such as `name` and `datasets` sit directly at the root alongside `version`, without a `semantic_model` wrapper. Converter and Python SDK support for this development format is follow-up work. See the [migration guidance](../core-spec/spec.md#migrating-earlier-document-shapes). The key constructs are: +The Ossie core specification (current version: **0.2.0.dev0**, latest released: **0.1.1**) defines a JSON/YAML format with one semantic model per document. Model properties such as `name` and `datasets` sit directly at the root alongside `version`, without a `semantic_model` wrapper. See the [migration guidance](../core-spec/spec.md#migrating-earlier-document-shapes). The key constructs are: | Construct | Description | |-----------|-------------| diff --git a/python/README.md b/python/README.md index a22c9b40..43f9e73b 100644 --- a/python/README.md +++ b/python/README.md @@ -21,6 +21,17 @@ The Apache Ossie Python package provides Pydantic v2 models for the Apache Ossie semantic model specification. It is the shared foundation used by Apache Ossie converters to parse, construct, validate, and serialize Ossie documents from Python application. +Each `OssieDocument` is one semantic model: `name`, `datasets`, `relationships`, +and `metrics` sit at the root alongside `version`. +Construct documents with `OssieDocument(name="sales", datasets=[...])` and access +their datasets as `document.datasets`. JSON and YAML serialization use the same +flat shape. The former `semantic_model` wrapper is rejected. Unwrap old +single-model documents and split multi-model documents into separate files, +preserving model contents and setting `version` in each file before loading them. + +`OssieSemanticModel` remains available for embedded models, such as ontology +components, which do not include document metadata. + ## Development ### Prerequisites @@ -35,4 +46,4 @@ uv sync ### Generating package distributions ```bash uv build -``` \ No newline at end of file +``` diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 7c73643c..a1825557 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -207,15 +207,12 @@ class OssieSemanticModel(BaseModel): custom_extensions: Optional[list[OssieCustomExtension]] = None -class OssieDocument(BaseModel): - """Root Ossie document.""" +class OssieDocument(OssieSemanticModel): + """A single semantic model with document metadata at the root.""" - model_config = ConfigDict(frozen=True) + model_config = ConfigDict(frozen=True, extra="forbid") version: str = "0.2.0.dev0" - dialects: Optional[list[OssieDialect]] = None - vendors: Optional[list[OssieVendor]] = None - semantic_model: list[OssieSemanticModel] def to_ossie_yaml(self, **kwargs: Any) -> str: """Serialize to Ossie-compliant YAML (uses field aliases and excludes None values).""" diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 74d2a320..1a115903 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -28,6 +28,7 @@ OssieDocument, OssieExpression, OssieField, + OssieSemanticModel, ) @@ -42,32 +43,28 @@ def _expression(value: str = "value") -> OssieExpression: def _document() -> dict: return { "version": "0.2.0.dev0", - "semantic_model": [ + "name": "typed_model", + "datasets": [ { - "name": "typed_model", - "datasets": [ + "name": "events", + "source": "catalog.schema.events", + "fields": [ { - "name": "events", - "source": "catalog.schema.events", - "fields": [ - { - "name": "occurred_at", - "expression": _expression_data("occurred_at"), - "dimension": {}, - "datatype": "DateTimeTz", - } - ], - } - ], - "metrics": [ - { - "name": "revenue", - "expression": _expression_data("SUM(events.revenue)"), - "datatype": "Decimal", + "name": "occurred_at", + "expression": _expression_data("occurred_at"), + "dimension": {}, + "datatype": "DateTimeTz", } ], } ], + "metrics": [ + { + "name": "revenue", + "expression": _expression_data("SUM(events.revenue)"), + "datatype": "Decimal", + } + ], } @@ -89,22 +86,96 @@ def test_data_type_enum_matches_core_schema() -> None: def test_field_and_metric_datatypes_survive_serialization() -> None: document = OssieDocument.model_validate(_document()) - field = document.semantic_model[0].datasets[0].fields[0] - metric = document.semantic_model[0].metrics[0] + field = document.datasets[0].fields[0] + metric = document.metrics[0] assert field.datatype is OssieDataType.DATE_TIME_TZ assert metric.datatype is OssieDataType.DECIMAL as_json = json.loads(document.to_ossie_json()) as_yaml = yaml.safe_load(document.to_ossie_yaml()) for serialized in (as_json, as_yaml): - model = serialized["semantic_model"][0] + model = serialized assert model["datasets"][0]["fields"][0]["datatype"] == "DateTimeTz" assert model["metrics"][0]["datatype"] == "Decimal" +def test_document_serialization_preserves_flat_model_and_metadata() -> None: + data = _document() + data.update( + description="A portable model", + ai_context="Use the event timestamp", + custom_extensions=[{"vendor_name": "SIGMA", "data": '{"id":"model-1"}'}], + relationships=[ + { + "name": "event_link", + "from": "events", + "to": "events", + "from_columns": ["id"], + "to_columns": ["id"], + } + ], + ) + document = OssieDocument.model_validate(data) + + for serialized in (json.loads(document.to_ossie_json()), yaml.safe_load(document.to_ossie_yaml())): + assert serialized == data + assert "semantic_model" not in serialized + assert OssieDocument.model_validate(serialized) == document + + +@pytest.mark.parametrize( + "legacy_value", + [ + None, + [], + {"name": "legacy", "datasets": []}, + [{"name": "legacy", "datasets": []}], + [{"name": "first", "datasets": []}, {"name": "second", "datasets": []}], + ], +) +@pytest.mark.parametrize("include_root_model", [False, True]) +def test_document_rejects_legacy_wrapper(legacy_value: object, include_root_model: bool) -> None: + data = _document() if include_root_model else {"version": "0.2.0.dev0"} + data["semantic_model"] = legacy_value + + with pytest.raises(ValidationError) as error: + OssieDocument.model_validate(data) + + assert any( + item["loc"] == ("semantic_model",) and item["type"] == "extra_forbidden" + for item in error.value.errors() + ) + + +@pytest.mark.parametrize("property_name", ["name", "datasets"]) +def test_document_requires_root_model_properties(property_name: str) -> None: + data = _document() + del data[property_name] + + with pytest.raises(ValidationError): + OssieDocument.model_validate(data) + + +@pytest.mark.parametrize("property_name", ["dialects", "vendors"]) +def test_document_rejects_removed_root_metadata(property_name: str) -> None: + data = _document() + data[property_name] = [] + with pytest.raises(ValidationError): + OssieDocument.model_validate(data) + + +def test_embedded_semantic_model_has_no_document_metadata() -> None: + data = _document() + del data["version"] + + embedded = OssieSemanticModel.model_validate(data) + + assert embedded.model_dump(by_alias=True, exclude_none=True, mode="json") == data + + def test_invalid_datatype_is_rejected() -> None: document = _document() - field = document["semantic_model"][0]["datasets"][0]["fields"][0] + field = document["datasets"][0]["fields"][0] field["datatype"] = "timestamp" with pytest.raises(ValidationError):