diff --git a/converters/databricks/src/ossie_databricks/_common.py b/converters/databricks/src/ossie_databricks/_common.py index 4f17a5b3..b77b6428 100644 --- a/converters/databricks/src/ossie_databricks/_common.py +++ b/converters/databricks/src/ossie_databricks/_common.py @@ -261,9 +261,16 @@ def validate_source(source, dataset_name): Accepts a 3-part `catalog.schema.table` identifier or a `SELECT`/`WITH` subquery. Raises ConversionError otherwise. """ - if not source or not str(source).strip(): + if not source: raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") - s = str(source).strip() + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise ConversionError( + f"Dataset '{dataset_name}': structured source kind {kind!r} is not supported by the Databricks converter" + ) + if not source.strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = source.strip() # A SELECT/WITH subquery source. `\b` after the keyword matches `WITH(...)` (no # space) too, but not an identifier like `WITHHELD`. if re.match(r"(?i)(select|with)\b", s): diff --git a/converters/databricks/tests/test_ossie_to_metric_view.py b/converters/databricks/tests/test_ossie_to_metric_view.py index 8b8808b2..33d37386 100644 --- a/converters/databricks/tests/test_ossie_to_metric_view.py +++ b/converters/databricks/tests/test_ossie_to_metric_view.py @@ -26,6 +26,13 @@ from _util import canon, load_fixture, parse +def test_structured_dataset_source_is_rejected(): + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(ConversionError, match="structured source kind.*file.*not supported"): + exporter.validate_source(source, "orders") + + def test_fixtureA_export_matches_expected(): out = exporter.convert_ossie_to_metric_view(load_fixture("fixtureA_ossie.yaml")) assert parse(out) == parse(load_fixture("fixtureA_metric_view.yaml")) diff --git a/converters/dbt/src/ossie_dbt/ossie_to_msi.py b/converters/dbt/src/ossie_dbt/ossie_to_msi.py index dfad90b0..46635bd2 100644 --- a/converters/dbt/src/ossie_dbt/ossie_to_msi.py +++ b/converters/dbt/src/ossie_dbt/ossie_to_msi.py @@ -25,6 +25,7 @@ OssieExpression, OssieField, OssieSemanticModel, + OssieSource, ) from ossie_dbt.converter_issues import ConverterResult from ossie_dbt.expression_utils import ( @@ -395,8 +396,13 @@ def _get_expression(self, ossie_expr: OssieExpression) -> str: return ossie_expr.dialects[0].expression if ossie_expr.dialects else "" @staticmethod - def _parse_source(source: str) -> PydanticNodeRelation: - """Parse `schema.table` or `db.schema.table` into a PydanticNodeRelation.""" + def _parse_source(source: OssieSource) -> PydanticNodeRelation: + """Parse a legacy string source into a PydanticNodeRelation.""" + if not isinstance(source, str): + kind = getattr(source, "kind", type(source).__name__) + raise TypeError( + f"Structured dataset source kind {kind!r} is not supported by the dbt converter" + ) parts = source.split(".") if len(parts) >= 3: database, schema, alias = parts[0], parts[1], ".".join(parts[2:]) diff --git a/converters/dbt/tests/test_ossie_to_msi.py b/converters/dbt/tests/test_ossie_to_msi.py index eaec212b..96e2a9f5 100644 --- a/converters/dbt/tests/test_ossie_to_msi.py +++ b/converters/dbt/tests/test_ossie_to_msi.py @@ -20,7 +20,7 @@ import pytest from syrupy.assertion import SnapshotAssertion -from ossie import OssieDataType, OssieDimension +from ossie import OssieDataType, OssieDimension, OssieFileSource from ossie_dbt.msi_to_ossie import MSIToOssieConverter from ossie_dbt.ossie_to_msi import OssieToMSIConverter from metricflow_semantic_interfaces.type_enums import ( @@ -37,6 +37,13 @@ ) +def test_structured_dataset_source_is_rejected() -> None: + source = OssieFileSource(kind="file", format="parquet", locations=["s3://bucket/orders.parquet"]) + + with pytest.raises(TypeError, match="Structured dataset source kind.*file.*not supported"): + OssieToMSIConverter._parse_source(source) + + class TestOssieToMSIBasicConversion: def test_empty_document_produces_empty_manifest(self) -> None: result = OssieToMSIConverter().convert(_ossie_doc()).output diff --git a/converters/gooddata/src/ossie_gooddata/ossie_to_gooddata.py b/converters/gooddata/src/ossie_gooddata/ossie_to_gooddata.py index 21a4a910..1bbadf5c 100644 --- a/converters/gooddata/src/ossie_gooddata/ossie_to_gooddata.py +++ b/converters/gooddata/src/ossie_gooddata/ossie_to_gooddata.py @@ -357,8 +357,13 @@ def _is_multivalue(rel: dict[str, Any]) -> bool: return bool(gd_ext and gd_ext.get("multivalue")) -def _parse_source_to_table_id(source: str, data_source_id: str) -> GdDataSourceTableId: - """Parse an Ossie source string into a GoodData DataSourceTableId.""" +def _parse_source_to_table_id(source: object, data_source_id: str) -> GdDataSourceTableId: + """Parse a legacy Ossie string source into a GoodData DataSourceTableId.""" + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise TypeError( + f"Structured dataset source kind {kind!r} is not supported by the GoodData converter" + ) parts = source.split(".") if len(parts) >= 3: # source_id.schema.table or more diff --git a/converters/gooddata/tests/test_ossie_to_gooddata.py b/converters/gooddata/tests/test_ossie_to_gooddata.py index 5f44be70..8e59570b 100644 --- a/converters/gooddata/tests/test_ossie_to_gooddata.py +++ b/converters/gooddata/tests/test_ossie_to_gooddata.py @@ -26,6 +26,7 @@ from ossie_gooddata.ossie_to_gooddata import ( _convert_to_attribute, _convert_to_fact, + _parse_source_to_table_id, ossie_to_gooddata, ) diff --git a/converters/honeydew/src/honeydew_ossie/converter.py b/converters/honeydew/src/honeydew_ossie/converter.py index ee6c092d..9477b837 100644 --- a/converters/honeydew/src/honeydew_ossie/converter.py +++ b/converters/honeydew/src/honeydew_ossie/converter.py @@ -453,7 +453,12 @@ def _is_simple_identifier(expr: str) -> bool: return bool(re.match(r"^[a-zA-Z_][a-zA-Z0-9_]*$", expr.strip())) -def _parse_ossie_source(source: str) -> tuple[str, str]: +def _parse_ossie_source(source: object) -> tuple[str, str]: + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise HoneydewConversionError( + f"Structured dataset source kind {kind!r} is not supported by the Honeydew converter" + ) source = (source or "").strip() if not source: return ("", "table") diff --git a/converters/honeydew/tests/test_honeydew_ossie_converter.py b/converters/honeydew/tests/test_honeydew_ossie_converter.py index 4f8a0a56..49b97478 100644 --- a/converters/honeydew/tests/test_honeydew_ossie_converter.py +++ b/converters/honeydew/tests/test_honeydew_ossie_converter.py @@ -50,6 +50,13 @@ OSSIE_VERSION = "0.2.0.dev0" +def test_structured_dataset_source_is_rejected() -> None: + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(HoneydewConversionError, match="Structured dataset source kind.*file.*not supported"): + _parse_ossie_source(source) + + def _ossie(model_dict): return yaml.dump( {"version": OSSIE_VERSION, "semantic_model": [model_dict]}, diff --git a/converters/nvidia/src/ossie_nvidia_gsf/native_converter.py b/converters/nvidia/src/ossie_nvidia_gsf/native_converter.py index f2c4fd7e..269aa4a6 100644 --- a/converters/nvidia/src/ossie_nvidia_gsf/native_converter.py +++ b/converters/nvidia/src/ossie_nvidia_gsf/native_converter.py @@ -1923,6 +1923,10 @@ def _parse_source( default_database: str | None, ) -> dict[str, str | None]: if isinstance(source, dict): + if "kind" in source: + raise GSFConversionError( + f"Structured dataset source kind {source.get('kind')!r} is not supported by the NVIDIA GSF converter" + ) database = source.get("database") or default_database schema = source.get("schema") table = source.get("table") diff --git a/converters/nvidia/tests/test_converter.py b/converters/nvidia/tests/test_converter.py index 70fd5e1c..73537073 100644 --- a/converters/nvidia/tests/test_converter.py +++ b/converters/nvidia/tests/test_converter.py @@ -50,6 +50,13 @@ SCHEMA = Path(__file__).resolve().parents[3] / "core-spec" / "ossie-schema.json" +def test_structured_dataset_source_is_rejected() -> None: + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(GSFConversionError, match="Structured dataset source kind.*file.*not supported"): + _parse_source(source, "analytics") + + def _ossie_yaml() -> str: return (FIXTURES / "sales.ossie.yaml").read_text(encoding="utf-8") diff --git a/converters/omni/src/ossie_omni/_common.py b/converters/omni/src/ossie_omni/_common.py index a8eb6935..0d127fb1 100644 --- a/converters/omni/src/ossie_omni/_common.py +++ b/converters/omni/src/ossie_omni/_common.py @@ -328,9 +328,16 @@ def parse_source(source, dataset_name): (parts may be double-quoted: `"Omni Views".channel_info`). Omni views require a `schema`, so a bare 1-part table name is rejected. """ - if not source or not str(source).strip(): + if not source: raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") - s = str(source).strip() + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise ConversionError( + f"Dataset '{dataset_name}': structured source kind {kind!r} is not supported by the Omni converter" + ) + if not source.strip(): + raise ConversionError(f"Dataset '{dataset_name}': missing/empty 'source'") + s = source.strip() if re.match(r"(?i)(select|with)\b", s): return ("sql", s) if not _SOURCE_PARTS_RE.match(s): diff --git a/converters/omni/tests/test_ossie_to_omni.py b/converters/omni/tests/test_ossie_to_omni.py index d6de14b1..9dbd6610 100644 --- a/converters/omni/tests/test_ossie_to_omni.py +++ b/converters/omni/tests/test_ossie_to_omni.py @@ -22,10 +22,17 @@ import pytest from ossie_omni import ConversionError, convert_ossie_to_omni -from ossie_omni._common import dump_yaml +from ossie_omni._common import dump_yaml, parse_source from _util import REPO_ROOT, load_fixture, load_fixture_dir, parse, parse_files +def test_structured_dataset_source_is_rejected(): + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(ConversionError, match="structured source kind.*file.*not supported"): + parse_source(source, "orders") + + def export(ossie_yaml, **kwargs): with warnings.catch_warnings(): warnings.simplefilter("ignore") diff --git a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py index 600d38cb..59b9d1ce 100644 --- a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py +++ b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py @@ -226,8 +226,14 @@ def _carry_foreign_extensions(ossie_exts: list[dict] | None, obml_target: dict[s {"vendor": vendor, "data": ext.get("data", "")} ) - def _parse_source(self, source: str) -> tuple[str, str, str]: - """Parse 'database.schema.table' into parts.""" + def _parse_source(self, source: object) -> tuple[str, str, str]: + """Parse a legacy string source into database/schema/table parts.""" + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise TypeError( + f"Structured dataset source kind {kind!r} is not supported " + "by the OrionBelt converter" + ) parts = source.split(".") if len(parts) == 3: return parts[0], parts[1], parts[2] diff --git a/converters/orionbelt/tests/test_ossie_v02_compat.py b/converters/orionbelt/tests/test_ossie_v02_compat.py index b097f341..c9a6d42f 100644 --- a/converters/orionbelt/tests/test_ossie_v02_compat.py +++ b/converters/orionbelt/tests/test_ossie_v02_compat.py @@ -43,6 +43,14 @@ # --------------------------------------------------------------------------- +def test_structured_dataset_source_is_rejected() -> None: + converter = conv.OssietoOBML({"version": "0.2.0.dev0", "semantic_model": []}) + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(TypeError, match="Structured dataset source kind.*file.*not supported"): + converter._parse_source(source) + + @pytest.fixture(scope="module") def schema_validator() -> Any: """Draft 2020-12 validator pinned to the resolved Ossie v0.2 core schema diff --git a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieModelParser.java b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieModelParser.java index d8378d44..18730c33 100644 --- a/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieModelParser.java +++ b/converters/polaris/src/main/java/org/apache/ossie/converter/polaris/OssieModelParser.java @@ -112,7 +112,13 @@ private SemanticModel parseSemanticModel(Map map) { private Dataset parseDataset(Map map) { Dataset ds = new Dataset(); ds.setName((String) map.get("name")); - ds.setSource((String) map.get("source")); + Object source = map.get("source"); + if (source != null && !(source instanceof String)) { + Object kind = source instanceof Map sourceMap ? sourceMap.get("kind") : source.getClass().getSimpleName(); + throw new IllegalArgumentException( + "Structured dataset source kind '" + kind + "' is not supported by the Polaris converter"); + } + ds.setSource((String) source); ds.setDescription((String) map.get("description")); List pk = (List) map.get("primary_key"); diff --git a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OssiePolarisConverterTest.java b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OssiePolarisConverterTest.java index 62ba18d2..af65894c 100644 --- a/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OssiePolarisConverterTest.java +++ b/converters/polaris/src/test/java/org/apache/ossie/converter/polaris/OssiePolarisConverterTest.java @@ -120,6 +120,27 @@ void testParseMinimalModel() { assertEquals(1, sm.getMetrics().size()); } + @Test + void testStructuredDatasetSourceIsRejected() { + String structuredSourceModel = + "version: \"0.2.0.dev0\"\n" + + "semantic_model:\n" + + " - name: test_model\n" + + " datasets:\n" + + " - name: orders\n" + + " source:\n" + + " kind: file\n" + + " format: parquet\n" + + " locations: [s3://bucket/orders.parquet]\n"; + + IllegalArgumentException error = assertThrows( + IllegalArgumentException.class, + () -> new OssieModelParser().parse( + new ByteArrayInputStream(structuredSourceModel.getBytes(StandardCharsets.UTF_8)))); + + assertTrue(error.getMessage().contains("Structured dataset source kind 'file' is not supported")); + } + @Test void testParseDatasetFields() { OssieModelParser parser = new OssieModelParser(); diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java index 044dfe4c..1daa8b97 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/ConverterConstants.java @@ -64,6 +64,7 @@ public enum Level { public static final String API_NAME = "apiName"; public static final String LABEL = "label"; public static final String DESCRIPTION = "description"; + public static final String SOURCE = "source"; public static final String DATA_TYPE = "dataType"; public static final String OSSIE_DATATYPE = "datatype"; public static final String AI_CONTEXT = "ai_context"; diff --git a/converters/salesforce/src/main/java/org/apache/ossie/converter/DatasetMappingHandler.java b/converters/salesforce/src/main/java/org/apache/ossie/converter/DatasetMappingHandler.java index eb82aeda..5588022a 100644 --- a/converters/salesforce/src/main/java/org/apache/ossie/converter/DatasetMappingHandler.java +++ b/converters/salesforce/src/main/java/org/apache/ossie/converter/DatasetMappingHandler.java @@ -69,6 +69,21 @@ public void execute(Map sourceData, Map outputDa private void mapOssieToSalesforce( Map sourceData, Map outputData, Map mappings) { + List osiDatasets = getList(sourceData, DATASETS); + if (osiDatasets != null) { + streamMaps(osiDatasets).forEach(dataset -> { + Object source = dataset.get(SOURCE); + if (source != null && !(source instanceof String)) { + Object kind = source instanceof Map sourceMap + ? sourceMap.get("kind") + : source.getClass().getSimpleName(); + throw new org.apache.ossie.exception.ConversionException( + "Structured dataset source kind '" + kind + + "' is not supported by the Salesforce converter"); + } + }); + } + Map datasetMappings = MappingUtils.filterMappingsByPrefix(mappings, DATASETS); var mappedData = GenericMappingEngine.applyMappings(sourceData, datasetMappings); diff --git a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java index 6d5c1237..32f60834 100644 --- a/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java +++ b/converters/salesforce/src/test/java/org/apache/ossie/OssieToSalesforceConverterTest.java @@ -25,6 +25,7 @@ import org.apache.ossie.converter.ConverterFactory; import org.apache.ossie.converter.ConversionDirection; import org.apache.ossie.converter.CustomExtensionHandler; +import org.apache.ossie.exception.ConversionException; import org.apache.ossie.validator.SchemaValidator; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -94,6 +95,23 @@ void setUp() throws IOException { ossieYaml = ossieYamlAnsiSql; } + @Test + void testStructuredDatasetSourceIsRejected() { + String structuredSourceModel = + "version: \"0.2.0.dev0\"\n" + + "semantic_model:\n" + + " - name: test_model\n" + + " datasets:\n" + + " - name: orders\n" + + " source:\n" + + " kind: file\n" + + " format: parquet\n" + + " locations: [s3://bucket/orders.parquet]\n"; + + ConversionException error = assertThrows(ConversionException.class, () -> converter.convert(structuredSourceModel)); + assertTrue(error.getMessage().contains("Structured dataset source kind 'file' is not supported")); + } + @Test void testCompleteConversion() throws Exception { List results = converter.convert(ossieYaml); diff --git a/converters/snowflake/src/ossie_snowflake/converter.py b/converters/snowflake/src/ossie_snowflake/converter.py index d8588826..ed8e15a9 100644 --- a/converters/snowflake/src/ossie_snowflake/converter.py +++ b/converters/snowflake/src/ossie_snowflake/converter.py @@ -446,8 +446,13 @@ def _parse_source(source): """ if not source: return None + if not isinstance(source, str): + kind = source.get("kind") if isinstance(source, dict) else type(source).__name__ + raise OssieConversionError( + f"Structured dataset source kind {kind!r} is not supported by the Snowflake converter" + ) - source_stripped = str(source).strip() + source_stripped = source.strip() if not source_stripped: return None diff --git a/converters/snowflake/tests/test_ossie_to_snowflake_yaml_converter.py b/converters/snowflake/tests/test_ossie_to_snowflake_yaml_converter.py index bcc59fd4..ca542cac 100644 --- a/converters/snowflake/tests/test_ossie_to_snowflake_yaml_converter.py +++ b/converters/snowflake/tests/test_ossie_to_snowflake_yaml_converter.py @@ -38,6 +38,13 @@ ) +def test_structured_dataset_source_is_rejected(): + source = {"kind": "file", "format": "parquet", "locations": ["s3://bucket/orders.parquet"]} + + with pytest.raises(OssieConversionError, match="Structured dataset source kind.*file.*not supported"): + _parse_source(source) + + # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- diff --git a/converters/wisdom/src/ossie_wisdom/ossie_to_wisdom.py b/converters/wisdom/src/ossie_wisdom/ossie_to_wisdom.py index 55524cda..b37a20e1 100644 --- a/converters/wisdom/src/ossie_wisdom/ossie_to_wisdom.py +++ b/converters/wisdom/src/ossie_wisdom/ossie_to_wisdom.py @@ -40,6 +40,7 @@ OssieDocument, OssieExpression, OssieSemanticModel, + OssieSource, ) from ossie_wisdom.converter_issues import ConverterIssue, ConverterIssueType, ConverterResult @@ -270,7 +271,12 @@ def _ai_context_text(self, ai_context, element_name: str, issues: List[Converter return ai_context.instructions or "" return ai_context - def _split_source(self, source: str) -> Tuple[str, str, str]: + def _split_source(self, source: OssieSource) -> Tuple[str, str, str]: + if not isinstance(source, str): + kind = getattr(source, "kind", type(source).__name__) + raise TypeError( + f"Structured dataset source kind {kind!r} is not supported by the Wisdom converter" + ) parts = source.split(".") if len(parts) >= 3: return parts[0], parts[1], ".".join(parts[2:]) diff --git a/converters/wisdom/tests/test_ossie_to_wisdom.py b/converters/wisdom/tests/test_ossie_to_wisdom.py index e74947bc..b3e21cd0 100644 --- a/converters/wisdom/tests/test_ossie_to_wisdom.py +++ b/converters/wisdom/tests/test_ossie_to_wisdom.py @@ -27,6 +27,7 @@ OssieDocument, OssieExpression, OssieField, + OssieFileSource, OssieRelationship, OssieSemanticModel, ) @@ -35,6 +36,13 @@ FIXTURE = Path(__file__).parent / "fixtures" / "sample_export.json" +def test_structured_dataset_source_is_rejected() -> None: + source = OssieFileSource(kind="file", format="parquet", locations=["s3://bucket/orders.parquet"]) + + with pytest.raises(TypeError, match="Structured dataset source kind.*file.*not supported"): + OssieToWisdomConverter()._split_source(source) + + def _snowflake(expression): return OssieExpression(dialects=[OssieDialectExpression(dialect=OssieDialect.SNOWFLAKE, expression=expression)]) diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index edddac31..722de685 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -173,6 +173,40 @@ "required": ["name", "expression"], "additionalProperties": false }, + "FileSource": { + "type": "object", + "description": "Structured descriptor for a file-backed dataset source.", + "properties": { + "kind": { + "type": "string", + "const": "file", + "description": "Discriminator identifying a file-backed source." + }, + "format": { + "type": "string", + "minLength": 1, + "description": "Physical file format, for example parquet." + }, + "locations": { + "type": "array", + "items": { + "type": "string", + "minLength": 1 + }, + "minItems": 1, + "description": "One or more file, object-store, or HTTPS locations." + } + }, + "required": ["kind", "format", "locations"], + "additionalProperties": false + }, + "Source": { + "description": "Dataset source. Legacy string sources remain supported for backward compatibility.", + "oneOf": [ + {"type": "string"}, + {"$ref": "#/$defs/FileSource"} + ] + }, "Dataset": { "type": "object", "description": "Logical dataset representing a business entity (fact or dimension table)", @@ -182,8 +216,7 @@ "description": "Unique identifier for the dataset" }, "source": { - "type": "string", - "description": "Reference to underlying physical table/view (database.schema.table) or query" + "$ref": "#/$defs/Source" }, "primary_key": { "type": "array", diff --git a/core-spec/spec.md b/core-spec/spec.md index 156cb1db..f6f57ddd 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -124,7 +124,7 @@ Logical datasets represent business entities or concepts (fact and dimension tab | Field | Type | Required | Description | |-------|------|----------|-------------| | `name` | string | Yes | Unique identifier for the dataset | -| `source` | string | Yes | Reference to underlying physical table/view (e.g., `database.schema.table`) or query | +| `source` | string/object | Yes | Legacy table/view/query string, or a structured file-backed source descriptor | | `primary_key` | array | No | Primary key columns that uniquely identify rows (single or composite) | | `unique_keys` | array of arrays | No | Array of unique key definitions (each can be single or composite) | | `description` | string | No | Human-readable description | @@ -132,6 +132,26 @@ Logical datasets represent business entities or concepts (fact and dimension tab | `fields` | array | No | Row-level attributes for grouping, filtering, and metric expressions | | `custom_extensions` | array | No | Vendor-specific attributes | +### Source Forms + +Existing string sources remain valid and preserve current behavior: + +```yaml +source: sales.public.orders +``` + +File-backed datasets may use an explicit structured form. `kind` is the discriminator; the initial portable file form requires a physical format and at least one location: + +```yaml +source: + kind: file + format: parquet + locations: + - s3://analytics-data/orders/*.parquet +``` + +Locations are metadata references only. The Ossie document does not imply that every converter or consumer can read the referenced storage system; unsupported source kinds must be handled explicitly rather than silently reinterpreted as table names. + ### Primary Key Examples ```yaml diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 9b21b444..984e2d5d 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -96,9 +96,17 @@ datasets: # Required: Unique identifier for the logical dataset - name: string - # Required: Reference to the underlying physical table/view or query - # Format should be either database_name.schema_name.table_name or query - source: string + # Required: Dataset source. Existing table/view/query strings remain valid. + # Legacy form: + # source: database_name.schema_name.table_name + # + # Structured file-backed form: + # source: + # kind: file + # format: parquet + # locations: + # - s3://bucket/path/*.parquet + source: string | object # Optional: Primary key definition that uniquely identifies rows in this dataset # Can be a single column or a composite of multiple columns diff --git a/docs/index.md b/docs/index.md index 3836092d..b8d2a3a1 100644 --- a/docs/index.md +++ b/docs/index.md @@ -52,7 +52,7 @@ The Ossie core specification (current version: **0.2.0.dev0**, latest released: | Construct | Description | |-----------|-------------| | **Semantic Model** | The top-level container representing a complete semantic model, including datasets, relationships, and metrics. | -| **Datasets** | Logical datasets representing business entities (fact and dimension tables), with fields, primary keys, and unique keys. | +| **Datasets** | Logical datasets representing business entities, with fields, keys, and a `source` that can be a legacy table/view/query string or a structured file-backed descriptor. | | **Fields** | Row-level attributes for grouping, filtering, and metric expressions. Fields support multiple SQL dialects for cross-platform compatibility. | | **Relationships** | Foreign key connections between datasets, supporting both simple and composite keys. | | **Metrics** | Quantitative measures (sums, averages, ratios, etc.) defined at the model level, capable of spanning multiple datasets. | @@ -61,7 +61,7 @@ The Ossie core specification (current version: **0.2.0.dev0**, latest released: The specification supports multiple SQL dialects (`ANSI_SQL`, `SNOWFLAKE`, `DATABRICKS`, `MDX`, `TABLEAU`) so that expressions can be tailored to each platform while maintaining a common model structure. -For the full specification, see [core-spec/spec.md](../core-spec/spec.md). For validation tooling, see [validation/validate.py](../validation/validate.py). For a complete example, see the [TPC-DS semantic model](../examples/tpcds_semantic_model.yaml). +For the full specification, see [core-spec/spec.md](../core-spec/spec.md). For validation tooling, see [validation/validate.py](../validation/validate.py). For examples, see the [TPC-DS semantic model](../examples/tpcds_semantic_model.yaml) and the [file-backed semantic model](../examples/file_backed_semantic_model.yaml). ### Participating Organizations diff --git a/examples/file_backed_semantic_model.yaml b/examples/file_backed_semantic_model.yaml new file mode 100644 index 00000000..bb4dbae0 --- /dev/null +++ b/examples/file_backed_semantic_model.yaml @@ -0,0 +1,37 @@ +# yaml-language-server: $schema=../core-spec/ossie-schema.json +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# File-backed Dataset Example +# Demonstrates a portable structured file source without requiring object-store access. + +version: 0.2.0.dev0 +semantic_model: + - name: file_backed_example + datasets: + - name: events + source: + kind: file + format: parquet + locations: + - s3://analytics-data/events/*.parquet + fields: + - name: event_id + expression: + dialects: + - dialect: ANSI_SQL + expression: event_id diff --git a/python/src/ossie/__init__.py b/python/src/ossie/__init__.py index d01b36e9..076559ba 100644 --- a/python/src/ossie/__init__.py +++ b/python/src/ossie/__init__.py @@ -27,9 +27,11 @@ OssieDocument, OssieExpression, OssieField, + OssieFileSource, OssieMetric, OssieRelationship, OssieSemanticModel, + OssieSource, OssieVendor, ) @@ -45,8 +47,10 @@ "OssieDocument", "OssieExpression", "OssieField", + "OssieFileSource", "OssieMetric", "OssieRelationship", "OssieSemanticModel", + "OssieSource", "OssieVendor", ] diff --git a/python/src/ossie/models.py b/python/src/ossie/models.py index 7ea515c3..c5ae4e72 100644 --- a/python/src/ossie/models.py +++ b/python/src/ossie/models.py @@ -16,7 +16,7 @@ # under the License. from enum import Enum -from typing import Any, Optional, Union +from typing import Annotated, Any, Literal, Optional, Union import yaml from pydantic import BaseModel, ConfigDict, Field @@ -147,13 +147,26 @@ def is_time_dimension(self) -> bool: return self.datatype in _TEMPORAL_DATA_TYPES +class OssieFileSource(BaseModel): + """Structured descriptor for a file-backed dataset source.""" + + model_config = ConfigDict(frozen=True, extra="forbid") + + kind: Literal["file"] + format: str = Field(min_length=1) + locations: list[Annotated[str, Field(min_length=1)]] = Field(min_length=1) + + +OssieSource = Union[str, OssieFileSource] + + class OssieDataset(BaseModel): """Logical dataset representing a business entity (fact or dimension table).""" model_config = ConfigDict(frozen=True) name: str - source: str + source: OssieSource primary_key: Optional[list[str]] = None unique_keys: Optional[list[list[str]]] = None description: Optional[str] = None diff --git a/python/tests/test_models.py b/python/tests/test_models.py index 74d2a320..68ddeeb2 100644 --- a/python/tests/test_models.py +++ b/python/tests/test_models.py @@ -27,6 +27,7 @@ OssieDimension, OssieDocument, OssieExpression, + OssieFileSource, OssieField, ) @@ -135,3 +136,81 @@ def test_effective_time_dimension_role( ) assert field.is_time_dimension() is expected + + +def test_legacy_dataset_source_remains_a_string() -> None: + document = OssieDocument.model_validate(_document()) + assert document.semantic_model[0].datasets[0].source == "catalog.schema.events" + + +def test_file_dataset_source_survives_serialization() -> None: + data = _document() + data["semantic_model"][0]["datasets"][0]["source"] = { + "kind": "file", + "format": "parquet", + "locations": ["s3://analytics/events/*.parquet"], + } + + document = OssieDocument.model_validate(data) + source = document.semantic_model[0].datasets[0].source + assert isinstance(source, OssieFileSource) + assert source.kind == "file" + assert source.format == "parquet" + assert source.locations == ["s3://analytics/events/*.parquet"] + + for serialized in (json.loads(document.to_ossie_json()), yaml.safe_load(document.to_ossie_yaml())): + source_data = serialized["semantic_model"][0]["datasets"][0]["source"] + assert source_data == { + "kind": "file", + "format": "parquet", + "locations": ["s3://analytics/events/*.parquet"], + } + + +def test_file_dataset_source_requires_at_least_one_location() -> None: + data = _document() + data["semantic_model"][0]["datasets"][0]["source"] = { + "kind": "file", + "format": "parquet", + "locations": [], + } + + with pytest.raises(ValidationError): + OssieDocument.model_validate(data) + + +def test_file_source_definition_matches_core_schema() -> None: + schema_path = Path(__file__).parents[2] / "core-spec" / "ossie-schema.json" + schema = json.loads(schema_path.read_text()) + + assert schema["$defs"]["Source"]["oneOf"] == [ + {"type": "string"}, + {"$ref": "#/$defs/FileSource"}, + ] + assert schema["$defs"]["FileSource"]["required"] == [ + "kind", + "format", + "locations", + ] + + +@pytest.mark.parametrize( + "source", + [ + {"kind": "file", "format": "", "locations": ["s3://bucket/events.parquet"]}, + {"kind": "file", "format": "parquet", "locations": [""]}, + {"kind": "table", "format": "parquet", "locations": ["s3://bucket/events.parquet"]}, + { + "kind": "file", + "format": "parquet", + "locations": ["s3://bucket/events.parquet"], + "unknown": True, + }, + ], +) +def test_invalid_file_dataset_source_is_rejected(source: dict) -> None: + data = _document() + data["semantic_model"][0]["datasets"][0]["source"] = source + + with pytest.raises(ValidationError): + OssieDocument.model_validate(data)