diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..227cea2 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +2.0.0 diff --git a/design/core/DESIGN_LANGUAGE.md b/design/core/DESIGN_LANGUAGE.md index b77838f..370f02e 100644 --- a/design/core/DESIGN_LANGUAGE.md +++ b/design/core/DESIGN_LANGUAGE.md @@ -258,7 +258,7 @@ A design document lists the capabilities it requires. Each platform implementati | `LineageCapture` | Record the origin of an entity instance: source system, source record, and the run that produced it. | insert into the Observability lineage entities. | | `AgentOutcomeCapture` | Record which tables and agent touched, when, and with what outcome, as a queryable event separate from the tables themselves. | insert into the Observability `AgentOutcome` entity. | | `QualityScore` | Obtain a reliability score in `0.00`-`1.00` for an entity instance or set, decomposed into the rule categories that produced it, so an agent can judge fitness before use. | quality metric entity plus a scoring view. | -| `MetadataCoverageCheck` | Confirm programmatically that every attribute carries metadata. | catalogue query returning uncommented columns. | +| `MetadataCoverageCheck` | Confirm programmatically that every container, object and attribute of the product carries metadata. The sweep covers everything the product owns, not only what its declared entities name. | catalogue query returning every uncommented container, table, view, routine or column. | | `SemanticRegistration` | Register the module's entities, columns, and relationships in the product's Semantic map so agents can discover them. | inserts into the Semantic discovery entities. | | `DocumentationCapture` | Record the module's design decisions, glossary terms, and change history in the product's Memory store. | inserts into the Memory documentation entities. | | `AgentContinuity` | Carry agent state across sessions: retrieve prior sessions, interactions, learned strategies, and preferences for a given scope, so an agent resumes rather than restarts. | inserts and reads over the Memory runtime entities. | diff --git a/design/core/DESIGN_SPECIFICATION.md b/design/core/DESIGN_SPECIFICATION.md index 441b240..cb74e58 100644 --- a/design/core/DESIGN_SPECIFICATION.md +++ b/design/core/DESIGN_SPECIFICATION.md @@ -93,6 +93,8 @@ An entity with no business identifier of its own (an embedding, a feature row, a | Qualifier | Meaning | |---|---| | `[-> A \| B]` | A reference whose target is one of several entities. Requires `[discriminator: ]`. | +| `[-> A.]` | A join on a named attribute of the target. The named attributes of one entity that point at the same target form one join, paired in declaration order, and must be exactly one key of the target (its surrogate or its natural key) with the same logical types. A bare `[-> A]` stays a whole join on the target's identity. | +| `[join: ]` | Names a join of several named attributes, so one entity can hold two joins to the same target. Without it the attributes pointing at one target are one join. | | `[discriminator: ]` | Names the `Enum` attribute recording which target a reference points at. Its members are the target entity names in upper case. | | `[derive: ]` | The attribute is computed, in the expression vocabulary of section 5. | | `[synonyms: , ]` | Business terms consumers use for this attribute. | diff --git a/design/modules/memory.md b/design/modules/memory.md index d30a2ef..d911ceb 100644 --- a/design/modules/memory.md +++ b/design/modules/memory.md @@ -94,12 +94,12 @@ Entity: AgentSession [kind: Record] [profile: CURRENT_STATE] [alloc session_key: NaturalKey [required] // business session identifier agent_key: ShortText [required] // which agent instance user_key: ShortText [optional] // which user - session_start_dts: Timestamp [required] + session_start_dts: Timestamp [required] // When the session began. session_end_dts: Timestamp [optional] // null while active - session_status: Enum{ACTIVE|COMPLETED|ABANDONED} - session_goal: Text [optional] + session_status: Enum{ACTIVE|COMPLETED|ABANDONED} // Whether the session is active, completed or abandoned. + session_goal: Text [optional] // What the session set out to do. session_context: Json [optional] // flexible context, processed by the consumer - scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] + scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] // Whose scope the record belongs to: a user, a team, the organisation or an agent. scope_identifier: ShortText [required] // user/team/org/agent key matching scope_level Keys: @@ -107,59 +107,59 @@ Entity: AgentSession [kind: Record] [profile: CURRENT_STATE] [alloc natural: session_key Entity: AgentInteraction [kind: Record] [profile: EVENT_APPEND_ONLY] - interaction_id: Identifier - session_id: Reference [required] [-> AgentSession] + interaction_id: Identifier // Generated identifier of the row. + session_id: Reference [required] [-> AgentSession] // The session the interaction belongs to. interaction_seq: Integer [required] // order within the session - interaction_type: Enum{QUERY|ACTION|DECISION|EXPLANATION} - interaction_dts: Timestamp [required] + interaction_type: Enum{QUERY|ACTION|DECISION|EXPLANATION} // Kind of interaction: a query, an action, a decision or an explanation. + interaction_dts: Timestamp [required] // When the interaction happened. user_input: Text [optional] [pii-incidental] // what the user asked, as written - agent_response: Text [optional] [pii-incidental] - action_taken: Text [optional] + agent_response: Text [optional] [pii-incidental] // What the agent replied; it can incidentally contain personal information. + action_taken: Text [optional] // The action the agent took, when it took one. referenced_tables: Text [optional] // logical entity names (.), comma-separated; ENTITY-LEVEL only, resolved through Semantic (INV-MEMORY-001) query_executed: Text [optional] // the query text, not its results query_result_count: Integer [optional] // aggregate count only, never the ids - execution_time_ms: Integer [optional] - outcome_status: Enum{SUCCESS|PARTIAL|FAILED} - user_feedback: Enum{POSITIVE|NEUTRAL|NEGATIVE} [optional] - scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] - scope_identifier: ShortText [required] + execution_time_ms: Integer [optional] // How long the interaction took, in milliseconds. + outcome_status: Enum{SUCCESS|PARTIAL|FAILED} // Whether the interaction succeeded, partly succeeded or failed. + user_feedback: Enum{POSITIVE|NEUTRAL|NEGATIVE} [optional] // The user's reaction, when one was given. + scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] // Whose scope the record belongs to: a user, a team, the organisation or an agent. + scope_identifier: ShortText [required] // The user, team, organisation or agent that the scope level names. Entity: LearnedStrategy [kind: Record] [profile: CURRENT_STATE] - strategy_id: Identifier - strategy_name: ShortText [required] + strategy_id: Identifier // Generated identifier of the row. + strategy_name: ShortText [required] // Name of the strategy the agent learned. strategy_category: Enum{QUERY_OPTIMIZATION|FEATURE_SELECTION|ERROR_HANDLING} // fixed by this standard strategy_pattern: Text [optional] // the pattern/approach, described - strategy_metadata: Json [optional] + strategy_metadata: Json [optional] // Details of the strategy, as JSON. success_rate: Decimal(5,4) [optional] // 0.0-1.0 - times_used: Integer [optional] + times_used: Integer [optional] // How often the strategy has been applied. is_active: Flag // lifecycle state: the strategy is offered to agents; cleared when retired is_validated: Flag // the strategy has been confirmed against observed outcomes - scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] - scope_identifier: ShortText [required] + scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] // Whose scope the record belongs to: a user, a team, the organisation or an agent. + scope_identifier: ShortText [required] // The user, team, organisation or agent that the scope level names. Entity: UserPreference [kind: Record] [profile: CURRENT_STATE] - preference_id: Identifier - user_key: ShortText [required] - preference_category: Enum{REPORT_FORMAT|DATA_FILTER|AGGREGATION_LEVEL|VISUALIZATION_TYPE} - preference_name: ShortText [required] - preference_value: Text [optional] - preference_json: Json [optional] - confidence: Decimal(5,4) [optional] + preference_id: Identifier // Generated identifier of the row. + user_key: ShortText [required] // The user the preference belongs to. + preference_category: Enum{REPORT_FORMAT|DATA_FILTER|AGGREGATION_LEVEL|VISUALIZATION_TYPE} // What the preference is about: report format, data filter, aggregation level or visualisation type. + preference_name: ShortText [required] // Name of the preference. + preference_value: Text [optional] // The preference as text. + preference_json: Json [optional] // The preference as JSON, where it is structured. + confidence: Decimal(5,4) [optional] // How confident the agent is in the preference, from 0 to 1. is_active: Flag // lifecycle state: the preference is applied; cleared when the user withdraws it - scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] - scope_identifier: ShortText [required] + scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] // Whose scope the record belongs to: a user, a team, the organisation or an agent. + scope_identifier: ShortText [required] // The user, team, organisation or agent that the scope level names. Entity: DiscoveredPattern [kind: Record] [profile: CURRENT_STATE] - pattern_id: Identifier - pattern_name: ShortText [required] - pattern_type: Enum{CORRELATION|TEMPORAL|TABLE_RELATIONSHIP|ANOMALY} - pattern_definition: Json [optional] + pattern_id: Identifier // Generated identifier of the row. + pattern_name: ShortText [required] // Name of the pattern. + pattern_type: Enum{CORRELATION|TEMPORAL|TABLE_RELATIONSHIP|ANOMALY} // Kind of pattern: a correlation, a temporal pattern, a table relationship or an anomaly. + pattern_definition: Json [optional] // The pattern's definition, as JSON. sample_size: Integer [optional] // how many records analysed (summary, not the records) - confidence_score: Decimal(5,4) [optional] + confidence_score: Decimal(5,4) [optional] // How confident the agent is in the pattern, from 0 to 1. involved_tables: Text [optional] // logical entity names only, resolved through Semantic (INV-MEMORY-001) is_validated: Flag // the pattern has been confirmed - scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] - scope_identifier: ShortText [required] + scope_level: Enum{USER|TEAM|ORGANIZATION|AGENT} [required] // Whose scope the record belongs to: a user, a team, the organisation or an agent. + scope_identifier: ShortText [required] // The user, team, organisation or agent that the scope level names. ``` All runtime entities apply `temporal-lifecycle-metadata`, `object-placement` and `access-layer`, and require `RichMetadata`. @@ -180,14 +180,14 @@ Each has one identity, declared in its `Keys:` section. A record whose business ``` Entity: ModuleRegistry [kind: History] [profile: SCD2_HISTORY] [allocation: inline] - module_registry_id: Identifier - module_name: Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY} [required] + module_registry_id: Identifier // Generated identifier of the module entry. + module_name: Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY} [required] // The module the entry describes. container_name: ShortText [required] // where the module is deployed; filled by the build from the build context, never by the design - deployment_status: Enum{DEPLOYED|PLANNED|DEPRECATED} [required] - module_version: ShortText [required] - module_purpose: LongText [required] + deployment_status: Enum{DEPLOYED|PLANNED|DEPRECATED} [required] // Whether the module is deployed, planned or deprecated. + module_version: ShortText [required] // The deployed version of the module. + module_purpose: LongText [required] // What the module supplies. key_entities: Text [optional] // logical entity names - dependencies: Text [optional] + dependencies: Text [optional] // Other modules this module depends on. Keys: surrogate: module_registry_id @@ -196,14 +196,14 @@ Entity: ModuleRegistry [kind: History] [profile: SCD2_HISTORY] [alloc Entity: DesignDecision [kind: History] [profile: SCD2_HISTORY] [allocation: natural-key] decision_id: NaturalKey [required] // DD-{MODULE}-{NNN} decision_version: Integer [required] // revision number; increases with each superseding version - decision_title: ShortText [required] - context: LongText [optional] - alternatives: LongText [optional] - rationale: LongText [optional] - consequences: LongText [optional] - decision_status: Enum{PROPOSED|ACCEPTED|SUPERSEDED|DEPRECATED} [required] - decision_category: Enum{ARCHITECTURE|SCHEMA|NAMING|PERFORMANCE|SECURITY|INTEGRATION|OPERATIONAL} [required] - source_module: ShortText [required] + decision_title: ShortText [required] // Short title of the decision. + context: LongText [optional] // Why the decision was needed. + alternatives: LongText [optional] // The options that were considered. + rationale: LongText [optional] // Why the chosen option was chosen. + consequences: LongText [optional] // What follows from the decision. + decision_status: Enum{PROPOSED|ACCEPTED|SUPERSEDED|DEPRECATED} [required] // Where the decision stands: proposed, accepted, superseded or deprecated. + decision_category: Enum{ARCHITECTURE|SCHEMA|NAMING|PERFORMANCE|SECURITY|INTEGRATION|OPERATIONAL} [required] // The area the decision concerns. + source_module: ShortText [required] // Module the decision belongs to. superseded_by: Reference [optional] [-> DesignDecision] // the superseding decision's id Keys: @@ -211,43 +211,43 @@ Entity: DesignDecision [kind: History] [profile: SCD2_HISTORY] [alloc Entity: BusinessGlossary [kind: History] [profile: SCD2_HISTORY] [allocation: natural-key] term: NaturalKey [required] // the Glossary block's name - term_category: Enum{ENTITY|ATTRIBUTE|METRIC|BUSINESS_RULE|CLASSIFICATION|REFERENCE_CODE} [required] - definition: LongText [required] - source_module: ShortText [required] + term_category: Enum{ENTITY|ATTRIBUTE|METRIC|BUSINESS_RULE|CLASSIFICATION|REFERENCE_CODE} [required] // Kind of term: entity, attribute, metric, business rule, classification or reference code. + definition: LongText [required] // What the term means. + source_module: ShortText [required] // Module the term belongs to. Keys: natural: term Entity: QueryCookbook [kind: History] [profile: SCD2_HISTORY] [allocation: natural-key] recipe_id: NaturalKey [required] // QC-{MODULE}-{NNN} - recipe_title: ShortText [required] - use_case: ShortText [required] - target_module: Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY|CROSS} [required] + recipe_title: ShortText [required] // Title of the recipe. + use_case: ShortText [required] // The question the recipe answers. + target_module: Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY|CROSS} [required] // Module the recipe queries, or CROSS when it spans several. query_template: LongText [required] // parameterised query in the platform's dialect, generated by the binding; consumed by agents - complexity: Enum{SIMPLE|MODERATE|COMPLEX|ADVANCED} [required] + complexity: Enum{SIMPLE|MODERATE|COMPLEX|ADVANCED} [required] // How hard the recipe is: simple, moderate, complex or advanced. is_batch: Flag // set: batch only; clear: safe for interactive agent use - source_module: ShortText [required] + source_module: ShortText [required] // Module the recipe belongs to. Keys: natural: recipe_id Entity: ImplementationNote [kind: History] [profile: SCD2_HISTORY] [allocation: natural-key] note_id: NaturalKey [required] // IN-{MODULE}-{NNN} - note_title: ShortText [required] - note_content: LongText [required] - note_category: Enum{DEPLOYMENT|WORKAROUND|KNOWN_ISSUE|PERFORMANCE_TIP|OPERATIONAL|SECURITY} [required] - severity: Enum{LOW|MEDIUM|HIGH|CRITICAL} [optional] - source_module: ShortText [required] + note_title: ShortText [required] // Short title of the note. + note_content: LongText [required] // The note itself. + note_category: Enum{DEPLOYMENT|WORKAROUND|KNOWN_ISSUE|PERFORMANCE_TIP|OPERATIONAL|SECURITY} [required] // Kind of note: deployment, workaround, known issue, performance tip, operational or security. + severity: Enum{LOW|MEDIUM|HIGH|CRITICAL} [optional] // How serious the issue is, where the note describes one. + source_module: ShortText [required] // Module the note belongs to. Keys: natural: note_id Entity: ChangeLog [kind: History] [profile: SCD2_HISTORY] [allocation: natural-key] change_id: NaturalKey [required] // CL-{MODULE}-{NNN} - version_number: ShortText [required] - change_title: ShortText [required] - change_type: Enum{INITIAL_RELEASE|SCHEMA_CHANGE|FEATURE_ADDITION|BUG_FIX|PERFORMANCE|DEPRECATION} [required] - source_module: ShortText [required] + version_number: ShortText [required] // Version the change belongs to. + change_title: ShortText [required] // Short title of the change. + change_type: Enum{INITIAL_RELEASE|SCHEMA_CHANGE|FEATURE_ADDITION|BUG_FIX|PERFORMANCE|DEPRECATION} [required] // Kind of change: initial release, schema change, feature addition, bug fix, performance or deprecation. + source_module: ShortText [required] // Module the change belongs to. related_decision_id: Reference [optional] [-> DesignDecision] // the decision's id Keys: diff --git a/design/modules/observability.md b/design/modules/observability.md index fbb3a1f..48ab671 100644 --- a/design/modules/observability.md +++ b/design/modules/observability.md @@ -63,21 +63,21 @@ Every entity header states its temporal profile, because a `Record` has no defau ``` Entity: ChangeEvent [kind: Record] [profile: EVENT_APPEND_ONLY] - change_event_id: Identifier - container_name: ShortText [optional] + change_event_id: Identifier // Generated identifier of the row. + container_name: ShortText [optional] // Container that holds the changed table. table_name: ShortText [required] // TABLE-LEVEL, not individual records - change_type: Enum{INSERT|UPDATE|DELETE|MERGE|TRUNCATE} [required] - change_dts: Timestamp [required] - changed_by: ShortText [required] - change_source: Enum{ETL|API|MANUAL|AGENT} [optional] + change_type: Enum{INSERT|UPDATE|DELETE|MERGE|TRUNCATE} [required] // Kind of change: insert, update, delete, merge or truncate. + change_dts: Timestamp [required] // When the change happened. + changed_by: ShortText [required] // Who or what made the change. + change_source: Enum{ETL|API|MANUAL|AGENT} [optional] // Where the change came from: ETL, an API, a manual edit or an agent. records_affected: Integer [optional] // aggregate count, never the keys - columns_changed: Text [optional] - batch_key: ShortText [optional] + columns_changed: Text [optional] // The columns the change touched. + batch_key: ShortText [optional] // Identifier of the batch the change belongs to. Entity: DataQualityMetric [kind: Record] [profile: EVENT_APPEND_ONLY] - quality_metric_id: Identifier - container_name: ShortText [optional] - table_name: ShortText [required] + quality_metric_id: Identifier // Generated identifier of the row. + container_name: ShortText [optional] // Container that holds the measured table. + table_name: ShortText [required] // Table the measurement describes. column_name: ShortText [optional] // null for table-level metrics quality_dimension: Enum{COMPLETENESS|VALIDITY|CONSISTENCY|ACCURACY|TIMELINESS} [required] // the category the rule scores into check_function: Enum{COMPLETENESS|VALIDITY|CONSISTENCY|INTEGRITY|UNIQUENESS|FRESHNESS} [required] // the Quality block rule's check @@ -86,59 +86,59 @@ Entity: DataQualityMetric [kind: Record] [profile: EVENT_APPEND_ONLY] pass_threshold: Decimal(5,4) [optional] // a share rule's pass threshold warn_threshold: Decimal(5,4) [optional] // a share rule's warn threshold check_outcome: Enum{PASS|WARN|FAIL} [required] // per the Quality block's threshold mapping - measured_dts: Timestamp [required] - sample_size: Integer [optional] + measured_dts: Timestamp [required] // When the measurement was taken. + sample_size: Integer [optional] // Number of rows the measurement was taken over. Entity: DataLineage [kind: Record] [profile: CURRENT_STATE] // definitional; one row per flow - lineage_id: Identifier - source_container: ShortText [optional] - source_table: ShortText [optional] + lineage_id: Identifier // Generated identifier of the flow. + source_container: ShortText [optional] // Container of the source table, when there is one. + source_table: ShortText [optional] // The table the flow reads from, when it reads from one. source_column: ShortText [optional] // null for a table-level flow; set only for column-level lineage source_system: ShortText [optional] // external origin; null if internal - target_container: ShortText [optional] - target_table: ShortText [required] + target_container: ShortText [optional] // Container of the target table. + target_table: ShortText [required] // The table the flow writes to. target_column: ShortText [optional] // null for a table-level flow; set only for column-level lineage - job_name: ShortText [optional] - transformation_type: Enum{ETL|FEATURE_ENG|AGGREGATION|JOIN|EMBEDDING_GEN|FILTER|PIVOT} [optional] - transformation_logic: Text [optional] - openlineage_job_name: ShortText [optional] + job_name: ShortText [optional] // Job that performs the flow. + transformation_type: Enum{ETL|FEATURE_ENG|AGGREGATION|JOIN|EMBEDDING_GEN|FILTER|PIVOT} [optional] // Kind of transformation: ETL, feature engineering, aggregation, join, embedding generation, filter or pivot. + transformation_logic: Text [optional] // How the flow transforms the data. + openlineage_job_name: ShortText [optional] // The job's name in OpenLineage. openlineage_namespace: ShortText [optional] // organisation or deployment configuration, never the designer's is_active: Flag // the flow is live: set at registration, cleared when it retires and retired_dts is recorded - registered_dts: Timestamp [optional] - retired_dts: Timestamp [optional] + registered_dts: Timestamp [optional] // When the flow was registered. + retired_dts: Timestamp [optional] // When the flow was retired; null while it is in use. Entity: LineageRun [kind: Record] [profile: EVENT_APPEND_ONLY] // operational; one row per execution - lineage_run_id: Identifier - lineage_id: Reference [required] [-> DataLineage] - run_dts: Timestamp [required] - run_status: Enum{SUCCESS|FAILED|PARTIAL|RUNNING} [required] - run_duration_ms: Integer [optional] - records_read: Integer [optional] - records_written: Integer [optional] - records_rejected: Integer [optional] + lineage_run_id: Identifier // Generated identifier of the row. + lineage_id: Reference [required] [-> DataLineage] // The flow the run executed. + run_dts: Timestamp [required] // When the run happened. + run_status: Enum{SUCCESS|FAILED|PARTIAL|RUNNING} [required] // Outcome of the run: success, failed, partial or still running. + run_duration_ms: Integer [optional] // How long the run took, in milliseconds. + records_read: Integer [optional] // Number of records the run read. + records_written: Integer [optional] // Number of records the run wrote. + records_rejected: Integer [optional] // Number of records the run rejected. batch_key: ShortText [optional] // links to ChangeEvent.batch_key - openlineage_run_id: ShortText [optional] - error_message: Text [optional] + openlineage_run_id: ShortText [optional] // The run's identifier in OpenLineage. + error_message: Text [optional] // Why the run failed, when it did. Entity: ModelPerformance [kind: Record] [profile: EVENT_APPEND_ONLY] - performance_id: Identifier - model_key: ShortText [required] - model_version: ShortText [required] - metric_name: Enum{ACCURACY|PRECISION|RECALL|AUC|LATENCY_MS|DRIFT_SCORE} [required] - metric_value: Decimal(10,6) [optional] - evaluation_dts: Timestamp [required] - sample_size: Integer [optional] - is_sla_met: Flag + performance_id: Identifier // Generated identifier of the row. + model_key: ShortText [required] // The model that was measured. + model_version: ShortText [required] // Version of the model that was measured. + metric_name: Enum{ACCURACY|PRECISION|RECALL|AUC|LATENCY_MS|DRIFT_SCORE} [required] // What was measured: accuracy, precision, recall, AUC, latency or drift. + metric_value: Decimal(10,6) [optional] // The measured value. + evaluation_dts: Timestamp [required] // When the evaluation happened. + sample_size: Integer [optional] // Number of rows the evaluation used. + is_sla_met: Flag // True when the measurement meets the model's service level. Entity: AgentOutcome [kind: Record] [profile: EVENT_APPEND_ONLY] - outcome_id: Identifier - agent_key: ShortText [required] - session_key: ShortText [optional] - action_type: Enum{QUERY|RECOMMENDATION|DECISION|PREDICTION} [required] - action_dts: Timestamp [required] + outcome_id: Identifier // Generated identifier of the row. + agent_key: ShortText [required] // The agent that acted. + session_key: ShortText [optional] // The session the action belonged to, when it had one. + action_type: Enum{QUERY|RECOMMENDATION|DECISION|PREDICTION} [required] // Kind of action: a query, a recommendation, a decision or a prediction. + action_dts: Timestamp [required] // When the action happened. tables_accessed: Text [optional] // TABLE-LEVEL, comma-separated - outcome_status: Enum{SUCCESS|PARTIAL|FAILED} [required] - user_feedback: Enum{POSITIVE|NEUTRAL|NEGATIVE|CORRECTION} [optional] + outcome_status: Enum{SUCCESS|PARTIAL|FAILED} [required] // Whether the action succeeded, partly succeeded or failed. + user_feedback: Enum{POSITIVE|NEUTRAL|NEGATIVE|CORRECTION} [optional] // The user's reaction to the action, including a correction. records_processed: Integer [optional] // aggregate count ``` diff --git a/design/modules/prediction.md b/design/modules/prediction.md index 7c809c5..cbd420d 100644 --- a/design/modules/prediction.md +++ b/design/modules/prediction.md @@ -117,18 +117,18 @@ Windowed aggregates use `count_related`, `sum_related` and `days_since_last`, wh ``` Entity: FeatureValue [kind: History] [profile: SCD2_BITEMPORAL] [allocation: inline] // TALL format; standard-owned - feature_value_id: Identifier - entity_id: Reference [required] [-> | ] [discriminator: entity_kind] - entity_kind: Enum{|} [required] + feature_value_id: Identifier // Generated identifier of the row. + entity_id: Reference [required] [-> | ] [discriminator: entity_kind] // The entity the value describes; entity_kind says which. + entity_kind: Enum{|} [required] // Which entity entity_id points into. feature_name: ShortText [required] // the feature attribute's name feature_group: ShortText [required] // the feature group it is declared on - value_numeric: Decimal(18,4) [optional] - value_text: Text [optional] - value_json: Json [optional] - value_type: Enum{NUMERIC|TEXT|JSON|BOOLEAN} [required] - observation_dts: Timestamp [required] - is_current: Flag [current-flag] - feature_version: ShortText [required] + value_numeric: Decimal(18,4) [optional] // The value, when it is numeric. + value_text: Text [optional] // The value, when it is text. + value_json: Json [optional] // The value, when it is JSON. + value_type: Enum{NUMERIC|TEXT|JSON|BOOLEAN} [required] // Which of the value attributes holds the value. + observation_dts: Timestamp [required] // When the value was observed. + is_current: Flag [current-flag] // True for the version of the value in force. + feature_version: ShortText [required] // Version of the feature definition that produced the value. Keys: surrogate: feature_value_id @@ -139,18 +139,18 @@ Entity: FeatureValue [kind: History] [profile: SCD2_BITEMPORAL] [al ``` Entity: ModelPrediction [kind: History] [profile: SCD2_BITEMPORAL] [allocation: inline] - prediction_id: Identifier - entity_id: Reference [required] [-> | ] [discriminator: entity_kind] - entity_kind: Enum{|} [required] + prediction_id: Identifier // Generated identifier of the row. + entity_id: Reference [required] [-> | ] [discriminator: entity_kind] // The entity the prediction describes; entity_kind says which. + entity_kind: Enum{|} [required] // Which entity entity_id points into. model_key: ShortText [required] // the Model block's name - model_version: ShortText [required] + model_version: ShortText [required] // Version of the model that made the prediction. prediction_value: Decimal(10,6) [optional] // score / probability / continuous output prediction_class: ShortText [optional] // classification label prediction_json: Json [optional] // multi-class or structured output confidence_score: Decimal(5,4) [optional] // 0-1 - prediction_dts: Timestamp [required] + prediction_dts: Timestamp [required] // When the prediction was made. feature_observation_dts: Timestamp [optional] // links the prediction to its feature timestamp (reproducibility) - is_current: Flag [current-flag] + is_current: Flag [current-flag] // True for the prediction in force. Keys: surrogate: prediction_id diff --git a/design/modules/semantic.md b/design/modules/semantic.md index df27555..0d103da 100644 --- a/design/modules/semantic.md +++ b/design/modules/semantic.md @@ -59,27 +59,39 @@ Semantic's entities are the discovery catalogue. All apply `object-placement`, ` ``` Entity: EntityMetadata [kind: Record] [profile: CURRENT_STATE] - entity_metadata_id: Identifier - entity_name: ShortText [required] // business name (Party, Product) + entity_metadata_id: Identifier // Generated identifier of the entity's catalogue row. + entity_name: ShortText [required] // logical identity, module.entity (domain.ticket); never contains a container, so it survives re-placement logical_name: ShortText [required] // the name the specification or owning standard writes (Ticket, ValidationRun) entity_description: Text [required] // purpose and scope - entity_kind: Enum{HISTORY|REFERENCE|RELATIONSHIP|KEYMAP|RECORD} [required] - module_name: Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY} [required] + entity_kind: Enum{HISTORY|REFERENCE|RELATIONSHIP|KEYMAP|RECORD} [required] // The entity's kind as the specification declares it: history, reference, relationship, keymap or record. + module_name: Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY} [required] // Module that owns the entity. container_name: ShortText [optional] // where the table lives, from the build context table_name: ShortText [required] // physical name view_name: ShortText [optional] // physical name of the canonical consumable object - surrogate_key_column: ShortText [optional] - natural_key_column: ShortText [optional] temporal_pattern: Enum{CURRENT_STATE|EVENT_APPEND_ONLY|SCD2_HISTORY|SCD2_BITEMPORAL|ASSOCIATION_CURRENT|ASSOCIATION_SCD2|OPERATIONAL_LOG} [required] // the declared temporal profile current_flag_column: ShortText [optional] // physical name of the current flag deleted_flag_column: ShortText [optional] // physical name of the deleted flag assumed_zone: ShortText [optional] // IANA zone from the Product block; present only under zone-naive (TLM-18) industry_standard: ShortText [optional] // FIBO, HL7, CUSTOM, … - is_active: Flag + is_active: Flag // True while the entity is registered; false once it is retired. + +Entity: KeyDefinition [kind: Record] [profile: CURRENT_STATE] // one row per key of an entity + key_definition_id: Identifier // Generated identifier of the key row. + entity_id: Reference [required] [-> EntityMetadata] // the entity that owns the key + key_role: Enum{SURROGATE|NATURAL|ALTERNATE} [required] // the stable identity, the business key, or another unique key + key_name: ShortText [required] // the key within its entity: surrogate, natural, or alternate-1, alternate-2, ... + is_active: Flag // True while the row is in force; false once it is retired. + +Entity: KeyColumn [kind: Record] [profile: CURRENT_STATE] // the ordered columns of a key + key_column_id: Identifier // Generated identifier of the key-column row. + key_definition_id: Reference [required] [-> KeyDefinition] // the key the column belongs to + ordinal: Integer [required] // position in the key, from 1 with no gaps + column_name: ShortText [required] // physical name of a column of the owning entity + is_active: Flag // True while the row is in force; false once it is retired. Entity: ColumnMetadata [kind: Record] [profile: CURRENT_STATE] - column_metadata_id: Identifier - container_name: ShortText [required] + column_metadata_id: Identifier // Generated identifier of the column's catalogue row. + container_name: ShortText [required] // Container that holds the relation the column belongs to. table_name: ShortText [required] // physical name of the relation the column belongs to: a table or a view column_name: ShortText [required] // physical name logical_name: ShortText [required] // the name the specification or owning standard writes (valid_from_dts) @@ -87,14 +99,14 @@ Entity: ColumnMetadata [kind: Record] [profile: CURRENT_STATE] is_pii: Flag // set when sensitivity_marker is PII sensitivity_marker: Enum{PII|PII_INCIDENTAL} [optional] // the specification's marker, absent when it carries none data_classification: ShortText [optional] // the organisation's class, from the profile's Classification scheme - is_required: Flag + is_required: Flag // True when the column must hold a value. declared_type: ShortText [optional] // the declared data type, as text allowed_values: Json [optional] // permitted-value domain - is_active: Flag + is_active: Flag // True while the column is registered; false once it is retired. Entity: NamingStandard [kind: Record] [profile: CURRENT_STATE] - naming_standard_id: Identifier - standard_type: Enum{TEMPLATE|STANDARD_NAME|ABBREVIATION|SUFFIX|PREFIX|PATTERN} [required] + naming_standard_id: Identifier // Generated identifier of the naming rule. + standard_type: Enum{TEMPLATE|STANDARD_NAME|ABBREVIATION|SUFFIX|PREFIX|PATTERN} [required] // Kind of rule: a template, a standard-name mapping, an abbreviation, a suffix, a prefix or a pattern. applies_to: Enum{TABLE|BASE_VIEW|CONSUMER_VIEW|PROCEDURE|FUNCTION|SEQUENCE|ATTRIBUTE|GRAPH_KEY|ALL} [optional] // the object role or name a rule forms module_scope: Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY} [optional] // the module a TEMPLATE is narrowed to, when it is kind_scope: Enum{HISTORY|REFERENCE|RELATIONSHIP|KEYMAP|RECORD} [optional] // the entity kind a TEMPLATE is narrowed to, when it is @@ -102,23 +114,40 @@ Entity: NamingStandard [kind: Record] [profile: CURRENT_STATE] standard_value: ShortText [optional] // the element a SUFFIX, PREFIX or PATTERN names, the logical name a STANDARD_NAME maps, or the module an ABBREVIATION shortens mapped_value: ShortText [optional] // the physical name a STANDARD_NAME maps to, or the abbreviation meaning: Text [required] // what the rule means - is_active: Flag + is_active: Flag // True while the row is in force; false once it is retired. Entity: TableRelationship [kind: Record] [profile: CURRENT_STATE] - relationship_id: Identifier - relationship_name: ShortText [required] - source_container: ShortText [optional] - source_table: ShortText [required] - source_column: ShortText [required] // the referencing (foreign) key - target_container: ShortText [optional] - target_table: ShortText [required] - target_column: ShortText [required] // the referenced key - relationship_type: Enum{FOREIGN_KEY|HIERARCHY|ASSOCIATIVE} [required] - cardinality: Enum{ONE_TO_ONE|ONE_TO_MANY|MANY_TO_ONE|MANY_TO_MANY} [optional] - is_mandatory: Flag - is_active: Flag + relationship_id: Identifier // Generated identifier of the relationship, which is also the join group of its column pairs. + relationship_name: ShortText [required] // Name of the relationship: the referencing entity and attribute, for example Ticket.customer_id. + source_entity: ShortText [required] // EntityMetadata.entity_name of the referencing entity + target_entity: ShortText [required] // EntityMetadata.entity_name of the referenced entity + relationship_type: Enum{FOREIGN_KEY|HIERARCHY|ASSOCIATIVE} [required] // Kind of relationship: a foreign key, a hierarchy or an association. + cardinality: Enum{ONE_TO_ONE|ONE_TO_MANY|MANY_TO_ONE|MANY_TO_MANY} [optional] // How many rows on each side one join can match. + is_mandatory: Flag // True when every source row must resolve to a target row. + is_active: Flag // True while the row is in force; false once it is retired. + +Entity: RelationshipColumn [kind: Record] [profile: CURRENT_STATE] // the ordered column pairs of one relationship + relationship_column_id: Identifier // Generated identifier of the column-pair row. + relationship_id: Reference [required] [-> TableRelationship] // the join group the pair belongs to + ordinal: Integer [required] // position of the pair in the join, from 1 with no gaps + source_column: ShortText [required] // the referencing (foreign) key column + target_column: ShortText [required] // the referenced key column + is_active: Flag // True while the row is in force; false once it is retired. ``` +**Identities are logical and placement-independent (normative).** Every identity a consumer follows is built from the module anchor and a logical name, never from a container, product prefix or object-type affix, so deploying the same design with different placement changes no identity. The location is recorded beside it, in separate columns. + +| Kind | Form | Example | +|---|---|---| +| Entity | `module.entity` | `domain.ticket` | +| Entity surface | `module.entity:surface`, surface `base`, `governed`, `current` or `history` | `domain.ticket:current` | +| Module-owned object that is not an entity | `module:object` | `observability:trust_map` | +| Composite access object | `composite:name` | `composite:ticket_enriched` | + +`module` is the module anchor in lower case; `entity` and `object` are lower-case logical names in snake_case (`ValidationRun` becomes `validation_run`). Every relationship, key, mapping and composition names entities and objects by these identities (`INV-SEMANTIC-016`). + +**Ordered groups are rows (normative).** A key with several columns, a join with several column pairs and an access object with several output columns are each recorded as rows with an explicit ordinal, never as a list, array or delimited string. A key is the ordered rows of `KeyColumn`; a relationship is a join group whose pairs are the ordered rows of `RelationshipColumn`, so two pairs belonging to one join are grouped by construction. A single-column key or join is the one-row case. Every entity has a key: a `SURROGATE` key where one is allocated, otherwise its `NATURAL` key; a natural key is recorded when the entity has one, and lists its components in order where the identity is built from several attributes (`INV-SEMANTIC-017`, `INV-SEMANTIC-018`). + **Names are logical; the catalogue records both.** The specification and the standards name entities and attributes logically, and the organisation profile decides their physical names ([Organisation Profile Standard](../core/ORGANISATION_PROFILE.md) §6.3), including the physical name of any standard-owned logical name it maps. The build registers every relation and column under both: `logical_name` holds the name the specification or owning standard writes, and `table_name`, `view_name` and `column_name` hold the physical names. Canonical temporal and lifecycle columns and standard-owned relations are registered with their logical names, so a conformance rule written against a canonical name, such as the required columns of TLM-02, is judged by resolving that name through this metadata after the organisation's mapping, never by assuming the physical name. **Naming rules are the organisation's.** `NamingStandard` is populated by the build from the organisation profile's `Naming:` block, never supplied by a designer. It records how the organisation forms names, its templates per object role, its standard-name mappings and its abbreviations, so a reader can understand a name. No agent derives a name from it: objects are read from the registry verbatim (`INV-SEMANTIC-003`). @@ -133,7 +162,7 @@ Metrics are declared at model level rather than on an entity, because a metric m ``` Entity: Metric [kind: Record] [profile: CURRENT_STATE] - metric_id: Identifier + metric_id: Identifier // Generated identifier of the metric. metric_name: ShortText [required] [unique] // business name (Total Sales, Churn Rate): the Metric block's name metric_description: Text [required] // what it measures and what it excludes: the block's Description metric_definition: Text [required] // the block's Measure, Filter and Joins, in the specification's expression vocabulary @@ -142,22 +171,22 @@ Entity: Metric [kind: Record] [profile: CURRENT_STATE] unit: ShortText [optional] // currency code, percent, count, days: the block's Unit grain_description: Text [optional] // the level the metric is defined at: the block's Grain is_additive: Flag // may be summed across every dimension: the block's Additive - is_active: Flag + is_active: Flag // True while the metric is registered; false once it is retired. Entity: MetricExpression [kind: Record] [profile: CURRENT_STATE] - metric_expression_id: Identifier - metric_name: Reference [required] [-> Metric] + metric_expression_id: Identifier // Generated identifier of the expression row. + metric_name: Reference [required] [-> Metric] // The metric the expression computes. sql_dialect: Code [required] // the dialect this expression is written in expression_text: Text [required] // the calculation, rendered by the binding - is_active: Flag + is_active: Flag // True while the row is in force; false once it is retired. Entity: MetricDataset [kind: Record] [profile: CURRENT_STATE] - metric_dataset_id: Identifier - metric_name: Reference [required] [-> Metric] - container_name: ShortText [optional] + metric_dataset_id: Identifier // Generated identifier of the dataset row. + metric_name: Reference [required] [-> Metric] // The metric that reads the dataset. + container_name: ShortText [optional] // Container that holds the dataset the metric reads. table_name: ShortText [required] // an entity the expression reads: the block's Dataset or one of its Joins dataset_role: Enum{PRIMARY|JOINED} [required] // PRIMARY for the Dataset, JOINED for each of the Joins - is_active: Flag + is_active: Flag // True while the row is in force; false once it is retired. ``` **A metric is defined once, logically, and expressed once per dialect.** The design specification states each metric in a `Metric:` block ([Design Specification Standard](../core/DESIGN_SPECIFICATION.md) §4.4), in the platform-neutral expression vocabulary of its §5, over declared entities and attributes: what is measured, how it aggregates, what it filters, and how component metrics combine. It never states SQL. Each platform binding renders that definition into its own dialect at build time and registers the result as a `MetricExpression`, and may register further dialects for consuming tools that push down their own SQL. The product's platform, a consuming tool and a catalogue that only displays the metric then share one definition rather than three that drift, and the same design builds on any platform. @@ -168,11 +197,11 @@ Entity: MetricDataset [kind: Record] [profile: CURRENT_STATE] ``` Entity: SemanticSynonym [kind: Record] [profile: CURRENT_STATE] - synonym_id: Identifier - synonym_object: Reference [required] [-> EntityMetadata | ColumnMetadata | Metric] [discriminator: object_kind] + synonym_id: Identifier // Generated identifier of the synonym. + synonym_object: Reference [required] [-> EntityMetadata | ColumnMetadata | Metric] [discriminator: object_kind] // The entity, column or metric the synonym resolves to; object_kind says which. object_kind: Enum{ENTITYMETADATA|COLUMNMETADATA|METRIC} [required] // which catalogue the synonym_object points into synonym_text: ShortText [required] // the alternative term - is_active: Flag + is_active: Flag // True while the row is in force; false once it is retired. ``` The terms a business uses are not the terms a schema uses, and the gap is where natural-language questions fail. A synonym set is separate from the glossary Memory holds: a glossary term is a definition a person reads, a synonym is an alias a resolver matches on. Synonyms come from the specification: an entity's `Synonyms:` section, an attribute's `[synonyms:]` qualifier and a metric's `Synonyms` field. Each points at the catalogue row it aliases, so a synonym resolves through metadata rather than by matching a name. @@ -191,39 +220,48 @@ Entity: DataProductRegistry [kind: Record] [profile: CURRENT_STATE] // pr technical_contact: ShortText [optional] // Product block: Technical contact trust_authoritative_producer: ShortText [required] // Product block: Trust producer, read by the validation pattern manifest: Json [optional] // machine-readable orientation manifest - contract_uri: ShortText [optional] - policy_uri: ShortText [optional] - quality_uri: ShortText [optional] - lineage_uri: ShortText [optional] + contract_uri: ShortText [optional] // Where the product's contract is published, when it has one. + policy_uri: ShortText [optional] // Where the product's access policy is published, when it has one. + quality_uri: ShortText [optional] // Where the product's quality evidence is published, when it has any. + lineage_uri: ShortText [optional] // Where the product's lineage is published, when it has any. approved_entrypoint: ShortText [required] // Orientation block: Entrypoint, resolved by the build to the surface it names approved_access_mode: Enum{VIEW|MCP_TOOL|SEMANTIC_QUERY} [required] // Orientation block: Access mode - is_active: Flag - is_deleted: Flag [deleted-flag] - deleted_dts: Timestamp [optional] + is_active: Flag // True while the product is registered; false once it is retired. + is_deleted: Flag [deleted-flag] // True once the registration has been logically deleted; the row is retained. + deleted_dts: Timestamp [optional] // When the registration was logically deleted; null before deletion. Entity: DataProductMap [kind: Record] [profile: CURRENT_STATE] // module registry: one row per module and object type - module_id: Identifier - module_name: Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY} [required] + module_id: Identifier // Generated identifier of the module and object-type row. + module_name: Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY} [required] // Module the row describes. object_type: Enum{TABLE|BASE_VIEW|CONSUMER_VIEW|PROCEDURE|FUNCTION|SEQUENCE} [required] // the organisation profile's object role - module_purpose: Text [optional] + module_purpose: Text [optional] // What the module supplies in this product. container_name: ShortText [required] // where this module's objects of this type are placed, from the build context - module_version: ShortText [optional] - deployment_status: Enum{DEPLOYED|PLANNED|DEPRECATED} [required] + module_version: ShortText [optional] // Version of the module as deployed. + deployment_status: Enum{DEPLOYED|PLANNED|DEPRECATED} [required] // Whether the module is deployed, planned or deprecated. graph_key: ShortText [optional] // OBSERVABILITY rows only, when the graph-lineage facet is enabled; from the build context - is_active: Flag + is_active: Flag // True while the row is in force; false once it is retired. + +Entity: StandardDeclaration [kind: Record] [profile: CURRENT_STATE] // one row per implemented component + standard_declaration_id: Identifier // Generated identifier of the declaration row. + product_id: NaturalKey [required] // the product making the declaration (DataProductRegistry.product_id) + component_kind: Enum{STANDARD|MODULE|PATTERN} [required] // the standard as a whole, a deployed module, or an applied pattern + component_name: ShortText [required] // standard, a module anchor or a pattern anchor + platform: ShortText [required] // the platform binding that implements the component + standard_version: ShortText [required] // the release of the standard the component conforms to + is_active: Flag // True while the row is in force; false once it is retired. Entity: PrimaryObject [kind: Record] [profile: CURRENT_STATE] // one row per agent-facing object - primary_object_id: Identifier + primary_object_id: Identifier // Generated identifier of the primary-object row. module_id: Reference [required] [-> DataProductMap] // the module and object-type row the object belongs to container_name: ShortText [required] // the object's exact deployed container object_name: ShortText [required] // exact deployed name; used verbatim, never derived object_type: Enum{TABLE|BASE_VIEW|CONSUMER_VIEW|PROCEDURE|FUNCTION} [required] // the organisation profile's object role object_role: Enum{AGENT_ENTRYPOINT|ANALYTICAL_QUERY|REFERENCE_LOOKUP|RELATIONSHIP_BRIDGE|LINEAGE_EVIDENCE|OPERATIONAL_METRIC|WRITE_TARGET|INTERNAL_SUPPORT} [required] // derived by the build - usage_guidance: Text [optional] - is_active: Flag + usage_guidance: Text [optional] // How an agent should use the object. + is_active: Flag // True while the row is in force; false once it is retired. Entity: DataProductOrientation [kind: Record] [profile: CURRENT_STATE] // one ordered row per product resource - orientation_id: Identifier + orientation_id: Identifier // Generated identifier of the resource row. product_id: NaturalKey [required] // the product this resource belongs to (DataProductRegistry.product_id) resource_role: Enum{MANIFEST|TRUST_MAP|MODULE_MAP|OBJECT_CATALOGUE|ENTITY_CATALOGUE|COLUMN_CATALOGUE|RELATIONSHIP_CATALOGUE|RELATIONSHIP_PATHS|LINEAGE|QUERY_COOKBOOK|GLOSSARY|DESIGN_DECISIONS|POLICY|QUALITY} [required] // open vocabulary; one role per row, never a list; TRUST_GATE is the legacy spelling of TRUST_MAP container_name: ShortText [optional] // deployed container, for object-backed resources @@ -232,9 +270,11 @@ Entity: DataProductOrientation [kind: Record] [profile: CURRENT_STATE] // on usage_guidance: Text [optional] // how a consumer should use this resource is_required: Flag // a missing required resource is a conformance failure discovery_order: Integer [required] // ascending processing order; the trust map precedes every analytical resource - is_active: Flag + is_active: Flag // True while the row is in force; false once it is retired. ``` +**The deployment declares the standard it implements (normative).** `StandardDeclaration` records the release of the standard the product implements: one `STANDARD` row for the whole, one `MODULE` row per deployed module and one `PATTERN` row per applied pattern, each with the platform that implements it. The generated manifest repeats the `STANDARD` row's version as its last column, so the first read an agent makes says which contract it is reading. A declaration is an assertion by the build, not evidence: the trust map is the evidence. The release bundle published with each release states what an agent may rely on at that version. + **Where registry facts come from.** The registry's descriptive facts come from the specification's `Product:` block ([Design Specification Standard](../core/DESIGN_SPECIFICATION.md) §4.12): description, domain, owner, technical contact and trust producer. Its approved entrypoint and access mode come from the `Orientation:` block, which carries nothing else. Version and lifecycle status are the build's, never the design's. **Containers are recorded per module and object type.** The organisation profile may route the object roles of one module to different containers, so a module may span several. `DataProductMap` holds one row per module and object type, and the build records each container from the build context. A consumer reads where an object lives from these rows or from the object's own registry row, never from a container name. @@ -260,7 +300,8 @@ The catalogue above models entities and how they relate; it does not model the * ``` Entity: AccessObject [kind: Record] [profile: CURRENT_STATE] // one row per consumable object - access_object_id: Identifier + access_object_id: Identifier // Generated identifier of the access-object row. + object_identity: ShortText [required] // logical identity (module.entity:surface, module:object, composite:name); never contains a container container_name: ShortText [required] // where the object is deployed object_name: ShortText [required] // the queryable object object_type: Enum{TABLE|BASE_VIEW|CONSUMER_VIEW|PROCEDURE|FUNCTION} [required] // the organisation profile's object role @@ -271,19 +312,31 @@ Entity: AccessObject [kind: Record] [profile: CURRENT_STATE] // on is_primary: Flag // the canonical consumable object for its entity; at most one per entity resolves_to_object: ShortText [optional] // for 1:1 passthroughs, the object it maps straight through to access_note: Text [optional] // governance, filtering, or usage guidance - is_active: Flag + is_active: Flag // True while the row is in force; false once it is retired. Entity: AccessComposition [kind: Record] [profile: CURRENT_STATE] // one row per member of a COMPOSITE object - access_composition_id: Identifier - composite_container: ShortText [required] + access_composition_id: Identifier // Generated identifier of the composition member row. + composite_container: ShortText [required] // Container that holds the composite object. composite_object: ShortText [required] // the composite being described member_seq: Integer [required] // ordering of the member within the composite member_entity: ShortText [required] // entity the member represents (EntityMetadata.entity_name) member_role: Enum{ANCHOR|INNER|LEFT|RIGHT|FULL} [required] // join role within the composite - join_path: Text [optional] // entity-level join condition, same form as the path-discovery joins + via_relationship_id: Reference [optional] [-> TableRelationship] // the relationship that joins this member to a prior member; null for the anchor + via_direction: Enum{FORWARD|REVERSE} [optional] // FORWARD crosses the relationship from source to target; null for the anchor is_grain_contributor: Flag // whether this member changes the composite's grain - member_note: Text [optional] - is_active: Flag + member_note: Text [optional] // Anything a consumer should know about this member's place in the composite. + is_active: Flag // True while the row is in force; false once it is retired. + +Entity: AccessColumn [kind: Record] [profile: CURRENT_STATE] // one row per output column of an access object + access_column_id: Identifier // Generated identifier of the mapping row. + access_object_id: Reference [required] [-> AccessObject] // the access object the column belongs to + ordinal: Integer [required] // position of the output column, from 1 with no gaps + column_name: ShortText [required] // the output column as a consumer sees it + source_entity: ShortText [optional] // logical identity of the entity the column derives from; null when derived from several + source_column: ShortText [optional] // the source column; null when derived + mapping_kind: Enum{PASSTHROUGH|RENAMED|DERIVED} [required] // unchanged, renamed only, or computed + derivation_note: Text [optional] // how a derived column is computed + is_active: Flag // True while the row is in force; false once it is retired. ``` **Consumption contract (normative).** A consumer selecting data resolves through `AccessObject` — it chooses objects marked agent-consumable and reads `represents_entity` — rather than querying base tables directly. A `COMPOSITE` object is presented as a single unit; its internal structure is read from `AccessComposition`, never by parsing a definition or recomputing column lineage at consumption time. Emitted joins target consumable objects (access-resolved, §5), not base tables. **Object names are not a contract:** no consumer infers an object's role, layer, entity, or purpose from its name; the registry is the single source of truth (`INV-SEMANTIC-008`). @@ -319,7 +372,9 @@ This decision is bounded by the product. Discovery *across* products depends on ## 5. Multi-Hop Path Discovery -`TableRelationship` is the machine-readable entity-relationship model. From it, a **path-discovery surface** lets an agent find how to join any two entities, directly or through intermediate entities, in either direction, up to **four hops**, and returns the join conditions to use. A path never revisits an entity. The bound is the standard's, so every platform's surface returns the same paths for the same relationships. This is the single most important discovery capability: an agent cannot traverse a path it has no record of. +`TableRelationship` is the machine-readable entity-relationship model. From it, a **path-discovery surface** lets an agent find how to join any two entities, directly or through intermediate entities, in either direction, up to **four hops**, and returns the join as rows to assemble: one row per step and key position, never a concatenated condition. A path never revisits an entity. The bound is the standard's, so every platform's surface returns the same paths for the same relationships. This is the single most important discovery capability: an agent cannot traverse a path it has no record of. + +**The path surface is rows (normative).** Each row carries the path (an id built from the ordered relationship legs, each marked with its direction), the origin and destination entities, the number of steps, the step and the key position within it, the from and to entity and column of that pair, and the relationship and direction crossed. A path is bounded to four steps and visits no entity twice. A consumer assembles the join from the rows of one path, grouped by step and ordered by key position; it never parses a stored join string, and no such string is part of the contract. **Completeness requirement (`INV-SEMANTIC-005`).** `TableRelationship` must register **every** relationship an agent is expected to traverse: not only those with physical foreign keys: @@ -417,6 +472,11 @@ Semantic never becomes a dependency of the modules it describes: it observes and - `INV-SEMANTIC-013`: every registered metric carries a platform-neutral definition and at least one expression derived from it, each in a declared dialect, and no two expressions of one metric declare the same dialect. - `INV-SEMANTIC-014`: every dataset a metric names is an entity registered in the same product, and every metric names exactly one dataset in the primary role. - `INV-SEMANTIC-015`: every synonym resolves to a registered entity, column or metric; a synonym is unique within the object it resolves to. +- `INV-SEMANTIC-016`: every entity and access identity follows the identity grammar and contains no container; entity identities are unique. +- `INV-SEMANTIC-017`: every entity has a key, `SURROGATE` where one is allocated and otherwise `NATURAL`; the columns of every key are ordered 1 to n without gaps and exist on the entity. +- `INV-SEMANTIC-018`: every relationship has at least one column pair; pairs are ordered 1 to n without gaps and exist on their entities. +- `INV-SEMANTIC-019`: every consumer view over a registered entity has exactly one `AccessColumn` row per output column, and every non-derived source exists. +- `INV-SEMANTIC-020`: a composite's members are ordered 1 to n, and each non-anchor member joins to a prior member through a registered relationship in a stated direction. --- diff --git a/design/patterns/catalogue-interface.md b/design/patterns/catalogue-interface.md index 77f484d..d39fcf3 100644 --- a/design/patterns/catalogue-interface.md +++ b/design/patterns/catalogue-interface.md @@ -121,7 +121,7 @@ The state vocabulary is the organisation's. The four states above are advocated; ``` Entity: DataProductInterface [kind: History] - product_id: Reference [required] [-> DataProductRegistration] + product_id: Reference [required] [-> DataProductRegistration] // The registered product the row belongs to. interface_name: ShortText [required] // the object as a consumer names it container_name: ShortText [required] // exact deployed container; used verbatim object_name: ShortText [required] // exact deployed object; never derived @@ -132,7 +132,7 @@ Entity: DataProductInterface [kind: History] interface_purpose: Text [optional] // what this surface is for is_primary: Flag [required] // the recommended surface for its entity is_consumer_facing: Flag [required] // a consumer may query it - is_active: Flag [required] + is_active: Flag [required] // True while the row is in force. Keys: natural: product_id, container_name, object_name @@ -157,11 +157,11 @@ The interface feed is a projection. The authority for which objects a consumer m ``` Entity: DataProductContainer [kind: History] - product_id: Reference [required] [-> DataProductRegistration] + product_id: Reference [required] [-> DataProductRegistration] // The registered product the row belongs to. module_name: Code [required] // projected: DataProductMap; DOMAIN, SEMANTIC, MEMORY, OBSERVABILITY, SEARCH, PREDICTION, or organisation-defined layer_code: Code [required] // the profile's layer code for the object roles the container holds container_name: ShortText [required] // exact deployed container - is_active: Flag [required] + is_active: Flag [required] // True while the row is in force. Keys: natural: product_id, module_name, layer_code diff --git a/design/patterns/validation.md b/design/patterns/validation.md index fd6b991..44a5191 100644 --- a/design/patterns/validation.md +++ b/design/patterns/validation.md @@ -144,9 +144,9 @@ The trust-map entity is **`ValidationArea`**, and that entity name is its logica ``` Entity: ValidationArea [kind: Record] [profile: EVENT_APPEND_ONLY] - product_code: ShortText [required] // the run's product_code - producer_id: ShortText [required] // the run's producer - run_id: ShortText [required] // the run this entry belongs to + product_code: ShortText [required] [-> ValidationRun.product_code] [join: run] // the run's product_code + producer_id: ShortText [required] [-> ValidationRun.producer_id] [join: run] // the run's producer + run_id: ShortText [required] [-> ValidationRun.run_id] [join: run] // the run this entry belongs to scope_kind: Enum{MODULE|ENTITY|PATTERN|CAPABILITY|PRODUCT} [required] // the kind of area described scope_id: ShortText [required] // the area within its kind checks_expected: Integer [required] // checks the profile defines for this area; 0 means none diff --git a/examples/it-service-desk-data-product/design-output/design_specification.md b/examples/it-service-desk-data-product/design-output/design_specification.md index 5e1e76e..d5fd76e 100644 --- a/examples/it-service-desk-data-product/design-output/design_specification.md +++ b/examples/it-service-desk-data-product/design-output/design_specification.md @@ -467,7 +467,7 @@ Decision: DD-SEM-002 **Invariants:** `INV-SEMANTIC-001`, `INV-SEMANTIC-002`, `INV-SEMANTIC-003`, `INV-SEMANTIC-004`, `INV-SEMANTIC-005`, `INV-SEMANTIC-006`, `INV-SEMANTIC-007`, `INV-SEMANTIC-008`, `INV-SEMANTIC-009`, `INV-SEMANTIC-010`, `INV-SEMANTIC-011`, -`INV-SEMANTIC-012`, `INV-SEMANTIC-013`, `INV-SEMANTIC-014`, `INV-SEMANTIC-015`. +`INV-SEMANTIC-012`, `INV-SEMANTIC-013`, `INV-SEMANTIC-014`, `INV-SEMANTIC-015`, `INV-SEMANTIC-016`, `INV-SEMANTIC-017`, `INV-SEMANTIC-018`, `INV-SEMANTIC-019`, `INV-SEMANTIC-020`. --- diff --git a/implementation/duckdb/CONFORMANCE.md b/implementation/duckdb/CONFORMANCE.md index 9c9231d..380bee5 100644 --- a/implementation/duckdb/CONFORMANCE.md +++ b/implementation/duckdb/CONFORMANCE.md @@ -8,7 +8,7 @@ Engine evidence: DuckDB v1.5.6 through the duckdb 1.5.6 Python driver. | Outcome | Count | |---|---| -| Passed | 36 | +| Passed | 48 | | Failed | 0 | | Skipped | 0 | | Gaps reported by the builds | 14 | @@ -59,6 +59,7 @@ What no check can detect, declared in the `Gaps` field of [PLATFORM_PROFILE.md]( | Test | Outcome | |---|---| +| A composite join the standard declares is one ordered group | pass | | A fixed size vector rejects the wrong dimension | pass | | A legacy run is graded by its counts and capped | pass | | A product file reopens read only | pass | @@ -68,7 +69,10 @@ What no check can detect, declared in the `Gaps` field of [PLATFORM_PROFILE.md]( | Bitemporal maintenance keeps known history | pass | | Designated producer then cautious fallback | pass | | Errors and failures are never successes | pass | +| Every consumer view column has a mapping row | pass | +| Every relationship pairs a source column with a target column | pass | | Evidence publishes and the trust map reads it | pass | +| Identities are logical and a composite key is ordered rows | pass | | ITSD deploys and every check passes | pass | | ITSD source rows load and stay conformant | pass | | Key allocation is permanent and idempotent | pass | @@ -78,7 +82,15 @@ What no check can detect, declared in the `Gaps` field of [PLATFORM_PROFILE.md]( | No evidence is unknown not sound | pass | | Relationship paths follow the standard bound | pass | | Similarity ranks current vectors of its embedding | pass | +| The access columns check detects an unmapped output column | pass | +| The comment coverage check detects an uncommented view column | pass | +| The composition check detects a composite with no anchor | pass | +| The deployment declares the standard it implements | pass | | The engine reports its version | pass | +| The identity check detects a placement bound identity | pass | +| The keys check detects an entity with no key | pass | +| The reference check tests every pair of a composite join | pass | +| The relationships check detects a relationship with no columns | pass | ### Rendering and portability diff --git a/implementation/duckdb/PLATFORM_PROFILE.md b/implementation/duckdb/PLATFORM_PROFILE.md index b2489f3..ad33d2e 100644 --- a/implementation/duckdb/PLATFORM_PROFILE.md +++ b/implementation/duckdb/PLATFORM_PROFILE.md @@ -133,6 +133,12 @@ Binding: - relationship reference: patterns/validation/03-relationship-reference.sql.j2 module semantic: modules/semantic/validation.sql.j2 module memory: modules/memory/validation.sql.j2 + contract identity: modules/semantic/contract/identity.sql.j2 + contract keys: modules/semantic/contract/keys.sql.j2 + contract relationships: modules/semantic/contract/relationships.sql.j2 + contract access-columns: modules/semantic/contract/access-columns.sql.j2 + contract composition: modules/semantic/contract/composition.sql.j2 + product comment-coverage: patterns/validation/05-comment-coverage.sql.j2 ``` ``` diff --git a/implementation/duckdb/modules/search/02-similarity.sql.j2 b/implementation/duckdb/modules/search/02-similarity.sql.j2 index 2d7ebc4..9195158 100644 --- a/implementation/duckdb/modules/search/02-similarity.sql.j2 +++ b/implementation/duckdb/modules/search/02-similarity.sql.j2 @@ -30,4 +30,5 @@ WHERE ranked.similarity >= {{ b.threshold }} {%- endif %} ORDER BY ranked.similarity DESC NULLS LAST, ranked.entity_id LIMIT greatest(0, least(result_limit, 1000)); +COMMENT ON MACRO TABLE {{ b.objects.function|qualified }} IS {{ ('Similarity search over the ' ~ b.name ~ ' embedding of ' ~ b.entity ~ '. Takes a query vector and a result limit (maximum 1000).')|literal }}; {% endfor %} diff --git a/implementation/duckdb/modules/semantic/02-registration.sql.j2 b/implementation/duckdb/modules/semantic/02-registration.sql.j2 index c165dce..380719a 100644 --- a/implementation/duckdb/modules/semantic/02-registration.sql.j2 +++ b/implementation/duckdb/modules/semantic/02-registration.sql.j2 @@ -2,6 +2,12 @@ relationship catalogues, access objects, primary objects, the orientation sequence, metrics, synonyms and naming standards. Every row is read from the build context. #} {% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} +{# The generated id of one key of an entity, found by the entity's identity and the key's name. #} +{% macro key_ref(keydef, meta, x, k) -%} +(SELECT d.{{ keydef.key_columns[0]|ident }} FROM {{ keydef.objects.table|qualified }} d + WHERE d.{{ keydef.column_names.key_name|ident }} = {{ k.name|literal }} + AND d.{{ keydef.column_names.entity_id|ident }} = {{ m.ref(meta, {'entity_name': x.logical_id}) }}) +{%- endmacro %} {% set registry = named('DataProductRegistry') %}{% set manifest = named('DataProductManifest') %} {% if registry and manifest %} {{ m.insert(registry, { @@ -21,21 +27,37 @@ 'deployment_status': 'DEPLOYED'}) }} {% endfor %} {% endif %} +{% set declaration = named('StandardDeclaration') %} +{% if declaration %} +{% for d in standard.declarations %} +{{ m.insert(declaration, {'product_id': product.code, 'component_kind': d.kind, 'component_name': d.name, + 'platform': standard.platform, 'standard_version': standard.version}) }} +{% endfor %} +{% endif %} {% set meta = named('EntityMetadata') %}{% set colmeta = named('ColumnMetadata') %} {% for x in entities + relations if x.objects.table is defined %} {% set t = x.objects.table %} {% if meta %} {{ m.insert(meta, { - 'entity_name': x.name, 'logical_name': x.name, 'entity_description': x.description, + 'entity_name': x.logical_id, 'logical_name': x.name, 'entity_description': x.description, 'entity_kind': x.kind|upper, 'module_name': x.module|upper, 'container_name': t.container, 'table_name': t.name, 'view_name': (x.objects.consumer_view.name if x.objects.consumer_view is defined else none), - 'surrogate_key_column': (x.surrogate_columns|join(', ') or none), - 'natural_key_column': (x.natural_columns|join(', ') or none), 'temporal_pattern': x.profile, 'current_flag_column': (x.column_names.is_current if x.history else none), 'deleted_flag_column': (x.column_names.is_deleted if x.supports_deletion else none)}) }} {% endif %} +{% set keydef = named('KeyDefinition') %}{% set keycol = named('KeyColumn') %} +{% if keydef and keycol and meta %} +{% for k in x.key_defs %} +{{ m.insert(keydef, {'key_role': k.role, 'key_name': k.name}, + {'entity_id': m.ref(meta, {'entity_name': x.logical_id})}) }} +{% for column in k.columns %} +{{ m.insert(keycol, {'ordinal': loop.index, 'column_name': column}, + {'key_definition_id': key_ref(keydef, meta, x, k)}) }} +{% endfor %} +{% endfor %} +{% endif %} {% if colmeta %} {% for col in x.columns %} {{ m.insert(colmeta, { @@ -50,35 +72,47 @@ {% endfor %} {% set rel = named('TableRelationship') %} {% if rel %} +{% set relcol = named('RelationshipColumn') %} {% for r in relationships if r.source_table and r.target_table %} {{ m.insert(rel, { - 'relationship_name': r.source ~ '.' ~ r.source_attribute, - 'source_container': r.source_table.container, 'source_table': r.source_table.name, - 'source_column': r.source_column, 'target_container': r.target_table.container, - 'target_table': r.target_table.name, 'target_column': r.target_columns|join(', '), + 'relationship_name': r.name, 'source_entity': r.source_identity, 'target_entity': r.target_identity, 'relationship_type': 'FOREIGN_KEY', 'cardinality': 'MANY_TO_ONE', 'is_mandatory': r.mandatory}) }} +{% if relcol %} +{% for pair in r.pairs %} +{{ m.insert(relcol, {'ordinal': pair.ordinal, 'source_column': pair.source_column, 'target_column': pair.target_column}, + {'relationship_id': m.ref(rel, {'relationship_name': r.name})}) }} {% endfor %} {% endif %} -{% set access = named('AccessObject') %} +{% endfor %} +{% endif %} +{% set access = named('AccessObject') %}{% set acccol = named('AccessColumn') %} +{% set surface = {'table': 'base', 'base_view': 'governed', 'consumer_view': 'current', 'function': 'history', 'procedure': 'history'} %} {% if access %} {% for x in entities + relations %} {% for role, obj in x.objects|dictsort if obj and m.member(access, 'object_type', role|upper) %} {{ m.insert(access, { - 'container_name': obj.container, 'object_name': obj.name, 'object_type': role|upper, + 'object_identity': x.logical_id ~ ':' ~ surface[role], 'container_name': obj.container, 'object_name': obj.name, + 'object_type': role|upper, 'access_role': ('BASE' if role in ('table', 'base_view') else 'PASSTHROUGH'), - 'represents_entity': x.name, + 'represents_entity': x.logical_id, 'is_agent_consumable': role in ('consumer_view', 'function'), 'is_primary': role == 'consumer_view', - 'resolves_to_object': (x.objects.table.container ~ '.' ~ x.objects.table.name - if role != 'table' and x.objects.table is defined else none), + 'resolves_to_object': (x.logical_id ~ ':base' if role != 'table' and x.objects.table is defined else none), 'access_note': x.description}) }} +{% if role == 'consumer_view' and acccol %} +{% for vc in x.view_columns %} +{{ m.insert(acccol, {'ordinal': vc.ordinal, 'column_name': vc.physical_name, 'source_entity': x.logical_id, + 'source_column': vc.physical_name, 'mapping_kind': 'PASSTHROUGH'}, + {'access_object_id': m.ref(access, {'object_identity': x.logical_id ~ ':current'})}) }} +{% endfor %} +{% endif %} {% endfor %} {% endfor %} {% for b in embeddings %} {% for role, obj in b.objects|dictsort if role in ('function', 'procedure') %} {{ m.insert(access, { - 'container_name': obj.container, 'object_name': obj.name, - 'object_type': role|upper, 'access_role': 'PASSTHROUGH', 'represents_entity': b.entity, + 'object_identity': b.logical_id, 'container_name': obj.container, 'object_name': obj.name, + 'object_type': role|upper, 'access_role': 'PASSTHROUGH', 'represents_entity': none, 'is_agent_consumable': true, 'is_primary': false, 'access_note': 'Similarity search over the ' ~ b.name ~ ' embedding of ' ~ b.entity ~ '.'}) }} {% endfor %} @@ -127,7 +161,7 @@ {% for x in entities if x.objects.table is defined %} {% for s in x.synonyms|default([]) %} {{ m.insert(syn, {'object_kind': 'ENTITYMETADATA', 'synonym_text': s}, - {'synonym_object': m.ref(meta, {'entity_name': x.name})}) }} + {'synonym_object': m.ref(meta, {'entity_name': x.logical_id})}) }} {% endfor %} {% for col in x.columns if col.synonyms %} {% for s in col.synonyms %} diff --git a/implementation/duckdb/modules/semantic/03-discovery.sql.j2 b/implementation/duckdb/modules/semantic/03-discovery.sql.j2 index 923a987..32f671b 100644 --- a/implementation/duckdb/modules/semantic/03-discovery.sql.j2 +++ b/implementation/duckdb/modules/semantic/03-discovery.sql.j2 @@ -1,8 +1,14 @@ {# The standard-owned discovery views (Design Specification Standard §4.1): Semantic's column catalogue, manifest and relationship paths, and Observability's lineage views. Each is created only where the composition places it, and reads only relations the build context - places. #} + places. Every view and every column carries a comment. #} {% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} +{% macro describe(view, text, columns) -%} +COMMENT ON VIEW {{ view|qualified }} IS {{ text|literal }}; +{% for name, meaning in columns -%} +COMMENT ON COLUMN {{ view|qualified }}.{{ name|ident }} IS {{ meaning|literal }}; +{% endfor -%} +{%- endmacro %} {% set colmeta = named('ColumnMetadata') %}{% set catalogue = named('ColumnCatalogue') %} {% if catalogue and colmeta %}{% set c = colmeta.column_names %} CREATE VIEW {{ catalogue.objects.consumer_view|qualified }} AS @@ -19,9 +25,21 @@ LEFT JOIN {{ colmeta.objects.table|qualified }} m ON m.{{ c.container_name|ident }} = col.table_schema AND m.{{ c.table_name|ident }} = col.table_name AND m.{{ c.column_name|ident }} = col.column_name WHERE col.table_catalog = current_database() AND {{ m.in_product('col.table_schema', 'col.table_name') }}; +{{ describe(catalogue.objects.consumer_view, 'Column-level catalogue of every product container: the structural facts from the system catalogue joined to the authored column metadata.', [ + ('container_name', "Container (schema) that holds the column's table or view."), + ('table_name', 'Table or view that holds the column.'), ('column_name', 'Physical column name.'), + ('data_type', 'Declared platform data type of the column.'), ('is_nullable', 'YES when the column accepts nulls, otherwise NO.'), + ('logical_name', 'Name the specification or owning standard gives the column; null when none is registered.'), + ('business_description', 'Authored business meaning of the column; null when none is registered.'), + ('is_pii', 'True when the column holds personally identifiable information.'), + ('sensitivity_marker', "The specification's sensitivity marker for the column, when it carries one."), + ('data_classification', "Class the organisation's scheme assigns the column."), + ('declared_type', 'Logical type the specification declares for the column.'), + ('structural_source', 'Catalogue that supplied the structural facts.'), + ('description_source', 'authored when a business description is registered, missing otherwise.')]) }} {% endif %} {% set registry = named('DataProductRegistry') %}{% set orient = named('DataProductOrientation') %} -{% set manifest = named('DataProductManifest') %} +{% set manifest = named('DataProductManifest') %}{% set decl = named('StandardDeclaration') %} {% if manifest and registry and orient %}{% set p = registry.column_names %}{% set o = orient.column_names %} CREATE VIEW {{ manifest.objects.consumer_view|qualified }} AS SELECT r.{{ p.product_id|ident }} AS product_id, r.{{ p.product_name|ident }} AS product_name, @@ -32,34 +50,76 @@ SELECT r.{{ p.product_id|ident }} AS product_id, r.{{ p.product_name|ident }} AS {%- for res in orientation.resources %}, max(CASE WHEN o.{{ o.resource_role|ident }} = {{ res.role|literal }} THEN o.{{ o.container_name|ident }} || '.' || o.{{ o.object_name|ident }} END) AS {{ res.role|lower|ident }} {%- endfor %} +{%- if decl %}{% set d = decl.column_names %}, + (SELECT s.{{ d.standard_version|ident }} FROM {{ decl.objects.table|qualified }} s + WHERE s.{{ d.product_id|ident }} = r.{{ p.product_id|ident }} AND s.{{ d.component_kind|ident }} = 'STANDARD' + AND s.{{ d.is_active|ident }}) AS standard_version +{%- endif %} FROM {{ registry.objects.table|qualified }} r JOIN {{ orient.objects.table|qualified }} o ON o.{{ o.product_id|ident }} = r.{{ p.product_id|ident }} WHERE r.{{ p.is_active|ident }} AND NOT r.{{ p.is_deleted|ident }} AND o.{{ o.is_active|ident }} GROUP BY 1, 2, 3, 4, 5, 6, 7; +{{ describe(manifest.objects.consumer_view, 'Bootstrap manifest: the approved entrypoint, trust producer, one deployed object per discovery resource, and the standard version the product declares.', + [('product_id', 'Identifier of the data product.'), ('product_name', 'Name of the data product.'), + ('product_version', 'Published contract version of the data product.'), + ('product_status', 'Publication lifecycle state of the product.'), + ('approved_entrypoint', 'Consumer object an agent should start from.'), + ('approved_access_mode', 'How the approved entrypoint is reached.'), + ('trust_authoritative_producer', 'Validation producer whose evidence is authoritative for trust.')]) }} +{% for res in orientation.resources -%} +COMMENT ON COLUMN {{ manifest.objects.consumer_view|qualified }}.{{ res.role|lower|ident }} IS {{ ('Container and object of the ' ~ res.role ~ ' discovery resource, to use as given.')|literal }}; +{% endfor -%} +{%- if decl -%} +COMMENT ON COLUMN {{ manifest.objects.consumer_view|qualified }}.{{ 'standard_version'|ident }} IS {{ 'Release of the standard the deployment declares it implements; null when no declaration is registered.'|literal }}; +{%- endif %} {% endif %} -{% set rel = named('TableRelationship') %}{% set paths = named('RelationshipPaths') %} -{% if paths and rel %}{% set t = rel.column_names %} +{% set rel = named('TableRelationship') %}{% set relcol = named('RelationshipColumn') %}{% set paths = named('RelationshipPaths') %} +{% if paths and rel and relcol %}{% set t = rel.column_names %}{% set u = relcol.column_names %} CREATE VIEW {{ paths.objects.consumer_view|qualified }} AS WITH RECURSIVE edges AS ( - SELECT {{ t.source_container|ident }} || '.' || {{ t.source_table|ident }} AS from_object, {{ t.source_column|ident }} AS from_column, - {{ t.target_container|ident }} || '.' || {{ t.target_table|ident }} AS to_object, {{ t.target_column|ident }} AS to_column - FROM {{ rel.objects.table|qualified }} WHERE {{ t.is_active|ident }} + SELECT r.{{ t.relationship_id|ident }} AS relationship_id, c.{{ u.ordinal|ident }} AS key_ordinal, 'FORWARD' AS direction, + r.{{ t.source_entity|ident }} AS from_entity, c.{{ u.source_column|ident }} AS from_column, + r.{{ t.target_entity|ident }} AS to_entity, c.{{ u.target_column|ident }} AS to_column + FROM {{ rel.objects.table|qualified }} r + INNER JOIN {{ relcol.objects.table|qualified }} c ON c.{{ u.relationship_id|ident }} = r.{{ t.relationship_id|ident }} + WHERE r.{{ t.is_active|ident }} AND c.{{ u.is_active|ident }} UNION ALL - SELECT {{ t.target_container|ident }} || '.' || {{ t.target_table|ident }}, {{ t.target_column|ident }}, - {{ t.source_container|ident }} || '.' || {{ t.source_table|ident }}, {{ t.source_column|ident }} - FROM {{ rel.objects.table|qualified }} WHERE {{ t.is_active|ident }} -), paths AS ( - SELECT from_object, to_object, [from_object, to_object] AS visited, - from_object || '.' || from_column || ' = ' || to_object || '.' || to_column AS join_path, 1 AS hops + SELECT r.{{ t.relationship_id|ident }}, c.{{ u.ordinal|ident }}, 'REVERSE', + r.{{ t.target_entity|ident }}, c.{{ u.target_column|ident }}, + r.{{ t.source_entity|ident }}, c.{{ u.source_column|ident }} + FROM {{ rel.objects.table|qualified }} r + INNER JOIN {{ relcol.objects.table|qualified }} c ON c.{{ u.relationship_id|ident }} = r.{{ t.relationship_id|ident }} + WHERE r.{{ t.is_active|ident }} AND c.{{ u.is_active|ident }} +), steps AS ( + SELECT DISTINCT relationship_id, direction, from_entity, to_entity, + relationship_id::VARCHAR || CASE direction WHEN 'FORWARD' THEN '>' ELSE '<' END AS leg FROM edges +), walk AS ( + SELECT s.from_entity AS origin_entity, s.to_entity AS destination_entity, + [s.from_entity::VARCHAR, s.to_entity::VARCHAR] AS visited, [s.leg::VARCHAR] AS legs, 1 AS hops + FROM steps s UNION ALL - SELECT p.from_object, e.to_object, list_append(p.visited, e.to_object), - p.join_path || ' AND ' || e.from_object || '.' || e.from_column || ' = ' || e.to_object || '.' || e.to_column, - p.hops + 1 - FROM paths p JOIN edges e ON e.from_object = p.to_object - WHERE p.hops < {{ paths.max_hops }} AND NOT list_contains(p.visited, e.to_object) + SELECT w.origin_entity, s.to_entity, list_append(w.visited, s.to_entity::VARCHAR), list_append(w.legs, s.leg::VARCHAR), w.hops + 1 + FROM walk w INNER JOIN steps s ON s.from_entity = w.destination_entity + WHERE w.hops < {{ paths.max_hops }} AND NOT list_contains(w.visited, s.to_entity) ) -SELECT from_object, to_object, join_path, hops FROM paths; +SELECT array_to_string(w.legs, '') AS path_id, w.origin_entity, w.destination_entity, w.hops AS hop_count, h.hop, + e.key_ordinal, e.from_entity, e.from_column, e.to_entity, e.to_column, e.relationship_id, e.direction +FROM walk w +CROSS JOIN LATERAL (SELECT unnest(range(1, len(w.legs) + 1)) AS hop) h +INNER JOIN edges e + ON e.relationship_id::VARCHAR = substr(w.legs[h.hop], 1, length(w.legs[h.hop]) - 1) + AND e.direction = CASE substr(w.legs[h.hop], length(w.legs[h.hop]), 1) WHEN '>' THEN 'FORWARD' ELSE 'REVERSE' END; +{{ describe(paths.objects.consumer_view, 'Join paths of up to four steps between entities: one row per step and key position, ready for a consumer to assemble the join. A path never revisits an entity.', [ + ('path_id', 'Ordered legs of the path, each a relationship id followed by > for FORWARD or < for REVERSE.'), + ('origin_entity', 'Logical identity of the entity the path starts from.'), + ('destination_entity', 'Logical identity of the entity the path ends at.'), + ('hop_count', 'Number of relationship steps in the path.'), ('hop', 'Position of this step in the path, from 1.'), + ('key_ordinal', 'Position of the column pair within the join of this step, from 1.'), + ('from_entity', 'Logical identity of the entity the step leaves.'), ('from_column', 'Join column on the entity the step leaves.'), + ('to_entity', 'Logical identity of the entity the step arrives at.'), ('to_column', 'Join column on the entity the step arrives at.'), + ('relationship_id', 'Relationship, which is the join group, this step crosses.'), + ('direction', "FORWARD when crossing from the relationship's source to its target, REVERSE for the other way.")]) }} {% endif %} {% set lineage = named('DataLineage') %}{% set graph_view = named('LineageGraph') %} {% if graph_view and lineage %}{% set d = lineage.column_names %} @@ -70,6 +130,10 @@ FROM {{ lineage.objects.table|qualified }} WHERE {{ d.is_active|ident }} AND {{ UNION SELECT DISTINCT {{ d.job_name|ident }}, coalesce({{ d.target_container|ident }} || '.', '') || {{ d.target_table|ident }}, 'produces' FROM {{ lineage.objects.table|qualified }} WHERE {{ d.is_active|ident }} AND {{ d.job_name|ident }} IS NOT NULL; +{{ describe(graph_view.objects.consumer_view, 'Table-and-job lineage graph: one edge from each active source to its job and from the job to its target.', [ + ('source_node', 'Node the edge leaves: a source table or a job.'), + ('target_node', 'Node the edge arrives at: a job or a target table.'), + ('edge_type', 'Kind of relationship the edge records; produces for data flow.')]) }} {% endif %} {% set run = named('LineageRun') %}{% set latest = named('LineageRunLatest') %} {% if latest and lineage and run %}{% set d = lineage.column_names %}{% set r = run.column_names %} @@ -84,4 +148,9 @@ SELECT lineage_id, source_table, target_table, job_name, run_dts, run_status FRO LEFT JOIN {{ run.objects.table|qualified }} r ON r.{{ r.lineage_id|ident }} = d.{{ d.lineage_id|ident }} WHERE d.{{ d.is_active|ident }} ) ranked WHERE rank = 1; +{{ describe(latest.objects.consumer_view, 'Latest recorded run of each active lineage definition.', [ + ('lineage_id', 'Identifier of the lineage definition.'), ('source_table', 'Table the job reads.'), + ('target_table', 'Table the job writes.'), ('job_name', 'Name of the job that moves the data.'), + ('run_dts', 'When the latest run was recorded; null when never run.'), + ('run_status', 'Outcome of the latest run; null when never run.')]) }} {% endif %} diff --git a/implementation/duckdb/modules/semantic/README.md b/implementation/duckdb/modules/semantic/README.md index 9fdab0d..f41cc66 100644 --- a/implementation/duckdb/modules/semantic/README.md +++ b/implementation/duckdb/modules/semantic/README.md @@ -41,6 +41,8 @@ Composite access objects (`access_objects`) are not registered; the manifest rep `validation.sql.j2` is the `module semantic` check: every base table in a schema the product places objects in, keymaps included, is registered in entity metadata. +**The metadata contract (release 2.0.0).** Registration writes the logical identity of each entity (`module.entity`) and, beside it, its location; the key definitions and their ordered key columns; one relationship column pair per join column; one access object per placed object, identified by entity and surface; one access-column mapping row per output column of each consumer view over an entity (the context's `view_columns`, which the view itself selects); and the `StandardDeclaration` rows from `standard`. `RelationshipPaths` returns one row per path step and key position, bounded to four hops. The manifest's last column is `standard_version`. The five checks under [`contract/`](contract/) test `INV-SEMANTIC-016` to `020`; they are declared as `contract` checks and read only registered metadata and the catalogue. + Registration uses the placement and relationships the context carries, never name inference. Structural facts come from `information_schema`, filtered by schema name only, not by `current_database()`; meanings come from the specification. ## Capability bindings diff --git a/implementation/duckdb/modules/semantic/contract/00-macros.sql.j2 b/implementation/duckdb/modules/semantic/contract/00-macros.sql.j2 new file mode 100644 index 0000000..805d6d8 --- /dev/null +++ b/implementation/duckdb/modules/semantic/contract/00-macros.sql.j2 @@ -0,0 +1,9 @@ +{# Shared by the Semantic metadata-contract checks (INV-SEMANTIC-016 to 020). A standard-owned + relation is read through the physical names the build context places; nothing here composes one. #} +{% macro tbl(name) -%}{{ named(name).objects.table|qualified }}{%- endmacro %} +{% macro col(name, column) -%}{{ named(name).column_names[column]|ident }}{%- endmacro %} +{% macro pk(name) -%}{{ named(name).key_columns[0]|ident }}{%- endmacro %} +{# A run of ordinals is 1..n with no gaps and no repeats. #} +{% macro contiguous(alias, ordinal) -%} +count(DISTINCT {{ alias }}.{{ ordinal }}) <> count(*) OR min({{ alias }}.{{ ordinal }}) <> 1 OR max({{ alias }}.{{ ordinal }}) <> count(*) +{%- endmacro %} diff --git a/implementation/duckdb/modules/semantic/contract/access-columns.sql.j2 b/implementation/duckdb/modules/semantic/contract/access-columns.sql.j2 new file mode 100644 index 0000000..d29fb52 --- /dev/null +++ b/implementation/duckdb/modules/semantic/contract/access-columns.sql.j2 @@ -0,0 +1,42 @@ +{# INV-SEMANTIC-019: every consumer view has exactly one mapping row per output column, ordered 1..n, + and every non-derived source exists. Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT a.{{ q.col('AccessObject', 'object_identity') }} || '.' || i.column_name AS identity, 'output column has no mapping' AS defect +FROM {{ q.tbl('AccessObject') }} a +INNER JOIN information_schema.columns i + ON i.table_catalog = current_database() AND i.table_schema = a.{{ q.col('AccessObject', 'container_name') }} AND i.table_name = a.{{ q.col('AccessObject', 'object_name') }} +WHERE a.{{ q.col('AccessObject', 'is_active') }} AND a.{{ q.col('AccessObject', 'object_type') }} = 'CONSUMER_VIEW' + AND EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = a.{{ q.col('AccessObject', 'represents_entity') }}) + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('AccessColumn') }} m + WHERE m.{{ q.col('AccessColumn', 'access_object_id') }} = a.{{ q.pk('AccessObject') }} + AND m.{{ q.col('AccessColumn', 'column_name') }} = i.column_name AND m.{{ q.col('AccessColumn', 'is_active') }}) +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }} || '.' || m.{{ q.col('AccessColumn', 'column_name') }}, + 'mapping names a column the object does not have, or is repeated' +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} + AND (NOT EXISTS (SELECT 1 FROM information_schema.columns i + WHERE i.table_catalog = current_database() AND i.table_schema = a.{{ q.col('AccessObject', 'container_name') }} AND i.table_name = a.{{ q.col('AccessObject', 'object_name') }} + AND i.column_name = m.{{ q.col('AccessColumn', 'column_name') }}) + OR (SELECT count(*) FROM {{ q.tbl('AccessColumn') }} d + WHERE d.{{ q.col('AccessColumn', 'access_object_id') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} + AND d.{{ q.col('AccessColumn', 'column_name') }} = m.{{ q.col('AccessColumn', 'column_name') }} + AND d.{{ q.col('AccessColumn', 'is_active') }}) > 1) +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }}, 'mapping ordinals do not run 1..n' +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} +GROUP BY a.{{ q.col('AccessObject', 'object_identity') }} +HAVING {{ q.contiguous('m', q.col('AccessColumn', 'ordinal')) }} +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }} || '.' || m.{{ q.col('AccessColumn', 'column_name') }}, 'non-derived mapping source is absent' +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} AND m.{{ q.col('AccessColumn', 'mapping_kind') }} <> 'DERIVED' + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e + INNER JOIN information_schema.columns i + ON i.table_catalog = current_database() AND i.table_schema = e.{{ q.col('EntityMetadata', 'container_name') }} AND i.table_name = e.{{ q.col('EntityMetadata', 'table_name') }} + WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = m.{{ q.col('AccessColumn', 'source_entity') }} + AND i.column_name = m.{{ q.col('AccessColumn', 'source_column') }}) diff --git a/implementation/duckdb/modules/semantic/contract/composition.sql.j2 b/implementation/duckdb/modules/semantic/contract/composition.sql.j2 new file mode 100644 index 0000000..281f55f --- /dev/null +++ b/implementation/duckdb/modules/semantic/contract/composition.sql.j2 @@ -0,0 +1,40 @@ +{# INV-SEMANTIC-020: a composite has exactly one anchor, its members are ordered 1..n, and every other member + joins to a prior member through a registered relationship in a stated direction. + Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT c.{{ q.col('AccessComposition', 'composite_object') }}::TEXT AS identity, 'composite does not have exactly one anchor' AS defect +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} +GROUP BY c.{{ q.col('AccessComposition', 'composite_object') }} +HAVING count(*) FILTER (WHERE c.{{ q.col('AccessComposition', 'member_role') }} = 'ANCHOR') <> 1 +UNION ALL +SELECT c.{{ q.col('AccessComposition', 'composite_object') }}, 'member order does not run 1..n' +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} +GROUP BY c.{{ q.col('AccessComposition', 'composite_object') }} +HAVING {{ q.contiguous('c', q.col('AccessComposition', 'member_seq')) }} +UNION ALL +SELECT c.{{ q.col('AccessComposition', 'composite_object') }} || '.' || c.{{ q.col('AccessComposition', 'member_entity') }}, + 'member does not join to a prior member through its relationship' +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} AND c.{{ q.col('AccessComposition', 'member_role') }} <> 'ANCHOR' + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('TableRelationship') }} r + INNER JOIN {{ q.tbl('AccessComposition') }} p + ON p.{{ q.col('AccessComposition', 'composite_object') }} = c.{{ q.col('AccessComposition', 'composite_object') }} + AND p.{{ q.col('AccessComposition', 'member_seq') }} < c.{{ q.col('AccessComposition', 'member_seq') }} + WHERE r.{{ q.pk('TableRelationship') }} = c.{{ q.col('AccessComposition', 'via_relationship_id') }} + AND ((c.{{ q.col('AccessComposition', 'via_direction') }} = 'FORWARD' + AND r.{{ q.col('TableRelationship', 'source_entity') }} = p.{{ q.col('AccessComposition', 'member_entity') }} + AND r.{{ q.col('TableRelationship', 'target_entity') }} = c.{{ q.col('AccessComposition', 'member_entity') }}) + OR (c.{{ q.col('AccessComposition', 'via_direction') }} = 'REVERSE' + AND r.{{ q.col('TableRelationship', 'target_entity') }} = p.{{ q.col('AccessComposition', 'member_entity') }} + AND r.{{ q.col('TableRelationship', 'source_entity') }} = c.{{ q.col('AccessComposition', 'member_entity') }}))) +UNION ALL +SELECT c.{{ q.col('AccessComposition', 'composite_object') }} || '.' || c.{{ q.col('AccessComposition', 'member_entity') }}, + 'anchor carries a join, or the member names an unregistered entity' +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} + AND ((c.{{ q.col('AccessComposition', 'member_role') }} = 'ANCHOR' + AND (c.{{ q.col('AccessComposition', 'via_relationship_id') }} IS NOT NULL OR c.{{ q.col('AccessComposition', 'via_direction') }} IS NOT NULL)) + OR NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e + WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = c.{{ q.col('AccessComposition', 'member_entity') }})) diff --git a/implementation/duckdb/modules/semantic/contract/identity.sql.j2 b/implementation/duckdb/modules/semantic/contract/identity.sql.j2 new file mode 100644 index 0000000..3cce7ba --- /dev/null +++ b/implementation/duckdb/modules/semantic/contract/identity.sql.j2 @@ -0,0 +1,27 @@ +{# INV-SEMANTIC-016: identities follow the grammar and carry no placement. Every row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }}::TEXT AS identity, 'entity identity is not module.entity in its own module' AS defect +FROM {{ q.tbl('EntityMetadata') }} e +WHERE e.{{ q.col('EntityMetadata', 'is_active') }} + AND (NOT regexp_matches(e.{{ q.col('EntityMetadata', 'entity_name') }}, '^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*$') + OR e.{{ q.col('EntityMetadata', 'entity_name') }} NOT LIKE lower(e.{{ q.col('EntityMetadata', 'module_name') }}) || '.%') +UNION ALL +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }}::TEXT, 'entity identity is registered more than once' +FROM {{ q.tbl('EntityMetadata') }} e +WHERE e.{{ q.col('EntityMetadata', 'is_active') }} +GROUP BY e.{{ q.col('EntityMetadata', 'entity_name') }} +HAVING count(*) > 1 +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }}::TEXT, 'access identity breaks the grammar' +FROM {{ q.tbl('AccessObject') }} a +WHERE a.{{ q.col('AccessObject', 'is_active') }} + AND NOT (regexp_matches(a.{{ q.col('AccessObject', 'object_identity') }}, '^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*:(base|governed|current|history)$') + OR regexp_matches(a.{{ q.col('AccessObject', 'object_identity') }}, '^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*$') + OR regexp_matches(a.{{ q.col('AccessObject', 'object_identity') }}, '^composite:[a-z][a-z0-9_]*$')) +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }}::TEXT, 'single-entity identity does not match its entity and surface' +FROM {{ q.tbl('AccessObject') }} a +WHERE a.{{ q.col('AccessObject', 'is_active') }} AND a.{{ q.col('AccessObject', 'represents_entity') }} IS NOT NULL + AND a.{{ q.col('AccessObject', 'object_identity') }} <> a.{{ q.col('AccessObject', 'represents_entity') }} || ':' || + CASE a.{{ q.col('AccessObject', 'object_type') }} WHEN 'TABLE' THEN 'base' WHEN 'BASE_VIEW' THEN 'governed' + WHEN 'CONSUMER_VIEW' THEN 'current' ELSE 'history' END diff --git a/implementation/duckdb/modules/semantic/contract/keys.sql.j2 b/implementation/duckdb/modules/semantic/contract/keys.sql.j2 new file mode 100644 index 0000000..9c7a023 --- /dev/null +++ b/implementation/duckdb/modules/semantic/contract/keys.sql.j2 @@ -0,0 +1,34 @@ +{# INV-SEMANTIC-017: every entity has a key; the columns of a key are ordered 1..n and exist on the entity. + Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }}::TEXT AS identity, 'entity has no key definition' AS defect +FROM {{ q.tbl('EntityMetadata') }} e +WHERE e.{{ q.col('EntityMetadata', 'is_active') }} + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('KeyDefinition') }} k + WHERE k.{{ q.col('KeyDefinition', 'entity_id') }} = e.{{ q.pk('EntityMetadata') }} AND k.{{ q.col('KeyDefinition', 'is_active') }}) +UNION ALL +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }} || '#' || k.{{ q.col('KeyDefinition', 'key_name') }}, 'key has no columns' +FROM {{ q.tbl('KeyDefinition') }} k +INNER JOIN {{ q.tbl('EntityMetadata') }} e ON e.{{ q.pk('EntityMetadata') }} = k.{{ q.col('KeyDefinition', 'entity_id') }} +WHERE k.{{ q.col('KeyDefinition', 'is_active') }} + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('KeyColumn') }} c + WHERE c.{{ q.col('KeyColumn', 'key_definition_id') }} = k.{{ q.pk('KeyDefinition') }} AND c.{{ q.col('KeyColumn', 'is_active') }}) +UNION ALL +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }} || '#' || k.{{ q.col('KeyDefinition', 'key_name') }}, 'key column ordinals do not run 1..n' +FROM {{ q.tbl('KeyDefinition') }} k +INNER JOIN {{ q.tbl('EntityMetadata') }} e ON e.{{ q.pk('EntityMetadata') }} = k.{{ q.col('KeyDefinition', 'entity_id') }} +INNER JOIN {{ q.tbl('KeyColumn') }} c ON c.{{ q.col('KeyColumn', 'key_definition_id') }} = k.{{ q.pk('KeyDefinition') }} +WHERE k.{{ q.col('KeyDefinition', 'is_active') }} AND c.{{ q.col('KeyColumn', 'is_active') }} +GROUP BY e.{{ q.col('EntityMetadata', 'entity_name') }}, k.{{ q.col('KeyDefinition', 'key_name') }} +HAVING {{ q.contiguous('c', q.col('KeyColumn', 'ordinal')) }} +UNION ALL +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }} || '#' || k.{{ q.col('KeyDefinition', 'key_name') }} || '.' || c.{{ q.col('KeyColumn', 'column_name') }}, + 'key column is absent from its entity' +FROM {{ q.tbl('KeyColumn') }} c +INNER JOIN {{ q.tbl('KeyDefinition') }} k ON k.{{ q.pk('KeyDefinition') }} = c.{{ q.col('KeyColumn', 'key_definition_id') }} +INNER JOIN {{ q.tbl('EntityMetadata') }} e ON e.{{ q.pk('EntityMetadata') }} = k.{{ q.col('KeyDefinition', 'entity_id') }} +WHERE c.{{ q.col('KeyColumn', 'is_active') }} + AND NOT EXISTS (SELECT 1 FROM information_schema.columns i + WHERE i.table_catalog = current_database() AND i.table_schema = e.{{ q.col('EntityMetadata', 'container_name') }} + AND i.table_name = e.{{ q.col('EntityMetadata', 'table_name') }} + AND i.column_name = c.{{ q.col('KeyColumn', 'column_name') }}) diff --git a/implementation/duckdb/modules/semantic/contract/relationships.sql.j2 b/implementation/duckdb/modules/semantic/contract/relationships.sql.j2 new file mode 100644 index 0000000..b9a149e --- /dev/null +++ b/implementation/duckdb/modules/semantic/contract/relationships.sql.j2 @@ -0,0 +1,35 @@ +{# INV-SEMANTIC-018: every relationship has at least one column pair; pairs are ordered 1..n and + their columns exist on the entities. Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT r.{{ q.col('TableRelationship', 'relationship_name') }}::TEXT AS identity, 'relationship has no column pair' AS defect +FROM {{ q.tbl('TableRelationship') }} r +WHERE r.{{ q.col('TableRelationship', 'is_active') }} + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('RelationshipColumn') }} c + WHERE c.{{ q.col('RelationshipColumn', 'relationship_id') }} = r.{{ q.pk('TableRelationship') }} + AND c.{{ q.col('RelationshipColumn', 'is_active') }}) +UNION ALL +SELECT r.{{ q.col('TableRelationship', 'relationship_name') }}, 'relationship ordinals do not run 1..n' +FROM {{ q.tbl('TableRelationship') }} r +INNER JOIN {{ q.tbl('RelationshipColumn') }} c ON c.{{ q.col('RelationshipColumn', 'relationship_id') }} = r.{{ q.pk('TableRelationship') }} +WHERE r.{{ q.col('TableRelationship', 'is_active') }} AND c.{{ q.col('RelationshipColumn', 'is_active') }} +GROUP BY r.{{ q.col('TableRelationship', 'relationship_name') }} +HAVING {{ q.contiguous('c', q.col('RelationshipColumn', 'ordinal')) }} +UNION ALL +SELECT r.{{ q.col('TableRelationship', 'relationship_name') }}, 'relationship names an unregistered entity' +FROM {{ q.tbl('TableRelationship') }} r +WHERE r.{{ q.col('TableRelationship', 'is_active') }} + AND (NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'source_entity') }}) + OR NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'target_entity') }})) +UNION ALL +SELECT r.{{ q.col('TableRelationship', 'relationship_name') }} || '.' || c.{{ q.col('RelationshipColumn', 'ordinal') }}, 'relationship column is absent from its entity' +FROM {{ q.tbl('RelationshipColumn') }} c +INNER JOIN {{ q.tbl('TableRelationship') }} r ON r.{{ q.pk('TableRelationship') }} = c.{{ q.col('RelationshipColumn', 'relationship_id') }} +INNER JOIN {{ q.tbl('EntityMetadata') }} s ON s.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'source_entity') }} +INNER JOIN {{ q.tbl('EntityMetadata') }} t ON t.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'target_entity') }} +WHERE c.{{ q.col('RelationshipColumn', 'is_active') }} + AND (NOT EXISTS (SELECT 1 FROM information_schema.columns i + WHERE i.table_catalog = current_database() AND i.table_schema = s.{{ q.col('EntityMetadata', 'container_name') }} AND i.table_name = s.{{ q.col('EntityMetadata', 'table_name') }} + AND i.column_name = c.{{ q.col('RelationshipColumn', 'source_column') }}) + OR NOT EXISTS (SELECT 1 FROM information_schema.columns i + WHERE i.table_catalog = current_database() AND i.table_schema = t.{{ q.col('EntityMetadata', 'container_name') }} AND i.table_name = t.{{ q.col('EntityMetadata', 'table_name') }} + AND i.column_name = c.{{ q.col('RelationshipColumn', 'target_column') }})) diff --git a/implementation/duckdb/patterns/temporal-lifecycle-metadata/00-macros.sql.j2 b/implementation/duckdb/patterns/temporal-lifecycle-metadata/00-macros.sql.j2 index 148081f..589e92c 100644 --- a/implementation/duckdb/patterns/temporal-lifecycle-metadata/00-macros.sql.j2 +++ b/implementation/duckdb/patterns/temporal-lifecycle-metadata/00-macros.sql.j2 @@ -111,6 +111,7 @@ VALUES ({{ exprs|join(', ') }}); {%- for keymaps in (true, false) -%} {%- for x in entities + relations if x.module == module and x.relation == 'table' and (x.defined|default(true)) and not (x.adopted|default(false)) and ((x.kind == 'Keymap') == keymaps) %} {% if x.objects.sequence is defined %}CREATE SEQUENCE {{ x.objects.sequence|qualified }}; +COMMENT ON SEQUENCE {{ x.objects.sequence|qualified }} IS {{ ('Allocates the permanent identifiers of ' ~ x.name ~ '; never reset or recycled.')|literal }}; {% endif %}{{ table(x) }} {%- endfor -%} {%- endfor -%} diff --git a/implementation/duckdb/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 b/implementation/duckdb/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 index e6e058b..504d11e 100644 --- a/implementation/duckdb/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 +++ b/implementation/duckdb/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 @@ -6,12 +6,16 @@ {% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} {% for x in entities + relations if x.relation == 'table' and (x.defined|default(true)) and x.objects.consumer_view is defined %} CREATE VIEW {{ x.objects.consumer_view|qualified }} AS -SELECT {{ m.cols(x.columns|rejectattr('name', 'in', ('valid_to_dts', 'transaction_to_dts'))|map(attribute='physical_name')) }} +SELECT {{ m.cols(x.view_columns|map(attribute='physical_name')) }} FROM {{ x.objects.table|qualified }} WHERE {{ m.current(x) }}; COMMENT ON VIEW {{ x.objects.consumer_view|qualified }} IS {{ ('Current consumer surface: ' ~ x.description)|literal }}; +{% for vc in x.view_columns %}{% set col = x.columns|selectattr('physical_name', 'equalto', vc.physical_name)|first %} +COMMENT ON COLUMN {{ x.objects.consumer_view|qualified }}.{{ vc.physical_name|ident }} IS {{ (col.description or col.name)|literal }}; +{% endfor %} {% endfor %} {% for x in entities + relations if x.relation == 'table' and (x.defined|default(true)) and x.objects.function is defined %} CREATE MACRO {{ x.objects.function|qualified }}(valid_at{% if x.bitemporal %}, known_at{% endif %}) AS TABLE SELECT * FROM {{ x.objects.table|qualified }} WHERE {{ m.at(x) }}; +COMMENT ON MACRO TABLE {{ x.objects.function|qualified }} IS {{ ('Point-in-time reconstruction of ' ~ x.description ~ (' Takes the valid-time and known-time instants.' if x.bitemporal else ' Takes the valid-time instant.'))|literal }}; {% endfor %} diff --git a/implementation/duckdb/patterns/validation/03-relationship-reference.sql.j2 b/implementation/duckdb/patterns/validation/03-relationship-reference.sql.j2 index 99c7706..ed64169 100644 --- a/implementation/duckdb/patterns/validation/03-relationship-reference.sql.j2 +++ b/implementation/duckdb/patterns/validation/03-relationship-reference.sql.j2 @@ -1,7 +1,8 @@ {# Rendered per relationship, with `relationship`. Current-surface referential coverage, not - historical proof: every returned row is a current reference with no current target. A generic - reference is checked only for the rows its discriminator points at this target. #} + historical proof: every returned row is a current reference with no current target. A join of + several columns resolves only when every pair matches. A generic reference is checked only for + the rows its discriminator points at this target. #} {% set r = relationship %}{% set source = named(r.source) %} -SELECT s.{{ r.source_column|ident }} FROM {{ (r.source_view or r.source_table)|qualified }} s -LEFT JOIN {{ (r.target_view or r.target_table)|qualified }} t ON s.{{ r.source_column|ident }} = t.{{ r.target_columns[0]|ident }} -WHERE {% if r.discriminator %}s.{{ source.column_names[r.discriminator]|ident }} = {{ r.discriminator_value|literal }} AND {% endif %}{% if not r.mandatory %}s.{{ r.source_column|ident }} IS NOT NULL AND {% endif %}t.{{ r.target_columns[0]|ident }} IS NULL +SELECT {% for p in r.pairs %}s.{{ p.source_column|ident }}{% if not loop.last %}, {% endif %}{% endfor %} FROM {{ (r.source_view or r.source_table)|qualified }} s +LEFT JOIN {{ (r.target_view or r.target_table)|qualified }} t ON {% for p in r.pairs %}s.{{ p.source_column|ident }} = t.{{ p.target_column|ident }}{% if not loop.last %} AND {% endif %}{% endfor %} +WHERE {% if r.discriminator %}s.{{ source.column_names[r.discriminator]|ident }} = {{ r.discriminator_value|literal }} AND {% endif %}{% if not r.mandatory %}{% for p in r.pairs %}s.{{ p.source_column|ident }} IS NOT NULL AND {% endfor %}{% endif %}t.{{ r.pairs[0].target_column|ident }} IS NULL diff --git a/implementation/duckdb/patterns/validation/04-trust-map.sql.j2 b/implementation/duckdb/patterns/validation/04-trust-map.sql.j2 index 6eaa47e..6f4a9a8 100644 --- a/implementation/duckdb/patterns/validation/04-trust-map.sql.j2 +++ b/implementation/duckdb/patterns/validation/04-trust-map.sql.j2 @@ -69,4 +69,24 @@ SELECT p.product_code, p.producer_id, NULL, 'PRODUCT', p.product_code, 'no-evide 0, 0, NULL, 0, 0, 0, 0, 0, 'No published area evidence.', 'Run the validator before reporting confidence.', NULL, FALSE FROM product p WHERE NOT EXISTS (SELECT 1 FROM latest l WHERE l.product_code = p.product_code); +COMMENT ON VIEW {{ view.objects.consumer_view|qualified }} IS 'Trust map: per area, the latest evidence from the designated producer, read at unknown confidence once expired. Read it before relying on any other object.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.product_code IS 'Data product the evidence describes.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.producer_id IS 'Validation producer whose evidence is read; null when none is designated.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.run_id IS 'Validation run that supplied the area evidence; null when none exists.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.scope_kind IS 'Kind of scope the evidence covers, for example MODULE, ENTITY or PRODUCT.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.scope_id IS 'Identifier of the scope the evidence covers.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.area_status IS 'Outcome of the area: pass, fail, partial or no-evidence.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.confidence IS 'Trust confidence for the area; unknown once the evidence has expired.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.checks_expected IS 'Number of checks the area should have run.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.checks_ran IS 'Number of checks the area actually ran.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.coverage IS 'Fraction of expected checks that ran; null when none were expected.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.passed_count IS 'Number of checks that passed.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.failed_count IS 'Number of checks that failed.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.error_count IS 'Number of checks that errored before producing a result.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.critical_failure_count IS 'Number of failed checks of critical severity.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.error_failure_count IS 'Number of failed or errored checks counted against trust.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.open_gaps IS 'Known gaps in the evidence; states staleness when the evidence has expired.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.recommended_action IS 'Next step to restore or confirm trust.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.completed_dts IS 'When the area evidence was completed; null when none exists.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.derived IS 'True when the entry is derived from a legacy run that published no area records.'; {% endif %} diff --git a/implementation/duckdb/patterns/validation/05-comment-coverage.sql.j2 b/implementation/duckdb/patterns/validation/05-comment-coverage.sql.j2 new file mode 100644 index 0000000..0e1a152 --- /dev/null +++ b/implementation/duckdb/patterns/validation/05-comment-coverage.sql.j2 @@ -0,0 +1,31 @@ +{# MetadataCoverageCheck, catalogue-wide: every row returned is an object or column with no comment. + Sweeps every container the product places objects in, so an object the specification never mentions is + caught too. DuckDB cannot comment a schema, so schemas are not swept. #} +{% set in_containers -%} +{%- for c in containers if c.holds_objects and not c.adopted %}{{ c.name|literal }}{% if not loop.last %}, {% endif %}{% endfor -%} +{%- endset %} +SELECT 'table' AS object_kind, t.schema_name AS container_name, t.table_name AS object_name, CAST(NULL AS VARCHAR) AS column_name +FROM duckdb_tables() t +WHERE t.database_name = current_database() AND NOT t.internal AND t.schema_name IN ({{ in_containers }}) + AND nullif(trim(t.comment), '') IS NULL +UNION ALL +SELECT 'view', v.schema_name, v.view_name, NULL +FROM duckdb_views() v +WHERE v.database_name = current_database() AND NOT v.internal AND v.schema_name IN ({{ in_containers }}) + AND nullif(trim(v.comment), '') IS NULL +UNION ALL +SELECT 'column', c.schema_name, c.table_name, c.column_name +FROM duckdb_columns() c +WHERE c.database_name = current_database() AND NOT c.internal AND c.schema_name IN ({{ in_containers }}) + AND nullif(trim(c.comment), '') IS NULL +UNION ALL +SELECT 'sequence', s.schema_name, s.sequence_name, NULL +FROM duckdb_sequences() s +WHERE s.database_name = current_database() AND s.schema_name IN ({{ in_containers }}) + AND nullif(trim(s.comment), '') IS NULL +UNION ALL +SELECT 'macro', f.schema_name, f.function_name, NULL +FROM duckdb_functions() f +WHERE f.database_name = current_database() AND NOT f.internal AND f.function_type IN ('macro', 'table_macro') + AND f.schema_name IN ({{ in_containers }}) + AND nullif(trim(f.comment), '') IS NULL diff --git a/implementation/duckdb/patterns/validation/README.md b/implementation/duckdb/patterns/validation/README.md index 2927348..57da995 100644 --- a/implementation/duckdb/patterns/validation/README.md +++ b/implementation/duckdb/patterns/validation/README.md @@ -24,8 +24,14 @@ Each check is a query whose every row is a defect. The `Checks:` block of [PLATF | `relationship reference` | `03-relationship-reference.sql.j2` | each entry in `relationships` (`relationship`) | current source references with no current target, read through the consumer views (the tables where no view is placed); an optional reference may be null | | `module semantic` | [`modules/semantic/validation.sql.j2`](../../modules/semantic/README.md) | the Semantic module | unregistered base tables | | `module memory` | [`modules/memory/validation.sql.j2`](../../modules/memory/README.md) | the Memory module | modules below the capture floor | - -Because DuckDB enforces neither one open version nor non-overlap in its schema, the temporal check is the only proof of both. The reference check is current-surface coverage, not historical proof, and compares the first target column only. +| `contract identity` | [`modules/semantic/contract/identity.sql.j2`](../../modules/semantic/README.md) | the Semantic module | identities that break the grammar, carry the wrong module qualifier or are registered twice | +| `contract keys` | `modules/semantic/contract/keys.sql.j2` | the Semantic module | entities with no key, keys with no columns, ordinals that do not run 1 to n, and key columns the entity lacks | +| `contract relationships` | `modules/semantic/contract/relationships.sql.j2` | the Semantic module | relationships with no column pair, gaps in the ordinals, unregistered entities and columns the entities lack | +| `contract access-columns` | `modules/semantic/contract/access-columns.sql.j2` | the Semantic module | consumer-view columns with no mapping row, mappings for columns the view lacks, repeated or gapped ordinals, and absent sources | +| `contract composition` | `modules/semantic/contract/composition.sql.j2` | the Semantic module | composites without exactly one anchor, gapped member order, and members not joined to a prior member through a registered relationship | +| `product comment-coverage` | `05-comment-coverage.sql.j2` | the product | every container, object and column of the product's containers with no comment (the catalogue-wide `MetadataCoverageCheck`) | + +Because DuckDB enforces neither one open version nor non-overlap in its schema, the temporal check is the only proof of both. The reference check is current-surface coverage, not historical proof. A join of several columns resolves only when every pair matches, so the check tests all of them. ## Trust map diff --git a/implementation/postgres/PLATFORM_PROFILE.md b/implementation/postgres/PLATFORM_PROFILE.md index f08b7c8..3b8dd99 100644 --- a/implementation/postgres/PLATFORM_PROFILE.md +++ b/implementation/postgres/PLATFORM_PROFILE.md @@ -94,6 +94,12 @@ Binding: - relationship reference: patterns/validation/03-relationship-reference.sql.j2 module semantic: modules/semantic/validation.sql.j2 module memory: modules/memory/validation.sql.j2 + contract identity: modules/semantic/contract/identity.sql.j2 + contract keys: modules/semantic/contract/keys.sql.j2 + contract relationships: modules/semantic/contract/relationships.sql.j2 + contract access-columns: modules/semantic/contract/access-columns.sql.j2 + contract composition: modules/semantic/contract/composition.sql.j2 + product comment-coverage: patterns/validation/05-comment-coverage.sql.j2 ``` ``` diff --git a/implementation/postgres/modules/search/02-similarity.sql.j2 b/implementation/postgres/modules/search/02-similarity.sql.j2 index be69907..969a02f 100644 --- a/implementation/postgres/modules/search/02-similarity.sql.j2 +++ b/implementation/postgres/modules/search/02-similarity.sql.j2 @@ -35,4 +35,5 @@ WHERE ranked.similarity >= {{ b.threshold }} ORDER BY ranked.similarity DESC NULLS LAST, ranked.entity_id LIMIT greatest(0, least(result_limit, 1000)); $body$; +COMMENT ON FUNCTION {{ b.objects.function|qualified }}(DOUBLE PRECISION[], INTEGER) IS {{ ('Similarity search over the ' ~ b.name ~ ' embedding of ' ~ b.entity ~ '. Takes a query vector and a result limit (maximum 1000).')|literal }}; {% endfor %} diff --git a/implementation/postgres/modules/semantic/02-registration.sql.j2 b/implementation/postgres/modules/semantic/02-registration.sql.j2 index c165dce..380719a 100644 --- a/implementation/postgres/modules/semantic/02-registration.sql.j2 +++ b/implementation/postgres/modules/semantic/02-registration.sql.j2 @@ -2,6 +2,12 @@ relationship catalogues, access objects, primary objects, the orientation sequence, metrics, synonyms and naming standards. Every row is read from the build context. #} {% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} +{# The generated id of one key of an entity, found by the entity's identity and the key's name. #} +{% macro key_ref(keydef, meta, x, k) -%} +(SELECT d.{{ keydef.key_columns[0]|ident }} FROM {{ keydef.objects.table|qualified }} d + WHERE d.{{ keydef.column_names.key_name|ident }} = {{ k.name|literal }} + AND d.{{ keydef.column_names.entity_id|ident }} = {{ m.ref(meta, {'entity_name': x.logical_id}) }}) +{%- endmacro %} {% set registry = named('DataProductRegistry') %}{% set manifest = named('DataProductManifest') %} {% if registry and manifest %} {{ m.insert(registry, { @@ -21,21 +27,37 @@ 'deployment_status': 'DEPLOYED'}) }} {% endfor %} {% endif %} +{% set declaration = named('StandardDeclaration') %} +{% if declaration %} +{% for d in standard.declarations %} +{{ m.insert(declaration, {'product_id': product.code, 'component_kind': d.kind, 'component_name': d.name, + 'platform': standard.platform, 'standard_version': standard.version}) }} +{% endfor %} +{% endif %} {% set meta = named('EntityMetadata') %}{% set colmeta = named('ColumnMetadata') %} {% for x in entities + relations if x.objects.table is defined %} {% set t = x.objects.table %} {% if meta %} {{ m.insert(meta, { - 'entity_name': x.name, 'logical_name': x.name, 'entity_description': x.description, + 'entity_name': x.logical_id, 'logical_name': x.name, 'entity_description': x.description, 'entity_kind': x.kind|upper, 'module_name': x.module|upper, 'container_name': t.container, 'table_name': t.name, 'view_name': (x.objects.consumer_view.name if x.objects.consumer_view is defined else none), - 'surrogate_key_column': (x.surrogate_columns|join(', ') or none), - 'natural_key_column': (x.natural_columns|join(', ') or none), 'temporal_pattern': x.profile, 'current_flag_column': (x.column_names.is_current if x.history else none), 'deleted_flag_column': (x.column_names.is_deleted if x.supports_deletion else none)}) }} {% endif %} +{% set keydef = named('KeyDefinition') %}{% set keycol = named('KeyColumn') %} +{% if keydef and keycol and meta %} +{% for k in x.key_defs %} +{{ m.insert(keydef, {'key_role': k.role, 'key_name': k.name}, + {'entity_id': m.ref(meta, {'entity_name': x.logical_id})}) }} +{% for column in k.columns %} +{{ m.insert(keycol, {'ordinal': loop.index, 'column_name': column}, + {'key_definition_id': key_ref(keydef, meta, x, k)}) }} +{% endfor %} +{% endfor %} +{% endif %} {% if colmeta %} {% for col in x.columns %} {{ m.insert(colmeta, { @@ -50,35 +72,47 @@ {% endfor %} {% set rel = named('TableRelationship') %} {% if rel %} +{% set relcol = named('RelationshipColumn') %} {% for r in relationships if r.source_table and r.target_table %} {{ m.insert(rel, { - 'relationship_name': r.source ~ '.' ~ r.source_attribute, - 'source_container': r.source_table.container, 'source_table': r.source_table.name, - 'source_column': r.source_column, 'target_container': r.target_table.container, - 'target_table': r.target_table.name, 'target_column': r.target_columns|join(', '), + 'relationship_name': r.name, 'source_entity': r.source_identity, 'target_entity': r.target_identity, 'relationship_type': 'FOREIGN_KEY', 'cardinality': 'MANY_TO_ONE', 'is_mandatory': r.mandatory}) }} +{% if relcol %} +{% for pair in r.pairs %} +{{ m.insert(relcol, {'ordinal': pair.ordinal, 'source_column': pair.source_column, 'target_column': pair.target_column}, + {'relationship_id': m.ref(rel, {'relationship_name': r.name})}) }} {% endfor %} {% endif %} -{% set access = named('AccessObject') %} +{% endfor %} +{% endif %} +{% set access = named('AccessObject') %}{% set acccol = named('AccessColumn') %} +{% set surface = {'table': 'base', 'base_view': 'governed', 'consumer_view': 'current', 'function': 'history', 'procedure': 'history'} %} {% if access %} {% for x in entities + relations %} {% for role, obj in x.objects|dictsort if obj and m.member(access, 'object_type', role|upper) %} {{ m.insert(access, { - 'container_name': obj.container, 'object_name': obj.name, 'object_type': role|upper, + 'object_identity': x.logical_id ~ ':' ~ surface[role], 'container_name': obj.container, 'object_name': obj.name, + 'object_type': role|upper, 'access_role': ('BASE' if role in ('table', 'base_view') else 'PASSTHROUGH'), - 'represents_entity': x.name, + 'represents_entity': x.logical_id, 'is_agent_consumable': role in ('consumer_view', 'function'), 'is_primary': role == 'consumer_view', - 'resolves_to_object': (x.objects.table.container ~ '.' ~ x.objects.table.name - if role != 'table' and x.objects.table is defined else none), + 'resolves_to_object': (x.logical_id ~ ':base' if role != 'table' and x.objects.table is defined else none), 'access_note': x.description}) }} +{% if role == 'consumer_view' and acccol %} +{% for vc in x.view_columns %} +{{ m.insert(acccol, {'ordinal': vc.ordinal, 'column_name': vc.physical_name, 'source_entity': x.logical_id, + 'source_column': vc.physical_name, 'mapping_kind': 'PASSTHROUGH'}, + {'access_object_id': m.ref(access, {'object_identity': x.logical_id ~ ':current'})}) }} +{% endfor %} +{% endif %} {% endfor %} {% endfor %} {% for b in embeddings %} {% for role, obj in b.objects|dictsort if role in ('function', 'procedure') %} {{ m.insert(access, { - 'container_name': obj.container, 'object_name': obj.name, - 'object_type': role|upper, 'access_role': 'PASSTHROUGH', 'represents_entity': b.entity, + 'object_identity': b.logical_id, 'container_name': obj.container, 'object_name': obj.name, + 'object_type': role|upper, 'access_role': 'PASSTHROUGH', 'represents_entity': none, 'is_agent_consumable': true, 'is_primary': false, 'access_note': 'Similarity search over the ' ~ b.name ~ ' embedding of ' ~ b.entity ~ '.'}) }} {% endfor %} @@ -127,7 +161,7 @@ {% for x in entities if x.objects.table is defined %} {% for s in x.synonyms|default([]) %} {{ m.insert(syn, {'object_kind': 'ENTITYMETADATA', 'synonym_text': s}, - {'synonym_object': m.ref(meta, {'entity_name': x.name})}) }} + {'synonym_object': m.ref(meta, {'entity_name': x.logical_id})}) }} {% endfor %} {% for col in x.columns if col.synonyms %} {% for s in col.synonyms %} diff --git a/implementation/postgres/modules/semantic/03-discovery.sql.j2 b/implementation/postgres/modules/semantic/03-discovery.sql.j2 index 09a59fe..a230cd4 100644 --- a/implementation/postgres/modules/semantic/03-discovery.sql.j2 +++ b/implementation/postgres/modules/semantic/03-discovery.sql.j2 @@ -1,8 +1,14 @@ {# The standard-owned discovery views (Design Specification Standard §4.1): Semantic's column catalogue, manifest and relationship paths, and Observability's lineage views. Each is created only where the composition places it, and reads only relations the build context - places. #} + places. Every view and every column carries a comment. #} {% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} +{% macro describe(view, text, columns) -%} +COMMENT ON VIEW {{ view|qualified }} IS {{ text|literal }}; +{% for name, meaning in columns -%} +COMMENT ON COLUMN {{ view|qualified }}.{{ name|ident }} IS {{ meaning|literal }}; +{% endfor -%} +{%- endmacro %} {% set colmeta = named('ColumnMetadata') %}{% set catalogue = named('ColumnCatalogue') %} {% if catalogue and colmeta %}{% set c = colmeta.column_names %} CREATE VIEW {{ catalogue.objects.consumer_view|qualified }} AS @@ -19,9 +25,21 @@ LEFT JOIN {{ colmeta.objects.table|qualified }} m ON m.{{ c.container_name|ident }} = col.table_schema AND m.{{ c.table_name|ident }} = col.table_name AND m.{{ c.column_name|ident }} = col.column_name WHERE {{ m.in_product('col.table_schema', 'col.table_name') }}; +{{ describe(catalogue.objects.consumer_view, 'Column-level catalogue of every product container: the structural facts from the system catalogue joined to the authored column metadata.', [ + ('container_name', "Container (schema) that holds the column's table or view."), + ('table_name', 'Table or view that holds the column.'), ('column_name', 'Physical column name.'), + ('data_type', 'Declared platform data type of the column.'), ('is_nullable', 'YES when the column accepts nulls, otherwise NO.'), + ('logical_name', 'Name the specification or owning standard gives the column; null when none is registered.'), + ('business_description', 'Authored business meaning of the column; null when none is registered.'), + ('is_pii', 'True when the column holds personally identifiable information.'), + ('sensitivity_marker', "The specification's sensitivity marker for the column, when it carries one."), + ('data_classification', "Class the organisation's scheme assigns the column."), + ('declared_type', 'Logical type the specification declares for the column.'), + ('structural_source', 'Catalogue that supplied the structural facts.'), + ('description_source', 'authored when a business description is registered, missing otherwise.')]) }} {% endif %} {% set registry = named('DataProductRegistry') %}{% set orient = named('DataProductOrientation') %} -{% set manifest = named('DataProductManifest') %} +{% set manifest = named('DataProductManifest') %}{% set decl = named('StandardDeclaration') %} {% if manifest and registry and orient %}{% set p = registry.column_names %}{% set o = orient.column_names %} CREATE VIEW {{ manifest.objects.consumer_view|qualified }} AS SELECT r.{{ p.product_id|ident }} AS product_id, r.{{ p.product_name|ident }} AS product_name, @@ -32,34 +50,76 @@ SELECT r.{{ p.product_id|ident }} AS product_id, r.{{ p.product_name|ident }} AS {%- for res in orientation.resources %}, max(CASE WHEN o.{{ o.resource_role|ident }} = {{ res.role|literal }} THEN o.{{ o.container_name|ident }} || '.' || o.{{ o.object_name|ident }} END) AS {{ res.role|lower|ident }} {%- endfor %} +{%- if decl %}{% set d = decl.column_names %}, + (SELECT s.{{ d.standard_version|ident }} FROM {{ decl.objects.table|qualified }} s + WHERE s.{{ d.product_id|ident }} = r.{{ p.product_id|ident }} AND s.{{ d.component_kind|ident }} = 'STANDARD' + AND s.{{ d.is_active|ident }}) AS standard_version +{%- endif %} FROM {{ registry.objects.table|qualified }} r JOIN {{ orient.objects.table|qualified }} o ON o.{{ o.product_id|ident }} = r.{{ p.product_id|ident }} WHERE r.{{ p.is_active|ident }} AND NOT r.{{ p.is_deleted|ident }} AND o.{{ o.is_active|ident }} GROUP BY 1, 2, 3, 4, 5, 6, 7; +{{ describe(manifest.objects.consumer_view, 'Bootstrap manifest: the approved entrypoint, trust producer, one deployed object per discovery resource, and the standard version the product declares.', + [('product_id', 'Identifier of the data product.'), ('product_name', 'Name of the data product.'), + ('product_version', 'Published contract version of the data product.'), + ('product_status', 'Publication lifecycle state of the product.'), + ('approved_entrypoint', 'Consumer object an agent should start from.'), + ('approved_access_mode', 'How the approved entrypoint is reached.'), + ('trust_authoritative_producer', 'Validation producer whose evidence is authoritative for trust.')]) }} +{% for res in orientation.resources -%} +COMMENT ON COLUMN {{ manifest.objects.consumer_view|qualified }}.{{ res.role|lower|ident }} IS {{ ('Container and object of the ' ~ res.role ~ ' discovery resource, to use as given.')|literal }}; +{% endfor -%} +{%- if decl -%} +COMMENT ON COLUMN {{ manifest.objects.consumer_view|qualified }}.{{ 'standard_version'|ident }} IS {{ 'Release of the standard the deployment declares it implements; null when no declaration is registered.'|literal }}; +{%- endif %} {% endif %} -{% set rel = named('TableRelationship') %}{% set paths = named('RelationshipPaths') %} -{% if paths and rel %}{% set t = rel.column_names %} +{% set rel = named('TableRelationship') %}{% set relcol = named('RelationshipColumn') %}{% set paths = named('RelationshipPaths') %} +{% if paths and rel and relcol %}{% set t = rel.column_names %}{% set u = relcol.column_names %} CREATE VIEW {{ paths.objects.consumer_view|qualified }} AS WITH RECURSIVE edges AS ( - SELECT {{ t.source_container|ident }} || '.' || {{ t.source_table|ident }} AS from_object, {{ t.source_column|ident }} AS from_column, - {{ t.target_container|ident }} || '.' || {{ t.target_table|ident }} AS to_object, {{ t.target_column|ident }} AS to_column - FROM {{ rel.objects.table|qualified }} WHERE {{ t.is_active|ident }} + SELECT r.{{ t.relationship_id|ident }} AS relationship_id, c.{{ u.ordinal|ident }} AS key_ordinal, 'FORWARD' AS direction, + r.{{ t.source_entity|ident }} AS from_entity, c.{{ u.source_column|ident }} AS from_column, + r.{{ t.target_entity|ident }} AS to_entity, c.{{ u.target_column|ident }} AS to_column + FROM {{ rel.objects.table|qualified }} r + INNER JOIN {{ relcol.objects.table|qualified }} c ON c.{{ u.relationship_id|ident }} = r.{{ t.relationship_id|ident }} + WHERE r.{{ t.is_active|ident }} AND c.{{ u.is_active|ident }} UNION ALL - SELECT {{ t.target_container|ident }} || '.' || {{ t.target_table|ident }}, {{ t.target_column|ident }}, - {{ t.source_container|ident }} || '.' || {{ t.source_table|ident }}, {{ t.source_column|ident }} - FROM {{ rel.objects.table|qualified }} WHERE {{ t.is_active|ident }} -), paths AS ( - SELECT from_object, to_object, ARRAY[from_object, to_object] AS visited, - from_object || '.' || from_column || ' = ' || to_object || '.' || to_column AS join_path, 1 AS hops + SELECT r.{{ t.relationship_id|ident }}, c.{{ u.ordinal|ident }}, 'REVERSE', + r.{{ t.target_entity|ident }}, c.{{ u.target_column|ident }}, + r.{{ t.source_entity|ident }}, c.{{ u.source_column|ident }} + FROM {{ rel.objects.table|qualified }} r + INNER JOIN {{ relcol.objects.table|qualified }} c ON c.{{ u.relationship_id|ident }} = r.{{ t.relationship_id|ident }} + WHERE r.{{ t.is_active|ident }} AND c.{{ u.is_active|ident }} +), steps AS ( + SELECT DISTINCT relationship_id, direction, from_entity, to_entity, + relationship_id::TEXT || CASE direction WHEN 'FORWARD' THEN '>' ELSE '<' END AS leg FROM edges +), walk AS ( + SELECT s.from_entity AS origin_entity, s.to_entity AS destination_entity, + ARRAY[s.from_entity::TEXT, s.to_entity::TEXT] AS visited, ARRAY[s.leg::TEXT] AS legs, 1 AS hops + FROM steps s UNION ALL - SELECT p.from_object, e.to_object, p.visited || e.to_object, - p.join_path || ' AND ' || e.from_object || '.' || e.from_column || ' = ' || e.to_object || '.' || e.to_column, - p.hops + 1 - FROM paths p JOIN edges e ON e.from_object = p.to_object - WHERE p.hops < {{ paths.max_hops }} AND NOT e.to_object = ANY(p.visited) + SELECT w.origin_entity, s.to_entity, w.visited || s.to_entity::TEXT, w.legs || s.leg::TEXT, w.hops + 1 + FROM walk w INNER JOIN steps s ON s.from_entity = w.destination_entity + WHERE w.hops < {{ paths.max_hops }} AND NOT s.to_entity = ANY(w.visited) ) -SELECT from_object, to_object, join_path, hops FROM paths; +SELECT array_to_string(w.legs, '') AS path_id, w.origin_entity, w.destination_entity, w.hops AS hop_count, h.hop, + e.key_ordinal, e.from_entity, e.from_column, e.to_entity, e.to_column, e.relationship_id, e.direction +FROM walk w +CROSS JOIN LATERAL generate_series(1, cardinality(w.legs)) AS h(hop) +INNER JOIN edges e + ON e.relationship_id::TEXT = substr(w.legs[h.hop], 1, length(w.legs[h.hop]) - 1) + AND e.direction = CASE substr(w.legs[h.hop], length(w.legs[h.hop]), 1) WHEN '>' THEN 'FORWARD' ELSE 'REVERSE' END; +{{ describe(paths.objects.consumer_view, 'Join paths of up to four steps between entities: one row per step and key position, ready for a consumer to assemble the join. A path never revisits an entity.', [ + ('path_id', 'Ordered legs of the path, each a relationship id followed by > for FORWARD or < for REVERSE.'), + ('origin_entity', 'Logical identity of the entity the path starts from.'), + ('destination_entity', 'Logical identity of the entity the path ends at.'), + ('hop_count', 'Number of relationship steps in the path.'), ('hop', 'Position of this step in the path, from 1.'), + ('key_ordinal', 'Position of the column pair within the join of this step, from 1.'), + ('from_entity', 'Logical identity of the entity the step leaves.'), ('from_column', 'Join column on the entity the step leaves.'), + ('to_entity', 'Logical identity of the entity the step arrives at.'), ('to_column', 'Join column on the entity the step arrives at.'), + ('relationship_id', 'Relationship, which is the join group, this step crosses.'), + ('direction', "FORWARD when crossing from the relationship's source to its target, REVERSE for the other way.")]) }} {% endif %} {% set lineage = named('DataLineage') %}{% set graph_view = named('LineageGraph') %} {% if graph_view and lineage %}{% set d = lineage.column_names %} @@ -70,6 +130,10 @@ FROM {{ lineage.objects.table|qualified }} WHERE {{ d.is_active|ident }} AND {{ UNION SELECT DISTINCT {{ d.job_name|ident }}, coalesce({{ d.target_container|ident }} || '.', '') || {{ d.target_table|ident }}, 'produces' FROM {{ lineage.objects.table|qualified }} WHERE {{ d.is_active|ident }} AND {{ d.job_name|ident }} IS NOT NULL; +{{ describe(graph_view.objects.consumer_view, 'Table-and-job lineage graph: one edge from each active source to its job and from the job to its target.', [ + ('source_node', 'Node the edge leaves: a source table or a job.'), + ('target_node', 'Node the edge arrives at: a job or a target table.'), + ('edge_type', 'Kind of relationship the edge records; produces for data flow.')]) }} {% endif %} {% set run = named('LineageRun') %}{% set latest = named('LineageRunLatest') %} {% if latest and lineage and run %}{% set d = lineage.column_names %}{% set r = run.column_names %} @@ -84,4 +148,9 @@ SELECT lineage_id, source_table, target_table, job_name, run_dts, run_status FRO LEFT JOIN {{ run.objects.table|qualified }} r ON r.{{ r.lineage_id|ident }} = d.{{ d.lineage_id|ident }} WHERE d.{{ d.is_active|ident }} ) ranked WHERE rank = 1; +{{ describe(latest.objects.consumer_view, 'Latest recorded run of each active lineage definition.', [ + ('lineage_id', 'Identifier of the lineage definition.'), ('source_table', 'Table the job reads.'), + ('target_table', 'Table the job writes.'), ('job_name', 'Name of the job that moves the data.'), + ('run_dts', 'When the latest run was recorded; null when never run.'), + ('run_status', 'Outcome of the latest run; null when never run.')]) }} {% endif %} diff --git a/implementation/postgres/modules/semantic/README.md b/implementation/postgres/modules/semantic/README.md index 0c4f9cc..99a7c88 100644 --- a/implementation/postgres/modules/semantic/README.md +++ b/implementation/postgres/modules/semantic/README.md @@ -41,6 +41,8 @@ Composite access objects (`access_objects`) are not registered; the manifest rep `validation.sql.j2` is the `module semantic` check: every base table in a container the product places objects in, keymaps included, is registered in entity metadata. +**The metadata contract (release 2.0.0).** Registration writes the logical identity of each entity (`module.entity`) and, beside it, its location; the key definitions and their ordered key columns; one relationship column pair per join column; one access object per placed object, identified by entity and surface; one access-column mapping row per output column of each consumer view over an entity (the context's `view_columns`, which the view itself selects); and the `StandardDeclaration` rows from `standard`. `RelationshipPaths` returns one row per path step and key position, bounded to four hops. The manifest's last column is `standard_version`. The five checks under [`contract/`](contract/) test `INV-SEMANTIC-016` to `020`; they are declared as `contract` checks and read only registered metadata and the catalogue. + Registration uses the placement and relationships the context carries, never name inference. Structural facts come from `information_schema`; meanings come from the specification. ## Capability bindings diff --git a/implementation/postgres/modules/semantic/contract/00-macros.sql.j2 b/implementation/postgres/modules/semantic/contract/00-macros.sql.j2 new file mode 100644 index 0000000..805d6d8 --- /dev/null +++ b/implementation/postgres/modules/semantic/contract/00-macros.sql.j2 @@ -0,0 +1,9 @@ +{# Shared by the Semantic metadata-contract checks (INV-SEMANTIC-016 to 020). A standard-owned + relation is read through the physical names the build context places; nothing here composes one. #} +{% macro tbl(name) -%}{{ named(name).objects.table|qualified }}{%- endmacro %} +{% macro col(name, column) -%}{{ named(name).column_names[column]|ident }}{%- endmacro %} +{% macro pk(name) -%}{{ named(name).key_columns[0]|ident }}{%- endmacro %} +{# A run of ordinals is 1..n with no gaps and no repeats. #} +{% macro contiguous(alias, ordinal) -%} +count(DISTINCT {{ alias }}.{{ ordinal }}) <> count(*) OR min({{ alias }}.{{ ordinal }}) <> 1 OR max({{ alias }}.{{ ordinal }}) <> count(*) +{%- endmacro %} diff --git a/implementation/postgres/modules/semantic/contract/access-columns.sql.j2 b/implementation/postgres/modules/semantic/contract/access-columns.sql.j2 new file mode 100644 index 0000000..cb088f5 --- /dev/null +++ b/implementation/postgres/modules/semantic/contract/access-columns.sql.j2 @@ -0,0 +1,42 @@ +{# INV-SEMANTIC-019: every consumer view has exactly one mapping row per output column, ordered 1..n, + and every non-derived source exists. Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT a.{{ q.col('AccessObject', 'object_identity') }} || '.' || i.column_name AS identity, 'output column has no mapping' AS defect +FROM {{ q.tbl('AccessObject') }} a +INNER JOIN information_schema.columns i + ON i.table_schema = a.{{ q.col('AccessObject', 'container_name') }} AND i.table_name = a.{{ q.col('AccessObject', 'object_name') }} +WHERE a.{{ q.col('AccessObject', 'is_active') }} AND a.{{ q.col('AccessObject', 'object_type') }} = 'CONSUMER_VIEW' + AND EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = a.{{ q.col('AccessObject', 'represents_entity') }}) + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('AccessColumn') }} m + WHERE m.{{ q.col('AccessColumn', 'access_object_id') }} = a.{{ q.pk('AccessObject') }} + AND m.{{ q.col('AccessColumn', 'column_name') }} = i.column_name AND m.{{ q.col('AccessColumn', 'is_active') }}) +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }} || '.' || m.{{ q.col('AccessColumn', 'column_name') }}, + 'mapping names a column the object does not have, or is repeated' +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} + AND (NOT EXISTS (SELECT 1 FROM information_schema.columns i + WHERE i.table_schema = a.{{ q.col('AccessObject', 'container_name') }} AND i.table_name = a.{{ q.col('AccessObject', 'object_name') }} + AND i.column_name = m.{{ q.col('AccessColumn', 'column_name') }}) + OR (SELECT count(*) FROM {{ q.tbl('AccessColumn') }} d + WHERE d.{{ q.col('AccessColumn', 'access_object_id') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} + AND d.{{ q.col('AccessColumn', 'column_name') }} = m.{{ q.col('AccessColumn', 'column_name') }} + AND d.{{ q.col('AccessColumn', 'is_active') }}) > 1) +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }}, 'mapping ordinals do not run 1..n' +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} +GROUP BY a.{{ q.col('AccessObject', 'object_identity') }} +HAVING {{ q.contiguous('m', q.col('AccessColumn', 'ordinal')) }} +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }} || '.' || m.{{ q.col('AccessColumn', 'column_name') }}, 'non-derived mapping source is absent' +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} AND m.{{ q.col('AccessColumn', 'mapping_kind') }} <> 'DERIVED' + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e + INNER JOIN information_schema.columns i + ON i.table_schema = e.{{ q.col('EntityMetadata', 'container_name') }} AND i.table_name = e.{{ q.col('EntityMetadata', 'table_name') }} + WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = m.{{ q.col('AccessColumn', 'source_entity') }} + AND i.column_name = m.{{ q.col('AccessColumn', 'source_column') }}) diff --git a/implementation/postgres/modules/semantic/contract/composition.sql.j2 b/implementation/postgres/modules/semantic/contract/composition.sql.j2 new file mode 100644 index 0000000..281f55f --- /dev/null +++ b/implementation/postgres/modules/semantic/contract/composition.sql.j2 @@ -0,0 +1,40 @@ +{# INV-SEMANTIC-020: a composite has exactly one anchor, its members are ordered 1..n, and every other member + joins to a prior member through a registered relationship in a stated direction. + Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT c.{{ q.col('AccessComposition', 'composite_object') }}::TEXT AS identity, 'composite does not have exactly one anchor' AS defect +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} +GROUP BY c.{{ q.col('AccessComposition', 'composite_object') }} +HAVING count(*) FILTER (WHERE c.{{ q.col('AccessComposition', 'member_role') }} = 'ANCHOR') <> 1 +UNION ALL +SELECT c.{{ q.col('AccessComposition', 'composite_object') }}, 'member order does not run 1..n' +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} +GROUP BY c.{{ q.col('AccessComposition', 'composite_object') }} +HAVING {{ q.contiguous('c', q.col('AccessComposition', 'member_seq')) }} +UNION ALL +SELECT c.{{ q.col('AccessComposition', 'composite_object') }} || '.' || c.{{ q.col('AccessComposition', 'member_entity') }}, + 'member does not join to a prior member through its relationship' +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} AND c.{{ q.col('AccessComposition', 'member_role') }} <> 'ANCHOR' + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('TableRelationship') }} r + INNER JOIN {{ q.tbl('AccessComposition') }} p + ON p.{{ q.col('AccessComposition', 'composite_object') }} = c.{{ q.col('AccessComposition', 'composite_object') }} + AND p.{{ q.col('AccessComposition', 'member_seq') }} < c.{{ q.col('AccessComposition', 'member_seq') }} + WHERE r.{{ q.pk('TableRelationship') }} = c.{{ q.col('AccessComposition', 'via_relationship_id') }} + AND ((c.{{ q.col('AccessComposition', 'via_direction') }} = 'FORWARD' + AND r.{{ q.col('TableRelationship', 'source_entity') }} = p.{{ q.col('AccessComposition', 'member_entity') }} + AND r.{{ q.col('TableRelationship', 'target_entity') }} = c.{{ q.col('AccessComposition', 'member_entity') }}) + OR (c.{{ q.col('AccessComposition', 'via_direction') }} = 'REVERSE' + AND r.{{ q.col('TableRelationship', 'target_entity') }} = p.{{ q.col('AccessComposition', 'member_entity') }} + AND r.{{ q.col('TableRelationship', 'source_entity') }} = c.{{ q.col('AccessComposition', 'member_entity') }}))) +UNION ALL +SELECT c.{{ q.col('AccessComposition', 'composite_object') }} || '.' || c.{{ q.col('AccessComposition', 'member_entity') }}, + 'anchor carries a join, or the member names an unregistered entity' +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} + AND ((c.{{ q.col('AccessComposition', 'member_role') }} = 'ANCHOR' + AND (c.{{ q.col('AccessComposition', 'via_relationship_id') }} IS NOT NULL OR c.{{ q.col('AccessComposition', 'via_direction') }} IS NOT NULL)) + OR NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e + WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = c.{{ q.col('AccessComposition', 'member_entity') }})) diff --git a/implementation/postgres/modules/semantic/contract/identity.sql.j2 b/implementation/postgres/modules/semantic/contract/identity.sql.j2 new file mode 100644 index 0000000..e3da7b8 --- /dev/null +++ b/implementation/postgres/modules/semantic/contract/identity.sql.j2 @@ -0,0 +1,27 @@ +{# INV-SEMANTIC-016: identities follow the grammar and carry no placement. Every row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }}::TEXT AS identity, 'entity identity is not module.entity in its own module' AS defect +FROM {{ q.tbl('EntityMetadata') }} e +WHERE e.{{ q.col('EntityMetadata', 'is_active') }} + AND (e.{{ q.col('EntityMetadata', 'entity_name') }} !~ '^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*$' + OR e.{{ q.col('EntityMetadata', 'entity_name') }} NOT LIKE lower(e.{{ q.col('EntityMetadata', 'module_name') }}) || '.%') +UNION ALL +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }}::TEXT, 'entity identity is registered more than once' +FROM {{ q.tbl('EntityMetadata') }} e +WHERE e.{{ q.col('EntityMetadata', 'is_active') }} +GROUP BY e.{{ q.col('EntityMetadata', 'entity_name') }} +HAVING count(*) > 1 +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }}::TEXT, 'access identity breaks the grammar' +FROM {{ q.tbl('AccessObject') }} a +WHERE a.{{ q.col('AccessObject', 'is_active') }} + AND NOT (a.{{ q.col('AccessObject', 'object_identity') }} ~ '^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*:(base|governed|current|history)$' + OR a.{{ q.col('AccessObject', 'object_identity') }} ~ '^[a-z][a-z0-9_]*:[a-z][a-z0-9_]*$' + OR a.{{ q.col('AccessObject', 'object_identity') }} ~ '^composite:[a-z][a-z0-9_]*$') +UNION ALL +SELECT a.{{ q.col('AccessObject', 'object_identity') }}::TEXT, 'single-entity identity does not match its entity and surface' +FROM {{ q.tbl('AccessObject') }} a +WHERE a.{{ q.col('AccessObject', 'is_active') }} AND a.{{ q.col('AccessObject', 'represents_entity') }} IS NOT NULL + AND a.{{ q.col('AccessObject', 'object_identity') }} <> a.{{ q.col('AccessObject', 'represents_entity') }} || ':' || + CASE a.{{ q.col('AccessObject', 'object_type') }} WHEN 'TABLE' THEN 'base' WHEN 'BASE_VIEW' THEN 'governed' + WHEN 'CONSUMER_VIEW' THEN 'current' ELSE 'history' END diff --git a/implementation/postgres/modules/semantic/contract/keys.sql.j2 b/implementation/postgres/modules/semantic/contract/keys.sql.j2 new file mode 100644 index 0000000..7871e9a --- /dev/null +++ b/implementation/postgres/modules/semantic/contract/keys.sql.j2 @@ -0,0 +1,34 @@ +{# INV-SEMANTIC-017: every entity has a key; the columns of a key are ordered 1..n and exist on the entity. + Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }}::TEXT AS identity, 'entity has no key definition' AS defect +FROM {{ q.tbl('EntityMetadata') }} e +WHERE e.{{ q.col('EntityMetadata', 'is_active') }} + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('KeyDefinition') }} k + WHERE k.{{ q.col('KeyDefinition', 'entity_id') }} = e.{{ q.pk('EntityMetadata') }} AND k.{{ q.col('KeyDefinition', 'is_active') }}) +UNION ALL +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }} || '#' || k.{{ q.col('KeyDefinition', 'key_name') }}, 'key has no columns' +FROM {{ q.tbl('KeyDefinition') }} k +INNER JOIN {{ q.tbl('EntityMetadata') }} e ON e.{{ q.pk('EntityMetadata') }} = k.{{ q.col('KeyDefinition', 'entity_id') }} +WHERE k.{{ q.col('KeyDefinition', 'is_active') }} + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('KeyColumn') }} c + WHERE c.{{ q.col('KeyColumn', 'key_definition_id') }} = k.{{ q.pk('KeyDefinition') }} AND c.{{ q.col('KeyColumn', 'is_active') }}) +UNION ALL +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }} || '#' || k.{{ q.col('KeyDefinition', 'key_name') }}, 'key column ordinals do not run 1..n' +FROM {{ q.tbl('KeyDefinition') }} k +INNER JOIN {{ q.tbl('EntityMetadata') }} e ON e.{{ q.pk('EntityMetadata') }} = k.{{ q.col('KeyDefinition', 'entity_id') }} +INNER JOIN {{ q.tbl('KeyColumn') }} c ON c.{{ q.col('KeyColumn', 'key_definition_id') }} = k.{{ q.pk('KeyDefinition') }} +WHERE k.{{ q.col('KeyDefinition', 'is_active') }} AND c.{{ q.col('KeyColumn', 'is_active') }} +GROUP BY e.{{ q.col('EntityMetadata', 'entity_name') }}, k.{{ q.col('KeyDefinition', 'key_name') }} +HAVING {{ q.contiguous('c', q.col('KeyColumn', 'ordinal')) }} +UNION ALL +SELECT e.{{ q.col('EntityMetadata', 'entity_name') }} || '#' || k.{{ q.col('KeyDefinition', 'key_name') }} || '.' || c.{{ q.col('KeyColumn', 'column_name') }}, + 'key column is absent from its entity' +FROM {{ q.tbl('KeyColumn') }} c +INNER JOIN {{ q.tbl('KeyDefinition') }} k ON k.{{ q.pk('KeyDefinition') }} = c.{{ q.col('KeyColumn', 'key_definition_id') }} +INNER JOIN {{ q.tbl('EntityMetadata') }} e ON e.{{ q.pk('EntityMetadata') }} = k.{{ q.col('KeyDefinition', 'entity_id') }} +WHERE c.{{ q.col('KeyColumn', 'is_active') }} + AND NOT EXISTS (SELECT 1 FROM information_schema.columns i + WHERE i.table_schema = e.{{ q.col('EntityMetadata', 'container_name') }} + AND i.table_name = e.{{ q.col('EntityMetadata', 'table_name') }} + AND i.column_name = c.{{ q.col('KeyColumn', 'column_name') }}) diff --git a/implementation/postgres/modules/semantic/contract/relationships.sql.j2 b/implementation/postgres/modules/semantic/contract/relationships.sql.j2 new file mode 100644 index 0000000..0c137e1 --- /dev/null +++ b/implementation/postgres/modules/semantic/contract/relationships.sql.j2 @@ -0,0 +1,35 @@ +{# INV-SEMANTIC-018: every relationship has at least one column pair; pairs are ordered 1..n and + their columns exist on the entities. Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT r.{{ q.col('TableRelationship', 'relationship_name') }}::TEXT AS identity, 'relationship has no column pair' AS defect +FROM {{ q.tbl('TableRelationship') }} r +WHERE r.{{ q.col('TableRelationship', 'is_active') }} + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('RelationshipColumn') }} c + WHERE c.{{ q.col('RelationshipColumn', 'relationship_id') }} = r.{{ q.pk('TableRelationship') }} + AND c.{{ q.col('RelationshipColumn', 'is_active') }}) +UNION ALL +SELECT r.{{ q.col('TableRelationship', 'relationship_name') }}, 'relationship ordinals do not run 1..n' +FROM {{ q.tbl('TableRelationship') }} r +INNER JOIN {{ q.tbl('RelationshipColumn') }} c ON c.{{ q.col('RelationshipColumn', 'relationship_id') }} = r.{{ q.pk('TableRelationship') }} +WHERE r.{{ q.col('TableRelationship', 'is_active') }} AND c.{{ q.col('RelationshipColumn', 'is_active') }} +GROUP BY r.{{ q.col('TableRelationship', 'relationship_name') }} +HAVING {{ q.contiguous('c', q.col('RelationshipColumn', 'ordinal')) }} +UNION ALL +SELECT r.{{ q.col('TableRelationship', 'relationship_name') }}, 'relationship names an unregistered entity' +FROM {{ q.tbl('TableRelationship') }} r +WHERE r.{{ q.col('TableRelationship', 'is_active') }} + AND (NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'source_entity') }}) + OR NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'target_entity') }})) +UNION ALL +SELECT r.{{ q.col('TableRelationship', 'relationship_name') }} || '.' || c.{{ q.col('RelationshipColumn', 'ordinal') }}, 'relationship column is absent from its entity' +FROM {{ q.tbl('RelationshipColumn') }} c +INNER JOIN {{ q.tbl('TableRelationship') }} r ON r.{{ q.pk('TableRelationship') }} = c.{{ q.col('RelationshipColumn', 'relationship_id') }} +INNER JOIN {{ q.tbl('EntityMetadata') }} s ON s.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'source_entity') }} +INNER JOIN {{ q.tbl('EntityMetadata') }} t ON t.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'target_entity') }} +WHERE c.{{ q.col('RelationshipColumn', 'is_active') }} + AND (NOT EXISTS (SELECT 1 FROM information_schema.columns i + WHERE i.table_schema = s.{{ q.col('EntityMetadata', 'container_name') }} AND i.table_name = s.{{ q.col('EntityMetadata', 'table_name') }} + AND i.column_name = c.{{ q.col('RelationshipColumn', 'source_column') }}) + OR NOT EXISTS (SELECT 1 FROM information_schema.columns i + WHERE i.table_schema = t.{{ q.col('EntityMetadata', 'container_name') }} AND i.table_name = t.{{ q.col('EntityMetadata', 'table_name') }} + AND i.column_name = c.{{ q.col('RelationshipColumn', 'target_column') }})) diff --git a/implementation/postgres/patterns/object-placement/01-containers.sql.j2 b/implementation/postgres/patterns/object-placement/01-containers.sql.j2 index 9c8f243..6b29700 100644 --- a/implementation/postgres/patterns/object-placement/01-containers.sql.j2 +++ b/implementation/postgres/patterns/object-placement/01-containers.sql.j2 @@ -3,4 +3,5 @@ CREATE EXTENSION IF NOT EXISTS btree_gist; {% for c in containers if c.holds_objects and not c.adopted %} CREATE SCHEMA {{ c.name|ident }}; +COMMENT ON SCHEMA {{ c.name|ident }} IS {{ ('Container of the ' ~ product.name ~ ' data product, as the organisation profile places it.')|literal }}; {%- endfor %} diff --git a/implementation/postgres/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 b/implementation/postgres/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 index 4e3cb28..0cc9a0e 100644 --- a/implementation/postgres/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 +++ b/implementation/postgres/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 @@ -6,10 +6,13 @@ {% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} {% for x in entities + relations if x.relation == 'table' and (x.defined|default(true)) and x.objects.consumer_view is defined %} CREATE VIEW {{ x.objects.consumer_view|qualified }} AS -SELECT {{ m.cols(x.columns|rejectattr('name', 'in', ('valid_to_dts', 'transaction_to_dts'))|map(attribute='physical_name')) }} +SELECT {{ m.cols(x.view_columns|map(attribute='physical_name')) }} FROM {{ x.objects.table|qualified }} WHERE {{ m.current(x) }}; COMMENT ON VIEW {{ x.objects.consumer_view|qualified }} IS {{ ('Current consumer surface: ' ~ x.description)|literal }}; +{% for vc in x.view_columns %}{% set col = x.columns|selectattr('physical_name', 'equalto', vc.physical_name)|first %} +COMMENT ON COLUMN {{ x.objects.consumer_view|qualified }}.{{ vc.physical_name|ident }} IS {{ (col.description or col.name)|literal }}; +{% endfor %} {% endfor %} {% for x in entities + relations if x.relation == 'table' and (x.defined|default(true)) and x.objects.function is defined %} CREATE FUNCTION {{ x.objects.function|qualified }}(valid_at TIMESTAMPTZ{% if x.bitemporal %}, known_at TIMESTAMPTZ{% endif %}) @@ -17,4 +20,5 @@ RETURNS SETOF {{ x.objects.table|qualified }} LANGUAGE SQL STABLE SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $body$ SELECT * FROM {{ x.objects.table|qualified }} WHERE {{ m.at(x) }}; $body$; +COMMENT ON FUNCTION {{ x.objects.function|qualified }}(TIMESTAMPTZ{% if x.bitemporal %}, TIMESTAMPTZ{% endif %}) IS {{ ('Point-in-time reconstruction of ' ~ x.description ~ (' Takes the valid-time and known-time instants.' if x.bitemporal else ' Takes the valid-time instant.'))|literal }}; {% endfor %} diff --git a/implementation/postgres/patterns/validation/03-relationship-reference.sql.j2 b/implementation/postgres/patterns/validation/03-relationship-reference.sql.j2 index 99c7706..ed64169 100644 --- a/implementation/postgres/patterns/validation/03-relationship-reference.sql.j2 +++ b/implementation/postgres/patterns/validation/03-relationship-reference.sql.j2 @@ -1,7 +1,8 @@ {# Rendered per relationship, with `relationship`. Current-surface referential coverage, not - historical proof: every returned row is a current reference with no current target. A generic - reference is checked only for the rows its discriminator points at this target. #} + historical proof: every returned row is a current reference with no current target. A join of + several columns resolves only when every pair matches. A generic reference is checked only for + the rows its discriminator points at this target. #} {% set r = relationship %}{% set source = named(r.source) %} -SELECT s.{{ r.source_column|ident }} FROM {{ (r.source_view or r.source_table)|qualified }} s -LEFT JOIN {{ (r.target_view or r.target_table)|qualified }} t ON s.{{ r.source_column|ident }} = t.{{ r.target_columns[0]|ident }} -WHERE {% if r.discriminator %}s.{{ source.column_names[r.discriminator]|ident }} = {{ r.discriminator_value|literal }} AND {% endif %}{% if not r.mandatory %}s.{{ r.source_column|ident }} IS NOT NULL AND {% endif %}t.{{ r.target_columns[0]|ident }} IS NULL +SELECT {% for p in r.pairs %}s.{{ p.source_column|ident }}{% if not loop.last %}, {% endif %}{% endfor %} FROM {{ (r.source_view or r.source_table)|qualified }} s +LEFT JOIN {{ (r.target_view or r.target_table)|qualified }} t ON {% for p in r.pairs %}s.{{ p.source_column|ident }} = t.{{ p.target_column|ident }}{% if not loop.last %} AND {% endif %}{% endfor %} +WHERE {% if r.discriminator %}s.{{ source.column_names[r.discriminator]|ident }} = {{ r.discriminator_value|literal }} AND {% endif %}{% if not r.mandatory %}{% for p in r.pairs %}s.{{ p.source_column|ident }} IS NOT NULL AND {% endfor %}{% endif %}t.{{ r.pairs[0].target_column|ident }} IS NULL diff --git a/implementation/postgres/patterns/validation/04-trust-map.sql.j2 b/implementation/postgres/patterns/validation/04-trust-map.sql.j2 index e294736..150b192 100644 --- a/implementation/postgres/patterns/validation/04-trust-map.sql.j2 +++ b/implementation/postgres/patterns/validation/04-trust-map.sql.j2 @@ -69,4 +69,24 @@ SELECT p.product_code, p.producer_id, NULL, 'PRODUCT', p.product_code, 'no-evide 0, 0, NULL, 0, 0, 0, 0, 0, 'No published area evidence.', 'Run the validator before reporting confidence.', NULL, FALSE FROM product p WHERE NOT EXISTS (SELECT 1 FROM latest l WHERE l.product_code = p.product_code); +COMMENT ON VIEW {{ view.objects.consumer_view|qualified }} IS 'Trust map: per area, the latest evidence from the designated producer, read at unknown confidence once expired. Read it before relying on any other object.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.product_code IS 'Data product the evidence describes.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.producer_id IS 'Validation producer whose evidence is read; null when none is designated.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.run_id IS 'Validation run that supplied the area evidence; null when none exists.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.scope_kind IS 'Kind of scope the evidence covers, for example MODULE, ENTITY or PRODUCT.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.scope_id IS 'Identifier of the scope the evidence covers.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.area_status IS 'Outcome of the area: pass, fail, partial or no-evidence.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.confidence IS 'Trust confidence for the area; unknown once the evidence has expired.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.checks_expected IS 'Number of checks the area should have run.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.checks_ran IS 'Number of checks the area actually ran.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.coverage IS 'Fraction of expected checks that ran; null when none were expected.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.passed_count IS 'Number of checks that passed.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.failed_count IS 'Number of checks that failed.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.error_count IS 'Number of checks that errored before producing a result.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.critical_failure_count IS 'Number of failed checks of critical severity.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.error_failure_count IS 'Number of failed or errored checks counted against trust.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.open_gaps IS 'Known gaps in the evidence; states staleness when the evidence has expired.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.recommended_action IS 'Next step to restore or confirm trust.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.completed_dts IS 'When the area evidence was completed; null when none exists.'; +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.derived IS 'True when the entry is derived from a legacy run that published no area records.'; {% endif %} diff --git a/implementation/postgres/patterns/validation/05-comment-coverage.sql.j2 b/implementation/postgres/patterns/validation/05-comment-coverage.sql.j2 new file mode 100644 index 0000000..7607189 --- /dev/null +++ b/implementation/postgres/patterns/validation/05-comment-coverage.sql.j2 @@ -0,0 +1,31 @@ +{# MetadataCoverageCheck, catalogue-wide: every row returned is a container, object or column with no comment. + Sweeps every container the product places objects in, so an object the specification never mentions is + caught too. Objects owned by an extension are not the product's to document and are skipped. #} +{% set in_containers -%} +{%- for c in containers if c.holds_objects and not c.adopted %}{{ c.name|literal }}{% if not loop.last %}, {% endif %}{% endfor -%} +{%- endset %} +SELECT n.nspname AS container_name, 'schema' AS object_kind, n.nspname AS object_name, NULL::TEXT AS column_name +FROM pg_catalog.pg_namespace n +WHERE n.nspname IN ({{ in_containers }}) + AND nullif(btrim(pg_catalog.obj_description(n.oid, 'pg_namespace')), '') IS NULL +UNION ALL +SELECT n.nspname, CASE c.relkind WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'f' THEN 'foreign table' ELSE 'table' END, + c.relname, NULL +FROM pg_catalog.pg_class c INNER JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname IN ({{ in_containers }}) AND c.relkind IN ('r', 'p', 'v', 'm', 'f') + AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_depend d WHERE d.classid = 'pg_catalog.pg_class'::regclass AND d.objid = c.oid AND d.deptype = 'e') + AND nullif(btrim(pg_catalog.obj_description(c.oid, 'pg_class')), '') IS NULL +UNION ALL +SELECT n.nspname, 'column', c.relname, a.attname +FROM pg_catalog.pg_attribute a +INNER JOIN pg_catalog.pg_class c ON c.oid = a.attrelid +INNER JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace +WHERE n.nspname IN ({{ in_containers }}) AND c.relkind IN ('r', 'p', 'v', 'm', 'f') AND a.attnum > 0 AND NOT a.attisdropped + AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_depend d WHERE d.classid = 'pg_catalog.pg_class'::regclass AND d.objid = c.oid AND d.deptype = 'e') + AND nullif(btrim(pg_catalog.col_description(c.oid, a.attnum)), '') IS NULL +UNION ALL +SELECT n.nspname, 'function', p.proname, NULL +FROM pg_catalog.pg_proc p INNER JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace +WHERE n.nspname IN ({{ in_containers }}) + AND NOT EXISTS (SELECT 1 FROM pg_catalog.pg_depend d WHERE d.classid = 'pg_catalog.pg_proc'::regclass AND d.objid = p.oid AND d.deptype = 'e') + AND nullif(btrim(pg_catalog.obj_description(p.oid, 'pg_proc')), '') IS NULL diff --git a/implementation/postgres/patterns/validation/README.md b/implementation/postgres/patterns/validation/README.md index 32ce267..2d9d7ed 100644 --- a/implementation/postgres/patterns/validation/README.md +++ b/implementation/postgres/patterns/validation/README.md @@ -24,8 +24,14 @@ Each check is a query whose every row is a defect. The `Checks:` block of [PLATF | `relationship reference` | `03-relationship-reference.sql.j2` | each entry in `relationships` (`relationship`) | current source references with no current target, read through the consumer views (the tables where no view is placed); an optional reference may be null | | `module semantic` | [`modules/semantic/validation.sql.j2`](../../modules/semantic/README.md) | the Semantic module | unregistered base tables | | `module memory` | [`modules/memory/validation.sql.j2`](../../modules/memory/README.md) | the Memory module | modules below the capture floor | - -The reference check is current-surface coverage, not historical proof, and compares the first target column only. +| `contract identity` | [`modules/semantic/contract/identity.sql.j2`](../../modules/semantic/README.md) | the Semantic module | identities that break the grammar, carry the wrong module qualifier or are registered twice | +| `contract keys` | `modules/semantic/contract/keys.sql.j2` | the Semantic module | entities with no key, keys with no columns, ordinals that do not run 1 to n, and key columns the entity lacks | +| `contract relationships` | `modules/semantic/contract/relationships.sql.j2` | the Semantic module | relationships with no column pair, gaps in the ordinals, unregistered entities and columns the entities lack | +| `contract access-columns` | `modules/semantic/contract/access-columns.sql.j2` | the Semantic module | consumer-view columns with no mapping row, mappings for columns the view lacks, repeated or gapped ordinals, and absent sources | +| `contract composition` | `modules/semantic/contract/composition.sql.j2` | the Semantic module | composites without exactly one anchor, gapped member order, and members not joined to a prior member through a registered relationship | +| `product comment-coverage` | `05-comment-coverage.sql.j2` | the product | every container, object and column of the product's containers with no comment (the catalogue-wide `MetadataCoverageCheck`) | + +The reference check is current-surface coverage, not historical proof. A join of several columns resolves only when every pair matches, so the check tests all of them. ## Trust map diff --git a/implementation/teradata/PLATFORM_PROFILE.md b/implementation/teradata/PLATFORM_PROFILE.md index f85c76c..b2c999e 100644 --- a/implementation/teradata/PLATFORM_PROFILE.md +++ b/implementation/teradata/PLATFORM_PROFILE.md @@ -249,6 +249,12 @@ Binding: - relationship reference: patterns/validation/03-relationship-reference.sql.j2 module semantic: modules/semantic/validation.sql.j2 module memory: modules/memory/validation.sql.j2 + contract identity: modules/semantic/contract/identity.sql.j2 + contract keys: modules/semantic/contract/keys.sql.j2 + contract relationships: modules/semantic/contract/relationships.sql.j2 + contract access-columns: modules/semantic/contract/access-columns.sql.j2 + contract composition: modules/semantic/contract/composition.sql.j2 + product comment-coverage: patterns/validation/05-comment-coverage.sql.j2 ``` ``` diff --git a/implementation/teradata/modules/search/02-similarity.sql.j2 b/implementation/teradata/modules/search/02-similarity.sql.j2 index e930d64..352ae43 100644 --- a/implementation/teradata/modules/search/02-similarity.sql.j2 +++ b/implementation/teradata/modules/search/02-similarity.sql.j2 @@ -14,6 +14,7 @@ A database's creator is not granted CREATE PROCEDURE on it, so where the profile names the `Deploying user`, that user is granted it on each database that holds a procedure. #} +{% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} {#- A text literal written inside the dynamic SQL's own literal: its quotes doubled twice. -#} {% macro inner(text) -%}{{ (text|literal)|replace("'", "''") }}{%- endmacro %} {% set measures = {'cosine': 'cosine', 'dot': 'dotproduct', 'euclidean': 'euclidean', 'manhattan': 'manhattan'} %} @@ -60,4 +61,5 @@ BEGIN PREPARE prepared_query FROM query_text; OPEN results; END; +COMMENT ON PROCEDURE {{ b.objects.procedure|qualified }} IS {{ m.comment('Similarity search over the ' ~ b.name ~ ' embedding of ' ~ b.entity ~ '. Takes a query vector and a result limit (maximum 1000); returns entity_id and similarity.') }}; {% endfor %} diff --git a/implementation/teradata/modules/search/README.md b/implementation/teradata/modules/search/README.md index cc6d958..617d108 100644 --- a/implementation/teradata/modules/search/README.md +++ b/implementation/teradata/modules/search/README.md @@ -26,7 +26,7 @@ A `Vector[n]` binds to `VARBYTE(n × 8)`: its elements as little-endian 8-byte f - returns no rows unless the query vector is exactly `n × 8` bytes and the limit is positive; - keeps scores at or above the declared threshold and returns one result set of `(entity_id, similarity)`, ordered by score, then identifier, with the limit clamped to 0..1000. -`TD_VECTORDISTANCE` is a table operator: it cannot sit in a view or a SQL function, and it takes no bind or macro parameter inside its `ON` clauses. So the procedure writes the query vector into the request as a hex literal, left-padded because `FROM_BYTES` drops leading zero digits, and runs it as dynamic SQL. A database's creator is not granted `CREATE PROCEDURE` on it, so where the profile names the `Deploying user`, the phase grants it on each database that holds a procedure. The access layer grants the read and agent tiers `EXECUTE PROCEDURE`; the procedure returns identifiers only, and join-back to Domain is the caller's (the cookbook's similarity recipe calls it with a `?` vector). +`TD_VECTORDISTANCE` is a table operator: it cannot sit in a view or a SQL function, and it takes no bind or macro parameter inside its `ON` clauses. So the procedure writes the query vector into the request as a hex literal, left-padded because `FROM_BYTES` drops leading zero digits, and runs it as dynamic SQL. Each procedure carries a `COMMENT ON PROCEDURE`. A database's creator is not granted `CREATE PROCEDURE` on it, so where the profile names the `Deploying user`, the phase grants it on each database that holds a procedure. The access layer grants the read and agent tiers `EXECUTE PROCEDURE`; the procedure returns identifiers only, and join-back to Domain is the caller's (the cookbook's similarity recipe calls it with a `?` vector). Vectors come from the encoder the specification declares, outside the database or in it; no encoder is substituted. An in-database encoder's output must be converted to 8-byte floats first: `mldb.ONNXEmbeddings`' `VARBYTE` output is 4-byte floats. Validate query vectors for finite elements and nonzero norm at the encoder and query boundary. diff --git a/implementation/teradata/modules/semantic/02-registration.sql.j2 b/implementation/teradata/modules/semantic/02-registration.sql.j2 index c165dce..380719a 100644 --- a/implementation/teradata/modules/semantic/02-registration.sql.j2 +++ b/implementation/teradata/modules/semantic/02-registration.sql.j2 @@ -2,6 +2,12 @@ relationship catalogues, access objects, primary objects, the orientation sequence, metrics, synonyms and naming standards. Every row is read from the build context. #} {% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} +{# The generated id of one key of an entity, found by the entity's identity and the key's name. #} +{% macro key_ref(keydef, meta, x, k) -%} +(SELECT d.{{ keydef.key_columns[0]|ident }} FROM {{ keydef.objects.table|qualified }} d + WHERE d.{{ keydef.column_names.key_name|ident }} = {{ k.name|literal }} + AND d.{{ keydef.column_names.entity_id|ident }} = {{ m.ref(meta, {'entity_name': x.logical_id}) }}) +{%- endmacro %} {% set registry = named('DataProductRegistry') %}{% set manifest = named('DataProductManifest') %} {% if registry and manifest %} {{ m.insert(registry, { @@ -21,21 +27,37 @@ 'deployment_status': 'DEPLOYED'}) }} {% endfor %} {% endif %} +{% set declaration = named('StandardDeclaration') %} +{% if declaration %} +{% for d in standard.declarations %} +{{ m.insert(declaration, {'product_id': product.code, 'component_kind': d.kind, 'component_name': d.name, + 'platform': standard.platform, 'standard_version': standard.version}) }} +{% endfor %} +{% endif %} {% set meta = named('EntityMetadata') %}{% set colmeta = named('ColumnMetadata') %} {% for x in entities + relations if x.objects.table is defined %} {% set t = x.objects.table %} {% if meta %} {{ m.insert(meta, { - 'entity_name': x.name, 'logical_name': x.name, 'entity_description': x.description, + 'entity_name': x.logical_id, 'logical_name': x.name, 'entity_description': x.description, 'entity_kind': x.kind|upper, 'module_name': x.module|upper, 'container_name': t.container, 'table_name': t.name, 'view_name': (x.objects.consumer_view.name if x.objects.consumer_view is defined else none), - 'surrogate_key_column': (x.surrogate_columns|join(', ') or none), - 'natural_key_column': (x.natural_columns|join(', ') or none), 'temporal_pattern': x.profile, 'current_flag_column': (x.column_names.is_current if x.history else none), 'deleted_flag_column': (x.column_names.is_deleted if x.supports_deletion else none)}) }} {% endif %} +{% set keydef = named('KeyDefinition') %}{% set keycol = named('KeyColumn') %} +{% if keydef and keycol and meta %} +{% for k in x.key_defs %} +{{ m.insert(keydef, {'key_role': k.role, 'key_name': k.name}, + {'entity_id': m.ref(meta, {'entity_name': x.logical_id})}) }} +{% for column in k.columns %} +{{ m.insert(keycol, {'ordinal': loop.index, 'column_name': column}, + {'key_definition_id': key_ref(keydef, meta, x, k)}) }} +{% endfor %} +{% endfor %} +{% endif %} {% if colmeta %} {% for col in x.columns %} {{ m.insert(colmeta, { @@ -50,35 +72,47 @@ {% endfor %} {% set rel = named('TableRelationship') %} {% if rel %} +{% set relcol = named('RelationshipColumn') %} {% for r in relationships if r.source_table and r.target_table %} {{ m.insert(rel, { - 'relationship_name': r.source ~ '.' ~ r.source_attribute, - 'source_container': r.source_table.container, 'source_table': r.source_table.name, - 'source_column': r.source_column, 'target_container': r.target_table.container, - 'target_table': r.target_table.name, 'target_column': r.target_columns|join(', '), + 'relationship_name': r.name, 'source_entity': r.source_identity, 'target_entity': r.target_identity, 'relationship_type': 'FOREIGN_KEY', 'cardinality': 'MANY_TO_ONE', 'is_mandatory': r.mandatory}) }} +{% if relcol %} +{% for pair in r.pairs %} +{{ m.insert(relcol, {'ordinal': pair.ordinal, 'source_column': pair.source_column, 'target_column': pair.target_column}, + {'relationship_id': m.ref(rel, {'relationship_name': r.name})}) }} {% endfor %} {% endif %} -{% set access = named('AccessObject') %} +{% endfor %} +{% endif %} +{% set access = named('AccessObject') %}{% set acccol = named('AccessColumn') %} +{% set surface = {'table': 'base', 'base_view': 'governed', 'consumer_view': 'current', 'function': 'history', 'procedure': 'history'} %} {% if access %} {% for x in entities + relations %} {% for role, obj in x.objects|dictsort if obj and m.member(access, 'object_type', role|upper) %} {{ m.insert(access, { - 'container_name': obj.container, 'object_name': obj.name, 'object_type': role|upper, + 'object_identity': x.logical_id ~ ':' ~ surface[role], 'container_name': obj.container, 'object_name': obj.name, + 'object_type': role|upper, 'access_role': ('BASE' if role in ('table', 'base_view') else 'PASSTHROUGH'), - 'represents_entity': x.name, + 'represents_entity': x.logical_id, 'is_agent_consumable': role in ('consumer_view', 'function'), 'is_primary': role == 'consumer_view', - 'resolves_to_object': (x.objects.table.container ~ '.' ~ x.objects.table.name - if role != 'table' and x.objects.table is defined else none), + 'resolves_to_object': (x.logical_id ~ ':base' if role != 'table' and x.objects.table is defined else none), 'access_note': x.description}) }} +{% if role == 'consumer_view' and acccol %} +{% for vc in x.view_columns %} +{{ m.insert(acccol, {'ordinal': vc.ordinal, 'column_name': vc.physical_name, 'source_entity': x.logical_id, + 'source_column': vc.physical_name, 'mapping_kind': 'PASSTHROUGH'}, + {'access_object_id': m.ref(access, {'object_identity': x.logical_id ~ ':current'})}) }} +{% endfor %} +{% endif %} {% endfor %} {% endfor %} {% for b in embeddings %} {% for role, obj in b.objects|dictsort if role in ('function', 'procedure') %} {{ m.insert(access, { - 'container_name': obj.container, 'object_name': obj.name, - 'object_type': role|upper, 'access_role': 'PASSTHROUGH', 'represents_entity': b.entity, + 'object_identity': b.logical_id, 'container_name': obj.container, 'object_name': obj.name, + 'object_type': role|upper, 'access_role': 'PASSTHROUGH', 'represents_entity': none, 'is_agent_consumable': true, 'is_primary': false, 'access_note': 'Similarity search over the ' ~ b.name ~ ' embedding of ' ~ b.entity ~ '.'}) }} {% endfor %} @@ -127,7 +161,7 @@ {% for x in entities if x.objects.table is defined %} {% for s in x.synonyms|default([]) %} {{ m.insert(syn, {'object_kind': 'ENTITYMETADATA', 'synonym_text': s}, - {'synonym_object': m.ref(meta, {'entity_name': x.name})}) }} + {'synonym_object': m.ref(meta, {'entity_name': x.logical_id})}) }} {% endfor %} {% for col in x.columns if col.synonyms %} {% for s in col.synonyms %} diff --git a/implementation/teradata/modules/semantic/03-discovery.sql.j2 b/implementation/teradata/modules/semantic/03-discovery.sql.j2 index 7c03910..2908c9a 100644 --- a/implementation/teradata/modules/semantic/03-discovery.sql.j2 +++ b/implementation/teradata/modules/semantic/03-discovery.sql.j2 @@ -1,8 +1,15 @@ {# The standard-owned discovery views (Design Specification Standard §4.1): Semantic's column catalogue, manifest and relationship paths, and Observability's lineage views. Each is - created only where the composition places it. Teradata views take no ORDER BY, and a - union's text widths come from its first branch, so every text column is cast. #} + created only where the composition places it, and reads only relations the build context + places. Every view and every column carries a comment. Teradata views take no ORDER BY, and + a union's text widths come from its first branch, so every text column is cast. #} {% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} +{% macro describe(view, text, columns) -%} +COMMENT ON VIEW {{ view|qualified }} IS {{ m.comment(text) }}; +{% for name, meaning in columns -%} +COMMENT ON COLUMN {{ view|qualified }}.{{ name|ident }} IS {{ m.comment(meaning) }}; +{% endfor -%} +{%- endmacro %} {% set colmeta = named('ColumnMetadata') %}{% set catalogue = named('ColumnCatalogue') %} {% if catalogue and colmeta %}{% set c = colmeta.column_names %} REPLACE VIEW {{ catalogue.objects.consumer_view|qualified }} AS @@ -19,9 +26,22 @@ LEFT JOIN {{ colmeta.objects.table|qualified }} m ON m.{{ c.container_name|ident }} = TRIM(col.DatabaseName) AND m.{{ c.table_name|ident }} = TRIM(col.TableName) AND m.{{ c.column_name|ident }} = TRIM(col.ColumnName) WHERE {{ m.in_product('TRIM(col.DatabaseName)', 'TRIM(col.TableName)') }}; +{{ describe(catalogue.objects.consumer_view, 'Column-level catalogue of every product container: the structural facts from the system catalogue joined to the authored column metadata.', [ + ('container_name', "Container (database) that holds the column's table or view."), + ('table_name', 'Table or view that holds the column.'), ('column_name', 'Physical column name.'), + ('data_type', 'Declared platform type code of the column, as the dictionary reports it.'), + ('is_nullable', 'Y when the column accepts nulls, otherwise N.'), + ('logical_name', 'Name the specification or owning standard gives the column; null when none is registered.'), + ('business_description', 'Authored business meaning of the column; null when none is registered.'), + ('is_pii', 'True when the column holds personally identifiable information.'), + ('sensitivity_marker', "The specification's sensitivity marker for the column, when it carries one."), + ('data_classification', "Class the organisation's scheme assigns the column."), + ('declared_type', 'Logical type the specification declares for the column.'), + ('structural_source', 'Catalogue that supplied the structural facts.'), + ('description_source', 'authored when a business description is registered, missing otherwise.')]) }} {% endif %} {% set registry = named('DataProductRegistry') %}{% set orient = named('DataProductOrientation') %} -{% set manifest = named('DataProductManifest') %} +{% set manifest = named('DataProductManifest') %}{% set decl = named('StandardDeclaration') %} {% if manifest and registry and orient %}{% set p = registry.column_names %}{% set o = orient.column_names %} REPLACE VIEW {{ manifest.objects.consumer_view|qualified }} AS LOCKING ROW FOR ACCESS @@ -33,44 +53,92 @@ SELECT r.{{ p.product_id|ident }} AS product_id, r.{{ p.product_name|ident }} AS {%- for res in orientation.resources %}, MAX(CASE WHEN o.{{ o.resource_role|ident }} = {{ res.role|literal }} THEN o.{{ o.container_name|ident }} || '.' || o.{{ o.object_name|ident }} END) AS {{ res.role|lower|ident }} {%- endfor %} +{%- if decl %}{% set d = decl.column_names %}, + std.{{ d.standard_version|ident }} AS standard_version +{%- endif %} FROM {{ registry.objects.table|qualified }} r -JOIN {{ orient.objects.table|qualified }} o ON o.{{ o.product_id|ident }} = r.{{ p.product_id|ident }} +INNER JOIN {{ orient.objects.table|qualified }} o ON o.{{ o.product_id|ident }} = r.{{ p.product_id|ident }} +{%- if decl %} +LEFT OUTER JOIN {{ decl.objects.table|qualified }} std + ON std.{{ d.product_id|ident }} = r.{{ p.product_id|ident }} AND std.{{ d.component_kind|ident }} = 'STANDARD' + AND std.{{ d.is_active|ident }} = 1 +{%- endif %} WHERE r.{{ p.is_active|ident }} = 1 AND r.{{ p.is_deleted|ident }} = 0 AND o.{{ o.is_active|ident }} = 1 -GROUP BY 1, 2, 3, 4, 5, 6, 7; +GROUP BY r.{{ p.product_id|ident }}, r.{{ p.product_name|ident }}, r.{{ p.product_version|ident }}, r.{{ p.product_status|ident }}, + r.{{ p.approved_entrypoint|ident }}, r.{{ p.approved_access_mode|ident }}, r.{{ p.trust_authoritative_producer|ident }} +{%- if decl %}, std.{{ d.standard_version|ident }}{% endif %}; +{{ describe(manifest.objects.consumer_view, 'Bootstrap manifest: the approved entrypoint, trust producer, one deployed object per discovery resource, and the standard version the product declares.', + [('product_id', 'Identifier of the data product.'), ('product_name', 'Name of the data product.'), + ('product_version', 'Published contract version of the data product.'), + ('product_status', 'Publication lifecycle state of the product.'), + ('approved_entrypoint', 'Consumer object an agent should start from.'), + ('approved_access_mode', 'How the approved entrypoint is reached.'), + ('trust_authoritative_producer', 'Validation producer whose evidence is authoritative for trust.')]) }} +{% for res in orientation.resources -%} +COMMENT ON COLUMN {{ manifest.objects.consumer_view|qualified }}.{{ res.role|lower|ident }} IS {{ m.comment('Container and object of the ' ~ res.role ~ ' discovery resource, to use as given.') }}; +{% endfor -%} +{%- if decl -%} +COMMENT ON COLUMN {{ manifest.objects.consumer_view|qualified }}.{{ 'standard_version'|ident }} IS {{ m.comment('Release of the standard the deployment declares it implements; null when no declaration is registered.') }}; +{%- endif %} {% endif %} -{% set rel = named('TableRelationship') %}{% set paths = named('RelationshipPaths') %} -{% if paths and rel %}{% set t = rel.column_names %} +{% set rel = named('TableRelationship') %}{% set relcol = named('RelationshipColumn') %}{% set paths = named('RelationshipPaths') %} +{% if paths and rel and relcol %}{% set t = rel.column_names %}{% set u = relcol.column_names %} +{#- Every key position of every relationship, walked in both directions. -#} {% set edges -%} -(SELECT CAST(COALESCE({{ t.source_container|ident }} || '.', '') || {{ t.source_table|ident }} AS VARCHAR(400)) AS from_object, - CAST({{ t.source_column|ident }} AS VARCHAR(400)) AS from_column, - CAST(COALESCE({{ t.target_container|ident }} || '.', '') || {{ t.target_table|ident }} AS VARCHAR(400)) AS to_object, - CAST({{ t.target_column|ident }} AS VARCHAR(400)) AS to_column - FROM {{ rel.objects.table|qualified }} WHERE {{ t.is_active|ident }} = 1 +(SELECT r.{{ t.relationship_id|ident }} AS relationship_id, c.{{ u.ordinal|ident }} AS key_ordinal, + CAST('FORWARD' AS VARCHAR(7)) AS direction, + CAST(r.{{ t.source_entity|ident }} AS VARCHAR(400)) AS from_entity, CAST(c.{{ u.source_column|ident }} AS VARCHAR(400)) AS from_column, + CAST(r.{{ t.target_entity|ident }} AS VARCHAR(400)) AS to_entity, CAST(c.{{ u.target_column|ident }} AS VARCHAR(400)) AS to_column + FROM {{ rel.objects.table|qualified }} r + INNER JOIN {{ relcol.objects.table|qualified }} c ON c.{{ u.relationship_id|ident }} = r.{{ t.relationship_id|ident }} + WHERE r.{{ t.is_active|ident }} = 1 AND c.{{ u.is_active|ident }} = 1 UNION ALL - SELECT CAST(COALESCE({{ t.target_container|ident }} || '.', '') || {{ t.target_table|ident }} AS VARCHAR(400)), - CAST({{ t.target_column|ident }} AS VARCHAR(400)), - CAST(COALESCE({{ t.source_container|ident }} || '.', '') || {{ t.source_table|ident }} AS VARCHAR(400)), - CAST({{ t.source_column|ident }} AS VARCHAR(400)) - FROM {{ rel.objects.table|qualified }} WHERE {{ t.is_active|ident }} = 1) + SELECT r.{{ t.relationship_id|ident }}, c.{{ u.ordinal|ident }}, CAST('REVERSE' AS VARCHAR(7)), + CAST(r.{{ t.target_entity|ident }} AS VARCHAR(400)), CAST(c.{{ u.target_column|ident }} AS VARCHAR(400)), + CAST(r.{{ t.source_entity|ident }} AS VARCHAR(400)), CAST(c.{{ u.source_column|ident }} AS VARCHAR(400)) + FROM {{ rel.objects.table|qualified }} r + INNER JOIN {{ relcol.objects.table|qualified }} c ON c.{{ u.relationship_id|ident }} = r.{{ t.relationship_id|ident }} + WHERE r.{{ t.is_active|ident }} = 1 AND c.{{ u.is_active|ident }} = 1) +{%- endset %} +{#- One crossing of one relationship in one direction, with its leg of the path id: the + relationship id followed by > forward or < reverse. -#} +{% set steps -%} +(SELECT DISTINCT src.relationship_id, src.direction, src.from_entity, src.to_entity, + CAST(CAST(src.relationship_id AS VARCHAR(20)) || CASE src.direction WHEN 'FORWARD' THEN '>' ELSE '<' END AS VARCHAR(24)) AS leg + FROM {{ edges }} src) {%- endset %} -{% macro hop(a) -%}{{ a }}.from_object || '.' || {{ a }}.from_column || ' = ' || {{ a }}.to_object || '.' || {{ a }}.to_column{%- endmacro %} -{#- Teradata views take no recursion, so each path length up to the standard's bound is its - own branch; a path never returns to an entity it has passed through. -#} -REPLACE VIEW {{ paths.objects.consumer_view|qualified }} AS +{#- Teradata has no arrays, and a view built on a recursive query is not relied on here, so each path + length up to the standard's bound is its own branch and each position in the path its own + row set, one branch per (length, position). A path never returns to an entity it has passed + through: a step's destination differs from the origin and from every earlier destination. -#} +REPLACE VIEW {{ paths.objects.consumer_view|qualified }} +({% for name in ('path_id', 'origin_entity', 'destination_entity', 'hop_count', 'hop', 'key_ordinal', 'from_entity', 'from_column', 'to_entity', 'to_column', 'relationship_id', 'direction') %}{{ name|ident }}{% if not loop.last %}, {% endif %}{% endfor %}) AS LOCKING ROW FOR ACCESS -{%- for n in range(1, paths.max_hops + 1) %} -SELECT e1.from_object, e{{ n }}.to_object, - CAST({% for k in range(1, n + 1) %}{{ hop('e' ~ k) }}{% if not loop.last %} || ' AND ' || {% endif %}{% endfor %} AS VARCHAR(8000)) AS join_path, - {{ n }} AS hops -FROM {{ edges }} e1 +{%- for n in range(1, paths.max_hops + 1) %}{% for h in range(1, n + 1) %} +SELECT CAST({% for k in range(1, n + 1) %}stp{{ k }}.leg{% if not loop.last %} || {% endif %}{% endfor %} AS VARCHAR(200)) AS path_id, + stp1.from_entity AS origin_entity, stp{{ n }}.to_entity AS destination_entity, + CAST({{ n }} AS INTEGER) AS hop_count, CAST({{ h }} AS INTEGER) AS hop, + edg.key_ordinal, edg.from_entity, edg.from_column, edg.to_entity, edg.to_column, edg.relationship_id, edg.direction +FROM {{ steps }} stp1 {%- for k in range(2, n + 1) %} -JOIN {{ edges }} e{{ k }} ON e{{ k }}.from_object = e{{ k - 1 }}.to_object - AND e{{ k }}.to_object NOT IN ({% for j in range(1, k + 1) %}e{{ j }}.from_object{% if not loop.last %}, {% endif %}{% endfor %}) +INNER JOIN {{ steps }} stp{{ k }} ON stp{{ k }}.from_entity = stp{{ k - 1 }}.to_entity + AND stp{{ k }}.to_entity <> stp1.from_entity{% for j in range(1, k) %} AND stp{{ k }}.to_entity <> stp{{ j }}.to_entity{% endfor %} {%- endfor %} -{%- if not loop.last %} +INNER JOIN {{ edges }} edg ON edg.relationship_id = stp{{ h }}.relationship_id AND edg.direction = stp{{ h }}.direction +{%- if not (n == paths.max_hops and h == n) %} UNION ALL {%- endif %} -{%- endfor %}; +{%- endfor %}{% endfor %}; +{{ describe(paths.objects.consumer_view, 'Join paths of up to four steps between entities: one row per step and key position, ready for a consumer to assemble the join. A path never revisits an entity.', [ + ('path_id', 'Ordered legs of the path, each a relationship id followed by > for FORWARD or < for REVERSE.'), + ('origin_entity', 'Logical identity of the entity the path starts from.'), + ('destination_entity', 'Logical identity of the entity the path ends at.'), + ('hop_count', 'Number of relationship steps in the path.'), ('hop', 'Position of this step in the path, from 1.'), + ('key_ordinal', 'Position of the column pair within the join of this step, from 1.'), + ('from_entity', 'Logical identity of the entity the step leaves.'), ('from_column', 'Join column on the entity the step leaves.'), + ('to_entity', 'Logical identity of the entity the step arrives at.'), ('to_column', 'Join column on the entity the step arrives at.'), + ('relationship_id', 'Relationship, which is the join group, this step crosses.'), + ('direction', "FORWARD when crossing from the relationship's source to its target, REVERSE for the other way.")]) }} {% endif %} {% set lineage = named('DataLineage') %}{% set graph_view = named('LineageGraph') %} {% if graph_view and lineage %}{% set d = lineage.column_names %} @@ -83,6 +151,10 @@ UNION SELECT DISTINCT CAST({{ d.job_name|ident }} AS VARCHAR(400)), CAST(COALESCE({{ d.target_container|ident }} || '.', '') || {{ d.target_table|ident }} AS VARCHAR(400)), 'produces' FROM {{ lineage.objects.table|qualified }} WHERE {{ d.is_active|ident }} = 1 AND {{ d.job_name|ident }} IS NOT NULL; +{{ describe(graph_view.objects.consumer_view, 'Table-and-job lineage graph: one edge from each active source to its job and from the job to its target.', [ + ('source_node', 'Node the edge leaves: a source table or a job.'), + ('target_node', 'Node the edge arrives at: a job or a target table.'), + ('edge_type', 'Kind of relationship the edge records; produces for data flow.')]) }} {% endif %} {% set run = named('LineageRun') %}{% set latest = named('LineageRunLatest') %} {% if latest and lineage and run %}{% set d = lineage.column_names %}{% set r = run.column_names %} @@ -96,4 +168,9 @@ LEFT JOIN {{ run.objects.table|qualified }} r ON r.{{ r.lineage_id|ident }} = d. WHERE d.{{ d.is_active|ident }} = 1 QUALIFY ROW_NUMBER() OVER (PARTITION BY d.{{ d.lineage_id|ident }} ORDER BY r.{{ r.run_dts|ident }} DESC, r.{{ r.lineage_run_id|ident }} DESC) = 1; +{{ describe(latest.objects.consumer_view, 'Latest recorded run of each active lineage definition.', [ + ('lineage_id', 'Identifier of the lineage definition.'), ('source_table', 'Table the job reads.'), + ('target_table', 'Table the job writes.'), ('job_name', 'Name of the job that moves the data.'), + ('run_dts', 'When the latest run was recorded; null when never run.'), + ('run_status', 'Outcome of the latest run; null when never run.')]) }} {% endif %} diff --git a/implementation/teradata/modules/semantic/README.md b/implementation/teradata/modules/semantic/README.md index 9a1c94c..cf52587 100644 --- a/implementation/teradata/modules/semantic/README.md +++ b/implementation/teradata/modules/semantic/README.md @@ -1,67 +1,83 @@ ---- -title: Teradata Semantic Module Implementation -anchor: semantic -type: implementation -status: draft -version: 3.0 -normative: true -implements: semantic -platform: teradata ---- - -# Teradata: Semantic - -Binding of [Semantic](../../../../design/modules/semantic.md). The design remains the source of truth; the binding supplies reusable mechanisms, not a product schema. - -## Templates - -`01-tables.sql.j2` calls `module_tables('semantic')` from the [shared macros](../../patterns/temporal-lifecycle-metadata/README.md) for the catalogue relations the Semantic standard defines and the resolver places. - -`02-registration.sql.j2` writes the product's description of itself, each block only where its relation is placed: - -- the product registry row from `product` (code, name, description, domain, version, owner, trust producer) and `orientation.access_mode`, with the manifest's consumer view as the approved entry point; -- the module map from `module_containers`, for object types the relation's vocabulary permits; -- entity and column metadata for every placed relation in `entities` and `relations`: placement, kind, profile, surrogate and natural columns, current and deleted flag columns, and per column the logical name, description, sensitivity marker, `classification`, `required`, declared type and Enum members; -- table relationships from `relationships`, by base table; -- access objects for every placed object of each relation, with consumer views marked agent-consumable; -- primary objects for every placed object with a `primary_role`; -- the orientation sequence from `orientation.resources`; -- metrics from `metrics`, with the rendered measure and filter as a Teradata expression and the dataset's consumer view; -- entity and column synonyms; -- naming standards from `naming`: each object template, the attribute template and each standard-name mapping. - -Composite access objects (`access_objects`) are not registered; the manifest reports each as a gap. - -`03-discovery.sql.j2` creates the standard-owned discovery views with `REPLACE VIEW` and `LOCKING ROW FOR ACCESS`, each only where the composition places it and the relations it reads: - -- `ColumnCatalogue`: `DBC.ColumnsV` for the product's databases, left-joined to column metadata, with `description_source` `authored` or `missing`; -- `DataProductManifest`: the active registry row with one column per orientation resource role; -- `RelationshipPaths`: active table relationships in both directions, up to the Semantic standard's bound of `max_hops` hops (§5) without revisiting a table. Teradata views take no recursion, so each path length is its own branch of a `UNION ALL`, unrolled from the context's bound. Paths join base tables; resolve them through the access objects before issuing consumer SQL; -- `LineageGraph` and `LineageRunLatest` over Observability's active lineage definitions and their runs, the latest run chosen with `QUALIFY`. - -Every text column a union produces is cast, because a union takes its column types from its first branch. - -`validation.sql.j2` is the `module semantic` check: every table in a database the product places objects in (`DBC.TablesV`, kinds `T` and `O`), keymaps included, is registered in entity metadata. - -Registration uses the placement and relationships the context carries, never name inference. Structural facts come from the `DBC` dictionary views; meanings come from the specification. - -## Capability bindings - -| Capability | Binding | -|---|---| -| `SemanticRegistration` | Registration inserts for the product, modules, entities, columns, relationships, access and primary objects, orientation, metrics, synonyms and naming standards. | -| `RichMetadata` | `COMMENT ON` for physical objects, plus authored entity and column metadata and the hybrid column catalogue. | - -Templates read only the build context: see [tooling/build](../../../../tooling/build/README.md) and the [Platform Implementation Authoring Standard](../../../../design/core/IMPLEMENTATION_AUTHORING.md) §3 and §6. Phases, checks, types and expression fragments are declared in [PLATFORM_PROFILE.md](../../PLATFORM_PROFILE.md#binding-declarations); coverage and gaps are in [CONFORMANCE.md](../../CONFORMANCE.md). Rendering uses StrictUndefined. - -## Invariants and checks - -| Invariant | Evidence or outstanding check | -|---|---| -| `INV-SEMANTIC-001,002` | Standard-owned catalogue structures; registries hold datasets, never business instances. | -| `INV-SEMANTIC-003,006` | Registration at deployment; `module semantic` check, with an engine test that an unregistered table is found. | -| `INV-SEMANTIC-004,011,012` | Manifest view over the registry and orientation sequence; consumer workflow verification remains required. | -| `INV-SEMANTIC-005,007..010` | Declared relationships and access objects; composite access objects are not rendered. | -| `INV-SEMANTIC-013..015` | Metrics and synonyms registered from the specification; metric recipes execute in engine tests. Reference checks over the registrations remain to be written. | - -Physical types come from the `Types:` block of the [binding declarations](../../PLATFORM_PROFILE.md#binding-declarations). Temporal and lifecycle columns come only from the shared pattern. Semantic registration and Memory capture are soft dependencies and render nothing when those modules are absent. +--- +title: Teradata Semantic Module Implementation +anchor: semantic +type: implementation +status: draft +version: 3.0 +normative: true +implements: semantic +platform: teradata +--- + +# Teradata: Semantic + +Binding of [Semantic](../../../../design/modules/semantic.md). The design remains the source of truth; the binding supplies reusable mechanisms, not a product schema. + +## Templates + +`01-tables.sql.j2` calls `module_tables('semantic')` from the [shared macros](../../patterns/temporal-lifecycle-metadata/README.md) for the catalogue relations the Semantic standard defines and the resolver places. + +`02-registration.sql.j2` writes the product's description of itself, each block only where its relation is placed: + +- the product registry row from `product` (code, name, description, domain, version, owner, trust producer) and `orientation.access_mode`, with the manifest's consumer view as the approved entry point; +- the module map from `module_containers`, for object types the relation's vocabulary permits; +- the standard declaration: one `StandardDeclaration` row each for the standard, every deployed module and every applied pattern, from `standard.declarations`, with the platform and `standard.version`; +- entity and column metadata for every placed relation in `entities` and `relations`: the logical identity (`module.entity`, from `logical_id`, never a container or a prefix) as `entity_name`, placement, kind, profile, current and deleted flag columns, and per column the logical name, description, sensitivity marker, `classification`, `required`, declared type and Enum members; +- a `KeyDefinition` row for each of an entity's `key_defs` (surrogate, natural) and a `KeyColumn` row per ordered column, so a composite key is ordered rows and no delimited list; +- table relationships from `relationships`, between the logical identities of the source and target entities, and a `RelationshipColumn` row per ordered column pair (`pairs`). A nested `m.ref` is not supported inside a match, so the key of an entity is found by an explicit subselect (`key_ref`); +- access objects for every placed object of each relation, each with its `object_identity` (`module.entity:base`, `:governed` or `:current`; `module:object` for a similarity procedure), the logical `represents_entity` and `resolves_to_object`, with consumer views marked agent-consumable, and an `AccessColumn` row per column of each consumer view, from the same `view_columns` list the view selects; +- primary objects for every placed object with a `primary_role`; +- the orientation sequence from `orientation.resources`; +- metrics from `metrics`, with the rendered measure and filter as a Teradata expression and the dataset's consumer view; +- entity and column synonyms; +- naming standards from `naming`: each object template, the attribute template and each standard-name mapping. + +Composite access objects (`access_objects`) are not registered; the manifest reports each as a gap. + +`03-discovery.sql.j2` creates the standard-owned discovery views with `REPLACE VIEW` and `LOCKING ROW FOR ACCESS`, each only where the composition places it and the relations it reads: + +- `ColumnCatalogue`: `DBC.ColumnsV` for the product's databases, left-joined to column metadata, with `description_source` `authored` or `missing`; +- `DataProductManifest`: the active registry row with one column per orientation resource role; +- `RelationshipPaths`: rows, one per step and key position, in the standard's column order (`path_id`, `origin_entity`, `destination_entity`, `hop_count`, `hop`, `key_ordinal`, `from_entity`, `from_column`, `to_entity`, `to_column`, `relationship_id`, `direction`). A leg is the relationship id followed by `>` (forward) or `<` (reverse) and `path_id` is the legs joined; the bound is `max_hops` (four) and a path never revisits an entity. Teradata views take no recursion or `WITH` clause and have no arrays, so each (path length, position) is its own branch of a `UNION ALL`, unrolled from the context's bound, with `path_id` built as text by concatenating each step's leg. Paths join the logical entities; resolve them through the access objects before issuing consumer SQL; +- `DataProductManifest` ends with `standard_version`, the version of the active `STANDARD` declaration row; +- `LineageGraph` and `LineageRunLatest` over Observability's active lineage definitions and their runs, the latest run chosen with `QUALIFY`. + +Every text column a union produces is cast, because a union takes its column types from its first branch. Every view and every view column carries a comment. + +`contract/` holds the five metadata-contract checks, declared as `contract ` in the platform profile and rendered once for the Semantic module. Each returns one row per defect, as `object_identity` and `defect`: + +| Check | Template | Returns | +|---|---|---| +| `contract-identity` | `contract/identity.sql.j2` | an entity identity outside the `module.entity` grammar (`REGEXP_SIMILAR`, case-sensitive) or outside its own module, a duplicate identity, an access identity that breaks the grammar or does not match its entity and surface (`INV-SEMANTIC-016`) | +| `contract-keys` | `contract/keys.sql.j2` | an entity with no key, a key with no columns, ordinals not running 1 to n, a key column absent from the entity (`INV-SEMANTIC-017`) | +| `contract-relationships` | `contract/relationships.sql.j2` | a relationship with no column pair, ordinals not running 1 to n, an unregistered entity, a pair column absent from its entity (`INV-SEMANTIC-018`) | +| `contract-access-columns` | `contract/access-columns.sql.j2` | a consumer view column with no mapping, a mapping naming no column or repeated, ordinals not running 1 to n, an absent non-derived source (`INV-SEMANTIC-019`) | +| `contract-composition` | `contract/composition.sql.j2` | a composite without exactly one anchor, members not ordered 1 to n, a member not joined to a prior member, an anchor carrying a join (`INV-SEMANTIC-020`) | + +Columns of tables and views are read from `DBC.ColumnsV`; the shared macros are in `contract/00-macros.sql.j2`. The build does not execute Teradata, so these are verified by rendering, and by the shared engine tests when a system is attached. + +`validation.sql.j2` is the `module semantic` check: every table in a database the product places objects in (`DBC.TablesV`, kinds `T` and `O`), keymaps included, is registered in entity metadata. + +Registration uses the placement and relationships the context carries, never name inference. Structural facts come from the `DBC` dictionary views; meanings come from the specification. + +## Capability bindings + +| Capability | Binding | +|---|---| +| `SemanticRegistration` | Registration inserts for the product, modules, entities, columns, relationships, access and primary objects, orientation, metrics, synonyms and naming standards. | +| `RichMetadata` | `COMMENT ON` for physical objects, plus authored entity and column metadata and the hybrid column catalogue. | + +Templates read only the build context: see [tooling/build](../../../../tooling/build/README.md) and the [Platform Implementation Authoring Standard](../../../../design/core/IMPLEMENTATION_AUTHORING.md) §3 and §6. Phases, checks, types and expression fragments are declared in [PLATFORM_PROFILE.md](../../PLATFORM_PROFILE.md#binding-declarations); coverage and gaps are in [CONFORMANCE.md](../../CONFORMANCE.md). Rendering uses StrictUndefined. + +## Invariants and checks + +| Invariant | Evidence or outstanding check | +|---|---| +| `INV-SEMANTIC-001,002` | Standard-owned catalogue structures; registries hold datasets, never business instances. | +| `INV-SEMANTIC-003,006` | Registration at deployment; `module semantic` check, with an engine test that an unregistered table is found. | +| `INV-SEMANTIC-004,011,012` | Manifest view over the registry and orientation sequence; consumer workflow verification remains required. | +| `INV-SEMANTIC-005,007..010` | Declared relationships and access objects; composite access objects are not rendered. | +| `INV-SEMANTIC-016..020` | The five `contract-*` checks, one mutation test each in the shared engine tests (skipped without a Teradata system). | +| `INV-SEMANTIC-013..015` | Metrics and synonyms registered from the specification; metric recipes execute in engine tests. Reference checks over the registrations remain to be written. | + +Physical types come from the `Types:` block of the [binding declarations](../../PLATFORM_PROFILE.md#binding-declarations). Temporal and lifecycle columns come only from the shared pattern. Semantic registration and Memory capture are soft dependencies and render nothing when those modules are absent. diff --git a/implementation/teradata/modules/semantic/contract/00-macros.sql.j2 b/implementation/teradata/modules/semantic/contract/00-macros.sql.j2 new file mode 100644 index 0000000..1267e92 --- /dev/null +++ b/implementation/teradata/modules/semantic/contract/00-macros.sql.j2 @@ -0,0 +1,19 @@ +{# Shared by the Semantic metadata-contract checks (INV-SEMANTIC-016 to 020). A standard-owned + relation is read through the physical names the build context places; nothing here composes + one. Every check returns two columns, the object_identity at fault and the defect, each cast + to a stated width because a union takes its text widths from its first branch. #} +{% macro tbl(name) -%}{{ named(name).objects.table|qualified }}{%- endmacro %} +{% macro col(name, column) -%}{{ named(name).column_names[column]|ident }}{%- endmacro %} +{% macro pk(name) -%}{{ named(name).key_columns[0]|ident }}{%- endmacro %} +{# The object a defect is about, cast to the check's identity width. #} +{% macro who(expression) -%}CAST({{ expression }} AS VARCHAR(500)){%- endmacro %} +{# What is wrong with it. #} +{% macro what(text) -%}CAST({{ text|literal }} AS VARCHAR(100)){%- endmacro %} +{# A run of ordinals is 1..n with no gaps and no repeats. #} +{% macro contiguous(alias, ordinal) -%} +COUNT(DISTINCT {{ alias }}.{{ ordinal }}) <> COUNT(*) OR MIN({{ alias }}.{{ ordinal }}) <> 1 OR MAX({{ alias }}.{{ ordinal }}) <> COUNT(*) +{%- endmacro %} +{# A column of a table or view, from the dictionary: true when `alias` names one for the given container, object and column expressions. #} +{% macro has_column(alias, container, object, column) -%} +TRIM({{ alias }}.DatabaseName) = {{ container }} AND TRIM({{ alias }}.TableName) = {{ object }} AND TRIM({{ alias }}.ColumnName) = {{ column }} +{%- endmacro %} diff --git a/implementation/teradata/modules/semantic/contract/access-columns.sql.j2 b/implementation/teradata/modules/semantic/contract/access-columns.sql.j2 new file mode 100644 index 0000000..57d33f5 --- /dev/null +++ b/implementation/teradata/modules/semantic/contract/access-columns.sql.j2 @@ -0,0 +1,46 @@ +{# INV-SEMANTIC-019: every consumer view has exactly one mapping row per output column, ordered 1..n, + and every non-derived source exists. Every returned row is a defect. The view's columns are read + from the dictionary. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT {{ q.who('a.' ~ q.col('AccessObject', 'object_identity') ~ " || '.' || TRIM(i.ColumnName)") }} AS object_identity, {{ q.what('output column has no mapping') }} AS defect +FROM {{ q.tbl('AccessObject') }} a +INNER JOIN DBC.ColumnsV i + ON TRIM(i.DatabaseName) = a.{{ q.col('AccessObject', 'container_name') }} AND TRIM(i.TableName) = a.{{ q.col('AccessObject', 'object_name') }} +WHERE a.{{ q.col('AccessObject', 'is_active') }} = 1 AND a.{{ q.col('AccessObject', 'object_type') }} = 'CONSUMER_VIEW' + AND EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = a.{{ q.col('AccessObject', 'represents_entity') }}) + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('AccessColumn') }} m + WHERE m.{{ q.col('AccessColumn', 'access_object_id') }} = a.{{ q.pk('AccessObject') }} + AND m.{{ q.col('AccessColumn', 'column_name') }} = TRIM(i.ColumnName) AND m.{{ q.col('AccessColumn', 'is_active') }} = 1) +UNION ALL +SELECT {{ q.who('a.' ~ q.col('AccessObject', 'object_identity') ~ " || '.' || m." ~ q.col('AccessColumn', 'column_name')) }}, + {{ q.what('mapping names a column the object does not have') }} +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} = 1 + AND NOT EXISTS (SELECT 1 FROM DBC.ColumnsV i + WHERE {{ q.has_column('i', 'a.' ~ q.col('AccessObject', 'container_name'), 'a.' ~ q.col('AccessObject', 'object_name'), 'm.' ~ q.col('AccessColumn', 'column_name')) }}) +UNION ALL +SELECT {{ q.who('a.' ~ q.col('AccessObject', 'object_identity') ~ " || '.' || m." ~ q.col('AccessColumn', 'column_name')) }}, + {{ q.what('mapping column is repeated') }} +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} = 1 +GROUP BY a.{{ q.col('AccessObject', 'object_identity') }}, m.{{ q.col('AccessColumn', 'column_name') }} +HAVING COUNT(*) > 1 +UNION ALL +SELECT {{ q.who('a.' ~ q.col('AccessObject', 'object_identity')) }}, {{ q.what('mapping ordinals do not run 1..n') }} +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} = 1 +GROUP BY a.{{ q.col('AccessObject', 'object_identity') }} +HAVING {{ q.contiguous('m', q.col('AccessColumn', 'ordinal')) }} +UNION ALL +SELECT {{ q.who('a.' ~ q.col('AccessObject', 'object_identity') ~ " || '.' || m." ~ q.col('AccessColumn', 'column_name')) }}, {{ q.what('non-derived mapping source is absent') }} +FROM {{ q.tbl('AccessColumn') }} m +INNER JOIN {{ q.tbl('AccessObject') }} a ON a.{{ q.pk('AccessObject') }} = m.{{ q.col('AccessColumn', 'access_object_id') }} +WHERE m.{{ q.col('AccessColumn', 'is_active') }} = 1 AND m.{{ q.col('AccessColumn', 'mapping_kind') }} <> 'DERIVED' + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e + INNER JOIN DBC.ColumnsV i + ON TRIM(i.DatabaseName) = e.{{ q.col('EntityMetadata', 'container_name') }} AND TRIM(i.TableName) = e.{{ q.col('EntityMetadata', 'table_name') }} + WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = m.{{ q.col('AccessColumn', 'source_entity') }} + AND TRIM(i.ColumnName) = m.{{ q.col('AccessColumn', 'source_column') }}) diff --git a/implementation/teradata/modules/semantic/contract/composition.sql.j2 b/implementation/teradata/modules/semantic/contract/composition.sql.j2 new file mode 100644 index 0000000..4357fa5 --- /dev/null +++ b/implementation/teradata/modules/semantic/contract/composition.sql.j2 @@ -0,0 +1,39 @@ +{# INV-SEMANTIC-020: a composite has exactly one anchor, its members are ordered 1..n, and every other member + joins to a prior member through a registered relationship in a stated direction. + Every returned row is a defect. Outer references stay in WHERE, never in a subquery's ON. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT {{ q.who('c.' ~ q.col('AccessComposition', 'composite_object')) }} AS object_identity, {{ q.what('composite does not have exactly one anchor') }} AS defect +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} = 1 +GROUP BY c.{{ q.col('AccessComposition', 'composite_object') }} +HAVING SUM(CASE WHEN c.{{ q.col('AccessComposition', 'member_role') }} = 'ANCHOR' THEN 1 ELSE 0 END) <> 1 +UNION ALL +SELECT {{ q.who('c.' ~ q.col('AccessComposition', 'composite_object')) }}, {{ q.what('member order does not run 1..n') }} +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} = 1 +GROUP BY c.{{ q.col('AccessComposition', 'composite_object') }} +HAVING {{ q.contiguous('c', q.col('AccessComposition', 'member_seq')) }} +UNION ALL +SELECT {{ q.who('c.' ~ q.col('AccessComposition', 'composite_object') ~ " || '.' || c." ~ q.col('AccessComposition', 'member_entity')) }}, + {{ q.what('member does not join to a prior member through its relationship') }} +FROM {{ q.tbl('AccessComposition') }} c +LEFT OUTER JOIN {{ q.tbl('TableRelationship') }} r ON r.{{ q.pk('TableRelationship') }} = c.{{ q.col('AccessComposition', 'via_relationship_id') }} +WHERE c.{{ q.col('AccessComposition', 'is_active') }} = 1 AND c.{{ q.col('AccessComposition', 'member_role') }} <> 'ANCHOR' + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('AccessComposition') }} p + WHERE p.{{ q.col('AccessComposition', 'composite_object') }} = c.{{ q.col('AccessComposition', 'composite_object') }} + AND p.{{ q.col('AccessComposition', 'member_seq') }} < c.{{ q.col('AccessComposition', 'member_seq') }} + AND ((c.{{ q.col('AccessComposition', 'via_direction') }} = 'FORWARD' + AND r.{{ q.col('TableRelationship', 'source_entity') }} = p.{{ q.col('AccessComposition', 'member_entity') }} + AND r.{{ q.col('TableRelationship', 'target_entity') }} = c.{{ q.col('AccessComposition', 'member_entity') }}) + OR (c.{{ q.col('AccessComposition', 'via_direction') }} = 'REVERSE' + AND r.{{ q.col('TableRelationship', 'target_entity') }} = p.{{ q.col('AccessComposition', 'member_entity') }} + AND r.{{ q.col('TableRelationship', 'source_entity') }} = c.{{ q.col('AccessComposition', 'member_entity') }}))) +UNION ALL +SELECT {{ q.who('c.' ~ q.col('AccessComposition', 'composite_object') ~ " || '.' || c." ~ q.col('AccessComposition', 'member_entity')) }}, + {{ q.what('anchor carries a join, or the member names an unregistered entity') }} +FROM {{ q.tbl('AccessComposition') }} c +WHERE c.{{ q.col('AccessComposition', 'is_active') }} = 1 + AND ((c.{{ q.col('AccessComposition', 'member_role') }} = 'ANCHOR' + AND (c.{{ q.col('AccessComposition', 'via_relationship_id') }} IS NOT NULL OR c.{{ q.col('AccessComposition', 'via_direction') }} IS NOT NULL)) + OR NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e + WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = c.{{ q.col('AccessComposition', 'member_entity') }})) diff --git a/implementation/teradata/modules/semantic/contract/identity.sql.j2 b/implementation/teradata/modules/semantic/contract/identity.sql.j2 new file mode 100644 index 0000000..c174319 --- /dev/null +++ b/implementation/teradata/modules/semantic/contract/identity.sql.j2 @@ -0,0 +1,29 @@ +{# INV-SEMANTIC-016: identities follow the grammar and carry no placement. Every row is a defect. + REGEXP_SIMILAR matches the whole string, and 'c' makes the match case-sensitive. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT {{ q.who('e.' ~ q.col('EntityMetadata', 'entity_name')) }} AS object_identity, {{ q.what('entity identity is not module.entity in its own module') }} AS defect +FROM {{ q.tbl('EntityMetadata') }} e +WHERE e.{{ q.col('EntityMetadata', 'is_active') }} = 1 + AND (COALESCE(REGEXP_SIMILAR(e.{{ q.col('EntityMetadata', 'entity_name') }}, '[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*', 'c'), 0) <> 1 + OR SUBSTR(e.{{ q.col('EntityMetadata', 'entity_name') }}, 1, CHARACTER_LENGTH(e.{{ q.col('EntityMetadata', 'module_name') }}) + 1) + <> LOWER(e.{{ q.col('EntityMetadata', 'module_name') }}) || '.') +UNION ALL +SELECT {{ q.who('e.' ~ q.col('EntityMetadata', 'entity_name')) }}, {{ q.what('entity identity is registered more than once') }} +FROM {{ q.tbl('EntityMetadata') }} e +WHERE e.{{ q.col('EntityMetadata', 'is_active') }} = 1 +GROUP BY e.{{ q.col('EntityMetadata', 'entity_name') }} +HAVING COUNT(*) > 1 +UNION ALL +SELECT {{ q.who('a.' ~ q.col('AccessObject', 'object_identity')) }}, {{ q.what('access identity breaks the grammar') }} +FROM {{ q.tbl('AccessObject') }} a +WHERE a.{{ q.col('AccessObject', 'is_active') }} = 1 + AND COALESCE(REGEXP_SIMILAR(a.{{ q.col('AccessObject', 'object_identity') }}, '[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*:(base|governed|current|history)', 'c'), 0) <> 1 + AND COALESCE(REGEXP_SIMILAR(a.{{ q.col('AccessObject', 'object_identity') }}, '[a-z][a-z0-9_]*:[a-z][a-z0-9_]*', 'c'), 0) <> 1 + AND COALESCE(REGEXP_SIMILAR(a.{{ q.col('AccessObject', 'object_identity') }}, 'composite:[a-z][a-z0-9_]*', 'c'), 0) <> 1 +UNION ALL +SELECT {{ q.who('a.' ~ q.col('AccessObject', 'object_identity')) }}, {{ q.what('single-entity identity does not match its entity and surface') }} +FROM {{ q.tbl('AccessObject') }} a +WHERE a.{{ q.col('AccessObject', 'is_active') }} = 1 AND a.{{ q.col('AccessObject', 'represents_entity') }} IS NOT NULL + AND a.{{ q.col('AccessObject', 'object_identity') }} <> a.{{ q.col('AccessObject', 'represents_entity') }} || ':' || + CASE a.{{ q.col('AccessObject', 'object_type') }} WHEN 'TABLE' THEN 'base' WHEN 'BASE_VIEW' THEN 'governed' + WHEN 'CONSUMER_VIEW' THEN 'current' ELSE 'history' END diff --git a/implementation/teradata/modules/semantic/contract/keys.sql.j2 b/implementation/teradata/modules/semantic/contract/keys.sql.j2 new file mode 100644 index 0000000..4dee0f7 --- /dev/null +++ b/implementation/teradata/modules/semantic/contract/keys.sql.j2 @@ -0,0 +1,32 @@ +{# INV-SEMANTIC-017: every entity has a key; the columns of a key are ordered 1..n and exist on the entity. + Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT {{ q.who('e.' ~ q.col('EntityMetadata', 'entity_name')) }} AS object_identity, {{ q.what('entity has no key definition') }} AS defect +FROM {{ q.tbl('EntityMetadata') }} e +WHERE e.{{ q.col('EntityMetadata', 'is_active') }} = 1 + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('KeyDefinition') }} k + WHERE k.{{ q.col('KeyDefinition', 'entity_id') }} = e.{{ q.pk('EntityMetadata') }} AND k.{{ q.col('KeyDefinition', 'is_active') }} = 1) +UNION ALL +SELECT {{ q.who('e.' ~ q.col('EntityMetadata', 'entity_name') ~ " || '#' || k." ~ q.col('KeyDefinition', 'key_name')) }}, {{ q.what('key has no columns') }} +FROM {{ q.tbl('KeyDefinition') }} k +INNER JOIN {{ q.tbl('EntityMetadata') }} e ON e.{{ q.pk('EntityMetadata') }} = k.{{ q.col('KeyDefinition', 'entity_id') }} +WHERE k.{{ q.col('KeyDefinition', 'is_active') }} = 1 + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('KeyColumn') }} c + WHERE c.{{ q.col('KeyColumn', 'key_definition_id') }} = k.{{ q.pk('KeyDefinition') }} AND c.{{ q.col('KeyColumn', 'is_active') }} = 1) +UNION ALL +SELECT {{ q.who('e.' ~ q.col('EntityMetadata', 'entity_name') ~ " || '#' || k." ~ q.col('KeyDefinition', 'key_name')) }}, {{ q.what('key column ordinals do not run 1..n') }} +FROM {{ q.tbl('KeyDefinition') }} k +INNER JOIN {{ q.tbl('EntityMetadata') }} e ON e.{{ q.pk('EntityMetadata') }} = k.{{ q.col('KeyDefinition', 'entity_id') }} +INNER JOIN {{ q.tbl('KeyColumn') }} c ON c.{{ q.col('KeyColumn', 'key_definition_id') }} = k.{{ q.pk('KeyDefinition') }} +WHERE k.{{ q.col('KeyDefinition', 'is_active') }} = 1 AND c.{{ q.col('KeyColumn', 'is_active') }} = 1 +GROUP BY e.{{ q.col('EntityMetadata', 'entity_name') }}, k.{{ q.col('KeyDefinition', 'key_name') }} +HAVING {{ q.contiguous('c', q.col('KeyColumn', 'ordinal')) }} +UNION ALL +SELECT {{ q.who('e.' ~ q.col('EntityMetadata', 'entity_name') ~ " || '#' || k." ~ q.col('KeyDefinition', 'key_name') ~ " || '.' || c." ~ q.col('KeyColumn', 'column_name')) }}, + {{ q.what('key column is absent from its entity') }} +FROM {{ q.tbl('KeyColumn') }} c +INNER JOIN {{ q.tbl('KeyDefinition') }} k ON k.{{ q.pk('KeyDefinition') }} = c.{{ q.col('KeyColumn', 'key_definition_id') }} +INNER JOIN {{ q.tbl('EntityMetadata') }} e ON e.{{ q.pk('EntityMetadata') }} = k.{{ q.col('KeyDefinition', 'entity_id') }} +WHERE c.{{ q.col('KeyColumn', 'is_active') }} = 1 + AND NOT EXISTS (SELECT 1 FROM DBC.ColumnsV i + WHERE {{ q.has_column('i', 'e.' ~ q.col('EntityMetadata', 'container_name'), 'e.' ~ q.col('EntityMetadata', 'table_name'), 'c.' ~ q.col('KeyColumn', 'column_name')) }}) diff --git a/implementation/teradata/modules/semantic/contract/relationships.sql.j2 b/implementation/teradata/modules/semantic/contract/relationships.sql.j2 new file mode 100644 index 0000000..9742b3e --- /dev/null +++ b/implementation/teradata/modules/semantic/contract/relationships.sql.j2 @@ -0,0 +1,34 @@ +{# INV-SEMANTIC-018: every relationship has at least one column pair; pairs are ordered 1..n and + their columns exist on the entities. Every returned row is a defect. #} +{% import 'modules/semantic/contract/00-macros.sql.j2' as q %} +SELECT {{ q.who('r.' ~ q.col('TableRelationship', 'relationship_name')) }} AS object_identity, {{ q.what('relationship has no column pair') }} AS defect +FROM {{ q.tbl('TableRelationship') }} r +WHERE r.{{ q.col('TableRelationship', 'is_active') }} = 1 + AND NOT EXISTS (SELECT 1 FROM {{ q.tbl('RelationshipColumn') }} c + WHERE c.{{ q.col('RelationshipColumn', 'relationship_id') }} = r.{{ q.pk('TableRelationship') }} + AND c.{{ q.col('RelationshipColumn', 'is_active') }} = 1) +UNION ALL +SELECT {{ q.who('r.' ~ q.col('TableRelationship', 'relationship_name')) }}, {{ q.what('relationship ordinals do not run 1..n') }} +FROM {{ q.tbl('TableRelationship') }} r +INNER JOIN {{ q.tbl('RelationshipColumn') }} c ON c.{{ q.col('RelationshipColumn', 'relationship_id') }} = r.{{ q.pk('TableRelationship') }} +WHERE r.{{ q.col('TableRelationship', 'is_active') }} = 1 AND c.{{ q.col('RelationshipColumn', 'is_active') }} = 1 +GROUP BY r.{{ q.col('TableRelationship', 'relationship_name') }} +HAVING {{ q.contiguous('c', q.col('RelationshipColumn', 'ordinal')) }} +UNION ALL +SELECT {{ q.who('r.' ~ q.col('TableRelationship', 'relationship_name')) }}, {{ q.what('relationship names an unregistered entity') }} +FROM {{ q.tbl('TableRelationship') }} r +WHERE r.{{ q.col('TableRelationship', 'is_active') }} = 1 + AND (NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'source_entity') }}) + OR NOT EXISTS (SELECT 1 FROM {{ q.tbl('EntityMetadata') }} e WHERE e.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'target_entity') }})) +UNION ALL +SELECT {{ q.who('r.' ~ q.col('TableRelationship', 'relationship_name') ~ " || '.' || CAST(c." ~ q.col('RelationshipColumn', 'ordinal') ~ ' AS VARCHAR(10))') }}, + {{ q.what('relationship column is absent from its entity') }} +FROM {{ q.tbl('RelationshipColumn') }} c +INNER JOIN {{ q.tbl('TableRelationship') }} r ON r.{{ q.pk('TableRelationship') }} = c.{{ q.col('RelationshipColumn', 'relationship_id') }} +INNER JOIN {{ q.tbl('EntityMetadata') }} s ON s.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'source_entity') }} +INNER JOIN {{ q.tbl('EntityMetadata') }} t ON t.{{ q.col('EntityMetadata', 'entity_name') }} = r.{{ q.col('TableRelationship', 'target_entity') }} +WHERE c.{{ q.col('RelationshipColumn', 'is_active') }} = 1 + AND (NOT EXISTS (SELECT 1 FROM DBC.ColumnsV i + WHERE {{ q.has_column('i', 's.' ~ q.col('EntityMetadata', 'container_name'), 's.' ~ q.col('EntityMetadata', 'table_name'), 'c.' ~ q.col('RelationshipColumn', 'source_column')) }}) + OR NOT EXISTS (SELECT 1 FROM DBC.ColumnsV i + WHERE {{ q.has_column('i', 't.' ~ q.col('EntityMetadata', 'container_name'), 't.' ~ q.col('EntityMetadata', 'table_name'), 'c.' ~ q.col('RelationshipColumn', 'target_column')) }})) diff --git a/implementation/teradata/patterns/object-placement/01-containers.sql.j2 b/implementation/teradata/patterns/object-placement/01-containers.sql.j2 index 7f4d92a..d430671 100644 --- a/implementation/teradata/patterns/object-placement/01-containers.sql.j2 +++ b/implementation/teradata/patterns/object-placement/01-containers.sql.j2 @@ -1,12 +1,16 @@ {# Containers the build context places objects in, parents first. A parent database holds no objects and allocates its children's space; adopted containers already exist and are not created. Space comes from the `Container space` setting, per container that holds - objects. #} + objects. Each database created carries a comment. #} +{% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} {% set space = settings['Container space']|int %} +{% set purpose = m.comment('Container of the ' ~ product.name ~ ' data product, as the organisation profile places it.') %} {% for c in containers if not c.parent and not c.adopted %} {% set children = containers|selectattr('parent', 'equalto', c.name)|list %} CREATE DATABASE {{ c.name|ident }} AS PERM = {{ space * ([children|length, 1]|max if not c.holds_objects else children|length + 1) }}; +COMMENT ON DATABASE {{ c.name|ident }} IS {{ purpose }}; {%- endfor %} {% for c in containers if c.parent and not c.adopted %} CREATE DATABASE {{ c.name|ident }} FROM {{ c.parent|ident }} AS PERM = {{ space }}; +COMMENT ON DATABASE {{ c.name|ident }} IS {{ purpose }}; {%- endfor %} diff --git a/implementation/teradata/patterns/object-placement/README.md b/implementation/teradata/patterns/object-placement/README.md index 577fe5a..296e670 100644 --- a/implementation/teradata/patterns/object-placement/README.md +++ b/implementation/teradata/patterns/object-placement/README.md @@ -15,7 +15,7 @@ Binding of [Object Placement](../../../../design/patterns/object-placement.md). ## Template -`01-containers.sql.j2` is the first phase. It creates one database for each entry in the context's `containers` that is not adopted: parents first, then each child `FROM` its parent. Adopted containers already exist and are never created. A database that holds objects gets the `Container space` binding setting as permanent space (the organisation profile's `Settings:`, or the binding's declared default of 1 GB, recorded in `defaults_applied`); a parent that holds no objects gets enough to allocate its children. `CREATE DATABASE` fails on an existing name, so accidental reuse fails rather than blending two products. +`01-containers.sql.j2` is the first phase. It creates one database for each entry in the context's `containers` that is not adopted: parents first, then each child `FROM` its parent. Adopted containers already exist and are never created. A database that holds objects gets the `Container space` binding setting as permanent space (the organisation profile's `Settings:`, or the binding's declared default of 1 GB, recorded in `defaults_applied`); a parent that holds no objects gets enough to allocate its children. `CREATE DATABASE` fails on an existing name, so accidental reuse fails rather than blending two products. Each database it creates carries a `COMMENT ON DATABASE`, which the `comment-coverage` product check requires. Every database name, and the database of every object, comes from the organisation profile through the resolver: the profile is the organisation's object-placement implementation ([Organisation Profile Standard](../../../../design/core/ORGANISATION_PROFILE.md) §1). No name is inferred from a product prefix, and the example profiles are two possible placements, not a required convention. diff --git a/implementation/teradata/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 b/implementation/teradata/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 index 6e19b4c..db49e94 100644 --- a/implementation/teradata/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 +++ b/implementation/teradata/patterns/temporal-lifecycle-metadata/03-access-views.sql.j2 @@ -1,23 +1,34 @@ {# The governed base view of every persisted relation (this binding emits `base_view: all`): every column, every row, the full temporal contract. Then the consumer view: current rows - only, without the open interval ends, which are always open on this surface. Point-in-time - reads go through the base view with the profile's predicates; Teradata SQL has no table - functions. Runtime relations have no consumer view. Views read with an access lock so a - consumer never blocks a load. #} + only, without the open interval ends, which are always open on this surface. The consumer + view selects the build context's `view_columns`, the same list Semantic registers as the + view's AccessColumn rows, so the two cannot drift. Point-in-time reads go through the base + view with the profile's predicates; Teradata SQL has no table functions. Runtime relations + have no consumer view. Views read with an access lock so a consumer never blocks a load. + Every view and every view column carries the comment of the table column beneath it. #} {% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} +{% macro describe_columns(x, view, physical_names) -%} +{% for name in physical_names %}{% set col = x.columns|selectattr('physical_name', 'equalto', name)|first -%} +COMMENT ON COLUMN {{ view|qualified }}.{{ name|ident }} IS {{ m.comment(col.description or col.name) }}; +{% endfor -%} +{%- endmacro %} {% for x in entities + relations if x.relation == 'table' and (x.defined|default(true)) and x.objects.base_view is defined %} -REPLACE VIEW {{ x.objects.base_view|qualified }} AS +{% set names = x.columns|map(attribute='physical_name')|list %} +REPLACE VIEW {{ x.objects.base_view|qualified }} ({{ m.cols(names) }}) AS LOCKING ROW FOR ACCESS -SELECT {{ m.cols(x.columns|map(attribute='physical_name')) }} +SELECT {{ m.cols(names) }} FROM {{ x.objects.table|qualified }}; COMMENT ON VIEW {{ x.objects.base_view|qualified }} IS {{ m.comment('Governed full contract: ' ~ x.description) }}; +{{ describe_columns(x, x.objects.base_view, names) }} {% endfor %} {% for x in entities + relations if x.relation == 'table' and (x.defined|default(true)) and x.objects.consumer_view is defined %} {% set source = x.objects.base_view if x.objects.base_view is defined else x.objects.table %} -REPLACE VIEW {{ x.objects.consumer_view|qualified }} AS +{% set names = x.view_columns|map(attribute='physical_name')|list %} +REPLACE VIEW {{ x.objects.consumer_view|qualified }} ({{ m.cols(names) }}) AS LOCKING ROW FOR ACCESS -SELECT {{ m.cols(x.columns|rejectattr('name', 'in', ('valid_to_dts', 'transaction_to_dts'))|map(attribute='physical_name')) }} +SELECT {{ m.cols(names) }} FROM {{ source|qualified }} WHERE {{ m.current(x) }}; COMMENT ON VIEW {{ x.objects.consumer_view|qualified }} IS {{ m.comment('Current consumer surface: ' ~ x.description) }}; +{{ describe_columns(x, x.objects.consumer_view, names) }} {% endfor %} diff --git a/implementation/teradata/patterns/temporal-lifecycle-metadata/README.md b/implementation/teradata/patterns/temporal-lifecycle-metadata/README.md index a885c71..f739243 100644 --- a/implementation/teradata/patterns/temporal-lifecycle-metadata/README.md +++ b/implementation/teradata/patterns/temporal-lifecycle-metadata/README.md @@ -1,58 +1,60 @@ ---- -title: Teradata Temporal Lifecycle Metadata Pattern Implementation -anchor: temporal-lifecycle-metadata -type: implementation -status: draft -version: 3.0 -normative: true -implements: temporal-lifecycle-metadata -platform: teradata ---- - -# Teradata: Temporal Lifecycle Metadata - -Binding of [Temporal Lifecycle Metadata](../../../../design/patterns/temporal-lifecycle-metadata.md). The design remains the source of truth; the binding supplies reusable mechanisms, not a product schema. - -## Templates - -`00-macros.sql.j2` holds the macros every other template imports. Each takes a relation from the context (an entity or a standard-owned relation, with its `columns`, `column_names`, `key_columns`, `version_columns` and placed `objects`) and composes no name: - -| Macro | Emits | -|---|---| -| `sentinel` | The open end of an interval, `TIMESTAMP '9999-12-31 23:59:59.999999+00:00'`, written by every statement that opens a version and never as a `DEFAULT` (PLATFORM_PROFILE §7) | -| `cols`, `column`, `unicode`, `vector_pattern` | A quoted column list; one column definition with type, `CHARACTER SET UNICODE` for text, identity, `NOT NULL`, `UNIQUE`, and Enum, flag and vector-dimension checks | -| `primary_index`, `uniqueness`, `constraints` | The primary index chosen from the access path; uniqueness of each version (or a keymap's key); interval-order checks | -| `current` | The current-row predicate | -| `table` | `CREATE MULTISET TABLE`, the keymap foreign key (`WITH NO CHECK OPTION`) and the primary index | -| `module_tables` | Every persisted, defined, non-adopted relation of one module, keymaps first | -| `value`, `ref`, `insert`, `member` | A Python value as SQL; a subquery for a current row's key; an insert by logical attribute name with open-row values for temporal columns; whether an Enum permits a value | -| `comment` | A comment literal held to the dictionary's 255 characters | - -`02-comments.sql.j2` applies `COMMENT ON TABLE` and `COMMENT ON COLUMN` to every persisted relation from its `description` and each column's description (or its logical name), after the tables and before anything depends on them. - -`03-access-views.sql.j2` creates two views per persisted relation, each with `REPLACE VIEW` and `LOCKING ROW FOR ACCESS`, so a consumer never blocks a load: - -- the **governed base view**, for every relation with a `base_view` object (this binding emits `base_view: all`): every column and every row, the full temporal contract. Teradata SQL has no table functions, so a read as at an instant applies the profile's predicates to this view. For versioned relations it is granted to the read and agent tiers ([access-layer](../access-layer/README.md)); -- the **consumer view**, for every relation with a `consumer_view` object: its current rows (the `current` macro) over the base view, carrying every column except the open interval ends `valid_to_dts` and `transaction_to_dts`. Runtime relations have no consumer view. - -`04-maintenance.sql.j2` is caller-rendered with `entity`, `stable_key`, `effective_at`, `recorded_at` and `replacement_rows` (each keyed by physical column name). It is a transaction body: the caller controls the transaction and runs the checks before ending it. It: - -- for a bitemporal relation, closes every open knowledge interval of the identity at `recorded_at` and clears its current flag; -- otherwise, closes the version whose validity covers `effective_at` at that instant; -- inserts every replacement row with every column: an open end given as `infinity` is written as the sentinel, an instant is cast to its timestamp type, and any other value goes through the literal filter. - -It is not a change planner. The caller supplies complete, non-overlapping intervals (for a bitemporal correction, the whole new known timeline), detects unchanged replay, preserves unaffected segments and tombstones, and rejects non-monotonic correction instants. Teradata enforces none of the cross-row rules in its schema; the `entity temporal` check finds them. - -`05-statistics.sql.j2` collects statistics on every persisted relation's key and version columns and its currency, deletion and interval-start columns (PLATFORM_PROFILE §6). - -Validity and knowledge intervals are half-open; the sentinel appears only as an open end. For bitemporal history, overlap is prohibited only where both axes overlap. The current flag is true exactly when every applicable end is open. - -## Capability bindings - -| Capability | Binding | -|---|---| -| `CurrentStateFilter` | `current` macro behind the consumer views and reference checks. | -| `PointInTimeReconstruction` | The governed base view, granted to the read and agent tiers; the profile's predicates select the rows at an instant. | -| `SoftDelete` | `is_deleted` and `deleted_dts`; the `entity temporal` check finds a disagreement; deleted rows are excluded from current reads. | - -Templates read only the build context: see [tooling/build](../../../../tooling/build/README.md) and the [Platform Implementation Authoring Standard](../../../../design/core/IMPLEMENTATION_AUTHORING.md) §3 and §6. Phases, checks, types and expression fragments are declared in [PLATFORM_PROFILE.md](../../PLATFORM_PROFILE.md#binding-declarations); coverage and gaps are in [CONFORMANCE.md](../../CONFORMANCE.md). Rendering uses StrictUndefined. +--- +title: Teradata Temporal Lifecycle Metadata Pattern Implementation +anchor: temporal-lifecycle-metadata +type: implementation +status: draft +version: 3.0 +normative: true +implements: temporal-lifecycle-metadata +platform: teradata +--- + +# Teradata: Temporal Lifecycle Metadata + +Binding of [Temporal Lifecycle Metadata](../../../../design/patterns/temporal-lifecycle-metadata.md). The design remains the source of truth; the binding supplies reusable mechanisms, not a product schema. + +## Templates + +`00-macros.sql.j2` holds the macros every other template imports. Each takes a relation from the context (an entity or a standard-owned relation, with its `columns`, `column_names`, `key_columns`, `version_columns` and placed `objects`) and composes no name: + +| Macro | Emits | +|---|---| +| `sentinel` | The open end of an interval, `TIMESTAMP '9999-12-31 23:59:59.999999+00:00'`, written by every statement that opens a version and never as a `DEFAULT` (PLATFORM_PROFILE §7) | +| `cols`, `column`, `unicode`, `vector_pattern` | A quoted column list; one column definition with type, `CHARACTER SET UNICODE` for text, identity, `NOT NULL`, `UNIQUE`, and Enum, flag and vector-dimension checks | +| `primary_index`, `uniqueness`, `constraints` | The primary index chosen from the access path; uniqueness of each version (or a keymap's key); interval-order checks | +| `current` | The current-row predicate | +| `table` | `CREATE MULTISET TABLE`, the keymap foreign key (`WITH NO CHECK OPTION`) and the primary index | +| `module_tables` | Every persisted, defined, non-adopted relation of one module, keymaps first | +| `value`, `ref`, `insert`, `member` | A Python value as SQL; a subquery for a current row's key; an insert by logical attribute name with open-row values for temporal columns; whether an Enum permits a value | +| `comment` | A comment literal held to the dictionary's 255 characters | + +`02-comments.sql.j2` applies `COMMENT ON TABLE` and `COMMENT ON COLUMN` to every persisted relation from its `description` and each column's description (or its logical name), after the tables and before anything depends on them. + +`03-access-views.sql.j2` creates two views per persisted relation, each with `REPLACE VIEW` and `LOCKING ROW FOR ACCESS`, so a consumer never blocks a load: + +- the **governed base view**, for every relation with a `base_view` object (this binding emits `base_view: all`): every column and every row, the full temporal contract. Teradata SQL has no table functions, so a read as at an instant applies the profile's predicates to this view. For versioned relations it is granted to the read and agent tiers ([access-layer](../access-layer/README.md)); +- the **consumer view**, for every relation with a `consumer_view` object: its current rows (the `current` macro) over the base view, carrying the context's `view_columns` (every column except the open interval ends `valid_to_dts` and `transaction_to_dts`). Semantic registers the same list as the view's `AccessColumn` rows, so the two cannot drift. Runtime relations have no consumer view. + +Both views are created with a column list, and the view and each of its columns is commented from the entity description and the table column's description (or its logical name), each held to 255 characters. + +`04-maintenance.sql.j2` is caller-rendered with `entity`, `stable_key`, `effective_at`, `recorded_at` and `replacement_rows` (each keyed by physical column name). It is a transaction body: the caller controls the transaction and runs the checks before ending it. It: + +- for a bitemporal relation, closes every open knowledge interval of the identity at `recorded_at` and clears its current flag; +- otherwise, closes the version whose validity covers `effective_at` at that instant; +- inserts every replacement row with every column: an open end given as `infinity` is written as the sentinel, an instant is cast to its timestamp type, and any other value goes through the literal filter. + +It is not a change planner. The caller supplies complete, non-overlapping intervals (for a bitemporal correction, the whole new known timeline), detects unchanged replay, preserves unaffected segments and tombstones, and rejects non-monotonic correction instants. Teradata enforces none of the cross-row rules in its schema; the `entity temporal` check finds them. + +`05-statistics.sql.j2` collects statistics on every persisted relation's key and version columns and its currency, deletion and interval-start columns (PLATFORM_PROFILE §6). + +Validity and knowledge intervals are half-open; the sentinel appears only as an open end. For bitemporal history, overlap is prohibited only where both axes overlap. The current flag is true exactly when every applicable end is open. + +## Capability bindings + +| Capability | Binding | +|---|---| +| `CurrentStateFilter` | `current` macro behind the consumer views and reference checks. | +| `PointInTimeReconstruction` | The governed base view, granted to the read and agent tiers; the profile's predicates select the rows at an instant. | +| `SoftDelete` | `is_deleted` and `deleted_dts`; the `entity temporal` check finds a disagreement; deleted rows are excluded from current reads. | + +Templates read only the build context: see [tooling/build](../../../../tooling/build/README.md) and the [Platform Implementation Authoring Standard](../../../../design/core/IMPLEMENTATION_AUTHORING.md) §3 and §6. Phases, checks, types and expression fragments are declared in [PLATFORM_PROFILE.md](../../PLATFORM_PROFILE.md#binding-declarations); coverage and gaps are in [CONFORMANCE.md](../../CONFORMANCE.md). Rendering uses StrictUndefined. diff --git a/implementation/teradata/patterns/validation/03-relationship-reference.sql.j2 b/implementation/teradata/patterns/validation/03-relationship-reference.sql.j2 index 99c7706..ed64169 100644 --- a/implementation/teradata/patterns/validation/03-relationship-reference.sql.j2 +++ b/implementation/teradata/patterns/validation/03-relationship-reference.sql.j2 @@ -1,7 +1,8 @@ {# Rendered per relationship, with `relationship`. Current-surface referential coverage, not - historical proof: every returned row is a current reference with no current target. A generic - reference is checked only for the rows its discriminator points at this target. #} + historical proof: every returned row is a current reference with no current target. A join of + several columns resolves only when every pair matches. A generic reference is checked only for + the rows its discriminator points at this target. #} {% set r = relationship %}{% set source = named(r.source) %} -SELECT s.{{ r.source_column|ident }} FROM {{ (r.source_view or r.source_table)|qualified }} s -LEFT JOIN {{ (r.target_view or r.target_table)|qualified }} t ON s.{{ r.source_column|ident }} = t.{{ r.target_columns[0]|ident }} -WHERE {% if r.discriminator %}s.{{ source.column_names[r.discriminator]|ident }} = {{ r.discriminator_value|literal }} AND {% endif %}{% if not r.mandatory %}s.{{ r.source_column|ident }} IS NOT NULL AND {% endif %}t.{{ r.target_columns[0]|ident }} IS NULL +SELECT {% for p in r.pairs %}s.{{ p.source_column|ident }}{% if not loop.last %}, {% endif %}{% endfor %} FROM {{ (r.source_view or r.source_table)|qualified }} s +LEFT JOIN {{ (r.target_view or r.target_table)|qualified }} t ON {% for p in r.pairs %}s.{{ p.source_column|ident }} = t.{{ p.target_column|ident }}{% if not loop.last %} AND {% endif %}{% endfor %} +WHERE {% if r.discriminator %}s.{{ source.column_names[r.discriminator]|ident }} = {{ r.discriminator_value|literal }} AND {% endif %}{% if not r.mandatory %}{% for p in r.pairs %}s.{{ p.source_column|ident }} IS NOT NULL AND {% endfor %}{% endif %}t.{{ r.pairs[0].target_column|ident }} IS NULL diff --git a/implementation/teradata/patterns/validation/04-trust-map.sql.j2 b/implementation/teradata/patterns/validation/04-trust-map.sql.j2 index 4a9d9b6..aae51f0 100644 --- a/implementation/teradata/patterns/validation/04-trust-map.sql.j2 +++ b/implementation/teradata/patterns/validation/04-trust-map.sql.j2 @@ -7,6 +7,7 @@ producer's entry stands for each area. The same columns as every binding's trust map. Teradata views take no WITH clause or ORDER BY here, so the steps are derived tables and the latest-row choices are QUALIFY; a NULL in a union branch is typed, or the union fails. #} +{% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %} {% set view = named('TrustMap') %}{% set run = named('ValidationRun') %}{% set area = named('ValidationArea') %} {% set registry = named('DataProductRegistry') %} {% if view and run and area and registry %} @@ -82,4 +83,27 @@ SELECT p.product_code, p.producer_id, CAST(NULL AS VARCHAR(400)), 'PRODUCT', p.p FROM {{ product }} p WHERE NOT EXISTS (SELECT 1 FROM {{ evidence }} e WHERE e.product_code = p.product_code AND (p.producer_id IS NULL OR e.producer_id = p.producer_id)); +COMMENT ON VIEW {{ view.objects.consumer_view|qualified }} IS {{ m.comment('Trust map: per area, the latest evidence from the designated producer, read at unknown confidence once expired. Read it before relying on any other object.') }}; +{% for name, meaning in [ + ('product_code', 'Data product the evidence describes.'), + ('producer_id', 'Validation producer whose evidence is read; null when none is designated.'), + ('run_id', 'Validation run that supplied the area evidence; null when none exists.'), + ('scope_kind', 'Kind of scope the evidence covers, for example MODULE, ENTITY or PRODUCT.'), + ('scope_id', 'Identifier of the scope the evidence covers.'), + ('area_status', 'Outcome of the area: pass, fail, partial or no-evidence.'), + ('confidence', 'Trust confidence for the area; unknown once the evidence has expired.'), + ('checks_expected', 'Number of checks the area should have run.'), + ('checks_ran', 'Number of checks the area actually ran.'), + ('coverage', 'Fraction of expected checks that ran; null when none were expected.'), + ('passed_count', 'Number of checks that passed.'), + ('failed_count', 'Number of checks that failed.'), + ('error_count', 'Number of checks that errored before producing a result.'), + ('critical_failure_count', 'Number of failed checks of critical severity.'), + ('error_failure_count', 'Number of failed or errored checks counted against trust.'), + ('open_gaps', 'Known gaps in the evidence; states staleness when the evidence has expired.'), + ('recommended_action', 'Next step to restore or confirm trust.'), + ('completed_dts', 'When the area evidence was completed; null when none exists.'), + ('derived', '1 when the entry is derived from a legacy run that published no area records, otherwise 0.')] -%} +COMMENT ON COLUMN {{ view.objects.consumer_view|qualified }}.{{ name|ident }} IS {{ m.comment(meaning) }}; +{% endfor -%} {% endif %} diff --git a/implementation/teradata/patterns/validation/05-comment-coverage.sql.j2 b/implementation/teradata/patterns/validation/05-comment-coverage.sql.j2 new file mode 100644 index 0000000..339a9f7 --- /dev/null +++ b/implementation/teradata/patterns/validation/05-comment-coverage.sql.j2 @@ -0,0 +1,32 @@ +{# MetadataCoverageCheck, catalogue-wide: every row returned is a database, object or column with no comment. + Sweeps every container the product creates, and every object and column in a container that holds + objects, so an object the specification never mentions is caught too. A container the product + adopted is the organisation's, not the product's to document. Objects swept: tables, views, + macros, procedures and functions. Every text column is cast, because a union takes its widths + from its first branch. #} +{% set created -%} +{%- for c in containers if not c.adopted %}{{ c.name|literal }}{% if not loop.last %}, {% endif %}{% endfor -%} +{%- endset %} +{% set holding -%} +{%- for c in containers if c.holds_objects and not c.adopted %}{{ c.name|literal }}{% if not loop.last %}, {% endif %}{% endfor -%} +{%- endset %} +SELECT CAST(TRIM(d.DatabaseName) AS VARCHAR(128)) AS container_name, CAST('database' AS VARCHAR(20)) AS object_kind, + CAST(TRIM(d.DatabaseName) AS VARCHAR(128)) AS object_name, CAST(NULL AS VARCHAR(128)) AS column_name +FROM DBC.DatabasesV d +WHERE TRIM(d.DatabaseName) IN ({{ created }}) + AND COALESCE(TRIM(d.CommentString), '') = '' +UNION ALL +SELECT CAST(TRIM(t.DatabaseName) AS VARCHAR(128)), + CAST(CASE t.TableKind WHEN 'V' THEN 'view' WHEN 'M' THEN 'macro' WHEN 'P' THEN 'procedure' WHEN 'E' THEN 'procedure' + WHEN 'F' THEN 'function' WHEN 'R' THEN 'function' WHEN 'A' THEN 'function' ELSE 'table' END AS VARCHAR(20)), + CAST(TRIM(t.TableName) AS VARCHAR(128)), CAST(NULL AS VARCHAR(128)) +FROM DBC.TablesV t +WHERE TRIM(t.DatabaseName) IN ({{ holding }}) AND t.TableKind IN ('T', 'O', 'V', 'M', 'P', 'E', 'F', 'R', 'A') + AND COALESCE(TRIM(t.CommentString), '') = '' +UNION ALL +SELECT CAST(TRIM(c.DatabaseName) AS VARCHAR(128)), CAST('column' AS VARCHAR(20)), CAST(TRIM(c.TableName) AS VARCHAR(128)), + CAST(TRIM(c.ColumnName) AS VARCHAR(128)) +FROM DBC.ColumnsV c +INNER JOIN DBC.TablesV t ON t.DatabaseName = c.DatabaseName AND t.TableName = c.TableName +WHERE TRIM(c.DatabaseName) IN ({{ holding }}) AND t.TableKind IN ('T', 'O', 'V') + AND COALESCE(TRIM(c.CommentString), '') = '' diff --git a/implementation/teradata/patterns/validation/README.md b/implementation/teradata/patterns/validation/README.md index 548cce6..5f24325 100644 --- a/implementation/teradata/patterns/validation/README.md +++ b/implementation/teradata/patterns/validation/README.md @@ -24,8 +24,10 @@ Each check is a query whose every row is a defect. The `Checks:` block of [PLATF | `relationship reference` | `03-relationship-reference.sql.j2` | each entry in `relationships` (`relationship`) | current source references with no current target, read through the consumer views (the tables where no view is placed); an optional reference may be null; a generic reference is checked only where its discriminator names this target | | `module semantic` | [`modules/semantic/validation.sql.j2`](../../modules/semantic/README.md) | the Semantic module | unregistered tables | | `module memory` | [`modules/memory/validation.sql.j2`](../../modules/memory/README.md) | the Memory module | modules below the capture floor | +| `contract identity`, `contract keys`, `contract relationships`, `contract access-columns`, `contract composition` | [`modules/semantic/contract/`](../../modules/semantic/README.md) | once, for the Semantic module (`contract` scope); rendered as `contract-identity` and so on | the defects of the Semantic metadata contract (`INV-SEMANTIC-016` to `020`), as `object_identity` and `defect` | +| `product comment-coverage` | `05-comment-coverage.sql.j2` | once, for the product (`product` scope) | each database, table, view, macro, procedure, function and table or view column with no comment, read from `DBC.DatabasesV`, `DBC.TablesV` and `DBC.ColumnsV` over the containers the product creates (databases) and those that hold objects (objects and columns); an adopted container is not swept | -The reference check is current-surface coverage, not historical proof, and compares the first target column only. +The reference check is current-surface coverage, not historical proof. A join of several columns resolves only when every pair matches, so the check tests all of them. ## Trust map @@ -37,7 +39,7 @@ The reference check is current-surface coverage, not historical proof, and compa - where no producer is designated, the most cautious producer's entry stands for each area; - a product with no evidence at all reads `no-evidence` and `unknown`. -Teradata views take no `WITH` clause and no `ORDER BY`, so each step is a derived table, and a `NULL` in a union branch is cast to its column's type. +Teradata views take no `WITH` clause and no `ORDER BY`, so each step is a derived table, and a `NULL` in a union branch is cast to its column's type. The view and each of its columns carry a comment. ## Validator diff --git a/release/COMPATIBILITY.md b/release/COMPATIBILITY.md new file mode 100644 index 0000000..fb38cd1 --- /dev/null +++ b/release/COMPATIBILITY.md @@ -0,0 +1,97 @@ +# Compatibility policy and release index + +This directory holds the release bundle: the consumer contract of a deployed AI-Native Data +Product, versioned with the standard. It is not a module. Nothing here is deployed; the bundle +describes what a deployment implements and what an agent reading it can rely on. + +Each `vX.Y.Z/` directory is generated by `tooling/release/build_release.py` from the repository +and is checked for staleness by `tooling/release/tests`. Do not edit generated files. This +document and `migrations/` are authored. + +## Reading a deployment + +A deployment declares the standard version it implements in `StandardDeclaration`, and the manifest +view repeats it as its last column, `standard_version`. A declaration is an assertion, not evidence. +Evidence is the trust map. + +| Question | Where an agent looks | +|---|---| +| Which release does this deployment claim? | The manifest's `standard_version` | +| Which modules and patterns, on which platform? | The `StandardDeclaration` consumer view | +| What can I rely on in that release? | `release/vX.Y.Z/contract.json` for that version | +| What changed since the release I know? | The rows below and `migrations/` | + +## Versioning + +One semantic version covers the whole standard, held in the root `VERSION` file. A tag must equal +`v` plus that value. + +| Bump | When | +|---|---| +| Major | A consumer-facing surface changes incompatibly: a metadata relation, attribute or discovery view is removed or renamed, an attribute changes meaning or type, a discovery role changes the object it names, or the rules for identity, keys or join order change | +| Minor | A surface is added without changing an existing one: a new relation, view or attribute, a new check, a new optional capability | +| Patch | Comment text, documentation or tooling changes that alter no surface | + +## What is stable + +A consumer may rely on these across minor and patch releases: + +- **Logical identities.** Entities, access objects and composites are identified by the grammar in + `contract.json` (`identity_grammar`): `module.entity`, `module.entity:surface`, `module:object` and + `composite:name`. An identity contains no container, product prefix or object-type affix and does + not change when a container is renamed or the design is deployed with different placement. The + physical container and object name are separate columns. +- **Ordered groups are rows.** A key is the ordered rows of `KeyColumn`; a relationship is a join group + whose column pairs are the ordered rows of `RelationshipColumn`; a consumer view's output columns are + the rows of `AccessColumn`. Ordinals run 1 to n without gaps. No platform stores a group as a list, + an array or a delimited string. +- **Join paths are rows.** `RelationshipPaths` returns one row per step and key position, with the + columns and order listed in `contract.json`. A path has at most four steps and visits no entity + twice. The consumer assembles the join; no join string is part of the contract. +- **Discovery roles.** The roles in `contract.json` and the views they name, with their columns and the + order in which an agent reads them. +- **Standard relations and views.** The names and types of each relation in `contract.json`; the columns + of each discovery view. New columns are added only at the end of a view. Column meanings are in + `design/modules/semantic.md`. +- **Comments exist.** Every product container (where the platform allows), table, view, routine and + column carries a comment (the `comment-coverage` check). The wording of a comment is not stable. + +## Known limitations in 2.0.0 + +| Area | Limitation today | +|---|---| +| Physical keys | A key is described as ordered columns, but the Domain module still allocates one surrogate column per entity. A composite natural key is registered as ordered rows; this is metadata, not a change to how rows are keyed | +| Bare references to composite identities | A bare `[-> Target]` carries one source column. Where the target's identity has several columns it pairs the first and the build reports the rest as a gap. Name the target attributes (`[-> Target.attribute]`) to declare the whole join | +| Generated mappings | Bindings generate `PASSTHROUGH` access columns for consumer views over entities. `RENAMED` and `DERIVED` mappings and composite access objects are not generated by any binding | +| Discovery views | The standard-owned discovery views are consumer views over no registered entity. Their columns are published here, not as `AccessColumn` rows | +| Declaration | A declaration is an assertion. Nothing verifies it against the deployment | +| Column meanings | The release bundle lists the standard relations' attributes by name and type; their meanings are in the design | + +## Platform differences a consumer may meet + +The contract is the same on every platform: identity grammar, ordered-group rows and path columns. + +| Platform | Difference | +|---|---| +| duckdb | Schemas cannot be commented, so the comment-coverage sweep covers tables, views, columns, sequences and macros | +| teradata | The point-in-time surface is a view, not a callable function (declared in the binding's gaps) | + +## Release index + +| Version | Date | Type | Summary | Migration | +|---|---|---|---|---| +| 2.0.0 | prepared 2026-10-14, not yet tagged | Major | Logical identities, key definitions, relationship column groups, path rows and access-column mappings on all three platforms; the release bundle and declared standard version; comment coverage | [v2.0.0](migrations/v2.0.0.md) | +| 1.0.0 | 2026-05-30 | Initial | First tagged release; no bundle was published | none | + +A 1.1.0 release was prepared and never tagged. Its content (the release bundle, the declaration and +comment coverage) is part of 2.0.0. + +## Binding adoption + +| Platform | Declares the standard version | Notes | +|---|---|---| +| postgres | yes | Templates render and share registration with DuckDB; engine tests need a database | +| duckdb | yes | Reference products deployed and every check run in the test suite | +| teradata | yes | Templates render; execution evidence is recorded in the migration note | + +`release/vX.Y.Z/bindings.json` records the same status and is checked against the sources. diff --git a/release/migrations/v2.0.0.md b/release/migrations/v2.0.0.md new file mode 100644 index 0000000..7e5615d --- /dev/null +++ b/release/migrations/v2.0.0.md @@ -0,0 +1,72 @@ +# Migration note: 2.0.0 + +Type: **major, breaking.** The metadata an agent reads from a deployed product changes shape. Business +data tables, their keys and their temporal mechanics do not change. This release also carries the +content of a prepared 1.1.0 that was never tagged: the release bundle, the declared standard version +and comment coverage. + +## What changed + +### Breaking + +| Change | Detail | +|---|---| +| Logical identities | An entity is `module.entity` (`domain.ticket`); an access object is `module.entity:base`, `:governed`, `:current`, `:history`, `module:object` or `composite:name`. Identities no longer contain a container. Every attribute that held an entity or object identity now holds the logical form | +| Keys | `EntityMetadata` loses `surrogate_key_column` and `natural_key_column`. Keys are rows in the new `KeyDefinition` and `KeyColumn`. A composite natural key, which 1.x stored as one comma-joined string, is now its ordered rows | +| Relationships | `TableRelationship` loses its container, table and column attributes and names the logical `source_entity` and `target_entity`. The column pairs are rows in the new `RelationshipColumn`; the relationship is the join group | +| Composites | `AccessComposition.join_path` is removed. A member names `via_relationship_id` and `via_direction` | +| Paths | `RelationshipPaths` returns rows, not strings, with the columns `path_id, origin_entity, destination_entity, hop_count, hop, key_ordinal, from_entity, from_column, to_entity, to_column, relationship_id, direction`. The old `join_path` string and the `hops` column are gone on every platform | +| Access objects | `AccessObject` gains `object_identity`. `represents_entity` and `resolves_to_object` hold logical identities | + +### Additive + +| Change | Detail | +|---|---| +| Access-column mappings | New `AccessColumn`: one row per output column of a consumer view over an entity, with its source entity and column and a mapping kind | +| Declared standard version | New `StandardDeclaration`; `standard_version` is the last column of the manifest view | +| Release bundle | `release/v2.0.0/` publishes the contract, bindings, examples and conformance checks | +| Conformance checks | The five `contract` checks (`INV-SEMANTIC-016` to `020`) and the `comment-coverage` check on all three platforms | +| Comment parity | Views, view columns, functions or macros, sequences and (where the platform allows) schemas carry comments in rendered products | +| Composite joins | A specification can declare a multi-column join with `[-> Target.attribute]` on each source attribute, grouped by target and, where needed, `[join: name]`. The standard's `ValidationArea` to `ValidationRun` join is declared this way. The relationship reference check joins on every pair | +| Standard-owned relationships | The resolver now registers the joins of the standard-owned relations too (for example `LineageRun` to `DataLineage`), so a product's relationship catalogue and path view are larger than in 1.x | +| Catalogue column comments | Every attribute of every standard relation in the design now states what it holds, so the catalogue tables' column comments describe each column instead of repeating its name | + +## The old-to-new identity rule + +| Old | New | +|---|---| +| Entity named by its business name or `schema.table` | `module.entity`: the module anchor, a dot, the logical name in snake_case (`ValidationRun` becomes `validation_run`) | +| Access object named by `container.object` | `module.entity:surface` by the object's role (table `base`, base view `governed`, consumer view `current`, point-in-time object `history`) | + +The container and object name remain in their own columns. Every attribute that held an identity takes +the same rewrite. + +## What a deployment must do + +There is no migration script. Metadata is established by the registration step, so a 1.x deployment +re-registers against the 2.0.0 relations. + +| Step | Action | +|---|---| +| 1 | Create the new relations (`KeyDefinition`, `KeyColumn`, `RelationshipColumn`, `AccessColumn`, `StandardDeclaration`), drop the removed attributes, add the new ones, and replace the discovery views | +| 2 | Re-run registration so every identity uses the grammar and the new relations are populated | +| 3 | Rewrite authored rows that reference identities (metrics, synonyms) with the rule above, or re-capture them from the design | +| 4 | Run the five `contract` checks and `comment-coverage`; every query returns zero rows | + +## What an agent should do + +Read the manifest's `standard_version`. When it is `2.x`, follow the rows described above. When it is null +or the column is absent, the deployment is 1.x: read the 1.x surface and do not assume keys, identities or +paths follow this release. + +To join along a path in 2.x, take the rows of one `path_id`, group them by `hop` in ascending order, and +within a hop join the `from_entity` to the `to_entity` on every `from_column = to_column` pair, ordered by +`key_ordinal`. Resolve each identity to an object through `AccessObject` before issuing SQL. + +## Execution evidence + +| Platform | Evidence | +|---|---| +| duckdb | Reference products deployed and every rendered check run in the test suite, including a mutation test for each contract check and the coverage check, a composite natural key, a composite join and the declaration | +| postgres | Templates render and share registration with DuckDB. The engine tests are the same and need `POSTGRES_TEST_DSN`; they were not run when this note was written | +| teradata | Templates render under strict undefined variables and pass the repository's validation suite. The DDL, views and checks have not been run on a Teradata system | diff --git a/release/v2.0.0/bindings.json b/release/v2.0.0/bindings.json new file mode 100644 index 0000000..846e9fc --- /dev/null +++ b/release/v2.0.0/bindings.json @@ -0,0 +1,152 @@ +{ + "standard_version": "2.0.0", + "bindings": [ + { + "platform": "duckdb", + "modules": [ + "domain", + "memory", + "observability", + "prediction", + "search", + "semantic" + ], + "patterns": [ + "access-layer", + "object-placement", + "physical-storage", + "temporal-lifecycle-metadata", + "validation" + ], + "conformance_documents": [ + "CONFORMANCE.md", + "PLATFORM_PROFILE.md" + ], + "declared_checks": [ + "contract access-columns", + "contract composition", + "contract identity", + "contract keys", + "contract relationships", + "entity metadata", + "entity temporal", + "module memory", + "module semantic", + "product comment-coverage", + "relationship reference" + ], + "declared_gaps": [ + "Adoption", + "Catalogue scope", + "Composition", + "Concurrency", + "Cookbook recipes", + "Coverage not carried over", + "Embedding, model and operations", + "Historical references", + "Maintenance", + "Roles, grants and runtime privacy", + "Temporal enforcement" + ], + "declares_standard_version": true + }, + { + "platform": "postgres", + "modules": [ + "domain", + "memory", + "observability", + "prediction", + "search", + "semantic" + ], + "patterns": [ + "access-layer", + "object-placement", + "physical-storage", + "temporal-lifecycle-metadata", + "validation" + ], + "conformance_documents": [ + "CONFORMANCE.md", + "PLATFORM_PROFILE.md" + ], + "declared_checks": [ + "contract access-columns", + "contract composition", + "contract identity", + "contract keys", + "contract relationships", + "entity metadata", + "entity temporal", + "module memory", + "module semantic", + "product comment-coverage", + "relationship reference" + ], + "declared_gaps": [ + "Adoption", + "Composition", + "Concurrency", + "Cookbook recipes", + "Embedding, model and operations", + "Historical references", + "Maintenance", + "Principals and membership" + ], + "declares_standard_version": true + }, + { + "platform": "teradata", + "modules": [ + "domain", + "memory", + "observability", + "prediction", + "search", + "semantic" + ], + "patterns": [ + "access-layer", + "catalogue-interface", + "object-placement", + "physical-storage", + "temporal-lifecycle-metadata", + "validation" + ], + "conformance_documents": [ + "CONFORMANCE.md", + "PLATFORM_PROFILE.md" + ], + "declared_checks": [ + "contract access-columns", + "contract composition", + "contract identity", + "contract keys", + "contract relationships", + "entity metadata", + "entity temporal", + "module memory", + "module semantic", + "product comment-coverage", + "relationship reference" + ], + "declared_gaps": [ + "Adopted databases", + "Adoption", + "Composition", + "Concurrency", + "Cookbook recipes", + "Deployment", + "Embedding, model and operations", + "Graph lineage and catalogue interface", + "Historical references", + "Maintenance", + "Point-in-time call", + "Principals and membership", + "Temporal enforcement" + ], + "declares_standard_version": true + } + ] +} diff --git a/release/v2.0.0/checks.json b/release/v2.0.0/checks.json new file mode 100644 index 0000000..cf01893 --- /dev/null +++ b/release/v2.0.0/checks.json @@ -0,0 +1,131 @@ +{ + "standard_version": "2.0.0", + "rendered_checks": { + "duckdb": [ + { + "scope_kind": "ENTITY", + "check": "metadata" + }, + { + "scope_kind": "ENTITY", + "check": "reference" + }, + { + "scope_kind": "ENTITY", + "check": "temporal" + }, + { + "scope_kind": "MODULE", + "check": "contract-access-columns" + }, + { + "scope_kind": "MODULE", + "check": "contract-composition" + }, + { + "scope_kind": "MODULE", + "check": "contract-identity" + }, + { + "scope_kind": "MODULE", + "check": "contract-keys" + }, + { + "scope_kind": "MODULE", + "check": "contract-relationships" + }, + { + "scope_kind": "MODULE", + "check": "module" + }, + { + "scope_kind": "PRODUCT", + "check": "comment-coverage" + } + ], + "postgres": [ + { + "scope_kind": "ENTITY", + "check": "metadata" + }, + { + "scope_kind": "ENTITY", + "check": "reference" + }, + { + "scope_kind": "ENTITY", + "check": "temporal" + }, + { + "scope_kind": "MODULE", + "check": "contract-access-columns" + }, + { + "scope_kind": "MODULE", + "check": "contract-composition" + }, + { + "scope_kind": "MODULE", + "check": "contract-identity" + }, + { + "scope_kind": "MODULE", + "check": "contract-keys" + }, + { + "scope_kind": "MODULE", + "check": "contract-relationships" + }, + { + "scope_kind": "MODULE", + "check": "module" + }, + { + "scope_kind": "PRODUCT", + "check": "comment-coverage" + } + ], + "teradata": [ + { + "scope_kind": "ENTITY", + "check": "metadata" + }, + { + "scope_kind": "ENTITY", + "check": "reference" + }, + { + "scope_kind": "ENTITY", + "check": "temporal" + }, + { + "scope_kind": "MODULE", + "check": "contract-access-columns" + }, + { + "scope_kind": "MODULE", + "check": "contract-composition" + }, + { + "scope_kind": "MODULE", + "check": "contract-identity" + }, + { + "scope_kind": "MODULE", + "check": "contract-keys" + }, + { + "scope_kind": "MODULE", + "check": "contract-relationships" + }, + { + "scope_kind": "MODULE", + "check": "module" + }, + { + "scope_kind": "PRODUCT", + "check": "comment-coverage" + } + ] + } +} diff --git a/release/v2.0.0/contract.json b/release/v2.0.0/contract.json new file mode 100644 index 0000000..e461f83 --- /dev/null +++ b/release/v2.0.0/contract.json @@ -0,0 +1,3866 @@ +{ + "standard_version": "2.0.0", + "modules": [ + "memory", + "semantic", + "domain", + "observability", + "search", + "prediction" + ], + "patterns": [ + "object-placement", + "temporal-lifecycle-metadata", + "validation", + "access-layer" + ], + "identity_grammar": [ + { + "kind": "entity", + "form": "module.entity", + "pattern": "^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*$", + "example": "domain.ticket" + }, + { + "kind": "entity surface", + "form": "module.entity:surface (base, governed, current or history)", + "pattern": "^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*:(base|governed|current|history)$", + "example": "domain.ticket:current" + }, + { + "kind": "module-owned object", + "form": "module:object (composite is reserved)", + "pattern": "^(?!composite:)[a-z][a-z0-9_]*:[a-z][a-z0-9_]*$", + "example": "observability:trust_map" + }, + { + "kind": "composite access object", + "form": "composite:name", + "pattern": "^composite:[a-z][a-z0-9_]*$", + "example": "composite:ticket_enriched" + } + ], + "metadata_entities": { + "memory": [ + { + "name": "AgentInteraction", + "identity": "memory.agent_interaction", + "profile": "EVENT_APPEND_ONLY", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "interaction_id" + ] + } + ], + "columns": [ + { + "name": "interaction_id", + "type": "Identifier", + "required": false + }, + { + "name": "session_id", + "type": "Reference", + "required": true + }, + { + "name": "interaction_seq", + "type": "Integer", + "required": true + }, + { + "name": "interaction_type", + "type": "Enum{QUERY|ACTION|DECISION|EXPLANATION}", + "required": false + }, + { + "name": "interaction_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "user_input", + "type": "Text", + "required": false + }, + { + "name": "agent_response", + "type": "Text", + "required": false + }, + { + "name": "action_taken", + "type": "Text", + "required": false + }, + { + "name": "referenced_tables", + "type": "Text", + "required": false + }, + { + "name": "query_executed", + "type": "Text", + "required": false + }, + { + "name": "query_result_count", + "type": "Integer", + "required": false + }, + { + "name": "execution_time_ms", + "type": "Integer", + "required": false + }, + { + "name": "outcome_status", + "type": "Enum{SUCCESS|PARTIAL|FAILED}", + "required": false + }, + { + "name": "user_feedback", + "type": "Enum{POSITIVE|NEUTRAL|NEGATIVE}", + "required": false + }, + { + "name": "scope_level", + "type": "Enum{USER|TEAM|ORGANIZATION|AGENT}", + "required": true + }, + { + "name": "scope_identifier", + "type": "ShortText", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "AgentSession", + "identity": "memory.agent_session", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "session_id" + ] + }, + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "session_key" + ] + } + ], + "columns": [ + { + "name": "session_id", + "type": "Identifier", + "required": false + }, + { + "name": "session_key", + "type": "NaturalKey", + "required": true + }, + { + "name": "agent_key", + "type": "ShortText", + "required": true + }, + { + "name": "user_key", + "type": "ShortText", + "required": false + }, + { + "name": "session_start_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "session_end_dts", + "type": "Timestamp", + "required": false + }, + { + "name": "session_status", + "type": "Enum{ACTIVE|COMPLETED|ABANDONED}", + "required": false + }, + { + "name": "session_goal", + "type": "Text", + "required": false + }, + { + "name": "session_context", + "type": "Json", + "required": false + }, + { + "name": "scope_level", + "type": "Enum{USER|TEAM|ORGANIZATION|AGENT}", + "required": true + }, + { + "name": "scope_identifier", + "type": "ShortText", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "BusinessGlossary", + "identity": "memory.business_glossary", + "profile": "SCD2_HISTORY", + "keys": [ + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "term" + ] + } + ], + "columns": [ + { + "name": "term", + "type": "NaturalKey", + "required": true + }, + { + "name": "term_category", + "type": "Enum{ENTITY|ATTRIBUTE|METRIC|BUSINESS_RULE|CLASSIFICATION|REFERENCE_CODE}", + "required": true + }, + { + "name": "definition", + "type": "LongText", + "required": true + }, + { + "name": "source_module", + "type": "ShortText", + "required": true + }, + { + "name": "valid_from_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "valid_to_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "is_current", + "type": "Flag", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "ChangeLog", + "identity": "memory.change_log", + "profile": "SCD2_HISTORY", + "keys": [ + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "change_id" + ] + } + ], + "columns": [ + { + "name": "change_id", + "type": "NaturalKey", + "required": true + }, + { + "name": "version_number", + "type": "ShortText", + "required": true + }, + { + "name": "change_title", + "type": "ShortText", + "required": true + }, + { + "name": "change_type", + "type": "Enum{INITIAL_RELEASE|SCHEMA_CHANGE|FEATURE_ADDITION|BUG_FIX|PERFORMANCE|DEPRECATION}", + "required": true + }, + { + "name": "source_module", + "type": "ShortText", + "required": true + }, + { + "name": "related_decision_id", + "type": "Reference", + "required": false + }, + { + "name": "valid_from_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "valid_to_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "is_current", + "type": "Flag", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "DesignDecision", + "identity": "memory.design_decision", + "profile": "SCD2_HISTORY", + "keys": [ + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "decision_id" + ] + } + ], + "columns": [ + { + "name": "decision_id", + "type": "NaturalKey", + "required": true + }, + { + "name": "decision_version", + "type": "Integer", + "required": true + }, + { + "name": "decision_title", + "type": "ShortText", + "required": true + }, + { + "name": "context", + "type": "LongText", + "required": false + }, + { + "name": "alternatives", + "type": "LongText", + "required": false + }, + { + "name": "rationale", + "type": "LongText", + "required": false + }, + { + "name": "consequences", + "type": "LongText", + "required": false + }, + { + "name": "decision_status", + "type": "Enum{PROPOSED|ACCEPTED|SUPERSEDED|DEPRECATED}", + "required": true + }, + { + "name": "decision_category", + "type": "Enum{ARCHITECTURE|SCHEMA|NAMING|PERFORMANCE|SECURITY|INTEGRATION|OPERATIONAL}", + "required": true + }, + { + "name": "source_module", + "type": "ShortText", + "required": true + }, + { + "name": "superseded_by", + "type": "Reference", + "required": false + }, + { + "name": "valid_from_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "valid_to_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "is_current", + "type": "Flag", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "DiscoveredPattern", + "identity": "memory.discovered_pattern", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "pattern_id" + ] + } + ], + "columns": [ + { + "name": "pattern_id", + "type": "Identifier", + "required": false + }, + { + "name": "pattern_name", + "type": "ShortText", + "required": true + }, + { + "name": "pattern_type", + "type": "Enum{CORRELATION|TEMPORAL|TABLE_RELATIONSHIP|ANOMALY}", + "required": false + }, + { + "name": "pattern_definition", + "type": "Json", + "required": false + }, + { + "name": "sample_size", + "type": "Integer", + "required": false + }, + { + "name": "confidence_score", + "type": "Decimal(5,4)", + "required": false + }, + { + "name": "involved_tables", + "type": "Text", + "required": false + }, + { + "name": "is_validated", + "type": "Flag", + "required": false + }, + { + "name": "scope_level", + "type": "Enum{USER|TEAM|ORGANIZATION|AGENT}", + "required": true + }, + { + "name": "scope_identifier", + "type": "ShortText", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "ImplementationNote", + "identity": "memory.implementation_note", + "profile": "SCD2_HISTORY", + "keys": [ + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "note_id" + ] + } + ], + "columns": [ + { + "name": "note_id", + "type": "NaturalKey", + "required": true + }, + { + "name": "note_title", + "type": "ShortText", + "required": true + }, + { + "name": "note_content", + "type": "LongText", + "required": true + }, + { + "name": "note_category", + "type": "Enum{DEPLOYMENT|WORKAROUND|KNOWN_ISSUE|PERFORMANCE_TIP|OPERATIONAL|SECURITY}", + "required": true + }, + { + "name": "severity", + "type": "Enum{LOW|MEDIUM|HIGH|CRITICAL}", + "required": false + }, + { + "name": "source_module", + "type": "ShortText", + "required": true + }, + { + "name": "valid_from_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "valid_to_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "is_current", + "type": "Flag", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "LearnedStrategy", + "identity": "memory.learned_strategy", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "strategy_id" + ] + } + ], + "columns": [ + { + "name": "strategy_id", + "type": "Identifier", + "required": false + }, + { + "name": "strategy_name", + "type": "ShortText", + "required": true + }, + { + "name": "strategy_category", + "type": "Enum{QUERY_OPTIMIZATION|FEATURE_SELECTION|ERROR_HANDLING}", + "required": false + }, + { + "name": "strategy_pattern", + "type": "Text", + "required": false + }, + { + "name": "strategy_metadata", + "type": "Json", + "required": false + }, + { + "name": "success_rate", + "type": "Decimal(5,4)", + "required": false + }, + { + "name": "times_used", + "type": "Integer", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "is_validated", + "type": "Flag", + "required": false + }, + { + "name": "scope_level", + "type": "Enum{USER|TEAM|ORGANIZATION|AGENT}", + "required": true + }, + { + "name": "scope_identifier", + "type": "ShortText", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "ModuleRegistry", + "identity": "memory.module_registry", + "profile": "SCD2_HISTORY", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "module_registry_id" + ] + }, + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "module_name" + ] + } + ], + "columns": [ + { + "name": "module_registry_id", + "type": "Identifier", + "required": false + }, + { + "name": "module_name", + "type": "Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY}", + "required": true + }, + { + "name": "container_name", + "type": "ShortText", + "required": true + }, + { + "name": "deployment_status", + "type": "Enum{DEPLOYED|PLANNED|DEPRECATED}", + "required": true + }, + { + "name": "module_version", + "type": "ShortText", + "required": true + }, + { + "name": "module_purpose", + "type": "LongText", + "required": true + }, + { + "name": "key_entities", + "type": "Text", + "required": false + }, + { + "name": "dependencies", + "type": "Text", + "required": false + }, + { + "name": "valid_from_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "valid_to_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "is_current", + "type": "Flag", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "QueryCookbook", + "identity": "memory.query_cookbook", + "profile": "SCD2_HISTORY", + "keys": [ + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "recipe_id" + ] + } + ], + "columns": [ + { + "name": "recipe_id", + "type": "NaturalKey", + "required": true + }, + { + "name": "recipe_title", + "type": "ShortText", + "required": true + }, + { + "name": "use_case", + "type": "ShortText", + "required": true + }, + { + "name": "target_module", + "type": "Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY|CROSS}", + "required": true + }, + { + "name": "query_template", + "type": "LongText", + "required": true + }, + { + "name": "complexity", + "type": "Enum{SIMPLE|MODERATE|COMPLEX|ADVANCED}", + "required": true + }, + { + "name": "is_batch", + "type": "Flag", + "required": false + }, + { + "name": "source_module", + "type": "ShortText", + "required": true + }, + { + "name": "valid_from_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "valid_to_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "is_current", + "type": "Flag", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "UserPreference", + "identity": "memory.user_preference", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "preference_id" + ] + } + ], + "columns": [ + { + "name": "preference_id", + "type": "Identifier", + "required": false + }, + { + "name": "user_key", + "type": "ShortText", + "required": true + }, + { + "name": "preference_category", + "type": "Enum{REPORT_FORMAT|DATA_FILTER|AGGREGATION_LEVEL|VISUALIZATION_TYPE}", + "required": false + }, + { + "name": "preference_name", + "type": "ShortText", + "required": true + }, + { + "name": "preference_value", + "type": "Text", + "required": false + }, + { + "name": "preference_json", + "type": "Json", + "required": false + }, + { + "name": "confidence", + "type": "Decimal(5,4)", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "scope_level", + "type": "Enum{USER|TEAM|ORGANIZATION|AGENT}", + "required": true + }, + { + "name": "scope_identifier", + "type": "ShortText", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + } + ], + "semantic": [ + { + "name": "AccessColumn", + "identity": "semantic.access_column", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "access_column_id" + ] + } + ], + "columns": [ + { + "name": "access_column_id", + "type": "Identifier", + "required": false + }, + { + "name": "access_object_id", + "type": "Reference", + "required": true + }, + { + "name": "ordinal", + "type": "Integer", + "required": true + }, + { + "name": "column_name", + "type": "ShortText", + "required": true + }, + { + "name": "source_entity", + "type": "ShortText", + "required": false + }, + { + "name": "source_column", + "type": "ShortText", + "required": false + }, + { + "name": "mapping_kind", + "type": "Enum{PASSTHROUGH|RENAMED|DERIVED}", + "required": true + }, + { + "name": "derivation_note", + "type": "Text", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "AccessComposition", + "identity": "semantic.access_composition", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "access_composition_id" + ] + } + ], + "columns": [ + { + "name": "access_composition_id", + "type": "Identifier", + "required": false + }, + { + "name": "composite_container", + "type": "ShortText", + "required": true + }, + { + "name": "composite_object", + "type": "ShortText", + "required": true + }, + { + "name": "member_seq", + "type": "Integer", + "required": true + }, + { + "name": "member_entity", + "type": "ShortText", + "required": true + }, + { + "name": "member_role", + "type": "Enum{ANCHOR|INNER|LEFT|RIGHT|FULL}", + "required": true + }, + { + "name": "via_relationship_id", + "type": "Reference", + "required": false + }, + { + "name": "via_direction", + "type": "Enum{FORWARD|REVERSE}", + "required": false + }, + { + "name": "is_grain_contributor", + "type": "Flag", + "required": false + }, + { + "name": "member_note", + "type": "Text", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "AccessObject", + "identity": "semantic.access_object", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "access_object_id" + ] + } + ], + "columns": [ + { + "name": "access_object_id", + "type": "Identifier", + "required": false + }, + { + "name": "object_identity", + "type": "ShortText", + "required": true + }, + { + "name": "container_name", + "type": "ShortText", + "required": true + }, + { + "name": "object_name", + "type": "ShortText", + "required": true + }, + { + "name": "object_type", + "type": "Enum{TABLE|BASE_VIEW|CONSUMER_VIEW|PROCEDURE|FUNCTION}", + "required": true + }, + { + "name": "access_role", + "type": "Enum{BASE|PASSTHROUGH|COMPOSITE}", + "required": true + }, + { + "name": "represents_entity", + "type": "ShortText", + "required": false + }, + { + "name": "object_grain", + "type": "ShortText", + "required": false + }, + { + "name": "is_agent_consumable", + "type": "Flag", + "required": false + }, + { + "name": "is_primary", + "type": "Flag", + "required": false + }, + { + "name": "resolves_to_object", + "type": "ShortText", + "required": false + }, + { + "name": "access_note", + "type": "Text", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "ColumnMetadata", + "identity": "semantic.column_metadata", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "column_metadata_id" + ] + } + ], + "columns": [ + { + "name": "column_metadata_id", + "type": "Identifier", + "required": false + }, + { + "name": "container_name", + "type": "ShortText", + "required": true + }, + { + "name": "table_name", + "type": "ShortText", + "required": true + }, + { + "name": "column_name", + "type": "ShortText", + "required": true + }, + { + "name": "logical_name", + "type": "ShortText", + "required": true + }, + { + "name": "business_description", + "type": "Text", + "required": false + }, + { + "name": "is_pii", + "type": "Flag", + "required": false + }, + { + "name": "sensitivity_marker", + "type": "Enum{PII|PII_INCIDENTAL}", + "required": false + }, + { + "name": "data_classification", + "type": "ShortText", + "required": false + }, + { + "name": "is_required", + "type": "Flag", + "required": false + }, + { + "name": "declared_type", + "type": "ShortText", + "required": false + }, + { + "name": "allowed_values", + "type": "Json", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "DataProductMap", + "identity": "semantic.data_product_map", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "module_id" + ] + } + ], + "columns": [ + { + "name": "module_id", + "type": "Identifier", + "required": false + }, + { + "name": "module_name", + "type": "Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY}", + "required": true + }, + { + "name": "object_type", + "type": "Enum{TABLE|BASE_VIEW|CONSUMER_VIEW|PROCEDURE|FUNCTION|SEQUENCE}", + "required": true + }, + { + "name": "module_purpose", + "type": "Text", + "required": false + }, + { + "name": "container_name", + "type": "ShortText", + "required": true + }, + { + "name": "module_version", + "type": "ShortText", + "required": false + }, + { + "name": "deployment_status", + "type": "Enum{DEPLOYED|PLANNED|DEPRECATED}", + "required": true + }, + { + "name": "graph_key", + "type": "ShortText", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "DataProductOrientation", + "identity": "semantic.data_product_orientation", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "orientation_id" + ] + } + ], + "columns": [ + { + "name": "orientation_id", + "type": "Identifier", + "required": false + }, + { + "name": "product_id", + "type": "NaturalKey", + "required": true + }, + { + "name": "resource_role", + "type": "Enum{MANIFEST|TRUST_MAP|MODULE_MAP|OBJECT_CATALOGUE|ENTITY_CATALOGUE|COLUMN_CATALOGUE|RELATIONSHIP_CATALOGUE|RELATIONSHIP_PATHS|LINEAGE|QUERY_COOKBOOK|GLOSSARY|DESIGN_DECISIONS|POLICY|QUALITY}", + "required": true + }, + { + "name": "container_name", + "type": "ShortText", + "required": false + }, + { + "name": "object_name", + "type": "ShortText", + "required": false + }, + { + "name": "resource_uri", + "type": "ShortText", + "required": false + }, + { + "name": "usage_guidance", + "type": "Text", + "required": false + }, + { + "name": "is_required", + "type": "Flag", + "required": false + }, + { + "name": "discovery_order", + "type": "Integer", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "DataProductRegistry", + "identity": "semantic.data_product_registry", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "product_id" + ] + } + ], + "columns": [ + { + "name": "product_id", + "type": "NaturalKey", + "required": true + }, + { + "name": "product_name", + "type": "ShortText", + "required": true + }, + { + "name": "product_description", + "type": "Text", + "required": true + }, + { + "name": "product_domain", + "type": "ShortText", + "required": true + }, + { + "name": "product_version", + "type": "ShortText", + "required": true + }, + { + "name": "product_status", + "type": "Enum{DRAFT|ACTIVE|DEPRECATED|RETIRED}", + "required": true + }, + { + "name": "owner_team", + "type": "ShortText", + "required": true + }, + { + "name": "technical_contact", + "type": "ShortText", + "required": false + }, + { + "name": "trust_authoritative_producer", + "type": "ShortText", + "required": true + }, + { + "name": "manifest", + "type": "Json", + "required": false + }, + { + "name": "contract_uri", + "type": "ShortText", + "required": false + }, + { + "name": "policy_uri", + "type": "ShortText", + "required": false + }, + { + "name": "quality_uri", + "type": "ShortText", + "required": false + }, + { + "name": "lineage_uri", + "type": "ShortText", + "required": false + }, + { + "name": "approved_entrypoint", + "type": "ShortText", + "required": true + }, + { + "name": "approved_access_mode", + "type": "Enum{VIEW|MCP_TOOL|SEMANTIC_QUERY}", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "is_deleted", + "type": "Flag", + "required": false + }, + { + "name": "deleted_dts", + "type": "Timestamp", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "EntityMetadata", + "identity": "semantic.entity_metadata", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "entity_metadata_id" + ] + } + ], + "columns": [ + { + "name": "entity_metadata_id", + "type": "Identifier", + "required": false + }, + { + "name": "entity_name", + "type": "ShortText", + "required": true + }, + { + "name": "logical_name", + "type": "ShortText", + "required": true + }, + { + "name": "entity_description", + "type": "Text", + "required": true + }, + { + "name": "entity_kind", + "type": "Enum{HISTORY|REFERENCE|RELATIONSHIP|KEYMAP|RECORD}", + "required": true + }, + { + "name": "module_name", + "type": "Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY}", + "required": true + }, + { + "name": "container_name", + "type": "ShortText", + "required": false + }, + { + "name": "table_name", + "type": "ShortText", + "required": true + }, + { + "name": "view_name", + "type": "ShortText", + "required": false + }, + { + "name": "temporal_pattern", + "type": "Enum{CURRENT_STATE|EVENT_APPEND_ONLY|SCD2_HISTORY|SCD2_BITEMPORAL|ASSOCIATION_CURRENT|ASSOCIATION_SCD2|OPERATIONAL_LOG}", + "required": true + }, + { + "name": "current_flag_column", + "type": "ShortText", + "required": false + }, + { + "name": "deleted_flag_column", + "type": "ShortText", + "required": false + }, + { + "name": "assumed_zone", + "type": "ShortText", + "required": false + }, + { + "name": "industry_standard", + "type": "ShortText", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "KeyColumn", + "identity": "semantic.key_column", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "key_column_id" + ] + } + ], + "columns": [ + { + "name": "key_column_id", + "type": "Identifier", + "required": false + }, + { + "name": "key_definition_id", + "type": "Reference", + "required": true + }, + { + "name": "ordinal", + "type": "Integer", + "required": true + }, + { + "name": "column_name", + "type": "ShortText", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "KeyDefinition", + "identity": "semantic.key_definition", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "key_definition_id" + ] + } + ], + "columns": [ + { + "name": "key_definition_id", + "type": "Identifier", + "required": false + }, + { + "name": "entity_id", + "type": "Reference", + "required": true + }, + { + "name": "key_role", + "type": "Enum{SURROGATE|NATURAL|ALTERNATE}", + "required": true + }, + { + "name": "key_name", + "type": "ShortText", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "Metric", + "identity": "semantic.metric", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "metric_id" + ] + } + ], + "columns": [ + { + "name": "metric_id", + "type": "Identifier", + "required": false + }, + { + "name": "metric_name", + "type": "ShortText", + "required": true + }, + { + "name": "metric_description", + "type": "Text", + "required": true + }, + { + "name": "metric_definition", + "type": "Text", + "required": true + }, + { + "name": "metric_datatype", + "type": "Code", + "required": false + }, + { + "name": "aggregation_type", + "type": "Enum{SUM|AVERAGE|COUNT|COUNT_DISTINCT|MIN|MAX|RATIO|DERIVED}", + "required": false + }, + { + "name": "unit", + "type": "ShortText", + "required": false + }, + { + "name": "grain_description", + "type": "Text", + "required": false + }, + { + "name": "is_additive", + "type": "Flag", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "MetricDataset", + "identity": "semantic.metric_dataset", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "metric_dataset_id" + ] + } + ], + "columns": [ + { + "name": "metric_dataset_id", + "type": "Identifier", + "required": false + }, + { + "name": "metric_name", + "type": "Reference", + "required": true + }, + { + "name": "container_name", + "type": "ShortText", + "required": false + }, + { + "name": "table_name", + "type": "ShortText", + "required": true + }, + { + "name": "dataset_role", + "type": "Enum{PRIMARY|JOINED}", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "MetricExpression", + "identity": "semantic.metric_expression", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "metric_expression_id" + ] + } + ], + "columns": [ + { + "name": "metric_expression_id", + "type": "Identifier", + "required": false + }, + { + "name": "metric_name", + "type": "Reference", + "required": true + }, + { + "name": "sql_dialect", + "type": "Code", + "required": true + }, + { + "name": "expression_text", + "type": "Text", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "NamingStandard", + "identity": "semantic.naming_standard", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "naming_standard_id" + ] + } + ], + "columns": [ + { + "name": "naming_standard_id", + "type": "Identifier", + "required": false + }, + { + "name": "standard_type", + "type": "Enum{TEMPLATE|STANDARD_NAME|ABBREVIATION|SUFFIX|PREFIX|PATTERN}", + "required": true + }, + { + "name": "applies_to", + "type": "Enum{TABLE|BASE_VIEW|CONSUMER_VIEW|PROCEDURE|FUNCTION|SEQUENCE|ATTRIBUTE|GRAPH_KEY|ALL}", + "required": false + }, + { + "name": "module_scope", + "type": "Enum{DOMAIN|SEARCH|PREDICTION|OBSERVABILITY|SEMANTIC|MEMORY}", + "required": false + }, + { + "name": "kind_scope", + "type": "Enum{HISTORY|REFERENCE|RELATIONSHIP|KEYMAP|RECORD}", + "required": false + }, + { + "name": "name_template", + "type": "ShortText", + "required": false + }, + { + "name": "standard_value", + "type": "ShortText", + "required": false + }, + { + "name": "mapped_value", + "type": "ShortText", + "required": false + }, + { + "name": "meaning", + "type": "Text", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "PrimaryObject", + "identity": "semantic.primary_object", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "primary_object_id" + ] + } + ], + "columns": [ + { + "name": "primary_object_id", + "type": "Identifier", + "required": false + }, + { + "name": "module_id", + "type": "Reference", + "required": true + }, + { + "name": "container_name", + "type": "ShortText", + "required": true + }, + { + "name": "object_name", + "type": "ShortText", + "required": true + }, + { + "name": "object_type", + "type": "Enum{TABLE|BASE_VIEW|CONSUMER_VIEW|PROCEDURE|FUNCTION}", + "required": true + }, + { + "name": "object_role", + "type": "Enum{AGENT_ENTRYPOINT|ANALYTICAL_QUERY|REFERENCE_LOOKUP|RELATIONSHIP_BRIDGE|LINEAGE_EVIDENCE|OPERATIONAL_METRIC|WRITE_TARGET|INTERNAL_SUPPORT}", + "required": true + }, + { + "name": "usage_guidance", + "type": "Text", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "RelationshipColumn", + "identity": "semantic.relationship_column", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "relationship_column_id" + ] + } + ], + "columns": [ + { + "name": "relationship_column_id", + "type": "Identifier", + "required": false + }, + { + "name": "relationship_id", + "type": "Reference", + "required": true + }, + { + "name": "ordinal", + "type": "Integer", + "required": true + }, + { + "name": "source_column", + "type": "ShortText", + "required": true + }, + { + "name": "target_column", + "type": "ShortText", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "SemanticSynonym", + "identity": "semantic.semantic_synonym", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "synonym_id" + ] + } + ], + "columns": [ + { + "name": "synonym_id", + "type": "Identifier", + "required": false + }, + { + "name": "synonym_object", + "type": "Reference", + "required": true + }, + { + "name": "object_kind", + "type": "Enum{ENTITYMETADATA|COLUMNMETADATA|METRIC}", + "required": true + }, + { + "name": "synonym_text", + "type": "ShortText", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "StandardDeclaration", + "identity": "semantic.standard_declaration", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "standard_declaration_id" + ] + } + ], + "columns": [ + { + "name": "standard_declaration_id", + "type": "Identifier", + "required": false + }, + { + "name": "product_id", + "type": "NaturalKey", + "required": true + }, + { + "name": "component_kind", + "type": "Enum{STANDARD|MODULE|PATTERN}", + "required": true + }, + { + "name": "component_name", + "type": "ShortText", + "required": true + }, + { + "name": "platform", + "type": "ShortText", + "required": true + }, + { + "name": "standard_version", + "type": "ShortText", + "required": true + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "TableRelationship", + "identity": "semantic.table_relationship", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "relationship_id" + ] + } + ], + "columns": [ + { + "name": "relationship_id", + "type": "Identifier", + "required": false + }, + { + "name": "relationship_name", + "type": "ShortText", + "required": true + }, + { + "name": "source_entity", + "type": "ShortText", + "required": true + }, + { + "name": "target_entity", + "type": "ShortText", + "required": true + }, + { + "name": "relationship_type", + "type": "Enum{FOREIGN_KEY|HIERARCHY|ASSOCIATIVE}", + "required": true + }, + { + "name": "cardinality", + "type": "Enum{ONE_TO_ONE|ONE_TO_MANY|MANY_TO_ONE|MANY_TO_MANY}", + "required": false + }, + { + "name": "is_mandatory", + "type": "Flag", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + } + ], + "domain": [], + "observability": [ + { + "name": "AgentOutcome", + "identity": "observability.agent_outcome", + "profile": "EVENT_APPEND_ONLY", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "outcome_id" + ] + } + ], + "columns": [ + { + "name": "outcome_id", + "type": "Identifier", + "required": false + }, + { + "name": "agent_key", + "type": "ShortText", + "required": true + }, + { + "name": "session_key", + "type": "ShortText", + "required": false + }, + { + "name": "action_type", + "type": "Enum{QUERY|RECOMMENDATION|DECISION|PREDICTION}", + "required": true + }, + { + "name": "action_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "tables_accessed", + "type": "Text", + "required": false + }, + { + "name": "outcome_status", + "type": "Enum{SUCCESS|PARTIAL|FAILED}", + "required": true + }, + { + "name": "user_feedback", + "type": "Enum{POSITIVE|NEUTRAL|NEGATIVE|CORRECTION}", + "required": false + }, + { + "name": "records_processed", + "type": "Integer", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "ChangeEvent", + "identity": "observability.change_event", + "profile": "EVENT_APPEND_ONLY", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "change_event_id" + ] + } + ], + "columns": [ + { + "name": "change_event_id", + "type": "Identifier", + "required": false + }, + { + "name": "container_name", + "type": "ShortText", + "required": false + }, + { + "name": "table_name", + "type": "ShortText", + "required": true + }, + { + "name": "change_type", + "type": "Enum{INSERT|UPDATE|DELETE|MERGE|TRUNCATE}", + "required": true + }, + { + "name": "change_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "changed_by", + "type": "ShortText", + "required": true + }, + { + "name": "change_source", + "type": "Enum{ETL|API|MANUAL|AGENT}", + "required": false + }, + { + "name": "records_affected", + "type": "Integer", + "required": false + }, + { + "name": "columns_changed", + "type": "Text", + "required": false + }, + { + "name": "batch_key", + "type": "ShortText", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "DataLineage", + "identity": "observability.data_lineage", + "profile": "CURRENT_STATE", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "lineage_id" + ] + } + ], + "columns": [ + { + "name": "lineage_id", + "type": "Identifier", + "required": false + }, + { + "name": "source_container", + "type": "ShortText", + "required": false + }, + { + "name": "source_table", + "type": "ShortText", + "required": false + }, + { + "name": "source_column", + "type": "ShortText", + "required": false + }, + { + "name": "source_system", + "type": "ShortText", + "required": false + }, + { + "name": "target_container", + "type": "ShortText", + "required": false + }, + { + "name": "target_table", + "type": "ShortText", + "required": true + }, + { + "name": "target_column", + "type": "ShortText", + "required": false + }, + { + "name": "job_name", + "type": "ShortText", + "required": false + }, + { + "name": "transformation_type", + "type": "Enum{ETL|FEATURE_ENG|AGGREGATION|JOIN|EMBEDDING_GEN|FILTER|PIVOT}", + "required": false + }, + { + "name": "transformation_logic", + "type": "Text", + "required": false + }, + { + "name": "openlineage_job_name", + "type": "ShortText", + "required": false + }, + { + "name": "openlineage_namespace", + "type": "ShortText", + "required": false + }, + { + "name": "is_active", + "type": "Flag", + "required": false + }, + { + "name": "registered_dts", + "type": "Timestamp", + "required": false + }, + { + "name": "retired_dts", + "type": "Timestamp", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "DataQualityMetric", + "identity": "observability.data_quality_metric", + "profile": "EVENT_APPEND_ONLY", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "quality_metric_id" + ] + } + ], + "columns": [ + { + "name": "quality_metric_id", + "type": "Identifier", + "required": false + }, + { + "name": "container_name", + "type": "ShortText", + "required": false + }, + { + "name": "table_name", + "type": "ShortText", + "required": true + }, + { + "name": "column_name", + "type": "ShortText", + "required": false + }, + { + "name": "quality_dimension", + "type": "Enum{COMPLETENESS|VALIDITY|CONSISTENCY|ACCURACY|TIMELINESS}", + "required": true + }, + { + "name": "check_function", + "type": "Enum{COMPLETENESS|VALIDITY|CONSISTENCY|INTEGRITY|UNIQUENESS|FRESHNESS}", + "required": true + }, + { + "name": "rule_text", + "type": "Text", + "required": true + }, + { + "name": "metric_value", + "type": "Decimal(10,4)", + "required": false + }, + { + "name": "pass_threshold", + "type": "Decimal(5,4)", + "required": false + }, + { + "name": "warn_threshold", + "type": "Decimal(5,4)", + "required": false + }, + { + "name": "check_outcome", + "type": "Enum{PASS|WARN|FAIL}", + "required": true + }, + { + "name": "measured_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "sample_size", + "type": "Integer", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "LineageRun", + "identity": "observability.lineage_run", + "profile": "EVENT_APPEND_ONLY", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "lineage_run_id" + ] + } + ], + "columns": [ + { + "name": "lineage_run_id", + "type": "Identifier", + "required": false + }, + { + "name": "lineage_id", + "type": "Reference", + "required": true + }, + { + "name": "run_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "run_status", + "type": "Enum{SUCCESS|FAILED|PARTIAL|RUNNING}", + "required": true + }, + { + "name": "run_duration_ms", + "type": "Integer", + "required": false + }, + { + "name": "records_read", + "type": "Integer", + "required": false + }, + { + "name": "records_written", + "type": "Integer", + "required": false + }, + { + "name": "records_rejected", + "type": "Integer", + "required": false + }, + { + "name": "batch_key", + "type": "ShortText", + "required": false + }, + { + "name": "openlineage_run_id", + "type": "ShortText", + "required": false + }, + { + "name": "error_message", + "type": "Text", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "ModelPerformance", + "identity": "observability.model_performance", + "profile": "EVENT_APPEND_ONLY", + "keys": [ + { + "role": "SURROGATE", + "name": "surrogate", + "columns": [ + "performance_id" + ] + } + ], + "columns": [ + { + "name": "performance_id", + "type": "Identifier", + "required": false + }, + { + "name": "model_key", + "type": "ShortText", + "required": true + }, + { + "name": "model_version", + "type": "ShortText", + "required": true + }, + { + "name": "metric_name", + "type": "Enum{ACCURACY|PRECISION|RECALL|AUC|LATENCY_MS|DRIFT_SCORE}", + "required": true + }, + { + "name": "metric_value", + "type": "Decimal(10,6)", + "required": false + }, + { + "name": "evaluation_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "sample_size", + "type": "Integer", + "required": false + }, + { + "name": "is_sla_met", + "type": "Flag", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "ValidationArea", + "identity": "observability.validation_area", + "profile": "EVENT_APPEND_ONLY", + "keys": [ + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "product_code", + "producer_id", + "run_id", + "scope_kind", + "scope_id" + ] + } + ], + "columns": [ + { + "name": "product_code", + "type": "ShortText", + "required": true + }, + { + "name": "producer_id", + "type": "ShortText", + "required": true + }, + { + "name": "run_id", + "type": "ShortText", + "required": true + }, + { + "name": "scope_kind", + "type": "Enum{MODULE|ENTITY|PATTERN|CAPABILITY|PRODUCT}", + "required": true + }, + { + "name": "scope_id", + "type": "ShortText", + "required": true + }, + { + "name": "checks_expected", + "type": "Integer", + "required": true + }, + { + "name": "checks_ran", + "type": "Integer", + "required": true + }, + { + "name": "passed_count", + "type": "Integer", + "required": true + }, + { + "name": "failed_count", + "type": "Integer", + "required": true + }, + { + "name": "error_count", + "type": "Integer", + "required": true + }, + { + "name": "critical_failure_count", + "type": "Integer", + "required": true + }, + { + "name": "error_failure_count", + "type": "Integer", + "required": true + }, + { + "name": "area_status", + "type": "Enum{pass|fail|partial|not-validated|no-evidence}", + "required": true + }, + { + "name": "confidence", + "type": "Enum{strong|partial|weak|unknown}", + "required": true + }, + { + "name": "open_gaps", + "type": "Text", + "required": false + }, + { + "name": "recommended_action", + "type": "Text", + "required": false + }, + { + "name": "completed_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + }, + { + "name": "ValidationRun", + "identity": "observability.validation_run", + "profile": "EVENT_APPEND_ONLY", + "keys": [ + { + "role": "NATURAL", + "name": "natural", + "columns": [ + "product_code", + "producer_id", + "run_id" + ] + } + ], + "columns": [ + { + "name": "product_code", + "type": "ShortText", + "required": true + }, + { + "name": "producer_id", + "type": "ShortText", + "required": true + }, + { + "name": "producer_version", + "type": "ShortText", + "required": false + }, + { + "name": "profile_id", + "type": "ShortText", + "required": false + }, + { + "name": "profile_version", + "type": "ShortText", + "required": false + }, + { + "name": "source_format", + "type": "ShortText", + "required": true + }, + { + "name": "payload_schema_version", + "type": "ShortText", + "required": true + }, + { + "name": "run_id", + "type": "ShortText", + "required": true + }, + { + "name": "started_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "completed_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "trust_status", + "type": "Enum{TRUSTED|DEGRADED|UNTRUSTED}", + "required": true + }, + { + "name": "agent_use_allowed", + "type": "ShortText", + "required": false + }, + { + "name": "total_checks", + "type": "Integer", + "required": true + }, + { + "name": "passed_count", + "type": "Integer", + "required": true + }, + { + "name": "failed_count", + "type": "Integer", + "required": true + }, + { + "name": "error_count", + "type": "Integer", + "required": true + }, + { + "name": "critical_failure_count", + "type": "Integer", + "required": true + }, + { + "name": "error_failure_count", + "type": "Integer", + "required": true + }, + { + "name": "data_product_trust_score", + "type": "Decimal(5,2)", + "required": false + }, + { + "name": "performance_readiness_score", + "type": "Decimal(5,2)", + "required": false + }, + { + "name": "operational_readiness_score", + "type": "Decimal(5,2)", + "required": false + }, + { + "name": "repair_candidate_count", + "type": "Integer", + "required": true + }, + { + "name": "failed_checks_json", + "type": "Json", + "required": false + }, + { + "name": "repair_candidates_json", + "type": "Json", + "required": false + }, + { + "name": "evidence_expires_dts", + "type": "Timestamp", + "required": false + }, + { + "name": "created_dts", + "type": "Timestamp", + "required": true + }, + { + "name": "updated_dts", + "type": "Timestamp", + "required": true + } + ] + } + ], + "search": [], + "prediction": [] + }, + "access_surface": [ + { + "relation": "BusinessGlossary", + "identity": "memory.business_glossary:current", + "columns": [ + "term", + "term_category", + "definition", + "source_module", + "valid_from_dts", + "is_current", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "ChangeLog", + "identity": "memory.change_log:current", + "columns": [ + "change_id", + "version_number", + "change_title", + "change_type", + "source_module", + "related_decision_id", + "valid_from_dts", + "is_current", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "DesignDecision", + "identity": "memory.design_decision:current", + "columns": [ + "decision_id", + "decision_version", + "decision_title", + "context", + "alternatives", + "rationale", + "consequences", + "decision_status", + "decision_category", + "source_module", + "superseded_by", + "valid_from_dts", + "is_current", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "ImplementationNote", + "identity": "memory.implementation_note:current", + "columns": [ + "note_id", + "note_title", + "note_content", + "note_category", + "severity", + "source_module", + "valid_from_dts", + "is_current", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "ModuleRegistry", + "identity": "memory.module_registry:current", + "columns": [ + "module_registry_id", + "module_name", + "container_name", + "deployment_status", + "module_version", + "module_purpose", + "key_entities", + "dependencies", + "valid_from_dts", + "is_current", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "QueryCookbook", + "identity": "memory.query_cookbook:current", + "columns": [ + "recipe_id", + "recipe_title", + "use_case", + "target_module", + "query_template", + "complexity", + "is_batch", + "source_module", + "valid_from_dts", + "is_current", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "AgentOutcome", + "identity": "observability.agent_outcome:current", + "columns": [ + "outcome_id", + "agent_key", + "session_key", + "action_type", + "action_dts", + "tables_accessed", + "outcome_status", + "user_feedback", + "records_processed", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "ChangeEvent", + "identity": "observability.change_event:current", + "columns": [ + "change_event_id", + "container_name", + "table_name", + "change_type", + "change_dts", + "changed_by", + "change_source", + "records_affected", + "columns_changed", + "batch_key", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "DataLineage", + "identity": "observability.data_lineage:current", + "columns": [ + "lineage_id", + "source_container", + "source_table", + "source_column", + "source_system", + "target_container", + "target_table", + "target_column", + "job_name", + "transformation_type", + "transformation_logic", + "openlineage_job_name", + "openlineage_namespace", + "is_active", + "registered_dts", + "retired_dts", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "DataQualityMetric", + "identity": "observability.data_quality_metric:current", + "columns": [ + "quality_metric_id", + "container_name", + "table_name", + "column_name", + "quality_dimension", + "check_function", + "rule_text", + "metric_value", + "pass_threshold", + "warn_threshold", + "check_outcome", + "measured_dts", + "sample_size", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "LineageRun", + "identity": "observability.lineage_run:current", + "columns": [ + "lineage_run_id", + "lineage_id", + "run_dts", + "run_status", + "run_duration_ms", + "records_read", + "records_written", + "records_rejected", + "batch_key", + "openlineage_run_id", + "error_message", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "ModelPerformance", + "identity": "observability.model_performance:current", + "columns": [ + "performance_id", + "model_key", + "model_version", + "metric_name", + "metric_value", + "evaluation_dts", + "sample_size", + "is_sla_met", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "ValidationArea", + "identity": "observability.validation_area:current", + "columns": [ + "product_code", + "producer_id", + "run_id", + "scope_kind", + "scope_id", + "checks_expected", + "checks_ran", + "passed_count", + "failed_count", + "error_count", + "critical_failure_count", + "error_failure_count", + "area_status", + "confidence", + "open_gaps", + "recommended_action", + "completed_dts", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "ValidationRun", + "identity": "observability.validation_run:current", + "columns": [ + "product_code", + "producer_id", + "producer_version", + "profile_id", + "profile_version", + "source_format", + "payload_schema_version", + "run_id", + "started_dts", + "completed_dts", + "trust_status", + "agent_use_allowed", + "total_checks", + "passed_count", + "failed_count", + "error_count", + "critical_failure_count", + "error_failure_count", + "data_product_trust_score", + "performance_readiness_score", + "operational_readiness_score", + "repair_candidate_count", + "failed_checks_json", + "repair_candidates_json", + "evidence_expires_dts", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "LineageGraph", + "identity": "observability:lineage_graph", + "columns": [ + "source_node", + "target_node", + "edge_type" + ] + }, + { + "relation": "LineageRunLatest", + "identity": "observability:lineage_run_latest", + "columns": [ + "lineage_id", + "source_table", + "target_table", + "job_name", + "run_dts", + "run_status" + ] + }, + { + "relation": "TrustMap", + "identity": "observability:trust_map", + "columns": [ + "product_code", + "producer_id", + "run_id", + "scope_kind", + "scope_id", + "area_status", + "confidence", + "checks_expected", + "checks_ran", + "coverage", + "passed_count", + "failed_count", + "error_count", + "critical_failure_count", + "error_failure_count", + "open_gaps", + "recommended_action", + "completed_dts", + "derived" + ] + }, + { + "relation": "AccessColumn", + "identity": "semantic.access_column:current", + "columns": [ + "access_column_id", + "access_object_id", + "ordinal", + "column_name", + "source_entity", + "source_column", + "mapping_kind", + "derivation_note", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "AccessComposition", + "identity": "semantic.access_composition:current", + "columns": [ + "access_composition_id", + "composite_container", + "composite_object", + "member_seq", + "member_entity", + "member_role", + "via_relationship_id", + "via_direction", + "is_grain_contributor", + "member_note", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "AccessObject", + "identity": "semantic.access_object:current", + "columns": [ + "access_object_id", + "object_identity", + "container_name", + "object_name", + "object_type", + "access_role", + "represents_entity", + "object_grain", + "is_agent_consumable", + "is_primary", + "resolves_to_object", + "access_note", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "ColumnMetadata", + "identity": "semantic.column_metadata:current", + "columns": [ + "column_metadata_id", + "container_name", + "table_name", + "column_name", + "logical_name", + "business_description", + "is_pii", + "sensitivity_marker", + "data_classification", + "is_required", + "declared_type", + "allowed_values", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "DataProductMap", + "identity": "semantic.data_product_map:current", + "columns": [ + "module_id", + "module_name", + "object_type", + "module_purpose", + "container_name", + "module_version", + "deployment_status", + "graph_key", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "DataProductOrientation", + "identity": "semantic.data_product_orientation:current", + "columns": [ + "orientation_id", + "product_id", + "resource_role", + "container_name", + "object_name", + "resource_uri", + "usage_guidance", + "is_required", + "discovery_order", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "DataProductRegistry", + "identity": "semantic.data_product_registry:current", + "columns": [ + "product_id", + "product_name", + "product_description", + "product_domain", + "product_version", + "product_status", + "owner_team", + "technical_contact", + "trust_authoritative_producer", + "manifest", + "contract_uri", + "policy_uri", + "quality_uri", + "lineage_uri", + "approved_entrypoint", + "approved_access_mode", + "is_active", + "is_deleted", + "deleted_dts", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "EntityMetadata", + "identity": "semantic.entity_metadata:current", + "columns": [ + "entity_metadata_id", + "entity_name", + "logical_name", + "entity_description", + "entity_kind", + "module_name", + "container_name", + "table_name", + "view_name", + "temporal_pattern", + "current_flag_column", + "deleted_flag_column", + "assumed_zone", + "industry_standard", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "KeyColumn", + "identity": "semantic.key_column:current", + "columns": [ + "key_column_id", + "key_definition_id", + "ordinal", + "column_name", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "KeyDefinition", + "identity": "semantic.key_definition:current", + "columns": [ + "key_definition_id", + "entity_id", + "key_role", + "key_name", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "Metric", + "identity": "semantic.metric:current", + "columns": [ + "metric_id", + "metric_name", + "metric_description", + "metric_definition", + "metric_datatype", + "aggregation_type", + "unit", + "grain_description", + "is_additive", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "MetricDataset", + "identity": "semantic.metric_dataset:current", + "columns": [ + "metric_dataset_id", + "metric_name", + "container_name", + "table_name", + "dataset_role", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "MetricExpression", + "identity": "semantic.metric_expression:current", + "columns": [ + "metric_expression_id", + "metric_name", + "sql_dialect", + "expression_text", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "NamingStandard", + "identity": "semantic.naming_standard:current", + "columns": [ + "naming_standard_id", + "standard_type", + "applies_to", + "module_scope", + "kind_scope", + "name_template", + "standard_value", + "mapped_value", + "meaning", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "PrimaryObject", + "identity": "semantic.primary_object:current", + "columns": [ + "primary_object_id", + "module_id", + "container_name", + "object_name", + "object_type", + "object_role", + "usage_guidance", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "RelationshipColumn", + "identity": "semantic.relationship_column:current", + "columns": [ + "relationship_column_id", + "relationship_id", + "ordinal", + "source_column", + "target_column", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "SemanticSynonym", + "identity": "semantic.semantic_synonym:current", + "columns": [ + "synonym_id", + "synonym_object", + "object_kind", + "synonym_text", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "StandardDeclaration", + "identity": "semantic.standard_declaration:current", + "columns": [ + "standard_declaration_id", + "product_id", + "component_kind", + "component_name", + "platform", + "standard_version", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "TableRelationship", + "identity": "semantic.table_relationship:current", + "columns": [ + "relationship_id", + "relationship_name", + "source_entity", + "target_entity", + "relationship_type", + "cardinality", + "is_mandatory", + "is_active", + "created_dts", + "updated_dts" + ] + }, + { + "relation": "ColumnCatalogue", + "identity": "semantic:column_catalogue", + "columns": [ + "container_name", + "table_name", + "column_name", + "data_type", + "is_nullable", + "logical_name", + "business_description", + "is_pii", + "sensitivity_marker", + "data_classification", + "declared_type", + "structural_source", + "description_source" + ] + }, + { + "relation": "DataProductManifest", + "identity": "semantic:data_product_manifest", + "columns": [ + "product_id", + "product_name", + "product_version", + "product_status", + "approved_entrypoint", + "approved_access_mode", + "trust_authoritative_producer", + "manifest", + "trust_map", + "module_map", + "object_catalogue", + "entity_catalogue", + "column_catalogue", + "relationship_catalogue", + "relationship_paths", + "lineage", + "query_cookbook", + "glossary", + "design_decisions", + "quality", + "standard_version" + ] + }, + { + "relation": "RelationshipPaths", + "identity": "semantic:relationship_paths", + "columns": [ + "path_id", + "origin_entity", + "destination_entity", + "hop_count", + "hop", + "key_ordinal", + "from_entity", + "from_column", + "to_entity", + "to_column", + "relationship_id", + "direction" + ] + } + ], + "discovery_roles": [ + { + "role": "MANIFEST", + "relation": "DataProductManifest", + "required": true, + "discovery_order": 1 + }, + { + "role": "TRUST_MAP", + "relation": "TrustMap", + "required": true, + "discovery_order": 2 + }, + { + "role": "MODULE_MAP", + "relation": "DataProductMap", + "required": true, + "discovery_order": 3 + }, + { + "role": "OBJECT_CATALOGUE", + "relation": "AccessObject", + "required": true, + "discovery_order": 4 + }, + { + "role": "ENTITY_CATALOGUE", + "relation": "EntityMetadata", + "required": true, + "discovery_order": 5 + }, + { + "role": "COLUMN_CATALOGUE", + "relation": "ColumnCatalogue", + "required": true, + "discovery_order": 6 + }, + { + "role": "RELATIONSHIP_CATALOGUE", + "relation": "TableRelationship", + "required": true, + "discovery_order": 7 + }, + { + "role": "RELATIONSHIP_PATHS", + "relation": "RelationshipPaths", + "required": false, + "discovery_order": 8 + }, + { + "role": "LINEAGE", + "relation": "LineageGraph", + "required": false, + "discovery_order": 9 + }, + { + "role": "QUERY_COOKBOOK", + "relation": "QueryCookbook", + "required": false, + "discovery_order": 10 + }, + { + "role": "GLOSSARY", + "relation": "BusinessGlossary", + "required": false, + "discovery_order": 11 + }, + { + "role": "DESIGN_DECISIONS", + "relation": "DesignDecision", + "required": false, + "discovery_order": 12 + }, + { + "role": "QUALITY", + "relation": "DataQualityMetric", + "required": false, + "discovery_order": 13 + } + ] +} diff --git a/release/v2.0.0/examples.json b/release/v2.0.0/examples.json new file mode 100644 index 0000000..100f5d9 --- /dev/null +++ b/release/v2.0.0/examples.json @@ -0,0 +1,19 @@ +{ + "standard_version": "2.0.0", + "examples": [ + { + "example": "alation-interface", + "platforms": [], + "readme": "examples/alation-interface/README.md" + }, + { + "example": "it-service-desk-data-product", + "platforms": [ + "duckdb", + "postgres", + "teradata" + ], + "readme": "examples/it-service-desk-data-product/README.md" + } + ] +} diff --git a/tooling/bindings/render.py b/tooling/bindings/render.py index ecda776..78dea10 100644 --- a/tooling/bindings/render.py +++ b/tooling/bindings/render.py @@ -78,6 +78,10 @@ def walk(node, where): if unread: out.append(f"history: {', '.join(unread)} have no history surface consumers can be granted " f"(temporal-lifecycle-metadata §9)") + for r in context["relationships"]: + if r["unpaired_target_columns"]: + out.append(f"relationships[{r['name']}]: the target identity has {len(r['unpaired_target_columns']) + 1} " + f"columns and the reference carries one; only the first is paired") for a in context.get("access_objects", []): out.append(f"access_objects[{a['name']}]: composite access objects are not rendered") out += [f"capture: {g}" for g in context.get("capture", {}).get("gaps", [])] @@ -142,6 +146,10 @@ def add(name, scope_kind, scope_id, module, sql): add(name, "ENTITY", r["source"], r["source_module"], render(template, relationship=r)) elif scope == "module" and name in context["modules"]: add("module", "MODULE", name, name, render(template, module=name)) + elif scope == "contract" and "semantic" in context["modules"]: + add(f"contract-{name}", "MODULE", "semantic", "semantic", render(template)) + elif scope == "product": + add(name, "PRODUCT", context["product"]["code"], None, render(template)) def placed(name): x = next((x for x in context["relations"] if x["name"] == name), None) diff --git a/tooling/bindings/tests/test_bindings.py b/tooling/bindings/tests/test_bindings.py index ac9be80..abc46ee 100644 --- a/tooling/bindings/tests/test_bindings.py +++ b/tooling/bindings/tests/test_bindings.py @@ -303,7 +303,8 @@ def deploy(self, spec=ITSD, ctx=None): return self.manifest def q(self, sql, params=None): - return self.con.execute(sql, params or []).fetchall() + # No parameters means None: an empty list makes psycopg read a literal LIKE '%' as a placeholder. + return self.con.execute(sql, params or None).fetchall() @property def marker(self): @@ -470,7 +471,7 @@ def test_an_adopted_entity_is_used_where_it_stands(self): meta = named(self.ctx, "EntityMetadata")["column_names"] self.assertEqual(self.q(f"SELECT {self.binding.ident(meta['container_name'])}, {self.binding.ident(meta['table_name'])} " f"FROM {self.view('EntityMetadata')} WHERE {self.binding.ident(meta['entity_name'])} = " - f"{self.marker}", ["Category"]), [(container, name)]) + f"{self.marker}", [category["logical_id"]]), [(container, name)]) failing = [r for r in run_checks(self.con, self.manifest) if r["status"] != "PASSED"] self.assertEqual(failing, []) self.assertEqual(self.q(f"SELECT x FROM {unrelated}"), [(1,)]) @@ -590,16 +591,134 @@ def test_relationship_paths_follow_the_standard_bound(self): edges = [] for r in self.ctx["relationships"]: if r["source_table"] and r["target_table"]: - s, t = (f"{x['container']}.{x['name']}" for x in (r["source_table"], r["target_table"])) + s, t = r["source_identity"], r["target_identity"] edges += [(s, t), (t, s)] expected, frontier = {}, [[e] for e in edges] for hops in range(1, named(self.ctx, "RelationshipPaths")["max_hops"] + 1): expected[hops] = len(frontier) frontier = [path + [e] for path in frontier for e in edges if e[0] == path[-1][1] and e[1] not in [path[0][0]] + [p[1] for p in path]] - rows = dict(self.q(f"SELECT hops, COUNT(*) FROM {self.view('RelationshipPaths')} GROUP BY hops")) + rows = dict(self.q(f"SELECT hop_count, COUNT(DISTINCT path_id) FROM {self.view('RelationshipPaths')} GROUP BY hop_count")) self.assertEqual({int(k): int(v) for k, v in rows.items()}, {k: v for k, v in expected.items() if v}) + # ---- the metadata contract (release 2.0.0) ------------------------------------------------ + def col(self, relation, logical): + """The quoted physical column the context places for a standard-owned relation's attribute.""" + return self.binding.ident(named(self.ctx, relation)["column_names"][logical]) + + def pk(self, relation): + return self.binding.ident(named(self.ctx, relation)["key_columns"][0]) + + def register(self, relation, values, sql=None): + """Insert a catalogue row the way registration does, through the shared insert macro.""" + env = environment(self.binding, self.ctx) + text = env.from_string("{% import 'patterns/temporal-lifecycle-metadata/00-macros.sql.j2' as m with context %}" + "{{ m.insert(rel, values, sql) }}").render( + **dict(self.ctx, rel=named(self.ctx, relation), values=values, sql=sql or {})) + self.con.execute(text) + + def contract_check(self, name): + return next(c for c in self.manifest["checks"] if c["check"] == name) + + def test_identities_are_logical_and_a_composite_key_is_ordered_rows(self): + self.deploy() + names = [r[0] for r in self.q(f"SELECT {self.col('EntityMetadata', 'entity_name')} FROM {self.table('EntityMetadata')}")] + self.assertIn("domain.ticket", names) + for name in names: + self.assertRegex(name, r"^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*$") + embedding = named(self.ctx, "EntityEmbedding") + self.assertGreater(len(embedding["natural_columns"]), 1) + rows = self.q( + f"SELECT kc.{self.col('KeyColumn', 'column_name')} FROM {self.table('KeyColumn')} kc " + f"INNER JOIN {self.table('KeyDefinition')} kd ON kd.{self.pk('KeyDefinition')} = kc.{self.col('KeyColumn', 'key_definition_id')} " + f"INNER JOIN {self.table('EntityMetadata')} e ON e.{self.pk('EntityMetadata')} = kd.{self.col('KeyDefinition', 'entity_id')} " + f"WHERE e.{self.col('EntityMetadata', 'entity_name')} = {self.marker} AND kd.{self.col('KeyDefinition', 'key_name')} = {self.marker} " + f"ORDER BY kc.{self.col('KeyColumn', 'ordinal')}", [embedding["logical_id"], "natural"]) + self.assertEqual([r[0] for r in rows], embedding["natural_columns"]) + + def test_every_consumer_view_column_has_a_mapping_row(self): + self.deploy() + ticket = named(self.ctx, "Ticket") + rows = self.q( + f"SELECT m.{self.col('AccessColumn', 'ordinal')}, m.{self.col('AccessColumn', 'column_name')} FROM {self.table('AccessColumn')} m " + f"INNER JOIN {self.table('AccessObject')} a ON a.{self.pk('AccessObject')} = m.{self.col('AccessColumn', 'access_object_id')} " + f"WHERE a.{self.col('AccessObject', 'object_identity')} = {self.marker} ORDER BY 1", [ticket["logical_id"] + ":current"]) + self.assertEqual([r[1] for r in rows], [c["physical_name"] for c in ticket["view_columns"]]) + self.assertEqual([r[0] for r in rows], list(range(1, len(rows) + 1))) + + def test_the_deployment_declares_the_standard_it_implements(self): + self.deploy() + version = (REPO / "VERSION").read_text(encoding="utf-8").strip() + self.assertEqual(self.q(f"SELECT DISTINCT standard_version FROM {self.view('DataProductManifest')}"), [(version,)]) + kinds = dict(self.q(f"SELECT {self.col('StandardDeclaration', 'component_kind')}, COUNT(*) FROM {self.table('StandardDeclaration')} " + f"GROUP BY {self.col('StandardDeclaration', 'component_kind')}")) + self.assertEqual(int(kinds["STANDARD"]), 1) + self.assertEqual(int(kinds["MODULE"]), len(self.ctx["modules"])) + + def test_a_composite_join_the_standard_declares_is_one_ordered_group(self): + self.deploy() + join = next(r for r in self.ctx["relationships"] if r["source"] == "ValidationArea") + self.assertEqual((join["name"], join["join_kind"], join["target"]), ("ValidationArea.run", "named", "ValidationRun")) + self.assertEqual([p["source_column"] for p in join["pairs"]], ["product_code", "producer_id", "run_id"]) + self.assertEqual([p["ordinal"] for p in join["pairs"]], [1, 2, 3]) + rows = self.q(f"SELECT path_id, hop, key_ordinal, from_column, to_column FROM {self.view('RelationshipPaths')} " + f"WHERE origin_entity = 'observability.validation_area' AND destination_entity = 'observability.validation_run' " + f"AND hop_count = 1 ORDER BY key_ordinal") + self.assertEqual([(r[1], r[2], r[3], r[4]) for r in rows], + [(1, 1, "product_code", "product_code"), (1, 2, "producer_id", "producer_id"), (1, 3, "run_id", "run_id")]) + self.assertEqual(len({r[0] for r in rows}), 1) + + def test_the_reference_check_tests_every_pair_of_a_composite_join(self): + self.deploy() + validate(self.con, self.manifest) # publishes a run and its areas, which join on all three columns + check = next(c for c in self.manifest["checks"] if c["check"] == "reference" and c["scope_id"] == "ValidationArea") + self.assertEqual(self.q(check["sql"]), []) + area = named(self.ctx, "ValidationArea")["column_names"] + # Break only the second pair: a check that compared the first column alone would not notice. + self.con.execute(f"UPDATE {self.table('ValidationArea')} SET {self.binding.ident(area['producer_id'])} = 'someone-else'") + self.assertTrue(self.q(check["sql"])) + + def test_every_relationship_pairs_a_source_column_with_a_target_column(self): + self.deploy() + pairs = self.q(f"SELECT COUNT(*) FROM {self.table('RelationshipColumn')}")[0][0] + self.assertEqual(int(pairs), sum(len(r["pairs"]) for r in self.ctx["relationships"])) + self.assertEqual(int(self.q(f"SELECT COUNT(*) FROM {self.table('TableRelationship')}")[0][0]), len(self.ctx["relationships"])) + self.assertEqual(len({r["name"] for r in self.ctx["relationships"]}), len(self.ctx["relationships"])) + + def breaks(self, check, mutation): + """A contract check returns nothing on a clean deployment and something once one rule is broken.""" + self.deploy() + sql = self.contract_check(check)["sql"] + self.assertEqual(self.q(sql), []) + mutation() + self.assertTrue(self.q(sql)) + + def test_the_identity_check_detects_a_placement_bound_identity(self): + self.breaks("contract-identity", lambda: self.con.execute( + f"UPDATE {self.table('EntityMetadata')} SET {self.col('EntityMetadata', 'entity_name')} = 'itsd_domain.ticket' " + f"WHERE {self.col('EntityMetadata', 'entity_name')} = 'domain.ticket'")) + + def test_the_keys_check_detects_an_entity_with_no_key(self): + self.breaks("contract-keys", lambda: self.con.execute(f"DELETE FROM {self.table('KeyDefinition')}")) + + def test_the_relationships_check_detects_a_relationship_with_no_columns(self): + self.breaks("contract-relationships", lambda: self.con.execute(f"DELETE FROM {self.table('RelationshipColumn')}")) + + def test_the_access_columns_check_detects_an_unmapped_output_column(self): + self.breaks("contract-access-columns", lambda: self.con.execute( + f"DELETE FROM {self.table('AccessColumn')} WHERE {self.pk('AccessColumn')} = " + f"(SELECT MIN({self.pk('AccessColumn')}) FROM {self.table('AccessColumn')})")) + + def test_the_composition_check_detects_a_composite_with_no_anchor(self): + self.breaks("contract-composition", lambda: self.register("AccessComposition", { + "composite_container": "orphan_container", "composite_object": "composite:orphan", + "member_seq": 1, "member_entity": "domain.ticket", + "member_role": "INNER", "is_grain_contributor": True})) + + def test_the_comment_coverage_check_detects_an_uncommented_view_column(self): + self.breaks("comment-coverage", lambda: self.uncomment( + self.view("Ticket"), self.binding.ident(named(self.ctx, "Ticket")["view_columns"][0]["physical_name"]))) + def test_key_allocation_is_permanent_and_idempotent(self): self.deploy() first = self.allocate("TicketKeymap", "SRC-1") diff --git a/tooling/bindings/tests/test_teradata_dialect.py b/tooling/bindings/tests/test_teradata_dialect.py index 7438d12..682bdf0 100644 --- a/tooling/bindings/tests/test_teradata_dialect.py +++ b/tooling/bindings/tests/test_teradata_dialect.py @@ -222,20 +222,21 @@ class TeradataDialect(unittest.TestCase): platform = "teradata" @classmethod def setUpClass(cls): - cls.rendered, cls.long = {}, set() + cls.rendered, cls.long, cls.built = {}, set(), {} with tempfile.TemporaryDirectory() as tmp: for name, spec in (("ITSD", ITSD), ("Customer Orders", CUSTORD), ("long-named ITSD", long_named(ITSD, tmp))): for profile in PROFILES: ctx, findings = build_context.resolve(spec, profile) assert not findings, [str(f) for f in findings] label = f"{name} on {profile.name}" - cls.rendered[label] = cls.everything(ctx) + files = render_context(ctx) + cls.built[label] = (ctx, files) + cls.rendered[label] = cls.everything(ctx, files) if name.startswith("long"): cls.long.add(label) @staticmethod - def everything(ctx): - files = render_context(ctx) + def everything(ctx, files): sql = [text for name, text in files.items() if name.endswith(".sql")] sql += [c["sql"] for c in json.loads(files["manifest.json"])["checks"]] binding, _ = load_binding("teradata") @@ -283,6 +284,97 @@ def test_vectors_are_stored_as_eight_byte_floats(self): self.assertEqual(width, checked, column) self.assertEqual(int(width) % 8, 0, column) + # ---- the metadata contract (release 2.0.0) as Teradata renders it ----------------- + CONTRACT = ("contract-identity", "contract-keys", "contract-relationships", "contract-access-columns", + "contract-composition", "comment-coverage") + + def view_statement(self, label, name): + """The REPLACE VIEW statement the build renders for a placed standard-owned relation.""" + ctx, files = self.built[label] + view = next(x for x in ctx["relations"] if x["name"] == name)["objects"]["consumer_view"] + found = [s for s in statements("\n".join(t for n, t in files.items() if n.endswith(".sql") and n != "deploy.sql")) + if re.match(rf'REPLACE\s+VIEW\s+"{view["container"]}"\."{view["name"]}"', s, re.I)] + self.assertEqual(len(found), 1, name) + return found[0] + + def test_the_contract_checks_are_declared_in_every_build(self): + for label, (ctx, files) in self.built.items(): + with self.subTest(build=label): + declared = {c["check"] for c in json.loads(files["manifest.json"])["checks"]} + self.assertLessEqual(set(self.CONTRACT), declared) + + def test_relationship_paths_are_rows_of_a_stated_shape(self): + columns = ["path_id", "origin_entity", "destination_entity", "hop_count", "hop", "key_ordinal", + "from_entity", "from_column", "to_entity", "to_column", "relationship_id", "direction"] + for label in self.built: + with self.subTest(build=label): + s = self.view_statement(label, "RelationshipPaths") + self.assertEqual(re.findall(r'"(\w+)"', s.split(" AS\n", 1)[0].split("(", 1)[1]), columns) + self.assertNotIn("join_path", s) + self.assertNotRegex(s, r"\bNOT\s+IN\b") + # one branch per (length, position) up to the standard's four steps, and no fifth + self.assertEqual(len(re.findall(r"^SELECT CAST\(stp1\.leg", s, re.M)), 1 + 2 + 3 + 4) + self.assertIn("stp4.leg", s) + self.assertNotIn("stp5", s) + + def test_the_manifest_ends_with_the_standard_version(self): + for label in self.built: + with self.subTest(build=label): + s = self.view_statement(label, "DataProductManifest") + select_list = s.split("\nFROM ", 1)[0] + self.assertEqual(re.findall(r"\bAS\s+\"?(\w+)\"?\s*,?\s*$", select_list.strip())[-1], "standard_version") + self.assertIn("'STANDARD'", s) + + def test_a_consumer_view_selects_the_context_view_columns_and_registers_each(self): + for label, (ctx, files) in self.built.items(): + with self.subTest(build=label): + text = "\n".join(t for n, t in files.items() if n.endswith(".sql") and n != "deploy.sql") + access = next(x for x in ctx["relations"] if x["name"] == "AccessColumn") + registered = len(re.findall(rf'INSERT INTO "{access["objects"]["table"]["container"]}"\."{access["objects"]["table"]["name"]}"', text)) + expected = 0 + for x in ctx["entities"] + ctx["relations"]: + view = x.get("objects", {}).get("consumer_view") + if not view or "table" not in x["objects"]: + continue + listed = [v["physical_name"] for v in x["view_columns"]] + expected += len(listed) + m = re.search(rf'^REPLACE VIEW "{view["container"]}"\."{view["name"]}" \(([^)]*)\) AS', text, re.M) + self.assertIsNotNone(m, x["name"]) + self.assertEqual(re.findall(r'"([^"]+)"', m.group(1)), listed, x["name"]) + self.assertEqual(registered, expected) + + def test_identities_are_logical_and_keys_and_pairs_are_rows(self): + for label, (ctx, files) in self.built.items(): + with self.subTest(build=label): + text = files["11-registration.sql"] + self.assertNotIn("surrogate_key_column", text) + self.assertNotIn("natural_key_column", text) + for x in ctx["entities"] + ctx["relations"]: + if "table" in x["objects"]: + self.assertIn(f"'{x['logical_id']}'", text, x["name"]) + for r in ctx["relationships"]: + if r["source_table"] and r["target_table"]: + self.assertIn(f"'{r['source_identity']}'", text) + for name in ("KeyDefinition", "KeyColumn", "RelationshipColumn", "StandardDeclaration"): + t = next(x for x in ctx["relations"] if x["name"] == name)["objects"]["table"] + self.assertIn(f'INSERT INTO "{t["container"]}"."{t["name"]}"', text, name) + + def test_every_object_the_build_creates_carries_a_comment(self): + """The catalogue-wide sweep needs a comment on every database, table, view, procedure and view column.""" + for label, (ctx, files) in self.built.items(): + with self.subTest(build=label): + sql = "\n".join(t for n, t in files.items() if n.endswith(".sql") and n != "deploy.sql") + made = {(m.group(1).upper(), m.group(2)) for m in re.finditer( + r"^(?:CREATE|REPLACE)\s+(?:MULTISET\s+)?(TABLE|VIEW|PROCEDURE|DATABASE)\s+(\S+)", sql, re.M | re.I)} + commented = {(m.group(1).upper(), m.group(2)) for m in re.finditer( + r"^COMMENT ON (TABLE|VIEW|PROCEDURE|DATABASE)\s+(\S+) IS", sql, re.M | re.I)} + self.assertEqual(made - commented, set()) + columns = {} + for m in re.finditer(r'^COMMENT ON COLUMN\s+(\S+?)\.("[^"]+") IS', sql, re.M): + columns.setdefault(m.group(1), set()).add(m.group(2)) + for m in re.finditer(r"^REPLACE VIEW\s+(\S+)\s*\(([^)]*)\)\s*AS", sql, re.M): + self.assertEqual(set(re.findall(r'"[^"]+"', m.group(2))) - columns.get(m.group(1), set()), set(), m.group(1)) + def test_each_rule_catches_what_it_names(self): self.assertEqual(set(CAUGHT), set(RULES)) for rule, bad in CAUGHT.items(): diff --git a/tooling/build/binding.py b/tooling/build/binding.py index f27a415..c529d01 100644 --- a/tooling/build/binding.py +++ b/tooling/build/binding.py @@ -23,6 +23,9 @@ from spec_notation import field_tables, parse_blocks # noqa: E402 BLOCKS = ("Binding", "Types", "Functions") +# entity and relationship run once per item; module once per included module; contract once for the +# Semantic module's metadata contract (INV-SEMANTIC-016 to 020); product once for the whole product. +CHECK_SCOPES = ("entity", "relationship", "module", "contract", "product") EXTRA_FRAGMENTS = {"duration", "map case", "true", "false", "null", "flag test"} @@ -122,9 +125,9 @@ def load_binding(platform: str, repo_root: Path = None) -> (Optional[Binding], L b.deploy = fields.get("Deploy") for key in b.checks: scope = key.split(" ")[0] - if scope not in ("entity", "relationship", "module") or len(key.split(" ")) != 2: + if scope not in CHECK_SCOPES or len(key.split(" ")) != 2: findings.append(Finding(str(path), blocks["Binding"].line, "binding-declaration", - f"Binding: check '{key}' is not ' '")) + f"Binding: check '{key}' is not '<{'|'.join(CHECK_SCOPES)}> '")) templates = root / "implementation" / platform for name in list(b.phases.values()) + list(b.checks.values()) + ([b.deploy] if b.deploy else []): if not (templates / name).is_file(): diff --git a/tooling/build/build_context.py b/tooling/build/build_context.py index 798bcf3..9a70850 100644 --- a/tooling/build/build_context.py +++ b/tooling/build/build_context.py @@ -50,6 +50,7 @@ from expressions import ExpressionError, Node, Unrenderable, functions_used, parse as parse_expression # noqa: E402 from expressions import references as expression_references, render as render_expression # noqa: E402 from org_profile import _snake # noqa: E402 +from joins import join_groups # noqa: E402 from spec_lint import Corpus, _keys, facet_names, lint_spec, standard # noqa: E402 from spec_notation import entity_module, parse_specification # noqa: E402 @@ -96,6 +97,10 @@ def default(self, where: str, value, source: str): # Semantic §5: the most hops a path-discovery surface returns. PATH_HOPS = 4 +# Patterns every binding applies; a deployment declares them beside its modules. +STANDARD_PATTERNS = ("object-placement", "temporal-lifecycle-metadata", "validation", "access-layer") +# The open interval ends are always open on a consumer view, so the view does not carry them. +HIDDEN_ON_VIEW = ("valid_to_dts", "transaction_to_dts") # Semantic, the orientation relation: role, serving relation, required. Trust map second, # before every analytical resource. @@ -258,7 +263,9 @@ def shape_attribute(entity_name, a): "description": a.description, "required": "required" in a.qualifiers, "unique": "unique" in a.qualifiers, - "references": [t.strip() for t in target.split("|")] if target else [], + "references": [t.strip().partition(".")[0] for t in target.split("|")] if target else [], + "reference": target, + "join": a.qualifiers.get("join"), "discriminator": a.qualifiers.get("discriminator"), "derive": a.qualifiers.get("derive"), "synonyms": [s.strip() for s in a.qualifiers.get("synonyms", "").split(",") if s.strip()], @@ -553,23 +560,76 @@ def identity_column(target): names = ident["surrogate"] or ident["natural"] return [physical.get((target["name"], n), n) for n in names] + # A reference to an entity keyed only by a natural key holds that key, so it takes its type + # (DesignDecision.superseded_by is a decision_id, not a BIGINT surrogate). + for x in entities + relations: + for column in x.get("columns", []) + x["attributes"]: + targets = [by_name[t] for t in column.get("references", []) if t in by_name] + if column["type"] != "Reference" or len(targets) != 1: + continue + ident = targets[0]["identity"] + if ident["surrogate"] or len(ident["natural"]) != 1: + continue + natural = next(c for c in targets[0].get("columns", []) if c["name"] == ident["natural"][0]) + column["physical_type"] = natural["physical_type"] + relationships = [] - for x in entities: - for a in x["attributes"]: - for target_name in a["references"]: + for x in entities + relations: + by_attribute = {a["name"]: a for a in x["attributes"]} + for group in join_groups((a["name"], a["reference"], a["join"]) for a in x["attributes"]): + sources = [by_attribute[n] for n in group["attributes"]] + for target_name in group["targets"]: target = by_name.get(target_name) if target is None: continue # a module entity this composition does not include + if group["kind"] == "named": + target_columns = [physical.get((target_name, t), t) for t in group["target_attributes"]] + else: + target_columns = identity_column(target) relationships.append({ - "source": x["name"], "source_attribute": a["name"], + "source": x["name"], "source_attribute": group["name"], "join_kind": group["kind"], "source_module": x["module"], "target_module": target["module"], "source_table": x["objects"].get("table"), "target_table": target["objects"].get("table"), "source_view": x["objects"].get("consumer_view"), "target_view": target["objects"].get("consumer_view"), - "source_column": a["physical_name"], "target": target_name, - "target_columns": identity_column(target), "mandatory": a["required"], - "discriminator": a["discriminator"], - "discriminator_value": target_name.upper() if a["discriminator"] else None}) + "source_column": sources[0]["physical_name"], + "source_columns": [a["physical_name"] for a in sources], "target": target_name, + "target_columns": target_columns, "mandatory": all(a["required"] for a in sources), + "discriminator": sources[0]["discriminator"], + "discriminator_value": target_name.upper() if sources[0]["discriminator"] else None}) + + # ---- the metadata contract (release 2.0.0): identities, keys, joins, view columns ------ + for x in entities + relations: + x["logical_id"] = f"{x['module']}.{_snake(x['name'])}" + keys = [] + if x.get("surrogate_columns"): + keys.append({"name": "surrogate", "role": "SURROGATE", "columns": list(x["surrogate_columns"])}) + if x.get("natural_columns"): + keys.append({"name": "natural", "role": "NATURAL", "columns": list(x["natural_columns"])}) + x["key_defs"] = keys + x["view_columns"] = [{"ordinal": n, "name": c["name"], "physical_name": c["physical_name"]} + for n, c in enumerate((c for c in x.get("columns", []) + if c["name"] not in HIDDEN_ON_VIEW), 1)] + for rel in relationships: + rel["name"] = f"{rel['source']}.{rel['source_attribute']}" + ( + f"->{rel['target']}" if rel["join_kind"] == "multi" else "") + rel["source_identity"] = by_name[rel["source"]]["logical_id"] + rel["target_identity"] = by_name[rel["target"]]["logical_id"] + if rel["join_kind"] == "named": + # A named join pairs each source column with the target attribute it names, in order. + rel["pairs"] = [{"ordinal": n, "source_column": s, "target_column": t} + for n, (s, t) in enumerate(zip(rel["source_columns"], rel["target_columns"]), 1)] + rel["unpaired_target_columns"] = [] + else: + # A bare reference carries one source column, so it pairs with the first target column; + # any further column of a composite target identity is reported as a gap, never dropped. + rel["pairs"] = [{"ordinal": 1, "source_column": rel["source_column"], + "target_column": rel["target_columns"][0]}] + rel["unpaired_target_columns"] = list(rel["target_columns"][1:]) + for name, count in Counter(rel["name"] for rel in relationships).items(): + if count > 1: + r.fail("duplicate-relationship", f"relationship '{name}' is declared {count} times; " + f"name the joins apart with [join: ]") # ---- expressions, rendered in the binding's dialect --------------------------------- measures = {b.name: b.fields.get("Measure") for b in spec.blocks_of("Metric")} @@ -623,6 +683,8 @@ def expression(text, condition=False): a["derive"] = expression(a["derive"]) embeddings = blocks("Embedding") + for b in embeddings: + b["logical_id"] = f"search:{_snake(b['name'])}" for b in embeddings: b["source"] = expression(b.get("source")) b["where"] = expression(b.get("where"), condition=True) @@ -943,6 +1005,11 @@ def module_recipe(module): for kind in ("Storage", "Files", "Partitioning", "Lifecycle", "StorageAccess") if kind in profile.blocks} or None, "settings": settings, + "standard": {"version": (REPO_ROOT / "VERSION").read_text(encoding="utf-8").strip(), + "platform": binding.platform, + "declarations": ([{"kind": "STANDARD", "name": "standard"}] + + [{"kind": "MODULE", "name": m} for m in modules] + + [{"kind": "PATTERN", "name": p} for p in STANDARD_PATTERNS])}, "defaults_applied": r.defaults, } return (context if not r.findings else {}), r.findings diff --git a/tooling/build/build_context.schema.json b/tooling/build/build_context.schema.json index 399f981..beb61cd 100644 --- a/tooling/build/build_context.schema.json +++ b/tooling/build/build_context.schema.json @@ -41,7 +41,8 @@ "catalogue", "storage", "settings", - "defaults_applied" + "defaults_applied", + "standard" ], "additionalProperties": false, "$defs": { @@ -506,7 +507,10 @@ "history", "bitemporal", "supports_deletion", - "temporal" + "temporal", + "logical_id", + "key_defs", + "view_columns" ], "properties": { "name": { @@ -635,6 +639,62 @@ "items": { "$ref": "#/$defs/temporal" } + }, + "logical_id": { + "type": "string" + }, + "key_defs": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "role", + "columns" + ], + "properties": { + "name": { + "type": "string" + }, + "role": { + "enum": [ + "SURROGATE", + "NATURAL", + "ALTERNATE" + ] + }, + "columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + } + } + }, + "view_columns": { + "type": "array", + "items": { + "type": "object", + "required": [ + "ordinal", + "name", + "physical_name" + ], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 1 + }, + "name": { + "type": "string" + }, + "physical_name": { + "type": "string" + } + } + } } } } @@ -654,7 +714,10 @@ "identity", "attributes", "temporal", - "objects" + "objects", + "logical_id", + "key_defs", + "view_columns" ], "properties": { "name": { @@ -744,6 +807,62 @@ }, "objects": { "$ref": "#/$defs/objects" + }, + "logical_id": { + "type": "string" + }, + "key_defs": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "role", + "columns" + ], + "properties": { + "name": { + "type": "string" + }, + "role": { + "enum": [ + "SURROGATE", + "NATURAL", + "ALTERNATE" + ] + }, + "columns": { + "type": "array", + "items": { + "type": "string" + }, + "minItems": 1 + } + } + } + }, + "view_columns": { + "type": "array", + "items": { + "type": "object", + "required": [ + "ordinal", + "name", + "physical_name" + ], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 1 + }, + "name": { + "type": "string" + }, + "physical_name": { + "type": "string" + } + } + } } } } @@ -936,7 +1055,14 @@ "source_column", "target", "target_columns", - "mandatory" + "mandatory", + "name", + "source_identity", + "target_identity", + "pairs", + "unpaired_target_columns", + "source_columns", + "join_kind" ], "properties": { "target_columns": { @@ -947,6 +1073,59 @@ }, "mandatory": { "type": "boolean" + }, + "name": { + "type": "string" + }, + "source_identity": { + "type": "string" + }, + "target_identity": { + "type": "string" + }, + "pairs": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": [ + "ordinal", + "source_column", + "target_column" + ], + "properties": { + "ordinal": { + "type": "integer", + "minimum": 1 + }, + "source_column": { + "type": "string" + }, + "target_column": { + "type": "string" + } + } + } + }, + "unpaired_target_columns": { + "type": "array", + "items": { + "type": "string" + } + }, + "source_columns": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + }, + "join_kind": { + "enum": [ + "bare", + "named", + "multi" + ] } } } @@ -1069,6 +1248,45 @@ } } } + }, + "standard": { + "type": "object", + "required": [ + "version", + "platform", + "declarations" + ], + "properties": { + "version": { + "type": "string", + "pattern": "^[0-9]+[.][0-9]+[.][0-9]+$" + }, + "platform": { + "type": "string" + }, + "declarations": { + "type": "array", + "items": { + "type": "object", + "required": [ + "kind", + "name" + ], + "properties": { + "kind": { + "enum": [ + "STANDARD", + "MODULE", + "PATTERN" + ] + }, + "name": { + "type": "string" + } + } + } + } + } } } } diff --git a/tooling/evals/README.md b/tooling/evals/README.md index 30c3e1f..c6b5c2f 100644 --- a/tooling/evals/README.md +++ b/tooling/evals/README.md @@ -39,6 +39,10 @@ Exit code is `0` when the specification conforms, `1` when it does not. The refe | `unrecorded-departure` | an entity's profile falls outside its kind's defaults, or its allocation differs from the product's, with no `Decision:` applying to it. | | `unresolved-reference` | a reference target is neither in the specification nor defined by an included module. | | `missing-discriminator` | a reference with several targets has no `Enum` discriminator listing them. | +| `composite-join-target` | a `[-> Target.attribute]` names an attribute the target lacks, or names one twice in a join. | +| `composite-join-key` | the target attributes of a named join are not exactly one key (surrogate or natural) of the target. | +| `composite-join-type` | a pair in a named join has different logical types (`Reference` counts as `Identifier`). | +| `multi-target-join` | a reference with several targets names target attributes. | | `invalid-expression` | a `[derive:]` expression uses a function outside the vocabulary, an aggregate in a row context, or an attribute that does not resolve. | | `timestamp-name` / `prohibited-name` | a `Timestamp` is not named `_dts`, or an attribute takes a name the temporal pattern prohibits. | | `missing-section` | an entity lacks a section its kind requires (`Volume:`), or a feature group lacks `Features:`. | diff --git a/tooling/evals/joins.py b/tooling/evals/joins.py new file mode 100644 index 0000000..44f970b --- /dev/null +++ b/tooling/evals/joins.py @@ -0,0 +1,48 @@ +"""Join groups: how an entity's `[-> Target]` attributes combine into joins. + +A bare `[-> Target]` is a whole join on its own, paired with the target's identity. A named +`[-> Target.attribute]` pairs its attribute with that attribute of the target, and the named +attributes of one entity that point at the same target (and share a `[join: name]`, if any) +form one join group, in declaration order. The specification lint and the build-context +resolver both read references through this one function, so they cannot disagree. + +Stdlib only. +""" +from typing import Dict, Iterable, List, Optional, Tuple + +Item = Tuple[str, Optional[str], Optional[str]] # (attribute, the `->` value, the `join` value) + + +def join_groups(items: Iterable[Item]) -> List[Dict]: + """The joins declared by an entity's attributes, in declaration order. + + Each group has: `name` (the join name, else its first attribute), `target` (the entity; + None for a multi-target reference), `targets` (every entity named), `attributes` (the + source attributes in order), `target_attributes` (the paired target attributes, or None + for a bare reference) and `kind`: `bare`, `named` or `multi`. + """ + groups: List[Dict] = [] + named: Dict[Tuple[str, str], Dict] = {} + for attribute, raw, join in items: + if raw is None: + continue + targets = [t.strip() for t in raw.split("|") if t.strip()] + if len(targets) != 1: + groups.append({"name": attribute, "target": None, "targets": [t.partition(".")[0] for t in targets], + "attributes": [attribute], "target_attributes": None, "kind": "multi"}) + continue + target, dot, target_attribute = targets[0].partition(".") + if not dot: + groups.append({"name": attribute, "target": target, "targets": [target], "attributes": [attribute], + "target_attributes": None, "kind": "bare"}) + continue + key = (target, join or "") + group = named.get(key) + if group is None: + group = {"name": join or attribute, "target": target, "targets": [target], "attributes": [], + "target_attributes": [], "kind": "named"} + named[key] = group + groups.append(group) + group["attributes"].append(attribute) + group["target_attributes"].append(target_attribute) + return groups diff --git a/tooling/evals/reference/customer-orders.md b/tooling/evals/reference/customer-orders.md index 39dee48..24df6dd 100644 --- a/tooling/evals/reference/customer-orders.md +++ b/tooling/evals/reference/customer-orders.md @@ -414,7 +414,7 @@ Orientation: - `INV-SEMANTIC-004`, `INV-SEMANTIC-005`, `INV-SEMANTIC-006`, `INV-SEMANTIC-007`, `INV-SEMANTIC-008`, `INV-SEMANTIC-009`, `INV-SEMANTIC-010`, `INV-SEMANTIC-011`, `INV-SEMANTIC-012`, `INV-SEMANTIC-013`, -`INV-SEMANTIC-014`, `INV-SEMANTIC-015`. +`INV-SEMANTIC-014`, `INV-SEMANTIC-015`, `INV-SEMANTIC-016`, `INV-SEMANTIC-017`, `INV-SEMANTIC-018`, `INV-SEMANTIC-019`, `INV-SEMANTIC-020`. --- diff --git a/tooling/evals/spec_lint.py b/tooling/evals/spec_lint.py index d8bbaa7..cd3a867 100644 --- a/tooling/evals/spec_lint.py +++ b/tooling/evals/spec_lint.py @@ -28,6 +28,7 @@ sys.path.insert(0, str(REPO_ROOT / "tooling" / "validation")) sys.path.insert(0, str(Path(__file__).resolve().parent)) +from joins import join_groups # noqa: E402 from design_lint import ( # noqa: E402 Finding, load_capability_catalogue, @@ -376,8 +377,31 @@ def __init__(self, fm: dict, spec: Specification, std: Standard, corpus: Corpus) for d in (fm.get("decisions") or []) if isinstance(d, dict)} self.anchors = set(corpus.modules) | std.patterns self.feature_groups = {b.fields.get("Features") for b in spec.blocks_of("Model")} - self.referenced = {t.strip() for e in spec.entities for a in e.attributes.values() + self.referenced = {t.strip().partition(".")[0] for e in spec.entities for a in e.attributes.values() for t in a.qualifiers.get("->", "").split("|") if t.strip()} + # Every entity a reference can name, with its attributes: the specification's own, then + # the included modules' and the patterns' definitions, so a join can be checked against a key. + self.defs = {} + for module in self.modules: + self.defs.update(std.module_entity_defs.get(module, {})) + self.defs.update(std.pattern_entity_defs) + self.defs.update({e.name: e for e in spec.entities}) + + def identity(self, entity: str): + """The surrogate and natural attribute lists of an entity, or None when its definition is unknown.""" + definition = self.defs.get(entity) + if definition is None: + return None + keys = _keys(definition) + surrogate = keys.get("surrogate") or [a.name for a in definition.attributes.values() if a.type == "Identifier"][:1] + natural = keys.get("natural") or ([] if surrogate else + [a.name for a in definition.attributes.values() if a.type == "NaturalKey"][:1]) + return surrogate, natural + + def attribute_type(self, entity: str, attribute: str): + definition = self.defs.get(entity) + found = definition.attributes.get(attribute) if definition is not None else None + return found.type if found is not None else None def resolves(self, entity: str, attribute: str = None) -> bool: if entity not in self.entities: @@ -595,10 +619,11 @@ def check_entity_notation(ctx: NotationContext, path) -> List[Finding]: f"{where} allocates '{alloc}' against the product's " f"DEC-SURROGATE-ALLOCATION with no Decision applying to it")) + findings.extend(check_joins(ctx, e, where, path)) for a in e.attributes.values(): target = a.qualifiers.get("->") if target is not None: - targets = [t.strip() for t in target.split("|")] + targets = [t.strip().partition(".")[0] for t in target.split("|")] for t in targets: if not ctx.resolves(t): findings.append(Finding(path, a.line, "unresolved-reference", @@ -787,6 +812,52 @@ def _keys(entity) -> Dict[str, List[str]]: return out +def _same_type(a: str, b: str) -> bool: + """Two logical types a join may pair: equal, or a Reference and the Identifier it points at.""" + def norm(t): + t = t.split("[")[0].strip() + return "Identifier" if t == "Reference" else ("Enum" if t.startswith("Enum") else t) + return norm(a) == norm(b) + + +def check_joins(ctx: NotationContext, e, where: str, path) -> List[Finding]: + """A named join (`[-> Target.attribute]`) pairs real attributes of one key of its target, of the same type.""" + findings = [] + for a in e.attributes.values(): + raw = a.qualifiers.get("->") + if raw and "|" in raw and "." in raw: + findings.append(Finding(path, a.line, "multi-target-join", + f"{where}.{a.name} names target attributes with several targets; " + f"a composite join has one target")) + for g in join_groups((a.name, a.qualifiers.get("->"), a.qualifiers.get("join")) for a in e.attributes.values()): + if g["kind"] != "named" or not ctx.resolves(g["target"]): + continue # a bare or multi-target reference; an unresolved target is reported elsewhere + target, names = g["target"], g["target_attributes"] + line = e.attributes[g["attributes"][0]].line + label = f"{where}: join '{g['name']}' to {target}" + missing = [t for t in names if not ctx.resolves(target, t)] + if missing: + findings.append(Finding(path, line, "composite-join-target", + f"{label} names {', '.join(missing)}, which {target} does not have")) + if len(set(names)) != len(names): + findings.append(Finding(path, line, "composite-join-target", + f"{label} names a target attribute more than once")) + identity = ctx.identity(target) + if identity and not missing and len(set(names)) == len(names): + keys = [set(k) for k in identity if k] + if set(names) not in keys: + findings.append(Finding(path, line, "composite-join-key", + f"{label} pairs {', '.join(names)}, which is not a whole key of {target} " + f"(its keys: {'; '.join(', '.join(sorted(k)) for k in keys) or 'none'})")) + for source, target_attribute in zip(g["attributes"], names): + theirs = ctx.attribute_type(target, target_attribute) + if theirs and not _same_type(e.attributes[source].type, theirs): + findings.append(Finding(path, e.attributes[source].line, "composite-join-type", + f"{label}: {source} is {e.attributes[source].type} but " + f"{target}.{target_attribute} is {theirs}")) + return findings + + def check_identity(ctx: NotationContext, path) -> List[Finding]: """Identity follows allocation (Design Specification Standard §3.3). diff --git a/tooling/release/build_release.py b/tooling/release/build_release.py new file mode 100644 index 0000000..98a842f --- /dev/null +++ b/tooling/release/build_release.py @@ -0,0 +1,193 @@ +"""Generate and check the release bundle: release/vX.Y.Z/*.json. + +The bundle describes the consumer contract of a deployed AI-Native Data Product for the +version in the root VERSION file. It is generated from the repository: the build context the +resolver produces for the reference product, the bindings' declarations, and the example and +binding directories. It is committed, so a tag checkout carries the contract. It is not a +module: it has no design document, no binding directory and no deployed container. + + python tooling/release/build_release.py # write release/vX.Y.Z/ + python tooling/release/build_release.py --check # exit 1 when committed files are stale +""" +import argparse +import json +from pathlib import Path +import re +import sys + +REPO = Path(__file__).resolve().parents[2] +sys.path.insert(0, str(REPO / "tooling" / "build")) +sys.path.insert(0, str(REPO / "tooling" / "bindings")) + +import build_context # noqa: E402 +from binding import load_binding # noqa: E402 +from build_context import STANDARD_PATTERNS # noqa: E402 +from org_profile import _snake # noqa: E402 +import render # noqa: E402 + +REFERENCE = REPO / "examples" / "it-service-desk-data-product" +REFERENCE_SPEC = REFERENCE / "design-output" / "design_specification.md" +CONTRACT_PLATFORM = "postgres" +MODULES = ("memory", "semantic", "domain", "observability", "search", "prediction") + +# The placement-independent identities every consumer follows (design/modules/semantic.md, INV-SEMANTIC-016). +IDENTITY_GRAMMAR = [ + {"kind": "entity", "form": "module.entity", "pattern": "^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*$", "example": "domain.ticket"}, + {"kind": "entity surface", "form": "module.entity:surface (base, governed, current or history)", + "pattern": "^[a-z][a-z0-9_]*[.][a-z][a-z0-9_]*:(base|governed|current|history)$", "example": "domain.ticket:current"}, + {"kind": "module-owned object", "form": "module:object (composite is reserved)", + "pattern": "^(?!composite:)[a-z][a-z0-9_]*:[a-z][a-z0-9_]*$", "example": "observability:trust_map"}, + {"kind": "composite access object", "form": "composite:name", "pattern": "^composite:[a-z][a-z0-9_]*$", + "example": "composite:ticket_enriched"}] + + +def read_version(repo=REPO): + """Return the standard version from VERSION, which must be MAJOR.MINOR.PATCH.""" + version = (repo / "VERSION").read_text(encoding="utf-8").strip() + if not re.fullmatch(r"\d+\.\d+\.\d+", version): + raise ValueError(f"VERSION must be MAJOR.MINOR.PATCH, not {version!r}") + return version + + +def _platforms(repo=REPO): + return sorted(p.name for p in (repo / "implementation").iterdir() if p.is_dir() and (p / "README.md").exists()) + + +def _build(platform, repo=REPO): + """Resolve and render the reference product on one platform: its context and rendered files.""" + context, findings = build_context.resolve(REFERENCE_SPEC, REFERENCE / "profiles" / f"{platform}.md") + if findings: + raise ValueError(f"the reference product does not resolve on {platform}: " + "; ".join(map(str, findings))) + return context, render.render_context(context) + + +def _view_columns_from_comments(sql): + """Column names per view, from the COMMENT statements. Every view column carries one.""" + columns = {} + for schema, view, quoted, bare in re.findall(r'COMMENT ON COLUMN "([^"]+)"\."([^"]+)"\.(?:"([^"]+)"|(\w+))', sql): + columns.setdefault((schema, view), []).append(quoted or bare) + return columns + + +def build_contract(repo=REPO): + """contract.json: what an agent reading a deployed product can rely on.""" + context, files = _build(CONTRACT_PLATFORM, repo) + sql = files["deploy.sql"] + comment_columns = _view_columns_from_comments(sql) + metadata = {m: [] for m in MODULES} + surface = [] + for x in context["relations"]: + view = x["objects"].get("consumer_view") + if x["relation"] == "table" and x.get("defined"): + metadata[x["module"]].append({ + "name": x["name"], "identity": x["logical_id"], "profile": x["profile"], + "keys": [{"role": k["role"], "name": k["name"], "columns": k["columns"]} for k in x["key_defs"]], + "columns": [{"name": a["name"], "type": a["type"], "required": bool(a["required"])} for a in x["attributes"]] + + [{"name": t["name"], "type": t["type"], "required": True} for t in x["temporal"]]}) + if view: + if x["relation"] == "table": + identity, columns = x["logical_id"] + ":current", [c["name"] for c in x["view_columns"]] + else: + identity = f"{x['module']}:{_snake(x['name'])}" + columns = comment_columns.get((view["container"], view["name"]), []) + surface.append({"relation": x["name"], "identity": identity, "columns": columns}) + resources = sorted(context["orientation"]["resources"], key=lambda r: r["order"]) + return {"standard_version": read_version(repo), "modules": list(MODULES), "patterns": list(STANDARD_PATTERNS), + "identity_grammar": IDENTITY_GRAMMAR, + "metadata_entities": {m: sorted(items, key=lambda i: i["name"]) for m, items in metadata.items()}, + "access_surface": sorted(surface, key=lambda s: s["identity"]), + "discovery_roles": [{"role": r["role"], "relation": r["relation"], "required": r["required"], + "discovery_order": r["order"]} for r in resources]} + + +def build_bindings(repo=REPO): + """bindings.json: per platform, what is present and whether it declares the version.""" + result = [] + for platform in _platforms(repo): + base = repo / "implementation" / platform + binding, _ = load_binding(platform) + declares = any("StandardDeclaration" in f.read_text(encoding="utf-8", errors="ignore") + for f in base.rglob("*.j2")) + result.append({ + "platform": platform, + "modules": sorted(p.name for p in (base / "modules").iterdir() if p.is_dir()) if (base / "modules").exists() else [], + "patterns": sorted(p.name for p in (base / "patterns").iterdir() if p.is_dir()) if (base / "patterns").exists() else [], + "conformance_documents": [n for n in ("CONFORMANCE.md", "PLATFORM_PROFILE.md") if (base / n).exists()], + "declared_checks": sorted(binding.checks), + "declared_gaps": sorted(binding.gaps), + "declares_standard_version": declares}) + return {"standard_version": read_version(repo), "bindings": result} + + +def build_examples(repo=REPO): + """examples.json: each example, the platforms it covers and its README.""" + platforms = _platforms(repo) + result = [] + for example in sorted(p for p in (repo / "examples").iterdir() if p.is_dir() and (p / "README.md").exists()): + covered = [p for p in platforms if (example / p).is_dir() or example.name.endswith("-" + p) + or (example / "profiles" / f"{p}.md").exists()] + result.append({"example": example.name, "platforms": covered, "readme": f"examples/{example.name}/README.md"}) + return {"standard_version": read_version(repo), "examples": result} + + +def build_checks(repo=REPO): + """checks.json: the checks a conforming deployment of the reference product runs, per platform.""" + rendered = {} + for platform in _platforms(repo): + if not (REFERENCE / "profiles" / f"{platform}.md").exists(): + continue + _, files = _build(platform, repo) + manifest = json.loads(files["manifest.json"]) + rendered[platform] = sorted({(c["scope_kind"], c["check"]) for c in manifest["checks"]}) + return {"standard_version": read_version(repo), + "rendered_checks": {p: [{"scope_kind": s, "check": c} for s, c in items] for p, items in rendered.items()}} + + +def render_bundle(repo=REPO): + """Return {relative path: text} for every generated bundle file.""" + directory = f"release/v{read_version(repo)}" + builders = {"contract.json": build_contract, "bindings.json": build_bindings, + "examples.json": build_examples, "checks.json": build_checks} + return {f"{directory}/{name}": json.dumps(build(repo), indent=2, ensure_ascii=False) + "\n" + for name, build in builders.items()} + + +def authored_problems(repo=REPO): + """Authored documents the current version must have; return a list of problems.""" + version = read_version(repo) + problems = [] + if not (repo / f"release/migrations/v{version}.md").exists(): + problems.append(f"release/migrations/v{version}.md is missing") + compat = repo / "release/COMPATIBILITY.md" + if not compat.exists() or not re.search(rf"^\|\s*{re.escape(version)}\s*\|", compat.read_text(encoding="utf-8"), re.M): + problems.append(f"release/COMPATIBILITY.md has no row for {version}") + return problems + + +def stale_files(repo=REPO): + """Bundle files whose committed content differs from what the sources generate.""" + stale = [] + for relative, text in render_bundle(repo).items(): + path = repo / relative + if not path.exists() or path.read_text(encoding="utf-8") != text: + stale.append(relative) + return stale + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--check", action="store_true", help="exit 1 when the committed bundle is stale or incomplete") + args = parser.parse_args() + if args.check: + problems = [f"stale: {f}" for f in stale_files()] + authored_problems() + print("\n".join(problems) if problems else "Release bundle is current.") + raise SystemExit(bool(problems)) + for relative, text in render_bundle().items(): + path = REPO / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8", newline="\n") + print("wrote", relative) + + +if __name__ == "__main__": + main() diff --git a/tooling/release/tests/test_release.py b/tooling/release/tests/test_release.py new file mode 100644 index 0000000..458c7b0 --- /dev/null +++ b/tooling/release/tests/test_release.py @@ -0,0 +1,152 @@ +"""Tests for the release bundle generator and the committed bundle. + +The committed bundle must match what the sources generate, and the current version must +have its authored documents. The release bundle must also never become a module. +""" +import json +from pathlib import Path +import re +import sys +import tempfile +import unittest + +REPO = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(REPO / "tooling" / "release")) +import build_release as release # noqa: E402 + +MODULES = ("memory", "semantic", "domain", "observability", "search", "prediction") + + +class CommittedBundle(unittest.TestCase): + def test_committed_bundle_is_current(self): + self.assertEqual(release.stale_files(), [], "run: python tooling/release/build_release.py") + + def test_current_version_has_authored_documents(self): + self.assertEqual(release.authored_problems(), []) + + def test_generation_is_deterministic(self): + self.assertEqual(release.render_bundle(), release.render_bundle()) + + def test_bundle_is_not_a_module(self): + self.assertFalse((REPO / "design/modules/release.md").exists()) + for platform in (REPO / "implementation").iterdir(): + self.assertFalse((platform / "modules/release").exists(), platform.name) + contract = release.build_contract() + self.assertEqual(contract["modules"], list(MODULES)) + self.assertNotIn("release", contract["metadata_entities"]) + + +class ContractContent(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.contract = release.build_contract() + cls.entities = {e["name"]: e for items in cls.contract["metadata_entities"].values() for e in items} + cls.surface = {s["relation"]: s for s in cls.contract["access_surface"]} + + def test_declares_its_own_version(self): + self.assertEqual(self.contract["standard_version"], release.read_version()) + + def test_declaration_surface_is_published(self): + self.assertIn("StandardDeclaration", self.entities) + self.assertEqual(self.entities["StandardDeclaration"]["identity"], "semantic.standard_declaration") + self.assertEqual(self.surface["DataProductManifest"]["columns"][-1], "standard_version") + + def test_identity_grammar_examples_match_their_patterns(self): + for rule in self.contract["identity_grammar"]: + with self.subTest(kind=rule["kind"]): + self.assertRegex(rule["example"], rule["pattern"]) + others = [r for r in self.contract["identity_grammar"] if r is not rule] + self.assertFalse(any(re.fullmatch(o["pattern"], rule["example"]) for o in others)) + + def test_every_published_identity_follows_the_grammar(self): + patterns = [re.compile(r["pattern"]) for r in self.contract["identity_grammar"]] + for identity in [e["identity"] for e in self.entities.values()] + [s["identity"] for s in self.surface.values()]: + with self.subTest(identity=identity): + self.assertTrue(any(p.fullmatch(identity) for p in patterns)) + + def test_ordered_group_surfaces_are_published(self): + columns = {name: [c["name"] for c in e["columns"]] for name, e in self.entities.items()} + self.assertEqual(columns["KeyColumn"][:4], ["key_column_id", "key_definition_id", "ordinal", "column_name"]) + self.assertEqual(columns["RelationshipColumn"][:5], + ["relationship_column_id", "relationship_id", "ordinal", "source_column", "target_column"]) + self.assertIn("source_entity", columns["AccessColumn"]) + for gone in ("source_column", "target_column", "source_table", "target_table"): + self.assertNotIn(gone, columns["TableRelationship"]) + for gone in ("surrogate_key_column", "natural_key_column"): + self.assertNotIn(gone, columns["EntityMetadata"]) + self.assertNotIn("join_path", columns["AccessComposition"]) + self.assertEqual(self.surface["RelationshipPaths"]["columns"], [ + "path_id", "origin_entity", "destination_entity", "hop_count", "hop", "key_ordinal", + "from_entity", "from_column", "to_entity", "to_column", "relationship_id", "direction"]) + + def test_every_entity_publishes_its_keys_in_order(self): + embedding = next(e for items in self.contract["metadata_entities"].values() for e in items + if e["name"] == "ValidationArea") + natural = next(k for k in embedding["keys"] if k["role"] == "NATURAL") + self.assertGreater(len(natural["columns"]), 1) + + def test_every_published_access_object_lists_columns(self): + for view in self.contract["access_surface"]: + with self.subTest(view=view["relation"]): + self.assertTrue(view["columns"]) + + def test_discovery_starts_at_the_trust_map(self): + roles = self.contract["discovery_roles"] + self.assertEqual(roles[0]["role"], "MANIFEST") + orders = [r["discovery_order"] for r in roles] + self.assertEqual(orders, sorted(orders)) + self.assertIn("TRUST_MAP", [r["role"] for r in roles[:2]]) + + def test_bindings_report_declaration_support(self): + support = {b["platform"]: b["declares_standard_version"] for b in release.build_bindings()["bindings"]} + self.assertTrue(all(support.values()), support) + + def test_checks_include_the_contract_and_comment_coverage(self): + rendered = release.build_checks()["rendered_checks"] + for platform, checks in rendered.items(): + names = {c["check"] for c in checks} + with self.subTest(platform=platform): + for expected in ("contract-identity", "contract-keys", "contract-relationships", + "contract-access-columns", "contract-composition", "comment-coverage"): + self.assertIn(expected, names) + + +class Generator(unittest.TestCase): + def test_version_must_be_semantic(self): + for bad in ("1.1", "v1.1.0", "1.1.0-rc1", ""): + with self.subTest(version=bad), tempfile.TemporaryDirectory() as tmp: + (Path(tmp) / "VERSION").write_text(bad + "\n", encoding="utf-8") + with self.assertRaises(ValueError): + release.read_version(Path(tmp)) + + def test_missing_authored_documents_are_reported(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "VERSION").write_text("2.0.0\n", encoding="utf-8") + self.assertEqual(len(release.authored_problems(root)), 2) + (root / "release/migrations").mkdir(parents=True) + (root / "release/migrations/v2.0.0.md").write_text("note\n", encoding="utf-8") + (root / "release/COMPATIBILITY.md").write_text("| 1.9.0 | x |\n", encoding="utf-8") + self.assertEqual(len(release.authored_problems(root)), 1) + (root / "release/COMPATIBILITY.md").write_text("| 2.0.0 | x |\n", encoding="utf-8") + self.assertEqual(release.authored_problems(root), []) + + def test_stale_file_is_detected(self): + bundle = release.render_bundle() + path = REPO / next(iter(bundle)) + original = path.read_text(encoding="utf-8") + try: + path.write_text(original + "\n", encoding="utf-8", newline="\n") + self.assertEqual(release.stale_files(), [str(path.relative_to(REPO)).replace("\\", "/")]) + finally: + path.write_text(original, encoding="utf-8", newline="\n") + self.assertEqual(release.stale_files(), []) + + def test_bundle_files_are_valid_json(self): + for relative, text in release.render_bundle().items(): + with self.subTest(file=relative): + json.loads(text) + + +if __name__ == "__main__": + unittest.main() diff --git a/tooling/validation/tests/test_binding_package.py b/tooling/validation/tests/test_binding_package.py index b9c8696..491ff07 100644 --- a/tooling/validation/tests/test_binding_package.py +++ b/tooling/validation/tests/test_binding_package.py @@ -75,7 +75,8 @@ def test_every_template_is_declared_or_caller_rendered(self): """A template no phase, check or caller reaches is dead weight that drifts.""" caller_rendered = {"modules/domain/02-key-allocation.sql.j2", "patterns/temporal-lifecycle-metadata/04-maintenance.sql.j2", - "patterns/temporal-lifecycle-metadata/00-macros.sql.j2"} + "patterns/temporal-lifecycle-metadata/00-macros.sql.j2", + "modules/semantic/contract/00-macros.sql.j2"} for platform in PLATFORMS: with self.subTest(platform=platform): b, _ = load_binding(platform) diff --git a/tooling/validation/tests/test_build_context.py b/tooling/validation/tests/test_build_context.py index 54e67d5..9b88f3f 100644 --- a/tooling/validation/tests/test_build_context.py +++ b/tooling/validation/tests/test_build_context.py @@ -462,6 +462,32 @@ class SharedBuildDecisions(unittest.TestCase): def setUpClass(cls): cls.ctx = Resolved(profile=POSTGRES_PROFILE).context + def test_the_standard_owned_relations_contribute_their_joins(self): + names = {r["name"] for r in self.ctx["relationships"]} + for expected in ("LineageRun.lineage_id", "KeyColumn.key_definition_id", "ValidationArea.run", + "Ticket.customer_id"): + self.assertIn(expected, names) + self.assertEqual(len(names), len(self.ctx["relationships"]), "relationship names are unique") + + def test_a_named_join_resolves_to_one_relationship_of_ordered_pairs(self): + join = next(r for r in self.ctx["relationships"] if r["name"] == "ValidationArea.run") + self.assertEqual((join["join_kind"], join["source"], join["target"]), ("named", "ValidationArea", "ValidationRun")) + self.assertEqual([(p["ordinal"], p["source_column"], p["target_column"]) for p in join["pairs"]], + [(1, "product_code", "product_code"), (2, "producer_id", "producer_id"), (3, "run_id", "run_id")]) + self.assertEqual(join["unpaired_target_columns"], []) + self.assertTrue(join["mandatory"]) + + def test_a_bare_reference_stays_one_pair_with_the_targets_identity(self): + join = next(r for r in self.ctx["relationships"] if r["name"] == "Ticket.customer_id") + self.assertEqual(join["join_kind"], "bare") + self.assertEqual(len(join["pairs"]), 1) + + def test_a_multi_target_reference_is_one_relationship_per_target(self): + synonyms = [r for r in self.ctx["relationships"] if r["source"] == "SemanticSynonym"] + self.assertEqual({r["target"] for r in synonyms}, {"EntityMetadata", "ColumnMetadata", "Metric"}) + self.assertEqual({r["join_kind"] for r in synonyms}, {"multi"}) + self.assertEqual(len({r["name"] for r in synonyms}), 3) + def test_orientation_publishes_the_trust_map_before_analytical_resources(self): resources = self.ctx["orientation"]["resources"] self.assertEqual([r["role"] for r in resources[:2]], ["MANIFEST", "TRUST_MAP"]) @@ -505,6 +531,16 @@ def test_tiers_are_enforced_outside_a_platform_without_principals(self): "ROLE_READ: create {product_code|lower}_reader")) self.assertIn("unsupported-principal", r.rules) + def test_a_reference_to_a_natural_key_takes_the_key_type(self): + """DesignDecision is keyed by the text decision_id, so a reference to it is not a BIGINT.""" + ctx = Resolved().context + decision = next(e for e in ctx["relations"] if e["name"] == "DesignDecision") + key = next(c for c in decision["columns"] if c["name"] == "decision_id") + for entity, name in (("DesignDecision", "superseded_by"), ("ChangeLog", "related_decision_id")): + holder = next(e for e in ctx["relations"] if e["name"] == entity) + column = next(c for c in holder["columns"] if c["name"] == name) + self.assertEqual(column["physical_type"], key["physical_type"], f"{entity}.{name}") + if __name__ == "__main__": unittest.main(verbosity=2) diff --git a/tooling/validation/tests/test_joins.py b/tooling/validation/tests/test_joins.py new file mode 100644 index 0000000..94c0f3e --- /dev/null +++ b/tooling/validation/tests/test_joins.py @@ -0,0 +1,43 @@ +"""How an entity's `[-> Target]` attributes combine into joins (tooling/evals/joins.py).""" +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / "evals")) +from joins import join_groups # noqa: E402 + + +class JoinGroups(unittest.TestCase): + def test_a_bare_reference_is_a_whole_join_on_its_own(self): + groups = join_groups([("customer_id", "Customer", None), ("note", None, None)]) + self.assertEqual(groups, [{"name": "customer_id", "target": "Customer", "targets": ["Customer"], + "attributes": ["customer_id"], "target_attributes": None, "kind": "bare"}]) + + def test_named_attributes_of_one_target_form_one_ordered_join(self): + groups = join_groups([("product_code", "Run.product_code", None), ("x", None, None), + ("producer_id", "Run.producer_id", None), ("run_id", "Run.run_id", None)]) + self.assertEqual(len(groups), 1) + self.assertEqual(groups[0]["attributes"], ["product_code", "producer_id", "run_id"]) + self.assertEqual(groups[0]["target_attributes"], ["product_code", "producer_id", "run_id"]) + self.assertEqual((groups[0]["name"], groups[0]["kind"]), ("product_code", "named")) + + def test_a_join_name_keeps_two_joins_to_one_target_apart(self): + groups = join_groups([("a_id", "T.id", "first"), ("b_id", "T.id", "second"), ("c_id", "T.k", "first")]) + self.assertEqual([(g["name"], g["attributes"], g["target_attributes"]) for g in groups], + [("first", ["a_id", "c_id"], ["id", "k"]), ("second", ["b_id"], ["id"])]) + + def test_a_join_without_a_name_groups_by_target(self): + groups = join_groups([("a_id", "T.id", None), ("b_id", "U.id", None), ("c_id", "T.k", None)]) + self.assertEqual([(g["target"], g["attributes"]) for g in groups], [("T", ["a_id", "c_id"]), ("U", ["b_id"])]) + + def test_a_bare_and_a_named_join_to_one_target_stay_separate(self): + groups = join_groups([("a_id", "T", None), ("b_id", "T.id", None)]) + self.assertEqual([g["kind"] for g in groups], ["bare", "named"]) + + def test_a_multi_target_reference_is_one_attribute(self): + groups = join_groups([("subject", "A | B", None)]) + self.assertEqual((groups[0]["kind"], groups[0]["targets"], groups[0]["target"]), ("multi", ["A", "B"], None)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tooling/validation/tests/test_spec_lint.py b/tooling/validation/tests/test_spec_lint.py index ab3f73a..5c80e99 100644 --- a/tooling/validation/tests/test_spec_lint.py +++ b/tooling/validation/tests/test_spec_lint.py @@ -270,6 +270,29 @@ def test_unresolved_reference_flagged(self): "[required] [-> Client] // the ordering customer") self.assertIn("unresolved-reference", rules_for(text)) + JOIN_RULES = ("composite-join-target", "composite-join-key", "composite-join-type", "multi-target-join") + + def join_rules(self, new): + text = broken("[required] [-> Customer] // the ordering customer", new + " // the ordering customer") + return [r for r in rules_for(text) if r in self.JOIN_RULES] + + def test_a_named_join_on_a_whole_key_of_the_target_is_accepted(self): + self.assertEqual(self.join_rules("[required] [-> Customer.customer_id]"), []) + + def test_a_named_join_to_an_attribute_the_target_lacks_is_flagged(self): + self.assertIn("composite-join-target", self.join_rules("[required] [-> Customer.nope]")) + + def test_a_named_join_that_is_not_a_key_of_the_target_is_flagged(self): + self.assertIn("composite-join-key", self.join_rules("[required] [-> Customer.region_code]")) + + def test_a_named_join_pairs_attributes_of_the_same_type(self): + text = broken("customer_id : Reference [required] [-> Customer] // the ordering customer", + "customer_id : ShortText [required] [-> Customer.customer_id] // the ordering customer") + self.assertIn("composite-join-type", rules_for(text)) + + def test_a_multi_target_reference_cannot_name_target_attributes(self): + self.assertIn("multi-target-join", self.join_rules("[required] [-> Customer.customer_id | Product]")) + def test_multi_target_reference_needs_a_discriminator(self): text = broken("[required] [-> Customer] // the ordering customer", "[required] [-> Customer | Product] // the ordering customer")