From 4a1c562b452b3ded51e6a91f9f69337a8bb70a42 Mon Sep 17 00:00:00 2001 From: Ning Zhou Date: Fri, 31 Jul 2026 16:05:09 +0800 Subject: [PATCH 1/2] feat: admit Chongqing real-source metadata baseline --- data_agent/chongqing_real_source_admission.py | 1082 ++++++++++++++++ .../test_chongqing_real_source_admission.py | 194 +++ ...hongqing-real-source-admission-manifest.md | 58 + ...qing-real-source-admission-2026-07-31.json | 1119 +++++++++++++++++ docs/roadmap.md | 3 +- scripts/chongqing-real-source-admission.sh | 22 + 6 files changed, 2477 insertions(+), 1 deletion(-) create mode 100644 data_agent/chongqing_real_source_admission.py create mode 100644 data_agent/test_chongqing_real_source_admission.py create mode 100644 docs/architecture-decisions/adr-074-chongqing-real-source-admission-manifest.md create mode 100644 docs/evidence/chongqing-real-source-admission-2026-07-31.json create mode 100755 scripts/chongqing-real-source-admission.sh diff --git a/data_agent/chongqing_real_source_admission.py b/data_agent/chongqing_real_source_admission.py new file mode 100644 index 00000000..8c631b6f --- /dev/null +++ b/data_agent/chongqing_real_source_admission.py @@ -0,0 +1,1082 @@ +"""Build a fail-closed admission manifest for the full Chongqing sample. + +M3-28 profiles the real planning-institute archive as an AR-2 source without +copying source payloads into the repository. The checked evidence is path-free, +content-addressed, and metadata-only. It never authorizes ingestion, +publication, scheduler submission, or provider mutation. +""" + +from __future__ import annotations + +import argparse +import csv +import hashlib +import json +import os +import re +import stat +import zipfile +import zlib +from collections import Counter +from collections.abc import Mapping +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +from .uwm.local_planning_zip_audit import scan_local_planning_zip_assets + +EVIDENCE_SCHEMA = "gda.chongqing_real_source_admission.v1" +VALIDATION_SCHEMA = "gda.chongqing_real_source_admission_validation.v1" +STATUS = "blocked_pending_source_governance" +SOURCE_ID = "chongqing-planning-institute-sample" + +REPO_ROOT = Path(__file__).resolve().parent.parent +DEFAULT_EVIDENCE_PATH = REPO_ROOT / "docs/evidence/chongqing-real-source-admission-2026-07-31.json" +RESEARCH_INVENTORY_PATH = ( + REPO_ROOT / "data/uwm_public_proxy/chongqing_central/" + "local_planning_zip_audit_2026_07_05/uwm_local_planning_zip_inventory.csv" +) + +EXPECTED_ARCHIVE_SHA256 = "2043b60c2f4f7f32a31388a634fae4ac28534990e205aa86b8df0e4b64dcbbca" +EXPECTED_ARCHIVE_SIZE_BYTES = 468_462_251 +EXPECTED_ARCHIVE_ENTRY_COUNT = 533 +EXPECTED_ARCHIVE_UNCOMPRESSED_SIZE_BYTES = 694_164_379 +EXPECTED_ARCHIVE_SCOPE_ENTRY_COUNT = 532 +EXPECTED_ARCHIVE_SCOPE_SIZE_BYTES = 694_147_946 +EXPECTED_EXTRACTED_FILE_COUNT = 584 +EXPECTED_EXTRACTED_SIZE_BYTES = 700_610_744 +EXPECTED_ARCHIVE_EXACT_MATCH_COUNT = 526 +EXPECTED_ARCHIVE_MODIFIED_COUNT = 6 +EXPECTED_ARCHIVE_MISSING_COUNT = 0 +EXPECTED_EXTRACTED_ADDITIONAL_COUNT = 52 +EXPECTED_EXTRACTED_PAYLOAD_SHA256 = ( + "e7e81e4f53f9f174792f500fbfdfde6bee30ec03beac8cbd91771fe09f548ea6" +) +EXPECTED_RESEARCH_INVENTORY_SHA256 = ( + "69b4167955f950041988dd174b75ea5376af146ef3e52f815aa57715fd24f70d" +) +EXPECTED_RESEARCH_INVENTORY_ROWS = 16 +EXPECTED_RESEARCH_ASSET_IDS_SHA256 = ( + "c7cb765c2b653dbf5619e2fe1cf027d108a5951b752b6dd4bfd60b4c00d91947" +) +EVIDENCE_FILE_SHA256 = "9b5c20369c235f7e0a2f2cb0a21cee77f86981aa273bac196605a4803b05ce83" + +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +SOURCE_REF_PATTERN = re.compile(rf"^source://{re.escape(SOURCE_ID)}/assets/[a-z0-9][a-z0-9_-]+$") +SENSITIVE_KEY_PATTERN = re.compile( + r"(^|[-_.])(password|passwd|secret|client[-_.]?secret|private[-_.]?key|" + r"access[-_.]?key|access[-_.]?token|refresh[-_.]?token|" + r"authorization[-_.]?header)($|[-_.])", + re.IGNORECASE, +) + +EVIDENCE_INVENTORY = { + "schema", + "status", + "captured_at", + "source_binding", + "research_audit_binding", + "source_groups", + "asset_profiles", + "admission_blockers", + "admission_policy", + "claims", + "evidence_sha256", +} +SOURCE_BINDING_INVENTORY = { + "source_id", + "archive_sha256", + "archive_size_bytes", + "archive_entry_count", + "archive_uncompressed_size_bytes", + "archive_source_scope_entry_count", + "archive_source_scope_size_bytes", + "archive_original_entry_exact_match_count", + "archive_original_entry_modified_count", + "archive_original_entry_missing_count", + "extracted_additional_file_count", + "archive_integrity_verified", + "archive_extracted_entry_multiset_verified", + "extracted_file_count", + "extracted_size_bytes", + "extracted_payload_sha256", + "logical_group_inventory_sha256", + "source_payload_in_repository", + "absolute_source_paths_in_evidence", +} +RESEARCH_AUDIT_INVENTORY = { + "path", + "file_sha256", + "row_count", + "asset_ids_sha256", + "authority", + "admission_authority", +} +SOURCE_GROUP_INVENTORY = { + "source_group_id", + "ordinal", + "file_count", + "size_bytes", + "extension_counts", + "content_manifest_sha256", + "data_domains", + "data_classification", + "license_status", + "admission_status", + "metadata_profiled", + "content_admitted", + "blockers", +} +ASSET_PROFILE_INVENTORY = { + "asset_id", + "source_group_id", + "source_ref", + "asset_kind", + "data_classification", + "admission_status", + "record_metrics", + "spatial_profile", + "schema_fields", + "roles", + "metadata_profiled", + "source_content_in_evidence", + "profile_sha256", +} +CLAIMS = { + "archive_integrity_verified", + "archive_extracted_entry_multiset_verified", + "full_source_metadata_profiled", + "source_governance_approved", + "source_content_admitted", + "source_publication_authorized", + "scheduler_submission_authorized", + "provider_mutation_authorized", + "production_ingestion_verified", + "production_ready", +} +SOURCE_BINDING_BLOCKERS = [ + "source_binding:extraction_derivation_provenance_missing", +] +ADMISSION_POLICY = { + "metadata_profiling_allowed": True, + "content_admission_requires_owner_license_retention_and_access_approval": True, + "restricted_fields_require_privacy_review": True, + "source_payload_copy_to_repository_forbidden": True, + "ci_requires_local_source_payload": False, + "fresh_protected_ingestion_required_after_admission": True, + "local_profile_is_not_production_admission": True, +} + + +SOURCE_GROUP_SPECS: tuple[dict[str, Any], ...] = ( + { + "ordinal": "01", + "source_group_id": "chongqing-dem-2020", + "data_domains": ["elevation", "terrain", "raster"], + "data_classification": "restricted_local_baseline", + "additional_blockers": ["source_vintage_unverified"], + }, + { + "ordinal": "02", + "source_group_id": "chongqing-osm-roads-2021", + "data_domains": ["transport_network", "vector"], + "data_classification": "restricted_local_open_data_copy", + "additional_blockers": ["odbl_attribution_policy_missing"], + }, + { + "ordinal": "03", + "source_group_id": "chongqing-clcd-2020", + "data_domains": ["land_cover", "remote_sensing", "raster"], + "data_classification": "restricted_local_remote_sensing", + "additional_blockers": ["classification_lineage_unverified"], + }, + { + "ordinal": "04", + "source_group_id": "chongqing-central-buildings-2021", + "data_domains": ["buildings", "urban_form", "vector"], + "data_classification": "restricted_local_built_environment", + "additional_blockers": ["source_vintage_unverified"], + }, + { + "ordinal": "05", + "source_group_id": "chongqing-historic-districts", + "data_domains": ["cultural_heritage", "planning_constraints", "vector"], + "data_classification": "restricted_local_cultural_planning", + "additional_blockers": ["source_vintage_unverified"], + }, + { + "ordinal": "07", + "source_group_id": "bishan-planning-materials", + "data_domains": ["land_use", "planning", "documents", "vector", "tables"], + "data_classification": "highly_restricted_planning", + "additional_blockers": ["planning_sensitivity_review_missing"], + }, + { + "ordinal": "08", + "source_group_id": "chongqing-district-population-2021", + "data_domains": ["population", "statistics", "table"], + "data_classification": "restricted_aggregate_population", + "additional_blockers": ["statistics_vintage_review_missing"], + }, + { + "ordinal": "09", + "source_group_id": "gaode-poi-2024", + "data_domains": ["poi", "commercial_location", "vector"], + "data_classification": "highly_restricted_commercial_location", + "additional_blockers": ["contact_field_privacy_review_missing"], + }, + { + "ordinal": "10", + "source_group_id": "baidu-aoi-2024", + "data_domains": ["aoi", "commercial_location", "vector"], + "data_classification": "highly_restricted_commercial_location", + "additional_blockers": ["contact_field_privacy_review_missing"], + }, + { + "ordinal": "11", + "source_group_id": "unicom-commuting-2023", + "data_domains": ["mobility", "population", "aggregate_signaling", "table"], + "data_classification": "highly_restricted_aggregate_mobility", + "additional_blockers": [ + "privacy_impact_assessment_missing", + "grid_geometry_dictionary_missing", + ], + }, + { + "ordinal": "12", + "source_group_id": "baidu-search-index-2023", + "data_domains": ["search_activity", "intercity_flow", "vector"], + "data_classification": "highly_restricted_commercial_activity", + "additional_blockers": ["commercial_terms_review_missing"], + }, +) + +GROUP_BY_ORDINAL = {spec["ordinal"]: spec for spec in SOURCE_GROUP_SPECS} +GROUP_BY_ID = {spec["source_group_id"]: spec for spec in SOURCE_GROUP_SPECS} + +EXPECTED_ASSET_BASELINES: dict[str, dict[str, Any]] = { + "chongqing_osm_roads_2021": { + "source_group_id": "chongqing-osm-roads-2021", + "asset_kind": "vector", + "record_metrics": {"feature_count": 50_366}, + "geometry_type": "LineString", + "crs": "EPSG:4326", + }, + "chongqing_central_buildings_2021": { + "source_group_id": "chongqing-central-buildings-2021", + "asset_kind": "vector", + "record_metrics": {"feature_count": 107_452}, + "geometry_type": "Polygon", + "crs": "EPSG:4326", + }, + "chongqing_historic_districts_local": { + "source_group_id": "chongqing-historic-districts", + "asset_kind": "vector", + "record_metrics": {"feature_count": 20}, + "geometry_type": "Polygon Z", + "crs": "EPSG:4490", + }, + "bishan_land_use_dltb_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "vector", + "record_metrics": {"feature_count": 101_657}, + "geometry_type": "MultiPolygon", + "crs": "EPSG:4610", + }, + "gaode_poi_2024": { + "source_group_id": "gaode-poi-2024", + "asset_kind": "vector", + "record_metrics": {"feature_count": 1_194_351}, + "geometry_type": "Point", + "crs": "EPSG:4490", + }, + "baidu_aoi_2024": { + "source_group_id": "baidu-aoi-2024", + "asset_kind": "vector", + "record_metrics": {"feature_count": 26_292}, + "geometry_type": "MultiPolygon", + "crs": "EPSG:4490", + }, + "baidu_search_index_2023_local": { + "source_group_id": "baidu-search-index-2023", + "asset_kind": "vector", + "record_metrics": {"feature_count": 325}, + "geometry_type": "MultiLineString", + "crs": "EPSG:4490", + }, + "bishan_admin_boundary_cjdcq_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "vector", + "record_metrics": {"feature_count": 1_488}, + "geometry_type": "MultiPolygon", + "crs": "EPSG:4523+EPSG:5737", + }, + "bishan_admin_boundary_xzq_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "vector", + "record_metrics": {"feature_count": 15}, + "geometry_type": "MultiPolygon", + "crs": "EPSG:4523+EPSG:5737", + }, + "fulu_village_planning_database_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "vector_collection", + "record_metrics": { + "feature_count": 8_050, + "layer_count": 28, + "nonempty_layer_count": 20, + }, + "geometry_type": "mixed", + "crs": "mixed_CGCS2000_GK_zone_35_EPSG4523", + }, + "chongqing_district_population_stats_2021_local": { + "source_group_id": "chongqing-district-population-2021", + "asset_kind": "workbook", + "record_metrics": {"row_count": 41, "sheet_count": 1}, + }, + "chongqing_unicom_commuting_2023_local": { + "source_group_id": "unicom-commuting-2023", + "asset_kind": "table", + "record_metrics": {"row_count": 2_120, "column_count": 7}, + }, + "clcd_classification_dictionary_local": { + "source_group_id": "chongqing-clcd-2020", + "asset_kind": "workbook", + "record_metrics": {"row_count": 10, "sheet_count": 1}, + }, + "bishan_land_development_ledger_2019_local": { + "source_group_id": "bishan-planning-materials", + "asset_kind": "workbook_collection", + "record_metrics": {"row_count": 1_438, "sheet_count": 4}, + }, + "chongqing_dem_80m": { + "source_group_id": "chongqing-dem-2020", + "asset_kind": "raster", + "record_metrics": { + "width": 1_766, + "height": 1_454, + "band_count": 1, + "pixel_count": 2_567_764, + }, + "crs": "EPSG:4490", + }, + "chongqing_clcd_2020": { + "source_group_id": "chongqing-clcd-2020", + "asset_kind": "raster", + "record_metrics": { + "width": 18_579, + "height": 15_082, + "band_count": 1, + "pixel_count": 280_208_478, + }, + "crs": "EPSG:4326", + }, +} + + +class ChongqingRealSourceAdmissionError(RuntimeError): + """The full-source admission snapshot failed closed.""" + + +def canonical_json_fingerprint(value: Any) -> str: + payload = json.dumps(value, ensure_ascii=True, sort_keys=True, separators=(",", ":")).encode( + "utf-8" + ) + return hashlib.sha256(payload).hexdigest() + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _load_json_object(path: Path) -> dict[str, Any]: + value = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError("JSON document is not an object") + return value + + +def _parse_time(value: Any) -> datetime: + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError as exc: + raise ChongqingRealSourceAdmissionError("captured_at is not a valid timestamp") from exc + if parsed.tzinfo is None or parsed.utcoffset() is None: + raise ChongqingRealSourceAdmissionError("captured_at must be timezone-aware") + return parsed.astimezone(UTC) + + +def _source_group_blockers(spec: Mapping[str, Any]) -> list[str]: + group_id = str(spec["source_group_id"]) + suffixes = [ + "owner_reference_missing", + "license_terms_unverified", + "retention_policy_missing", + "access_policy_missing", + *[str(item) for item in spec.get("additional_blockers", [])], + ] + return [f"source_group:{group_id}:{suffix}" for suffix in suffixes] + + +def _stat_identity(value: os.stat_result) -> tuple[int, int, int, int, int]: + return ( + value.st_dev, + value.st_ino, + value.st_size, + value.st_mtime_ns, + value.st_mode, + ) + + +def _hash_regular_file(path: Path) -> tuple[int, str, str]: + before = path.lstat() + if stat.S_ISLNK(before.st_mode) or not stat.S_ISREG(before.st_mode): + raise ChongqingRealSourceAdmissionError("source payload contains a non-regular file") + sha256 = hashlib.sha256() + crc32 = 0 + with path.open("rb") as stream: + opened = os.fstat(stream.fileno()) + if _stat_identity(opened) != _stat_identity(before): + raise ChongqingRealSourceAdmissionError( + "source payload file identity changed before hashing" + ) + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + sha256.update(chunk) + crc32 = zlib.crc32(chunk, crc32) + after = os.fstat(stream.fileno()) + if _stat_identity(after) != _stat_identity(before): + raise ChongqingRealSourceAdmissionError( + "source payload file identity changed while hashing" + ) + return before.st_size, sha256.hexdigest(), f"{crc32 & 0xFFFFFFFF:08x}" + + +def _scan_payload_files(source_root: Path) -> list[dict[str, Any]]: + root = source_root.resolve(strict=True) + root_stat = source_root.lstat() + if stat.S_ISLNK(root_stat.st_mode) or not stat.S_ISDIR(root_stat.st_mode): + raise ChongqingRealSourceAdmissionError("source root must be a real directory") + records: list[dict[str, Any]] = [] + for path in sorted(root.rglob("*"), key=lambda item: item.relative_to(root).as_posix()): + if path.is_symlink(): + raise ChongqingRealSourceAdmissionError("source payload contains a symbolic link") + if not path.is_file(): + continue + relative_path = path.relative_to(root).as_posix() + ordinal = relative_path.split("/", 1)[0][:2] + spec = GROUP_BY_ORDINAL.get(ordinal) + if spec is None: + raise ChongqingRealSourceAdmissionError( + f"source payload contains an unknown top-level group: {ordinal}" + ) + size, sha256, crc32 = _hash_regular_file(path) + suffix = path.suffix.lower().lstrip(".") or "[no_ext]" + records.append( + { + "relative_path": relative_path, + "source_group_id": spec["source_group_id"], + "size_bytes": size, + "extension": suffix, + "sha256": sha256, + "crc32": crc32, + } + ) + if not records: + raise ChongqingRealSourceAdmissionError("source payload is empty") + return records + + +def _payload_fingerprint(records: list[Mapping[str, Any]]) -> str: + material = [ + { + "relative_path": item["relative_path"], + "size_bytes": item["size_bytes"], + "sha256": item["sha256"], + } + for item in records + ] + return canonical_json_fingerprint(material) + + +def _build_source_groups(records: list[Mapping[str, Any]]) -> list[dict[str, Any]]: + groups: list[dict[str, Any]] = [] + for spec in SOURCE_GROUP_SPECS: + group_records = [ + item for item in records if item.get("source_group_id") == spec["source_group_id"] + ] + if not group_records: + raise ChongqingRealSourceAdmissionError( + f"source group is empty: {spec['source_group_id']}" + ) + manifest = [ + { + "relative_path": item["relative_path"], + "size_bytes": item["size_bytes"], + "sha256": item["sha256"], + } + for item in group_records + ] + extensions = Counter(str(item["extension"]) for item in group_records) + groups.append( + { + "source_group_id": spec["source_group_id"], + "ordinal": spec["ordinal"], + "file_count": len(group_records), + "size_bytes": sum(int(item["size_bytes"]) for item in group_records), + "extension_counts": dict(sorted(extensions.items())), + "content_manifest_sha256": canonical_json_fingerprint(manifest), + "data_domains": list(spec["data_domains"]), + "data_classification": spec["data_classification"], + "license_status": "unverified_restricted", + "admission_status": STATUS, + "metadata_profiled": True, + "content_admitted": False, + "blockers": _source_group_blockers(spec), + } + ) + return groups + + +def _archive_binding( + source_zip: Path, + payload_records: list[Mapping[str, Any]], +) -> dict[str, Any]: + archive_stat = source_zip.lstat() + if stat.S_ISLNK(archive_stat.st_mode) or not stat.S_ISREG(archive_stat.st_mode): + raise ChongqingRealSourceAdmissionError("source archive must be a regular file") + archive_sha256 = _file_sha256(source_zip) + with zipfile.ZipFile(source_zip) as archive: + bad_entry = archive.testzip() + if bad_entry is not None: + raise ChongqingRealSourceAdmissionError("source archive CRC validation failed") + all_entries = [item for item in archive.infolist() if not item.is_dir()] + scoped_entries = [ + (item, relative_path) + for item in archive.infolist() + if not item.is_dir() + and (relative_path := _source_scope_relative_path(item)) is not None + ] + extracted_by_path = {str(item["relative_path"]): item for item in payload_records} + scoped_paths = {relative_path for _, relative_path in scoped_entries} + if len(scoped_paths) != len(scoped_entries): + raise ChongqingRealSourceAdmissionError("source archive scope contains duplicate paths") + exact_match_count = 0 + modified_count = 0 + missing_count = 0 + for item, relative_path in scoped_entries: + extracted = extracted_by_path.get(relative_path) + if extracted is None: + missing_count += 1 + continue + if ( + int(extracted["size_bytes"]) == item.file_size + and str(extracted["crc32"]) == f"{item.CRC & 0xFFFFFFFF:08x}" + ): + exact_match_count += 1 + else: + modified_count += 1 + additional_count = len(set(extracted_by_path) - scoped_paths) + multiset_verified = not (modified_count or missing_count or additional_count) + return { + "archive_sha256": archive_sha256, + "archive_size_bytes": archive_stat.st_size, + "archive_entry_count": len(all_entries), + "archive_uncompressed_size_bytes": sum(item.file_size for item in all_entries), + "archive_source_scope_entry_count": len(scoped_entries), + "archive_source_scope_size_bytes": sum(item.file_size for item, _ in scoped_entries), + "archive_original_entry_exact_match_count": exact_match_count, + "archive_original_entry_modified_count": modified_count, + "archive_original_entry_missing_count": missing_count, + "extracted_additional_file_count": additional_count, + "archive_integrity_verified": True, + "archive_extracted_entry_multiset_verified": multiset_verified, + } + + +def _source_scope_relative_path(item: zipfile.ZipInfo) -> str | None: + name = item.filename + if not item.flag_bits & 0x800: + try: + name = name.encode("cp437").decode("gbk") + except UnicodeError: + pass + marker = "/01数据样例/" + if marker not in name: + return None + return name.split(marker, 1)[1] + + +def _normalise_crs(value: Any) -> str: + text = str(value or "") + if text.startswith("COMPD_CS[") and 'AUTHORITY["EPSG","4523"]' in text: + return "EPSG:4523+EPSG:5737" + return text + + +def _source_group_for_profile(profile: Mapping[str, Any], source_root: Path) -> str: + source_path = Path(str(profile.get("source_path") or "")) + try: + relative = source_path.resolve().relative_to(source_root.resolve()).as_posix() + except ValueError as exc: + raise ChongqingRealSourceAdmissionError("asset profile escaped the source root") from exc + ordinal = relative.split("/", 1)[0][:2] + spec = GROUP_BY_ORDINAL.get(ordinal) + if spec is None: + raise ChongqingRealSourceAdmissionError("asset profile has an unknown source group") + return str(spec["source_group_id"]) + + +def _asset_profile( + raw: Mapping[str, Any], + *, + source_root: Path, +) -> dict[str, Any]: + asset_id = str(raw.get("asset_id") or "") + source_group_id = _source_group_for_profile(raw, source_root) + group_spec = GROUP_BY_ID[source_group_id] + metric_names = ( + "feature_count", + "row_count", + "column_count", + "sheet_count", + "width", + "height", + "band_count", + "pixel_count", + "layer_count", + "nonempty_layer_count", + ) + metrics = {name: int(raw[name]) for name in metric_names if raw.get(name) is not None} + crs = _normalise_crs(raw.get("crs")) + spatial: dict[str, Any] | None = None + if raw.get("geometry_type") is not None or crs or raw.get("bounds") is not None: + spatial = { + "geometry_type": str(raw.get("geometry_type") or ""), + "crs": crs, + "bounds": ( + [float(item) for item in raw["bounds"]] if raw.get("bounds") is not None else None + ), + "dtype": [str(item) for item in raw.get("dtype", [])], + "nodata": raw.get("nodata"), + } + fields = raw.get("fields") or raw.get("columns") or [] + stable = { + "asset_id": asset_id, + "source_group_id": source_group_id, + "source_ref": f"source://{SOURCE_ID}/assets/{asset_id}", + "asset_kind": str(raw.get("asset_kind") or ""), + "data_classification": group_spec["data_classification"], + "admission_status": STATUS, + "record_metrics": metrics, + "spatial_profile": spatial, + "schema_fields": [str(item) for item in fields], + "roles": sorted(item for item in str(raw.get("uwm_roles") or "").split(";") if item), + "metadata_profiled": True, + "source_content_in_evidence": False, + } + return {**stable, "profile_sha256": canonical_json_fingerprint(stable)} + + +def _asset_profiles(source_root: Path, source_zip: Path, captured_at: str) -> list[dict[str, Any]]: + report = scan_local_planning_zip_assets( + source_root=source_root, + source_zip=source_zip, + created_at=captured_at, + ) + raw_profiles = [ + *report.get("vector_profiles", []), + *report.get("tabular_profiles", []), + *report.get("raster_profiles", []), + ] + profiles = [_asset_profile(raw, source_root=source_root) for raw in raw_profiles] + return sorted(profiles, key=lambda item: str(item["asset_id"])) + + +def _research_audit_binding(path: Path = RESEARCH_INVENTORY_PATH) -> dict[str, Any]: + with path.open(newline="", encoding="utf-8") as stream: + rows = list(csv.DictReader(stream)) + return { + "path": str(path.relative_to(REPO_ROOT)), + "file_sha256": _file_sha256(path), + "row_count": len(rows), + "asset_ids_sha256": canonical_json_fingerprint( + sorted(str(row.get("asset_id") or "") for row in rows) + ), + "authority": "research_inventory_only", + "admission_authority": False, + } + + +def build_evidence( + *, + source_root: Path, + source_zip: Path, + captured_at: datetime | None = None, + research_inventory_path: Path = RESEARCH_INVENTORY_PATH, +) -> dict[str, Any]: + timestamp = (captured_at or datetime.now(UTC)).astimezone(UTC) + captured_text = timestamp.isoformat().replace("+00:00", "Z") + payload_records = _scan_payload_files(source_root) + groups = _build_source_groups(payload_records) + archive = _archive_binding(source_zip, payload_records) + profiles = _asset_profiles(source_root, source_zip, captured_text) + blockers = sorted( + [ + *SOURCE_BINDING_BLOCKERS, + *(blocker for group in groups for blocker in group["blockers"]), + ] + ) + stable = { + "schema": EVIDENCE_SCHEMA, + "status": STATUS, + "captured_at": captured_text, + "source_binding": { + "source_id": SOURCE_ID, + **archive, + "extracted_file_count": len(payload_records), + "extracted_size_bytes": sum(int(item["size_bytes"]) for item in payload_records), + "extracted_payload_sha256": _payload_fingerprint(payload_records), + "logical_group_inventory_sha256": canonical_json_fingerprint(groups), + "source_payload_in_repository": False, + "absolute_source_paths_in_evidence": False, + }, + "research_audit_binding": _research_audit_binding(research_inventory_path), + "source_groups": groups, + "asset_profiles": profiles, + "admission_blockers": blockers, + "admission_policy": dict(ADMISSION_POLICY), + "claims": { + "archive_integrity_verified": archive["archive_integrity_verified"], + "archive_extracted_entry_multiset_verified": archive[ + "archive_extracted_entry_multiset_verified" + ], + "full_source_metadata_profiled": True, + "source_governance_approved": False, + "source_content_admitted": False, + "source_publication_authorized": False, + "scheduler_submission_authorized": False, + "provider_mutation_authorized": False, + "production_ingestion_verified": False, + "production_ready": False, + }, + } + return {**stable, "evidence_sha256": canonical_json_fingerprint(stable)} + + +def _sensitive_paths(value: Any, prefix: str = "") -> list[str]: + findings: list[str] = [] + if isinstance(value, Mapping): + for key, item in value.items(): + path = f"{prefix}.{key}" if prefix else str(key) + if SENSITIVE_KEY_PATTERN.search(str(key)): + findings.append(path) + findings.extend(_sensitive_paths(item, path)) + elif isinstance(value, list): + for index, item in enumerate(value): + findings.extend(_sensitive_paths(item, f"{prefix}[{index}]")) + return findings + + +def _asset_baseline_errors(profile: Mapping[str, Any]) -> list[str]: + asset_id = str(profile.get("asset_id") or "") + expected = EXPECTED_ASSET_BASELINES.get(asset_id) + if expected is None: + return [f"M3-28 asset profile is unknown: {asset_id}"] + errors: list[str] = [] + for key in ("source_group_id", "asset_kind"): + if profile.get(key) != expected[key]: + errors.append(f"M3-28 asset {key} does not match: {asset_id}") + metrics = profile.get("record_metrics") + if not isinstance(metrics, Mapping): + errors.append(f"M3-28 asset record metrics are invalid: {asset_id}") + else: + for key, value in expected["record_metrics"].items(): + if metrics.get(key) != value: + errors.append(f"M3-28 asset metric does not match: {asset_id}.{key}") + spatial = profile.get("spatial_profile") + for key in ("geometry_type", "crs"): + if key not in expected: + continue + if not isinstance(spatial, Mapping) or spatial.get(key) != expected[key]: + errors.append(f"M3-28 asset spatial profile does not match: {asset_id}.{key}") + return errors + + +def validate_evidence(evidence: Mapping[str, Any]) -> list[str]: + errors: list[str] = [] + if set(evidence) != EVIDENCE_INVENTORY: + errors.append("M3-28 evidence inventory does not match") + stable = {key: value for key, value in evidence.items() if key != "evidence_sha256"} + if evidence.get("evidence_sha256") != canonical_json_fingerprint(stable): + errors.append("M3-28 evidence fingerprint does not match") + if evidence.get("schema") != EVIDENCE_SCHEMA or evidence.get("status") != STATUS: + errors.append("M3-28 evidence schema or status does not match") + try: + _parse_time(evidence.get("captured_at")) + except ChongqingRealSourceAdmissionError as exc: + errors.append(str(exc)) + + source = evidence.get("source_binding") + if not isinstance(source, Mapping): + errors.append("M3-28 source binding is invalid") + source = {} + elif set(source) != SOURCE_BINDING_INVENTORY: + errors.append("M3-28 source binding inventory does not match") + expected_source_values = { + "source_id": SOURCE_ID, + "archive_sha256": EXPECTED_ARCHIVE_SHA256, + "archive_size_bytes": EXPECTED_ARCHIVE_SIZE_BYTES, + "archive_entry_count": EXPECTED_ARCHIVE_ENTRY_COUNT, + "archive_uncompressed_size_bytes": EXPECTED_ARCHIVE_UNCOMPRESSED_SIZE_BYTES, + "archive_source_scope_entry_count": EXPECTED_ARCHIVE_SCOPE_ENTRY_COUNT, + "archive_source_scope_size_bytes": EXPECTED_ARCHIVE_SCOPE_SIZE_BYTES, + "archive_original_entry_exact_match_count": EXPECTED_ARCHIVE_EXACT_MATCH_COUNT, + "archive_original_entry_modified_count": EXPECTED_ARCHIVE_MODIFIED_COUNT, + "archive_original_entry_missing_count": EXPECTED_ARCHIVE_MISSING_COUNT, + "extracted_additional_file_count": EXPECTED_EXTRACTED_ADDITIONAL_COUNT, + "archive_integrity_verified": True, + "archive_extracted_entry_multiset_verified": False, + "extracted_file_count": EXPECTED_EXTRACTED_FILE_COUNT, + "extracted_size_bytes": EXPECTED_EXTRACTED_SIZE_BYTES, + "source_payload_in_repository": False, + "absolute_source_paths_in_evidence": False, + } + for key, value in expected_source_values.items(): + if source.get(key) != value: + errors.append(f"M3-28 source binding does not match: {key}") + if ( + EXPECTED_EXTRACTED_PAYLOAD_SHA256 + and source.get("extracted_payload_sha256") != EXPECTED_EXTRACTED_PAYLOAD_SHA256 + ): + errors.append("M3-28 extracted payload fingerprint does not match") + for key in ( + "archive_sha256", + "extracted_payload_sha256", + "logical_group_inventory_sha256", + ): + if not SHA256_PATTERN.fullmatch(str(source.get(key) or "")): + errors.append(f"M3-28 source fingerprint is invalid: {key}") + + research = evidence.get("research_audit_binding") + if not isinstance(research, Mapping): + errors.append("M3-28 research audit binding is invalid") + research = {} + if ( + set(research) != RESEARCH_AUDIT_INVENTORY + or research.get("path") != str(RESEARCH_INVENTORY_PATH.relative_to(REPO_ROOT)) + or research.get("file_sha256") != EXPECTED_RESEARCH_INVENTORY_SHA256 + or research.get("row_count") != EXPECTED_RESEARCH_INVENTORY_ROWS + or research.get("asset_ids_sha256") != EXPECTED_RESEARCH_ASSET_IDS_SHA256 + or research.get("authority") != "research_inventory_only" + or research.get("admission_authority") is not False + ): + errors.append("M3-28 research audit binding does not match") + + if evidence.get("admission_policy") != ADMISSION_POLICY: + errors.append("M3-28 admission policy does not match") + + groups_value = evidence.get("source_groups") + groups = groups_value if isinstance(groups_value, list) else [] + if not isinstance(groups_value, list): + errors.append("M3-28 source groups are not a list") + group_ids: list[str] = [] + derived_blockers: list[str] = [] + file_count = 0 + size_bytes = 0 + for group in groups: + if not isinstance(group, Mapping): + errors.append("M3-28 source group is not an object") + continue + group_id = str(group.get("source_group_id") or "") + group_ids.append(group_id) + if set(group) != SOURCE_GROUP_INVENTORY: + errors.append(f"M3-28 source group inventory does not match: {group_id}") + spec = GROUP_BY_ID.get(group_id) + if spec is None: + errors.append(f"M3-28 source group is unknown: {group_id}") + continue + if ( + group.get("ordinal") != spec["ordinal"] + or group.get("data_domains") != spec["data_domains"] + or group.get("data_classification") != spec["data_classification"] + or group.get("blockers") != _source_group_blockers(spec) + ): + errors.append(f"M3-28 source group contract does not match: {group_id}") + for key, expected in ( + ("license_status", "unverified_restricted"), + ("admission_status", STATUS), + ("metadata_profiled", True), + ("content_admitted", False), + ): + if group.get(key) != expected: + errors.append(f"M3-28 source group claim does not match: {group_id}.{key}") + if not SHA256_PATTERN.fullmatch(str(group.get("content_manifest_sha256") or "")): + errors.append(f"M3-28 source group fingerprint is invalid: {group_id}") + extensions = group.get("extension_counts") + if not isinstance(extensions, Mapping) or sum( + int(value) for value in extensions.values() + ) != group.get("file_count"): + errors.append(f"M3-28 source group extension counts do not match: {group_id}") + file_count += int(group.get("file_count") or 0) + size_bytes += int(group.get("size_bytes") or 0) + derived_blockers.extend(str(item) for item in group.get("blockers", [])) + expected_group_ids = [str(spec["source_group_id"]) for spec in SOURCE_GROUP_SPECS] + if group_ids != expected_group_ids or len(group_ids) != len(set(group_ids)): + errors.append("M3-28 source group inventory is incomplete or reordered") + if file_count != EXPECTED_EXTRACTED_FILE_COUNT or size_bytes != EXPECTED_EXTRACTED_SIZE_BYTES: + errors.append("M3-28 source group physical totals do not match") + if source.get("logical_group_inventory_sha256") != canonical_json_fingerprint(groups): + errors.append("M3-28 logical group inventory fingerprint does not match") + + profiles_value = evidence.get("asset_profiles") + profiles = profiles_value if isinstance(profiles_value, list) else [] + if not isinstance(profiles_value, list): + errors.append("M3-28 asset profiles are not a list") + asset_ids: list[str] = [] + for profile in profiles: + if not isinstance(profile, Mapping): + errors.append("M3-28 asset profile is not an object") + continue + asset_id = str(profile.get("asset_id") or "") + asset_ids.append(asset_id) + if set(profile) != ASSET_PROFILE_INVENTORY: + errors.append(f"M3-28 asset profile inventory does not match: {asset_id}") + profile_stable = {key: value for key, value in profile.items() if key != "profile_sha256"} + if profile.get("profile_sha256") != canonical_json_fingerprint(profile_stable): + errors.append(f"M3-28 asset profile fingerprint does not match: {asset_id}") + if not SOURCE_REF_PATTERN.fullmatch(str(profile.get("source_ref") or "")): + errors.append(f"M3-28 asset source reference is invalid: {asset_id}") + group_spec = GROUP_BY_ID.get(str(profile.get("source_group_id") or "")) + if ( + group_spec is None + or profile.get("data_classification") != group_spec["data_classification"] + or profile.get("admission_status") != STATUS + or profile.get("metadata_profiled") is not True + or profile.get("source_content_in_evidence") is not False + ): + errors.append(f"M3-28 asset admission boundary does not match: {asset_id}") + errors.extend(_asset_baseline_errors(profile)) + expected_asset_ids = sorted(EXPECTED_ASSET_BASELINES) + if asset_ids != expected_asset_ids or len(asset_ids) != len(set(asset_ids)): + errors.append("M3-28 asset profile inventory is incomplete or reordered") + + blockers = evidence.get("admission_blockers") + expected_blockers = sorted([*SOURCE_BINDING_BLOCKERS, *derived_blockers]) + if blockers != expected_blockers or len(expected_blockers) != len(set(expected_blockers)): + errors.append("M3-28 admission blockers do not match source groups") + + claims = evidence.get("claims") + if not isinstance(claims, Mapping) or set(claims) != CLAIMS: + errors.append("M3-28 claims inventory does not match") + else: + expected_true = { + "archive_integrity_verified", + "full_source_metadata_profiled", + } + for claim in CLAIMS: + expected = claim in expected_true + if claims.get(claim) is not expected: + errors.append(f"M3-28 claim does not match: {claim}") + if _sensitive_paths(evidence): + errors.append("M3-28 evidence contains credential-bearing fields") + rendered = json.dumps(evidence, ensure_ascii=False, sort_keys=True) + for forbidden in ("/Users/", "Downloads/", ".tmp/twm_standard_1128"): + if forbidden in rendered: + errors.append("M3-28 evidence contains a local source path") + break + return sorted(set(errors)) + + +def build_validation_report( + evidence_path: Path = DEFAULT_EVIDENCE_PATH, +) -> dict[str, Any]: + try: + file_sha256 = _file_sha256(evidence_path) + evidence = _load_json_object(evidence_path) + errors = validate_evidence(evidence) + if EVIDENCE_FILE_SHA256 and file_sha256 != EVIDENCE_FILE_SHA256: + errors.append("M3-28 evidence file fingerprint does not match") + except (OSError, TypeError, ValueError, json.JSONDecodeError) as exc: + evidence = {} + file_sha256 = None + errors = [f"M3-28 evidence is unreadable: {type(exc).__name__}"] + source = evidence.get("source_binding") + return { + "schema": VALIDATION_SCHEMA, + "status": "valid" if not errors else "invalid", + "errors": sorted(set(errors)), + "evidence_file_sha256": file_sha256, + "evidence_sha256": evidence.get("evidence_sha256"), + "source_status": evidence.get("status"), + "extracted_file_count": ( + source.get("extracted_file_count") if isinstance(source, Mapping) else None + ), + "source_group_count": len(evidence.get("source_groups", [])), + "asset_profile_count": len(evidence.get("asset_profiles", [])), + "admission_blocker_count": len(evidence.get("admission_blockers", [])), + "source_content_admitted": ( + evidence.get("claims", {}).get("source_content_admitted") + if isinstance(evidence.get("claims"), Mapping) + else None + ), + } + + +def _write_json(path: Path, payload: Mapping[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n", + encoding="utf-8", + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + validate = subparsers.add_parser("validate") + validate.add_argument("--evidence", type=Path, default=DEFAULT_EVIDENCE_PATH) + snapshot = subparsers.add_parser("snapshot") + snapshot.add_argument("--source-root", type=Path, required=True) + snapshot.add_argument("--source-zip", type=Path, required=True) + snapshot.add_argument("--output", type=Path, default=DEFAULT_EVIDENCE_PATH) + snapshot.add_argument( + "--research-inventory", + type=Path, + default=RESEARCH_INVENTORY_PATH, + ) + args = parser.parse_args(argv) + try: + if args.command == "validate": + report = build_validation_report(args.evidence) + exit_code = 0 if report["status"] == "valid" else 1 + else: + report = build_evidence( + source_root=args.source_root, + source_zip=args.source_zip, + research_inventory_path=args.research_inventory, + ) + errors = validate_evidence(report) + if errors: + raise ChongqingRealSourceAdmissionError("; ".join(errors)) + _write_json(args.output, report) + exit_code = 0 + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + return exit_code + except ( + OSError, + TypeError, + ValueError, + json.JSONDecodeError, + zipfile.BadZipFile, + ChongqingRealSourceAdmissionError, + ) as exc: + print(f"Chongqing real-source admission: {exc}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/data_agent/test_chongqing_real_source_admission.py b/data_agent/test_chongqing_real_source_admission.py new file mode 100644 index 00000000..8d8585d9 --- /dev/null +++ b/data_agent/test_chongqing_real_source_admission.py @@ -0,0 +1,194 @@ +import json +import zipfile +from copy import deepcopy +from datetime import UTC, datetime + +from data_agent import chongqing_real_source_admission as admission + +CAPTURED_AT = datetime(2026, 7, 31, 9, 30, tzinfo=UTC) + + +def _checked_evidence() -> dict: + return json.loads(admission.DEFAULT_EVIDENCE_PATH.read_text(encoding="utf-8")) + + +def _rehash(value: dict) -> None: + for profile in value.get("asset_profiles", []): + stable = {key: item for key, item in profile.items() if key != "profile_sha256"} + profile["profile_sha256"] = admission.canonical_json_fingerprint(stable) + stable = {key: item for key, item in value.items() if key != "evidence_sha256"} + value["evidence_sha256"] = admission.canonical_json_fingerprint(stable) + + +def test_checked_full_source_evidence_is_valid_and_blocked(): + evidence = _checked_evidence() + report = admission.build_validation_report() + + assert admission.validate_evidence(evidence) == [] + assert report["status"] == "valid" + assert report["errors"] == [] + assert report["extracted_file_count"] == 584 + assert report["source_group_count"] == 11 + assert report["asset_profile_count"] == 16 + assert evidence["status"] == admission.STATUS + assert evidence["claims"]["source_content_admitted"] is False + assert evidence["claims"]["scheduler_submission_authorized"] is False + assert evidence["claims"]["provider_mutation_authorized"] is False + assert evidence["claims"]["production_ready"] is False + + +def test_checked_evidence_binds_real_archive_and_all_extracted_bytes(): + evidence = _checked_evidence() + source = evidence["source_binding"] + + assert source["archive_sha256"] == admission.EXPECTED_ARCHIVE_SHA256 + assert source["archive_size_bytes"] == 468_462_251 + assert source["archive_entry_count"] == 533 + assert source["archive_uncompressed_size_bytes"] == 694_164_379 + assert source["archive_source_scope_entry_count"] == 532 + assert source["archive_source_scope_size_bytes"] == 694_147_946 + assert source["extracted_file_count"] == 584 + assert source["extracted_size_bytes"] == 700_610_744 + assert source["archive_original_entry_exact_match_count"] == 526 + assert source["archive_original_entry_modified_count"] == 6 + assert source["archive_original_entry_missing_count"] == 0 + assert source["extracted_additional_file_count"] == 52 + assert source["archive_integrity_verified"] is True + assert source["archive_extracted_entry_multiset_verified"] is False + assert source["source_payload_in_repository"] is False + assert source["absolute_source_paths_in_evidence"] is False + + +def test_source_groups_cover_physical_inventory_and_stay_unadmitted(): + evidence = _checked_evidence() + groups = evidence["source_groups"] + + assert [group["source_group_id"] for group in groups] == [ + spec["source_group_id"] for spec in admission.SOURCE_GROUP_SPECS + ] + assert sum(group["file_count"] for group in groups) == 584 + assert sum(group["size_bytes"] for group in groups) == 700_610_744 + assert all(group["metadata_profiled"] is True for group in groups) + assert all(group["content_admitted"] is False for group in groups) + assert all(group["license_status"] == "unverified_restricted" for group in groups) + assert all(group["blockers"] for group in groups) + assert len(evidence["admission_blockers"]) == len(set(evidence["admission_blockers"])) + + +def test_asset_profiles_correct_old_village_layer_scope_and_bind_baselines(): + evidence = _checked_evidence() + profiles = {profile["asset_id"]: profile for profile in evidence["asset_profiles"]} + village = profiles["fulu_village_planning_database_local"] + + assert set(profiles) == set(admission.EXPECTED_ASSET_BASELINES) + assert village["record_metrics"] == { + "feature_count": 8050, + "layer_count": 28, + "nonempty_layer_count": 20, + } + assert profiles["gaode_poi_2024"]["record_metrics"]["feature_count"] == 1_194_351 + assert profiles["chongqing_clcd_2020"]["record_metrics"]["pixel_count"] == 280_208_478 + assert profiles["chongqing_unicom_commuting_2023_local"]["data_classification"] == ( + "highly_restricted_aggregate_mobility" + ) + assert all(profile["source_content_in_evidence"] is False for profile in profiles.values()) + + +def test_evidence_is_path_free_and_contains_no_source_values(): + rendered = admission.DEFAULT_EVIDENCE_PATH.read_text(encoding="utf-8") + evidence = json.loads(rendered) + + assert "/Users/" not in rendered + assert "Downloads/" not in rendered + assert ".tmp/twm_standard_1128" not in rendered + assert "geometry_values" not in rendered + assert "od_rows" not in rendered + assert "flow_rows" not in rendered + assert admission._sensitive_paths(evidence) == [] + assert all( + profile["source_ref"].startswith("source://chongqing-planning-institute-sample/assets/") + for profile in evidence["asset_profiles"] + ) + + +def test_outer_rehash_cannot_hide_admission_or_baseline_overclaim(): + evidence = deepcopy(_checked_evidence()) + evidence["claims"]["source_content_admitted"] = True + evidence["admission_policy"]["ci_requires_local_source_payload"] = True + evidence["source_binding"]["local_source_path"] = "/untrusted/source" + evidence["research_audit_binding"]["admission_authority"] = True + evidence["source_groups"][0]["content_admitted"] = True + profile = next( + item for item in evidence["asset_profiles"] if item["asset_id"] == "gaode_poi_2024" + ) + profile["record_metrics"]["feature_count"] = 1 + _rehash(evidence) + + errors = admission.validate_evidence(evidence) + + assert "M3-28 claim does not match: source_content_admitted" in errors + assert "M3-28 admission policy does not match" in errors + assert "M3-28 source binding inventory does not match" in errors + assert "M3-28 research audit binding does not match" in errors + assert any("source group claim does not match" in error for error in errors) + assert "M3-28 asset metric does not match: gaode_poi_2024.feature_count" in errors + + +def test_checked_file_fingerprint_rejects_rehashed_copy(tmp_path): + evidence = deepcopy(_checked_evidence()) + evidence["captured_at"] = "2026-07-31T10:00:00Z" + _rehash(evidence) + path = tmp_path / "evidence.json" + path.write_text(json.dumps(evidence, ensure_ascii=False), encoding="utf-8") + + report = admission.build_validation_report(path) + + assert report["status"] == "invalid" + assert "M3-28 evidence file fingerprint does not match" in report["errors"] + + +def test_archive_and_extracted_entry_multiset_is_verified(tmp_path): + root = tmp_path / "source" + group = root / "01-dem" + group.mkdir(parents=True) + payload = group / "tile.bin" + payload.write_bytes(b"real-source-content") + archive_path = tmp_path / "source.zip" + with zipfile.ZipFile(archive_path, "w") as archive: + archive.write(payload, "bundle/01数据样例/01-dem/tile.bin") + + records = admission._scan_payload_files(root) + binding = admission._archive_binding(archive_path, records) + + assert len(records) == 1 + assert binding["archive_entry_count"] == 1 + assert binding["archive_source_scope_entry_count"] == 1 + assert binding["archive_original_entry_exact_match_count"] == 1 + assert binding["archive_original_entry_modified_count"] == 0 + assert binding["extracted_additional_file_count"] == 0 + assert binding["archive_integrity_verified"] is True + assert binding["archive_extracted_entry_multiset_verified"] is True + + +def test_source_scan_rejects_symbolic_links(tmp_path): + root = tmp_path / "source" + group = root / "01-dem" + group.mkdir(parents=True) + target = tmp_path / "outside.bin" + target.write_bytes(b"outside") + (group / "linked.bin").symlink_to(target) + + try: + admission._scan_payload_files(root) + except admission.ChongqingRealSourceAdmissionError as exc: + assert "symbolic link" in str(exc) + else: + raise AssertionError("symbolic link was accepted") + + +def test_wrapper_is_strict_and_invokes_source_admission(): + wrapper = admission.REPO_ROOT / "scripts/chongqing-real-source-admission.sh" + text = wrapper.read_text(encoding="utf-8") + + assert "set -euo pipefail" in text + assert "chongqing_real_source_admission" in text diff --git a/docs/architecture-decisions/adr-074-chongqing-real-source-admission-manifest.md b/docs/architecture-decisions/adr-074-chongqing-real-source-admission-manifest.md new file mode 100644 index 00000000..f2af3501 --- /dev/null +++ b/docs/architecture-decisions/adr-074-chongqing-real-source-admission-manifest.md @@ -0,0 +1,58 @@ +# ADR-074: Chongqing real-source admission manifest + +**Status**: Accepted + +**Date**: 2026-07-31 + +## Context + +AR-2 needs representative real sources before it can define ingestion, quality and lakehouse contracts. The available Chongqing planning-institute sample spans elevation, roads, remote-sensing land cover, buildings, cultural-planning data, population, POI/AOI, commuting and search activity. Earlier work profiled 16 useful assets, but that research inventory did not bind the complete source payload and had no authority to admit content. + +The original ZIP and the current extracted working set are not identical. All 532 archive entries in the `01数据样例` scope are present, but only 526 match by size and CRC, 6 differ and 52 files exist only in the extracted set. The additional material includes expanded FileGDB content and generated sidecars. Without derivation provenance, treating the extracted set as a byte-exact extraction would create a false source claim. + +## Decision + +Adopt the checked M3-28 manifest as a path-free, content-addressed and metadata-only admission contract for the full Chongqing source set. + +The manifest binds: + +- the 468,462,251-byte source archive with SHA-256 `2043b60c2f4f7f32a31388a634fae4ac28534990e205aa86b8df0e4b64dcbbca`, 533 files and 694,164,379 uncompressed bytes; +- the 532-entry archive source scope with 694,147,946 bytes; +- the 584-file, 700,610,744-byte extracted working set with payload fingerprint `e7e81e4f53f9f174792f500fbfdfde6bee30ec03beac8cbd91771fe09f548ea6`; +- 11 complete physical source groups and 16 metadata asset profiles; +- the earlier 16-row research inventory as `research_inventory_only`, never as admission authority. + +The checked evidence contains aggregate physical inventories, schema field names, counts, CRS, bounds, roles and content fingerprints. It contains no absolute source path, source file, geometry, row value, credential or record-level payload. + +The validator freezes the archive/extracted comparison at 526 exact matches, 6 modified entries, no missing archive entry and 52 extracted-only files. It therefore requires `archive_integrity_verified=true` but `archive_extracted_entry_multiset_verified=false`, and retains `source_binding:extraction_derivation_provenance_missing` as an admission blocker. + +Every source group also remains blocked on owner, license, retention and access decisions, plus domain-specific privacy, commercial terms, attribution, vintage, lineage or sensitivity review where applicable. The manifest reports 57 unique blockers. + +## Authority boundary + +Metadata profiling is allowed; content admission is not. The manifest does not create a `ResourceVersion`, `PolicyDecision`, `Approval`, `PlatformRun`, scheduler command, provider mutation, landing object, lakehouse table or data product. + +The local ZIP remains the observed archive payload and the extracted directory remains a working set. Neither local path is a production source authority. The checked JSON is evidence about those bytes, not a copy of them and not authority for owner, license, retention, access, privacy or publication decisions. + +CI validates only the checked evidence and never requires the local source paths. Any future admission must resolve every blocker, preserve derivation provenance and execute a fresh protected ingestion. M3-24/M3-25 retained material cannot be promoted as a substitute. + +## Consequences + +**Positive**: AR-2 now has a reproducible real-source baseline covering the full available Chongqing corpus instead of a synthetic fixture or one 20-feature slice. + +**Positive**: archive integrity, extracted working-set identity and asset metadata are independently bound, so later ingestion can detect source drift without committing restricted data. + +**Positive**: the earlier village-planning scope is corrected to 28 Shapefile layers, 20 non-empty layers and 8,050 features; the prior count of 31 described all Shapefiles under the root, not the village subset. + +**Negative**: M3-28 intentionally admits no content and leaves 57 blockers unresolved. AR-2 is `in_progress`, not verified. + +**Negative**: the extracted working set cannot become an admitted landing source until its six modified and 52 additional files have documented derivation provenance. + +## Verification + +```bash +./scripts/chongqing-real-source-admission.sh validate +python -m pytest data_agent/test_chongqing_real_source_admission.py -q +``` + +The checked evidence fingerprint is `a2196495d845d61be939c7fc36a7f05c3567e365599d2d04be0aab9c568459c1`; its file SHA-256 is `9b5c20369c235f7e0a2f2cb0a21cee77f86981aa273bac196605a4803b05ce83`. diff --git a/docs/evidence/chongqing-real-source-admission-2026-07-31.json b/docs/evidence/chongqing-real-source-admission-2026-07-31.json new file mode 100644 index 00000000..e1115b9b --- /dev/null +++ b/docs/evidence/chongqing-real-source-admission-2026-07-31.json @@ -0,0 +1,1119 @@ +{ + "admission_blockers": [ + "source_binding:extraction_derivation_provenance_missing", + "source_group:baidu-aoi-2024:access_policy_missing", + "source_group:baidu-aoi-2024:contact_field_privacy_review_missing", + "source_group:baidu-aoi-2024:license_terms_unverified", + "source_group:baidu-aoi-2024:owner_reference_missing", + "source_group:baidu-aoi-2024:retention_policy_missing", + "source_group:baidu-search-index-2023:access_policy_missing", + "source_group:baidu-search-index-2023:commercial_terms_review_missing", + "source_group:baidu-search-index-2023:license_terms_unverified", + "source_group:baidu-search-index-2023:owner_reference_missing", + "source_group:baidu-search-index-2023:retention_policy_missing", + "source_group:bishan-planning-materials:access_policy_missing", + "source_group:bishan-planning-materials:license_terms_unverified", + "source_group:bishan-planning-materials:owner_reference_missing", + "source_group:bishan-planning-materials:planning_sensitivity_review_missing", + "source_group:bishan-planning-materials:retention_policy_missing", + "source_group:chongqing-central-buildings-2021:access_policy_missing", + "source_group:chongqing-central-buildings-2021:license_terms_unverified", + "source_group:chongqing-central-buildings-2021:owner_reference_missing", + "source_group:chongqing-central-buildings-2021:retention_policy_missing", + "source_group:chongqing-central-buildings-2021:source_vintage_unverified", + "source_group:chongqing-clcd-2020:access_policy_missing", + "source_group:chongqing-clcd-2020:classification_lineage_unverified", + "source_group:chongqing-clcd-2020:license_terms_unverified", + "source_group:chongqing-clcd-2020:owner_reference_missing", + "source_group:chongqing-clcd-2020:retention_policy_missing", + "source_group:chongqing-dem-2020:access_policy_missing", + "source_group:chongqing-dem-2020:license_terms_unverified", + "source_group:chongqing-dem-2020:owner_reference_missing", + "source_group:chongqing-dem-2020:retention_policy_missing", + "source_group:chongqing-dem-2020:source_vintage_unverified", + "source_group:chongqing-district-population-2021:access_policy_missing", + "source_group:chongqing-district-population-2021:license_terms_unverified", + "source_group:chongqing-district-population-2021:owner_reference_missing", + "source_group:chongqing-district-population-2021:retention_policy_missing", + "source_group:chongqing-district-population-2021:statistics_vintage_review_missing", + "source_group:chongqing-historic-districts:access_policy_missing", + "source_group:chongqing-historic-districts:license_terms_unverified", + "source_group:chongqing-historic-districts:owner_reference_missing", + "source_group:chongqing-historic-districts:retention_policy_missing", + "source_group:chongqing-historic-districts:source_vintage_unverified", + "source_group:chongqing-osm-roads-2021:access_policy_missing", + "source_group:chongqing-osm-roads-2021:license_terms_unverified", + "source_group:chongqing-osm-roads-2021:odbl_attribution_policy_missing", + "source_group:chongqing-osm-roads-2021:owner_reference_missing", + "source_group:chongqing-osm-roads-2021:retention_policy_missing", + "source_group:gaode-poi-2024:access_policy_missing", + "source_group:gaode-poi-2024:contact_field_privacy_review_missing", + "source_group:gaode-poi-2024:license_terms_unverified", + "source_group:gaode-poi-2024:owner_reference_missing", + "source_group:gaode-poi-2024:retention_policy_missing", + "source_group:unicom-commuting-2023:access_policy_missing", + "source_group:unicom-commuting-2023:grid_geometry_dictionary_missing", + "source_group:unicom-commuting-2023:license_terms_unverified", + "source_group:unicom-commuting-2023:owner_reference_missing", + "source_group:unicom-commuting-2023:privacy_impact_assessment_missing", + "source_group:unicom-commuting-2023:retention_policy_missing" + ], + "admission_policy": { + "ci_requires_local_source_payload": false, + "content_admission_requires_owner_license_retention_and_access_approval": true, + "fresh_protected_ingestion_required_after_admission": true, + "local_profile_is_not_production_admission": true, + "metadata_profiling_allowed": true, + "restricted_fields_require_privacy_review": true, + "source_payload_copy_to_repository_forbidden": true + }, + "asset_profiles": [ + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "baidu_aoi_2024", + "asset_kind": "vector", + "data_classification": "highly_restricted_commercial_location", + "metadata_profiled": true, + "profile_sha256": "2b6ce471d518ff7205eb9d5690e40cf7de39de1328260276b39673afad4ad9bc", + "record_metrics": { + "feature_count": 26292 + }, + "roles": [ + "baseline", + "planner_targeting", + "service_accessibility", + "urban_form" + ], + "schema_fields": [ + "uid", + "名称", + "地址", + "省份", + "城市", + "区县", + "街镇乡", + "类型", + "第一分类", + "第二分类", + "评分", + "更新时间", + "评论数", + "开业时间", + "人均价格_元", + "街道id", + "电话", + "创建时间", + "其他", + "高德分类", + "经度wgs84", + "纬度wgs84", + "SHAPE_Length", + "SHAPE_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "baidu-aoi-2024", + "source_ref": "source://chongqing-planning-institute-sample/assets/baidu_aoi_2024", + "spatial_profile": { + "bounds": [ + 105.31552411900009, + 28.18493382400004, + 110.16574626100004, + 32.16731932000005 + ], + "crs": "EPSG:4490", + "dtype": [], + "geometry_type": "MultiPolygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "baidu_search_index_2023_local", + "asset_kind": "vector", + "data_classification": "highly_restricted_commercial_activity", + "metadata_profiled": true, + "profile_sha256": "b6bc2eeaa4e0b6626551a9e7732632ee5983ae5572709d109d4996665dbf3202", + "record_metrics": { + "feature_count": 325 + }, + "roles": [ + "mmfe_alignment", + "mobility_activity", + "planner_targeting", + "simulator_context", + "urban_activity_proxy" + ], + "schema_fields": [ + "id", + "ODJSMC", + "DDJSMC", + "PCSSCS", + "YDSSCS", + "SSZS", + "Shape_Length" + ], + "source_content_in_evidence": false, + "source_group_id": "baidu-search-index-2023", + "source_ref": "source://chongqing-planning-institute-sample/assets/baidu_search_index_2023_local", + "spatial_profile": { + "bounds": [ + 102.66382204100006, + 27.822151572568785, + 111.13525443382999, + 33.09055455500004 + ], + "crs": "EPSG:4490", + "dtype": [], + "geometry_type": "MultiLineString", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "bishan_admin_boundary_cjdcq_local", + "asset_kind": "vector", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "0c7607470d2b0049e23b86639eba4e5109a0909e7f40bbeb357639ccb6e2ed9c", + "record_metrics": { + "feature_count": 1488 + }, + "roles": [ + "administrative_units", + "baseline", + "land_use_context", + "planning_constraints" + ], + "schema_fields": [ + "BSM", + "YSDM", + "ZLDWDM", + "ZLDWMC", + "DCMJ", + "JSMJ", + "MSSM", + "HDMC", + "BZ", + "SHAPE_Length", + "SHAPE_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/bishan_admin_boundary_cjdcq_local", + "spatial_profile": { + "bounds": [ + 35601015.4985, + 3241091.9637, + 35632312.22315, + 3308361.5723 + ], + "crs": "EPSG:4523+EPSG:5737", + "dtype": [], + "geometry_type": "MultiPolygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "bishan_admin_boundary_xzq_local", + "asset_kind": "vector", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "384675e6d9888d701b9ae88875fcc11488f45ee7778bdf729a31cba8e8270d70", + "record_metrics": { + "feature_count": 15 + }, + "roles": [ + "administrative_units", + "baseline", + "land_use_context", + "planning_constraints" + ], + "schema_fields": [ + "BSM", + "YSDM", + "XZQDM", + "XZQMC", + "DCMJ", + "JSMJ", + "MSSM", + "HDMC", + "BZ", + "SHAPE_Length", + "SHAPE_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/bishan_admin_boundary_xzq_local", + "spatial_profile": { + "bounds": [ + 35601015.4985, + 3241091.9637, + 35632312.22315, + 3308361.5723 + ], + "crs": "EPSG:4523+EPSG:5737", + "dtype": [], + "geometry_type": "MultiPolygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "bishan_land_development_ledger_2019_local", + "asset_kind": "workbook_collection", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "b62a42e2cf0c1a3a269ecaf53edb583ce5ce066ffd95e1e6cc28ca9357435cb7", + "record_metrics": { + "row_count": 1438, + "sheet_count": 4 + }, + "roles": [ + "land_development_pressure", + "planner_constraints", + "planning_context", + "simulator_context" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/bishan_land_development_ledger_2019_local", + "spatial_profile": null + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "bishan_land_use_dltb_local", + "asset_kind": "vector", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "dd0f4eab16f69cc59fe320eeeff3aed747ed8de35a948a5cb87ee649ae3813c9", + "record_metrics": { + "feature_count": 101657 + }, + "roles": [ + "baseline", + "land_use_context", + "planning_constraints", + "simulator_context", + "urban_form" + ], + "schema_fields": [ + "BSM", + "YSDM", + "DLBM", + "DLMC", + "QSDWDM", + "QSDWMC", + "ZLDWDM", + "ZLDWMC", + "TBMJ", + "SHAPE_Length", + "SHAPE_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/bishan_land_use_dltb_local", + "spatial_profile": { + "bounds": [ + 106.04001576200005, + 29.282454706000067, + 106.36873229600008, + 29.887474954000083 + ], + "crs": "EPSG:4610", + "dtype": [], + "geometry_type": "MultiPolygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_central_buildings_2021", + "asset_kind": "vector", + "data_classification": "restricted_local_built_environment", + "metadata_profiled": true, + "profile_sha256": "2e4997ea793c5a841a03a4da56e6dffc2962f196cbbaa1f8a3e8b5bc091dbce7", + "record_metrics": { + "feature_count": 107452 + }, + "roles": [ + "baseline", + "renderer", + "urban_form" + ], + "schema_fields": [ + "Id", + "Floor" + ], + "source_content_in_evidence": false, + "source_group_id": "chongqing-central-buildings-2021", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_central_buildings_2021", + "spatial_profile": { + "bounds": [ + 106.20951745600001, + 29.212573738600042, + 106.821612684, + 29.831229147900103 + ], + "crs": "EPSG:4326", + "dtype": [], + "geometry_type": "Polygon", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_clcd_2020", + "asset_kind": "raster", + "data_classification": "restricted_local_remote_sensing", + "metadata_profiled": true, + "profile_sha256": "653f7c175ff192b3fc4135eee60aa0754e389fe0de7b8301718f66148bf63571", + "record_metrics": { + "band_count": 1, + "height": 15082, + "pixel_count": 280208478, + "width": 18579 + }, + "roles": [ + "baseline", + "remote_sensing_state", + "urban_form" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "chongqing-clcd-2020", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_clcd_2020", + "spatial_profile": { + "bounds": [ + 105.21303750199797, + 28.140836217657494, + 110.21997740109495, + 32.20535355218468 + ], + "crs": "EPSG:4326", + "dtype": [ + "uint8" + ], + "geometry_type": "", + "nodata": 15.0 + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_dem_80m", + "asset_kind": "raster", + "data_classification": "restricted_local_baseline", + "metadata_profiled": true, + "profile_sha256": "a145f700de59f65a895bcace518250648db4ee130c147e7ff262fb529976893e", + "record_metrics": { + "band_count": 1, + "height": 1454, + "pixel_count": 2567764, + "width": 1766 + }, + "roles": [ + "heat_exposure", + "renderer" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "chongqing-dem-2020", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_dem_80m", + "spatial_profile": { + "bounds": [ + 105.28983805000004, + 28.165268865555987, + 110.19539360948005, + 32.204157757675986 + ], + "crs": "EPSG:4490", + "dtype": [ + "int16" + ], + "geometry_type": "", + "nodata": 32767.0 + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_district_population_stats_2021_local", + "asset_kind": "workbook", + "data_classification": "restricted_aggregate_population", + "metadata_profiled": true, + "profile_sha256": "d16f10876967f0c31d2dba850bda33fe82c0df2dde71945a7d239504a6a83840", + "record_metrics": { + "row_count": 41, + "sheet_count": 1 + }, + "roles": [ + "equity_evaluation", + "population_vulnerability" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "chongqing-district-population-2021", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_district_population_stats_2021_local", + "spatial_profile": null + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_historic_districts_local", + "asset_kind": "vector", + "data_classification": "restricted_local_cultural_planning", + "metadata_profiled": true, + "profile_sha256": "d51869b52b11220da25690d7e890c14da7c0188b2ef381716daa07b58fceb385", + "record_metrics": { + "feature_count": 20 + }, + "roles": [ + "cultural_heritage", + "livability_context", + "planner_targeting", + "renderer", + "service_accessibility", + "urban_form" + ], + "schema_fields": [ + "OBJECTID", + "Jqmc", + "Fwlx", + "fwmc", + "Xzqmc", + "Fwkzyq", + "Fwmj", + "Bhbkydwwsl", + "Bhbkydwwzj", + "Bhlsjzsl", + "Bhlsjzmc", + "Bhlsjzzjzm", + "Bhlshjyssl", + "Bhlshjysmc", + "Bhlsjxsl", + "Bhlsjxzcd", + "Bhlsjxmcjc", + "Bhfwzwhycm", + "Bhqtfwzycs", + "Bhqtfwzycm", + "Bz", + "Jj", + "Bsm", + "Bhctfmjzsl", + "Bhbkydwwmc", + "Bhctfmjzzj", + "Bhgsmmsl", + "Bhgsmmmc", + "Bhfwzwhy_1", + "Tymj", + "Tycd", + "Shape_Leng", + "Shape_Area" + ], + "source_content_in_evidence": false, + "source_group_id": "chongqing-historic-districts", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_historic_districts_local", + "spatial_profile": { + "bounds": [ + 106.37987914500007, + 29.558008447000077, + 106.59532712300008, + 29.877271985000025 + ], + "crs": "EPSG:4490", + "dtype": [], + "geometry_type": "Polygon Z", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_osm_roads_2021", + "asset_kind": "vector", + "data_classification": "restricted_local_open_data_copy", + "metadata_profiled": true, + "profile_sha256": "ff1129bc6b54ae32891e8dbeea0262942155ea4a08e89393720fd1cbd799fbe6", + "record_metrics": { + "feature_count": 50366 + }, + "roles": [ + "baseline", + "mobility_activity", + "mobility_graph", + "renderer" + ], + "schema_fields": [ + "osm_id", + "code", + "fclass", + "name", + "ref", + "oneway", + "maxspeed", + "layer", + "bridge", + "tunnel" + ], + "source_content_in_evidence": false, + "source_group_id": "chongqing-osm-roads-2021", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_osm_roads_2021", + "spatial_profile": { + "bounds": [ + 105.30804952870155, + 28.163572226020626, + 110.1732225690069, + 32.1562025 + ], + "crs": "EPSG:4326", + "dtype": [], + "geometry_type": "LineString", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "chongqing_unicom_commuting_2023_local", + "asset_kind": "table", + "data_classification": "highly_restricted_aggregate_mobility", + "metadata_profiled": true, + "profile_sha256": "d7d0661e3e97356ca74d2e601d76c4b30184691f0266e8a27c8284eaade7008b", + "record_metrics": { + "column_count": 7, + "row_count": 2120 + }, + "roles": [ + "commuting_od", + "mobility_activity", + "planner_targeting", + "population_vulnerability", + "simulator_context" + ], + "schema_fields": [ + "居住格网", + "工作格网", + "职住格网是否重合", + "性别", + "年龄", + "扩样前人口", + "扩样后人口" + ], + "source_content_in_evidence": false, + "source_group_id": "unicom-commuting-2023", + "source_ref": "source://chongqing-planning-institute-sample/assets/chongqing_unicom_commuting_2023_local", + "spatial_profile": null + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "clcd_classification_dictionary_local", + "asset_kind": "workbook", + "data_classification": "restricted_local_remote_sensing", + "metadata_profiled": true, + "profile_sha256": "5e2dead48038cf37d37ddcf69ee28be06d5bc636c5c390d9602a0ee2c60bc21d", + "record_metrics": { + "row_count": 10, + "sheet_count": 1 + }, + "roles": [ + "baseline_context", + "remote_sensing_state", + "renderer" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "chongqing-clcd-2020", + "source_ref": "source://chongqing-planning-institute-sample/assets/clcd_classification_dictionary_local", + "spatial_profile": null + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "fulu_village_planning_database_local", + "asset_kind": "vector_collection", + "data_classification": "highly_restricted_planning", + "metadata_profiled": true, + "profile_sha256": "a7046403283e32c6972382b882940407f3a6205a4d6f63561863d8e1cb24d743", + "record_metrics": { + "feature_count": 8050, + "layer_count": 28, + "nonempty_layer_count": 20 + }, + "roles": [ + "land_use_context", + "planning_constraints", + "simulator_context", + "village_livability_context" + ], + "schema_fields": [], + "source_content_in_evidence": false, + "source_group_id": "bishan-planning-materials", + "source_ref": "source://chongqing-planning-institute-sample/assets/fulu_village_planning_database_local", + "spatial_profile": { + "bounds": null, + "crs": "mixed_CGCS2000_GK_zone_35_EPSG4523", + "dtype": [], + "geometry_type": "mixed", + "nodata": null + } + }, + { + "admission_status": "blocked_pending_source_governance", + "asset_id": "gaode_poi_2024", + "asset_kind": "vector", + "data_classification": "highly_restricted_commercial_location", + "metadata_profiled": true, + "profile_sha256": "fc8e70f04bfab1c9305a1edff49d2cc39d9c1eee4f71d5b8bfe99780ed57dd5c", + "record_metrics": { + "feature_count": 1194351 + }, + "roles": [ + "baseline", + "planner_targeting", + "service_accessibility" + ], + "schema_fields": [ + "ID", + "名称", + "地址", + "电话", + "类型", + "区域ID", + "经度wgs84", + "纬度wgs84", + "百度经度", + "百度纬度", + "更新时间" + ], + "source_content_in_evidence": false, + "source_group_id": "gaode-poi-2024", + "source_ref": "source://chongqing-planning-institute-sample/assets/gaode_poi_2024", + "spatial_profile": { + "bounds": [ + 105.28943500000003, + 28.164064000000053, + 110.18358100000006, + 32.190248000000054 + ], + "crs": "EPSG:4490", + "dtype": [], + "geometry_type": "Point", + "nodata": null + } + } + ], + "captured_at": "2026-07-31T07:54:06.279487Z", + "claims": { + "archive_extracted_entry_multiset_verified": false, + "archive_integrity_verified": true, + "full_source_metadata_profiled": true, + "production_ingestion_verified": false, + "production_ready": false, + "provider_mutation_authorized": false, + "scheduler_submission_authorized": false, + "source_content_admitted": false, + "source_governance_approved": false, + "source_publication_authorized": false + }, + "evidence_sha256": "a2196495d845d61be939c7fc36a7f05c3567e365599d2d04be0aab9c568459c1", + "research_audit_binding": { + "admission_authority": false, + "asset_ids_sha256": "c7cb765c2b653dbf5619e2fe1cf027d108a5951b752b6dd4bfd60b4c00d91947", + "authority": "research_inventory_only", + "file_sha256": "69b4167955f950041988dd174b75ea5376af146ef3e52f815aa57715fd24f70d", + "path": "data/uwm_public_proxy/chongqing_central/local_planning_zip_audit_2026_07_05/uwm_local_planning_zip_inventory.csv", + "row_count": 16 + }, + "schema": "gda.chongqing_real_source_admission.v1", + "source_binding": { + "absolute_source_paths_in_evidence": false, + "archive_entry_count": 533, + "archive_extracted_entry_multiset_verified": false, + "archive_integrity_verified": true, + "archive_original_entry_exact_match_count": 526, + "archive_original_entry_missing_count": 0, + "archive_original_entry_modified_count": 6, + "archive_sha256": "2043b60c2f4f7f32a31388a634fae4ac28534990e205aa86b8df0e4b64dcbbca", + "archive_size_bytes": 468462251, + "archive_source_scope_entry_count": 532, + "archive_source_scope_size_bytes": 694147946, + "archive_uncompressed_size_bytes": 694164379, + "extracted_additional_file_count": 52, + "extracted_file_count": 584, + "extracted_payload_sha256": "e7e81e4f53f9f174792f500fbfdfde6bee30ec03beac8cbd91771fe09f548ea6", + "extracted_size_bytes": 700610744, + "logical_group_inventory_sha256": "5aa5f1aebd135956c550f392deb5965e7c088d4cb09070b60b4db45e5ce9503b", + "source_id": "chongqing-planning-institute-sample", + "source_payload_in_repository": false + }, + "source_groups": [ + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-dem-2020:owner_reference_missing", + "source_group:chongqing-dem-2020:license_terms_unverified", + "source_group:chongqing-dem-2020:retention_policy_missing", + "source_group:chongqing-dem-2020:access_policy_missing", + "source_group:chongqing-dem-2020:source_vintage_unverified" + ], + "content_admitted": false, + "content_manifest_sha256": "9796f99f214d6ae1e9cef659ffe341465ec48a93a25bba1b966c4ba3052630b0", + "data_classification": "restricted_local_baseline", + "data_domains": [ + "elevation", + "terrain", + "raster" + ], + "extension_counts": { + "cpg": 1, + "dbf": 1, + "ovr": 1, + "tfw": 1, + "tif": 1, + "xml": 2 + }, + "file_count": 7, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "01", + "size_bytes": 2360205, + "source_group_id": "chongqing-dem-2020" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-osm-roads-2021:owner_reference_missing", + "source_group:chongqing-osm-roads-2021:license_terms_unverified", + "source_group:chongqing-osm-roads-2021:retention_policy_missing", + "source_group:chongqing-osm-roads-2021:access_policy_missing", + "source_group:chongqing-osm-roads-2021:odbl_attribution_policy_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "8298af4d695e701f1346e5792f9bca491c0be490d8a3c0cf20baa5eed941da7a", + "data_classification": "restricted_local_open_data_copy", + "data_domains": [ + "transport_network", + "vector" + ], + "extension_counts": { + "cpg": 1, + "dbf": 1, + "prj": 1, + "sbn": 1, + "sbx": 1, + "shp": 1, + "shx": 1, + "xml": 1 + }, + "file_count": 8, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "02", + "size_bytes": 25416298, + "source_group_id": "chongqing-osm-roads-2021" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-clcd-2020:owner_reference_missing", + "source_group:chongqing-clcd-2020:license_terms_unverified", + "source_group:chongqing-clcd-2020:retention_policy_missing", + "source_group:chongqing-clcd-2020:access_policy_missing", + "source_group:chongqing-clcd-2020:classification_lineage_unverified" + ], + "content_admitted": false, + "content_manifest_sha256": "e3bd232f24c41c6b77e067690b4c70d68b20da6600f8be59c60de5868077420e", + "data_classification": "restricted_local_remote_sensing", + "data_domains": [ + "land_cover", + "remote_sensing", + "raster" + ], + "extension_counts": { + "cpg": 1, + "dbf": 1, + "tfw": 1, + "tif": 1, + "xlsx": 1, + "xml": 2 + }, + "file_count": 7, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "03", + "size_bytes": 9879957, + "source_group_id": "chongqing-clcd-2020" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-central-buildings-2021:owner_reference_missing", + "source_group:chongqing-central-buildings-2021:license_terms_unverified", + "source_group:chongqing-central-buildings-2021:retention_policy_missing", + "source_group:chongqing-central-buildings-2021:access_policy_missing", + "source_group:chongqing-central-buildings-2021:source_vintage_unverified" + ], + "content_admitted": false, + "content_manifest_sha256": "a9343b5f8971c731fefee08864fd6b586d989e4f52d0cca584d6d70152de5039", + "data_classification": "restricted_local_built_environment", + "data_domains": [ + "buildings", + "urban_form", + "vector" + ], + "extension_counts": { + "dbf": 1, + "prj": 1, + "sbn": 1, + "sbx": 1, + "shp": 1, + "shx": 1, + "xml": 1 + }, + "file_count": 7, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "04", + "size_bytes": 22006362, + "source_group_id": "chongqing-central-buildings-2021" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-historic-districts:owner_reference_missing", + "source_group:chongqing-historic-districts:license_terms_unverified", + "source_group:chongqing-historic-districts:retention_policy_missing", + "source_group:chongqing-historic-districts:access_policy_missing", + "source_group:chongqing-historic-districts:source_vintage_unverified" + ], + "content_admitted": false, + "content_manifest_sha256": "4ea511bb915b54fd21bd1addedad93cfa65de6f87a2940bdbddb0cf6f95b6f96", + "data_classification": "restricted_local_cultural_planning", + "data_domains": [ + "cultural_heritage", + "planning_constraints", + "vector" + ], + "extension_counts": { + "cpg": 1, + "dbf": 1, + "prj": 1, + "sbn": 1, + "sbx": 1, + "shp": 1, + "shx": 1, + "xml": 1 + }, + "file_count": 8, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "05", + "size_bytes": 372603, + "source_group_id": "chongqing-historic-districts" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:bishan-planning-materials:owner_reference_missing", + "source_group:bishan-planning-materials:license_terms_unverified", + "source_group:bishan-planning-materials:retention_policy_missing", + "source_group:bishan-planning-materials:access_policy_missing", + "source_group:bishan-planning-materials:planning_sensitivity_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "1d2ff573f09b991fffb79dbc85f630e0176192b3a98ab86aba1706fef414877c", + "data_classification": "highly_restricted_planning", + "data_domains": [ + "land_use", + "planning", + "documents", + "vector", + "tables" + ], + "extension_counts": { + "[no_ext]": 4, + "atx": 34, + "cpg": 18, + "dbf": 28, + "doc": 6, + "docx": 6, + "dwg": 17, + "dwl": 2, + "dwl2": 2, + "freelist": 3, + "gdbindexes": 15, + "gdbtable": 17, + "gdbtablx": 17, + "horizon": 5, + "jpg": 34, + "pdf": 8, + "png": 2, + "prj": 28, + "sbn": 28, + "sbx": 28, + "shp": 28, + "shx": 28, + "spx": 4, + "xls": 5, + "xlsx": 12, + "xml": 33, + "zip": 1 + }, + "file_count": 413, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "07", + "size_bytes": 389617168, + "source_group_id": "bishan-planning-materials" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:chongqing-district-population-2021:owner_reference_missing", + "source_group:chongqing-district-population-2021:license_terms_unverified", + "source_group:chongqing-district-population-2021:retention_policy_missing", + "source_group:chongqing-district-population-2021:access_policy_missing", + "source_group:chongqing-district-population-2021:statistics_vintage_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "8ac775d79db7c98fa25e0f697ee5f86ee8e35a095a0fd906890251c664684757", + "data_classification": "restricted_aggregate_population", + "data_domains": [ + "population", + "statistics", + "table" + ], + "extension_counts": { + "xlsx": 1 + }, + "file_count": 1, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "08", + "size_bytes": 13036, + "source_group_id": "chongqing-district-population-2021" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:gaode-poi-2024:owner_reference_missing", + "source_group:gaode-poi-2024:license_terms_unverified", + "source_group:gaode-poi-2024:retention_policy_missing", + "source_group:gaode-poi-2024:access_policy_missing", + "source_group:gaode-poi-2024:contact_field_privacy_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "a5def66308dc5d3bee6a7be235fce935fd5b8a0182d6669551493f392d0e8918", + "data_classification": "highly_restricted_commercial_location", + "data_domains": [ + "poi", + "commercial_location", + "vector" + ], + "extension_counts": { + "[no_ext]": 2, + "atx": 17, + "gdbindexes": 7, + "gdbtable": 8, + "gdbtablx": 8, + "spx": 2 + }, + "file_count": 44, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "09", + "size_bytes": 233667334, + "source_group_id": "gaode-poi-2024" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:baidu-aoi-2024:owner_reference_missing", + "source_group:baidu-aoi-2024:license_terms_unverified", + "source_group:baidu-aoi-2024:retention_policy_missing", + "source_group:baidu-aoi-2024:access_policy_missing", + "source_group:baidu-aoi-2024:contact_field_privacy_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "09e1d3034316e936300607637ae3f220444d89004629aa9e2ea8ccfefbe6100a", + "data_classification": "highly_restricted_commercial_location", + "data_domains": [ + "aoi", + "commercial_location", + "vector" + ], + "extension_counts": { + "[no_ext]": 2, + "atx": 17, + "gdbindexes": 7, + "gdbtable": 8, + "gdbtablx": 8, + "spx": 2 + }, + "file_count": 44, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "10", + "size_bytes": 17006652, + "source_group_id": "baidu-aoi-2024" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:unicom-commuting-2023:owner_reference_missing", + "source_group:unicom-commuting-2023:license_terms_unverified", + "source_group:unicom-commuting-2023:retention_policy_missing", + "source_group:unicom-commuting-2023:access_policy_missing", + "source_group:unicom-commuting-2023:privacy_impact_assessment_missing", + "source_group:unicom-commuting-2023:grid_geometry_dictionary_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "8c494e7fce9e43208ab8ec09872cf0febd48bebe5b425629b71278ad285fae32", + "data_classification": "highly_restricted_aggregate_mobility", + "data_domains": [ + "mobility", + "population", + "aggregate_signaling", + "table" + ], + "extension_counts": { + "csv": 1 + }, + "file_count": 1, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "11", + "size_bytes": 69667, + "source_group_id": "unicom-commuting-2023" + }, + { + "admission_status": "blocked_pending_source_governance", + "blockers": [ + "source_group:baidu-search-index-2023:owner_reference_missing", + "source_group:baidu-search-index-2023:license_terms_unverified", + "source_group:baidu-search-index-2023:retention_policy_missing", + "source_group:baidu-search-index-2023:access_policy_missing", + "source_group:baidu-search-index-2023:commercial_terms_review_missing" + ], + "content_admitted": false, + "content_manifest_sha256": "87db57d0821164492800c4271f817165ce884d6b261b46595c6e3f1e6b4ee1cf", + "data_classification": "highly_restricted_commercial_activity", + "data_domains": [ + "search_activity", + "intercity_flow", + "vector" + ], + "extension_counts": { + "[no_ext]": 2, + "atx": 17, + "gdbindexes": 7, + "gdbtable": 8, + "gdbtablx": 8, + "spx": 2 + }, + "file_count": 44, + "license_status": "unverified_restricted", + "metadata_profiled": true, + "ordinal": "12", + "size_bytes": 201462, + "source_group_id": "baidu-search-index-2023" + } + ], + "status": "blocked_pending_source_governance" +} diff --git a/docs/roadmap.md b/docs/roadmap.md index df27586b..aadc8b35 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -439,6 +439,7 @@ AR-0 Architecture / Schema / Runtime Truth 交付： +- 真实源 admission contract：以不可变 archive checksum、解压 payload fingerprint、source-group manifest、metadata profile 和治理 blocker 建立准入基线；证据不得含源 payload、绝对路径、记录值或 geometry，profiling 不等于 content admission。 - SourceDefinition、CredentialReference、SourceCapability、SyncDefinition/Version、SyncRun、Cursor/Watermark、SchemaDriftEvent 和 Reconciliation。 - 数据库、对象存储/空间文件、HTTP/STAC 三类代表 source 的连接、凭据、连通、发现、preview、profile 和 owner 登记。 - 全量/增量微批的 Append/Overwrite/Merge 策略，以及至少一个真实 CDC 或事件流 source 通过 Flink 写入版本化 Bronze；覆盖 watermark/offset、checkpoint、迟到/乱序、源端删除、幂等、对账、重放和失败恢复。 @@ -686,7 +687,7 @@ Golden checks 至少覆盖： 3. 冻结 ResourceURN、ResourceVersion、PlatformDefinition/PlatformRun/FrameworkAttemptObservation/Artifact/LineageEvent、SubjectContext 与 storage/table/compute provider 最小合同。 4. 分阶段实现 `gda-metadata-fabric-bridge`：M1 只读 mapping/reconciliation、M2a 本地 foundation/重启连续性、M2b-1 本地三存储恢复、M2b-2 隔离 versioned/Object-Locked repository round-trip、M2b-3 本机双集群 + Kubernetes 外 COMPLIANCE repository + 独立 writer/reader、M2c-1 provider-native metrics、M2c-2 临时 OTel Collector + JSON Exporter 的双周期本地 pipeline、M2c-3 本地单 job scrape 故障检测/配置恢复/完整清理 evidence、M2c-4 绑定 source revision 的 production observability readiness contract，以及 M2d-1 本地 kindnet 跨节点 NetworkPolicy enforcement 已验证；M2c-4 当前仍有 20 项 blockers，M2d-1 也未验证生产 provider policy 或 tenant isolation。下一步完成 source host/cluster 外的生产 bucket、KMS/TLS/workload identity、source-loss recovery 与 RPO/RTO，并批准 metrics backend、retention、OTel/TLS、tenant、alert/SLO/owner 后在受保护环境验证持续采集、存储、查询、真实告警投递、runbook 响应和 provider NetworkPolicy；再推进 OIDC、upgrade/rollback、registry provenance 和 owner/runbook；之后才进入 M3 ingestion/OpenLineage/conformance。 5. 实现 `gda-orchestration-gateway`、DolphinScheduler process/task/schedule/complement/worker-group、Spark/Flink provider task adapter 和故障注入；不再开发新的 lease/queue/scheduler。 -6. 冻结首条地类图斑数据、标准版本、敏感级别、owner、SLO 和 golden result。 +6. M3-28 已冻结全量重庆真实源的 path-free physical/metadata admission baseline；下一步补齐解压派生 provenance，并由 owner 决定首条地类图斑源的 license、retention、access、privacy/sensitivity、标准版本、SLO 和 golden result，未获批准前不得 content admission。 7. 冻结 Default Lakehouse、Cloud Managed、Lightweight Integrated profiles；以统一 Run 完成默认 MinIO/Iceberg/Spark/Flink、轻量 PostGIS/DuckDB 和 Azure 代表 adapter 的 conformance smoke。 8. 实现跨 profile 的 Raw -> ODS -> DIM/DWD -> DWS -> ADS 通用生产、质量、发布、回滚和 golden equivalence。 9. 建立 DataProductBlueprint、模型版本和 Visual/SQL/Notebook 共用 definition 的 Build 工作台，打通 preview、test、publish、approval 和 rollback。 diff --git a/scripts/chongqing-real-source-admission.sh b/scripts/chongqing-real-source-admission.sh new file mode 100755 index 00000000..90ce3dec --- /dev/null +++ b/scripts/chongqing-real-source-admission.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +common_git_dir="$(git -C "$repo_root" rev-parse --path-format=absolute --git-common-dir 2>/dev/null || true)" +shared_root="" +if [ -n "$common_git_dir" ]; then + shared_root="$(cd "$common_git_dir/.." && pwd)" +fi + +if [ -n "${PYTHON:-}" ]; then + : +elif [ -x "$repo_root/.venv/bin/python" ]; then + PYTHON="$repo_root/.venv/bin/python" +elif [ -n "$shared_root" ] && [ -x "$shared_root/.venv/bin/python" ]; then + PYTHON="$shared_root/.venv/bin/python" +else + PYTHON="python" +fi + +cd "$repo_root" +exec "$PYTHON" -m data_agent.chongqing_real_source_admission "$@" From 3c47be3eb4a4ea44ccf81f70774e28c6d02ecd58 Mon Sep 17 00:00:00 2001 From: Ning Zhou Date: Mon, 17 Aug 2026 12:03:43 +0400 Subject: [PATCH 2/2] ci: gate real-source admission evidence --- .github/workflows/cd-staging.yml | 1 + .github/workflows/ci.yml | 4 ++++ docs/roadmap.md | 2 +- docs/system-of-record-matrix-2026-07-24.md | 9 ++++++--- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index c7f2ceda..b78d8725 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -165,6 +165,7 @@ jobs: data_agent/test_adk_compat.py \ data_agent/test_arcpy_mcp_configuration.py \ data_agent/test_auth.py \ + data_agent/test_chongqing_real_source_admission.py \ data_agent/test_dolphinscheduler_adapter.py \ data_agent/test_dolphinscheduler_command_consumer.py \ data_agent/test_dolphinscheduler_command_worker.py \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee3b037c..07eccd6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,6 +124,9 @@ jobs: - name: Validate metadata fabric production identity gate run: python -m data_agent.metadata_fabric_identity_gate validate + - name: Validate Chongqing real-source admission evidence + run: python -m data_agent.chongqing_real_source_admission validate + - name: Validate DolphinScheduler adapter boundary run: python -m data_agent.dolphinscheduler_adapter validate @@ -159,6 +162,7 @@ jobs: data_agent/test_adk_compat.py \ data_agent/test_arcpy_mcp_configuration.py \ data_agent/test_auth.py \ + data_agent/test_chongqing_real_source_admission.py \ data_agent/test_dolphinscheduler_adapter.py \ data_agent/test_dolphinscheduler_command_consumer.py \ data_agent/test_dolphinscheduler_command_worker.py \ diff --git a/docs/roadmap.md b/docs/roadmap.md index aadc8b35..de2eeddd 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -728,7 +728,7 @@ AR-4 parity/control gate 退出前暂停以下主线扩张： |---|---|---| | AR-0 Architecture/Schema/Runtime Truth Freeze | `in_progress` | 全环境 schema/config fingerprint、迁移 fail-closed、事实清单、storage/compute/GIS serving provider profile/capability、ADR-017 benchmark、owner/SLO 和首条数据/服务验收集冻结 | | AR-1 Unified Metadata + Orchestration Control Planes | `in_progress` | controlled gateway、DolphinScheduler adapter、Metadata Fabric M1/M2a/M2b、M2c-1 provider metrics、M2c-2 本地临时 OTel pipeline、M2c-3 本地 scrape failure/recovery、M2c-4 production observability readiness contract 与 M2d-1 本地跨节点 NetworkPolicy enforcement 已验证，生产观测和生产 policy/tenant isolation 仍 blocked；下一证据是 source host/cluster 外的生产 recovery、持久 metrics backend/TLS/tenant/真实 alert delivery/SLO、OIDC、受保护 provider NetworkPolicy、升级回滚/registry provenance，以及受控 ingestion/replay 与无双写验收 | -| AR-2 Source/Ingestion + Geospatial Lakehouse Vertical Slice | `planned` | 三类代表源、`DriveTransfer` 云盘客户端和大文件恢复通过统一控制面；默认湖仓、轻量存算一体及 Azure 代表 adapter 通过 provider conformance 与 Raw -> ADS 验收 | +| AR-2 Source/Ingestion + Geospatial Lakehouse Vertical Slice | `in_progress` | M3-28 已完成重庆真实源 metadata-only admission baseline；下一证据是解压派生 provenance，以及 owner 对 license、retention、access、privacy/sensitivity、标准版本、SLO 和 golden result 的治理决策，随后才可授权 immutable Landing authority、content admission 和 ingestion；最终仍需三类代表源、`DriveTransfer` 云盘客户端、大文件恢复、默认湖仓、轻量存算一体及 Azure 代表 adapter 通过 provider conformance 与 Raw -> ADS 验收 | | AR-3 Data Product Engineering + Governance Workbench | `planned` | Blueprint、模型、Visual/SQL/Notebook、DataOps CI/CD、质量/安全/审批共用 definition 和产品生命周期 | | AR-4 Asset/GIS Service/Spatial Experience Operations | `planned` | Service Control Plane、Features/Tiles/MVT/COG/STAC/export 及条件 legacy OGC/3D/EDR provider、Gateway/权限/缓存、原子切换/回滚、Discover/Operate/Govern 和无 LLM 多入口通过 conformance/parity/control gate | | AR-5 AgentOps Runtime + UX Uplift | `planned` | DataOps parity/control 通过；Agent bundle eval、deployment、online observation、incident/rollback 和 uplift gate | diff --git a/docs/system-of-record-matrix-2026-07-24.md b/docs/system-of-record-matrix-2026-07-24.md index 199c6ef7..db52a75c 100644 --- a/docs/system-of-record-matrix-2026-07-24.md +++ b/docs/system-of-record-matrix-2026-07-24.md @@ -1,8 +1,8 @@ # GIS Data Agent System-of-Record 矩阵 -日期：2026-07-28 +日期：2026-08-17 -阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery、M3-5 local OpenMetadata bounded identity、M3-6 local Gravitino Basic bounded identity 与 M3-7 production identity readiness contract 已验证，生产 provider ingestion、生产观测、生产 policy/tenant isolation、生产 identity attestation 和生产切换仍 `in_progress` +阶段：AR-0 `in_progress`；AR-1 gateway、成功终局 evidence gate、DolphinScheduler adapter sandbox POC、Metadata Fabric M1/M2、M2c-4/M2d-2 production readiness contracts、M3-1/M3-2、M3-3 local binding ledger、M3-4 local OpenLineage wire delivery、M3-5 local OpenMetadata bounded identity、M3-6 local Gravitino Basic bounded identity 与 M3-7 production identity readiness contract 已验证；AR-2 M3-28 重庆真实源 metadata-only admission baseline 已建立，内容准入、Landing authority、生产 provider ingestion、生产观测、生产 policy/tenant isolation、生产 identity attestation 和生产切换仍 `in_progress` 适用分支：`main` @@ -21,7 +21,7 @@ | 部署配置策略 | Compose/K8s/进程环境；`platform_truth.CONFIG_SPECS` 定义关键类型与策略；DolphinScheduler worker 有默认零副本、外部 ConfigMap/Secret 驱动的 Kustomize 模板、静态 validator、staging activation preflight 和受保护的单副本 activation admission/workflow | `.env` 仅补默认；脱敏 snapshot、Secret key attestation、未扩容 Deployment、`ready_for_activation` 和未执行的 activation workflow 都是观测/变更能力 | 版本化 DeploymentProfile + secret reference；部署环境始终优先；模板、preflight 或 admission 通过都不等于环境已启用 | Platform/SRE/Security | AR-0，部分实现；worker 激活合同已验证、真实运行待审批 | | 环境发布与晋级 | publisher `31862363442`、protected verifier `31862984294` 与 staging deploy/observe `31863077257` 已将 `main@5fffc85`、GHCR digest、attested release、cluster/namespace identity 和 live revision 绑定；schema/config/runtime/health/rollout 通过，golden slice 缺失使 promotion fail closed | 旧 mainline、feature branch、CI artifact、JSON、离线 report、单独的 staging deployment 或人工批准都不能成为 production 发布权威 | 由受保护 environment 的 DeploymentRevision 绑定 OCI、provenance artifact、release manifest、golden slice 与全部 live verdict | Platform/SRE/Security/Repository Owner | AR-1 真实 staging 已部署 -> golden slice/production exit gates 待完成 | | 后台运行时清单 | `platform_truth.RUNTIME_INVENTORY` 是代码层登记；`gda_control` 已有受控 PlatformRun 写入口；DolphinScheduler adapter、tenant-scoped managed command worker 与受保护的单副本激活边界已有代码和测试，但 worker 尚未在 staging 运行；M2b recovery runner、M2c-1 provider probe、M2c-2 `_OtelPortForward`、M2c-3 failure rehearsal 与 M2d-1 NetworkPolicy rehearsal 均登记为 `local_verification_only`，不是 scheduler、worker、持续监控、生产 policy controller 或状态权威 | AST primitive report、worker status JSON、FrameworkAttemptObservation、DolphinScheduler instance state、本地 recovery/metrics/network-policy evidence | PlatformRun ledger 唯一登记最终状态；framework/provider attempt 只能回报观测；worker status 仅为进程健康投影；本地演练进程与 evidence 不得变成生产控制器、监控后端或 tenant-isolation 权威 | Platform Architecture | AR-1 adapter/worker/activation 合同已验证 -> staging 运行待审批；metadata recovery/metrics/policy runner 仅本地验证 | -| 原始文件/对象 | 当前 local uploads、S3/MinIO/OBS 均可能被直接写入，权威边界未统一 | 临时上传、下载缓存、预览文件 | Landing object 以 immutable URI + checksum + retention 为权威；本地 scratch 可删除 | Data Platform | AR-2 | +| 原始文件/对象 | M3-28 已以 path-free metadata manifest 记录重庆真实源 archive/extracted fingerprints、source groups、asset profiles 与治理 blockers；`source_content_admitted=false`，因此尚无内容写入权威 | admission evidence JSON、archive/extracted comparison、临时上传、下载缓存、预览文件 | 在 owner/license/retention/access/privacy 等决策和派生 provenance 进入正式 authority 后，才可由 immutable Landing object URI + checksum + retention 建立内容权威；本地 scratch 与 checked evidence 均不可替代 Landing | Data Platform | AR-2 `in_progress` | | 湖仓表与 snapshot | Iceberg/STAC/S3A 有局部实现，尚无通用发布权威 | STAC item、GeoParquet export | Iceberg catalog snapshot 是分析表版本权威；对象是物理内容，STAC 是发现投影 | Data Platform | AR-2 | | 在线空间数据 | PostGIS 业务表是当前编辑/查询事实，部分临时表混入 | Martin MVT、API JSON、导出文件 | 已批准 DataProductVersion 物化到 PostGIS；不能由瓦片或临时表反向定义产品版本 | GIS/Data Platform | AR-2 -> AR-4 | | 数据资产身份与版本 | `gda_control.resource/resource_version` 已实现 identity、hash、predecessor、tenant FK 和幂等 gateway 写入；`agent_data_assets`、`agent_asset_versions` 仍是兼容写路径 | UI catalog、search index、STAC | GDA ledger 管身份与版本绑定；旧行只有在 tenant、authority identity、checksum 和 version evidence 完整时才可形成 eligible plan；OpenMetadata 管治理目录，Gravitino 管技术对象映射 | Metadata Platform | AR-1 gateway 已验证 -> 生产切换待验收 | @@ -62,6 +62,7 @@ 16. Metadata Fabric M3-3 只将同一 source evidence 经 PlatformGateway 写入 tenant-scoped、append-only binding ledger，且不调用 provider、不写 legacy；binding replay 必须幂等、跨租户读取必须拒绝、UPDATE/DELETE 必须拒绝。本地临时 ledger、bootstrap admin、未认证 Gravitino、memory catalog、合成 attestation 和 local evidence 都不等于生产最小权限/OIDC、持久 catalog/binding、live OpenLineage、tenant isolation、alert/SLO、生产 ingestion/conformance 或生产写权威。 17. Metadata Fabric M3-4 只经 tenant-scoped outbox 向无认证 loopback receiver 投递精确 candidate，并验证 at-least-once + receiver idempotency；outbox 只拥有投递状态，receiver 只拥有本地接收状态，不反写 ResourceVersion、Run 或审批权威。loopback receiver、临时 outbox、合成 attestation 和 local evidence 都不等于受保护 production receiver、TLS/OIDC、生产 OpenLineage、生产 ingestion/conformance 或生产写权威。 18. Metadata Fabric M3-5 只证明临时 OpenMetadata bot 在 provider 强制 `DefaultBotRole` 之上的项目新增 grant 是 `table/Create`，以及 policy-create 拒绝、本地 JWT 轮换/吊销和 cleanup；bootstrap provisioner、临时 identity、合成 attestation 和 local evidence 都不等于双 provider/生产最小权限、protected workload identity/OIDC、持久 credential delivery、生产 tenant isolation、生产 ingestion/conformance 或生产写权威。 +19. M3-28 只允许 path-free、metadata-only 的重庆真实源 admission；archive/extracted fingerprint、source-group manifest、asset profile 和 57 个治理 blocker 形成准入观察，不形成 Landing object、ResourceVersion、PlatformRun、授权 Artifact、scheduler submission、provider mutation、content admission 或 production ingestion 权威；checked evidence 不得被编辑成批准。 ## 已建立的 AR-0/AR-1 entry 证据 @@ -93,6 +94,7 @@ - Metadata Fabric M3-5 已创建临时 OpenMetadata 非管理员 bot，effective roles 只有 provider 强制 `DefaultBotRole` 与项目 role，项目 policy 只有 `table/Create`；table create/read 为 201/200，policy create 为 403，旧 JWT 轮换后为 401，新 JWT 吊销后为 401，六项 provider cleanup 与 port-forward cleanup 全部通过。evidence SHA 为 `61b6a3429ae948f563bfc2bd012d8b586be581704cec646fd5e74b991243f03f`。`local_openmetadata_minimum_privilege_verified=true` 只描述强制 default role 之上的项目新增 grant；`provider_minimum_privilege_verified=false`、`protected_workload_identity_verified=false`、`oidc_verified=false`、`gravitino_authentication_verified=false`、`production_identity_verified=false`、`production_ready=false`。 - Metadata Fabric M3-6 已在隔离 Gravitino `1.3.0` Basic IdP 中将 bounded user 限定为 `lakehouse` 的 `USE_CATALOG` 与 `lakehouse.published` 的 `USE_SCHEMA`/`CREATE_TABLE`；table create/read 为 200/200，catalog create 为 403，旧密码轮换后为 401，用户删除后替换密码为 401，临时 provider 对象、namespace 和 port-forward 全部清理。evidence SHA 为 `f0b0de1f80f079d43318937e0a0cc151a8546e9e307bef204738b1367f9b29fd`。`local_gravitino_minimum_privilege_verified=true` 只描述本地 Basic rehearsal；OIDC、TLS、持久 catalog 和 production identity 均仍为 `false`。 - Metadata Fabric M3-7 已建立 production identity profile/attestation gate；checked-in profile fingerprint 为 `2e9d5cac3560b853820f923669f6794ead63bcb36a528639fc0e9539e148ee2f`，report fingerprint 为 `c607589ee25a87acc8a1ab71372618a9a4c10c1e8ebff15b8db7e78b37600b9f`，`profile_valid=true`，40 项 federation/provider/TLS/catalog/tenancy/operations 外部输入以 blockers 暴露，`ready_for_protected_verification=false`、`production_identity_gate_passed=false`、`production_ready=false`。合成完整 attestation 只验证门禁逻辑，不计入生产证据。 +- AR-2 M3-28 已完成重庆真实源 metadata-only admission baseline：evidence SHA 为 `a2196495d845d61be939c7fc36a7f05c3567e365599d2d04be0aab9c568459c1`，evidence file SHA 为 `9b5c20369c235f7e0a2f2cb0a21cee77f86981aa273bac196605a4803b05ce83`；archive SHA 为 `2043b60c2f4f7f32a31388a634fae4ac28534990e205aa86b8df0e4b64dcbbca`，extracted payload SHA 为 `e7e81e4f53f9f174792f500fbfdfde6bee30ec03beac8cbd91771fe09f548ea6`；11 个 source groups、16 个 asset profiles、57 个 admission blockers、584 个 extracted files，archive/extracted comparison 为 526 exact、6 modified、0 missing、52 additional；`source_content_admitted=false`。该证据不含 source payload、绝对路径、记录值或 geometry，也不证明 Landing authority、ResourceVersion、Run、授权 Artifact、scheduler/provider mutation、生产 ingestion 或 `production_ready`。 ## 下一验收证据 @@ -101,4 +103,5 @@ - 为受保护 activation 提供真实 ConfigMap snapshot、Secret key attestation、provider identity 和 reviewer approval，随后完成 managed outbox worker/provider callback 单副本实际部署、唯一 worker ID、status/lease 故障恢复和无双写证据； - 首条真实图斑链对 golden slice 的 output hash、独立质量结果/evidence、血缘、发布 revision 和 rollback 演练； - OpenMetadata/Gravitino 的 source host/cluster 外生产 backup account/bucket、KMS/TLS/workload identity、PITR/source-loss recovery、RPO/RTO、OIDC、受保护环境 provider NetworkPolicy/tenant isolation、upgrade/rollback、registry provenance、持续 metrics backend/retention/query、真实 alert delivery/SLO owner/runbook，以及受保护 PolicyDecision/Approval、双 provider 最小权限 ingestion、生产持久 binding、受保护 production OpenLineage receiver、无双写 read-back 和 conformance；M1 fixture、M2 本地 evidence/readiness contracts、M3-1 projection candidate、M3-2 local replay、M3-3 临时 binding ledger、M3-4 loopback delivery、M3-5/M3-6 本地临时 provider identity 与 M3-7 pending profile/合成 attestation 均不计入生产退出门； +- M3-28 的下一证据必须将解压派生 provenance，以及 owner/license、retention、access、privacy/sensitivity、标准版本、SLO 和 golden result 决策纳入正式 authority；metadata-only admission evidence 是不可变检查结果，不能被编辑或重解释为内容批准、Landing authority 或 ingestion authorization； - DolphinScheduler/Temporal sandbox 的独立数据库、备份恢复、身份、版本和升级责任证明；DolphinScheduler standalone/H2 不计入此退出门。