From bf88f47cf23e927d640ef9c527dc7be692496c44 Mon Sep 17 00:00:00 2001 From: Rahul Krishna Date: Thu, 27 Aug 2026 13:30:56 -0400 Subject: [PATCH] feat(neo4j)!: schema v2 graph projection; v2 becomes the default schema MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the canonical schema v2 → Neo4j projection (graph contract 2.0.0), mirroring codeanalyzer-python's vocabulary with J/J_ namespacing: V2GraphProjector (pure Analysis → GraphRows), V2SchemaCatalog, and the L3 CPG overlay (J_HAS_BODY_NODE, J_CFG_NEXT/J_CDG/J_DDG with the _k MERGE discriminant). Per the convergence spec there are no call-site/parameter/ comment nodes: call sites are :JBodyNode rows, parameters flatten to parameters_json, javadoc collapses to docstring. Breaking (3.0.0): - --schema defaults to v2 everywhere; --schema v1 selects the legacy JSON and legacy graph (v1 node/edge statements unchanged). - --emit neo4j at v2 is always full depth (forces L3 + external symbols); -a/--graph-field-depth alongside it exit non-zero. - --emit schema always emits the 2.0.0 catalog; schema.neo4j.json regenerated as the v2 artifact. The wipe/prune traversal spans both generations (J_HAS_UNIT|J_HAS_MODULE + union DESCENDANTS) and sweeps orphaned degree-0 symbols, so a v2 push replaces a same-app v1 graph completely — verified against Neo4j 5: v1 load → v2 load leaves 0 v1 leftovers, reload is idempotent (30223 nodes / 158704 rels unchanged), and graph counts equal the run's analysis.json exact-set for modules/callables/body/cfg/cdg/ddg. Spec: docs/design/specs/2026-08-27-v2-neo4j-projection.md Closes #198 --- gradle.properties | 2 +- schema.neo4j.json | 371 +++++-------- src/main/java/com/ibm/cldk/CodeAnalyzer.java | 48 +- .../java/com/ibm/cldk/neo4j/BoltWriter.java | 40 +- .../java/com/ibm/cldk/neo4j/CypherWriter.java | 27 +- .../java/com/ibm/cldk/neo4j/GraphRows.java | 12 + .../java/com/ibm/cldk/neo4j/Neo4jEmitter.java | 19 +- .../java/com/ibm/cldk/neo4j/RowBuilder.java | 15 +- src/main/java/com/ibm/cldk/neo4j/Schema.java | 6 +- .../com/ibm/cldk/neo4j/V2GraphProjector.java | 492 ++++++++++++++++++ .../com/ibm/cldk/neo4j/V2SchemaCatalog.java | 226 ++++++++ .../com/ibm/cldk/CodeAnalyzerV2CliTest.java | 59 ++- .../neo4j/Neo4jSchemaConformanceTest.java | 3 +- .../neo4j/V2Neo4jSchemaConformanceTest.java | 177 +++++++ 14 files changed, 1220 insertions(+), 277 deletions(-) create mode 100644 src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java create mode 100644 src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java create mode 100644 src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java diff --git a/gradle.properties b/gradle.properties index 58089caa..4950f0de 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=2.4.1 +version=3.0.0 diff --git a/schema.neo4j.json b/schema.neo4j.json index 09fff6b6..338c2b23 100644 --- a/schema.neo4j.json +++ b/schema.neo4j.json @@ -1,5 +1,5 @@ { - "schema_version": "1.0.0", + "schema_version": "2.0.0", "generator": "codeanalyzer-java", "marker_labels": [ "JEntrypoint" @@ -11,20 +11,20 @@ "key": "name", "properties": { "name": "string", - "schema_version": "string" + "schema_version": "string", + "analyzer_name": "string", + "analyzer_version": "string" } }, { - "label": "JCompilationUnit", - "merge_label": "JCompilationUnit", - "key": "file_key", + "label": "JModule", + "merge_label": "JModule", + "key": "id", "properties": { + "id": "string", "file_key": "string", - "file_path": "string", - "package_name": "string", + "package": "string", "content_hash": "string", - "comment_count": "integer", - "is_modified": "boolean", "_module": "string" } }, @@ -35,21 +35,15 @@ "properties": { "id": "string", "name": "string", - "fqn": "string", "kind": "string", "modifiers": "string[]", - "annotations": "string[]", - "extends_list": "string[]", - "implements_list": "string[]", - "nested_type_declarations": "string[]", - "is_interface": "boolean", - "is_nested_type": "boolean", - "is_inner_class": "boolean", - "is_local_class": "boolean", - "is_entrypoint_class": "boolean", - "parent_type": "string", + "base_types": "string[]", + "interfaces": "string[]", "docstring": "string", - "_module": "string" + "is_entrypoint": "boolean", + "_module": "string", + "start_line": "integer", + "end_line": "integer" } }, { @@ -60,60 +54,49 @@ "id": "string", "name": "string", "signature": "string", - "file_path": "string", + "kind": "string", "declaration": "string", "return_type": "string", + "parameters_json": "string", "modifiers": "string[]", - "annotations": "string[]", - "thrown_exceptions": "string[]", - "parameter_types": "string[]", - "referenced_types": "string[]", - "accessed_fields": "string[]", + "error_channel": "string[]", "code": "string", - "code_start_line": "integer", - "start_line": "integer", - "end_line": "integer", + "docstring": "string", "cyclomatic_complexity": "integer", - "is_constructor": "boolean", + "referenced_types": "string[]", + "accessed_fields": "string[]", "is_implicit": "boolean", "is_entrypoint": "boolean", - "docstring": "string", - "_module": "string" + "_module": "string", + "start_line": "integer", + "end_line": "integer" } }, { - "label": "JField", - "merge_label": "JField", + "label": "JExternal", + "merge_label": "JSymbol", "key": "id", "properties": { "id": "string", - "name": "string", - "type": "string", - "modifiers": "string[]", - "annotations": "string[]", - "variables": "string[]", - "variable_initializers_json": "string", - "start_line": "integer", - "end_line": "integer", - "docstring": "string", - "_module": "string" + "kind": "string", + "signature": "string", + "declaring_type": "string" } }, { - "label": "JParameter", - "merge_label": "JParameter", + "label": "JField", + "merge_label": "JField", "key": "id", "properties": { "id": "string", "name": "string", "type": "string", - "annotations": "string[]", + "initializer": "string", "modifiers": "string[]", + "docstring": "string", + "_module": "string", "start_line": "integer", - "end_line": "integer", - "start_column": "integer", - "end_column": "integer", - "_module": "string" + "end_line": "integer" } }, { @@ -125,39 +108,9 @@ "name": "string", "type": "string", "initializer": "string", + "_module": "string", "start_line": "integer", - "end_line": "integer", - "start_column": "integer", - "end_column": "integer", - "docstring": "string", - "_module": "string" - } - }, - { - "label": "JCallSite", - "merge_label": "JCallSite", - "key": "id", - "properties": { - "id": "string", - "method_name": "string", - "receiver_expr": "string", - "receiver_type": "string", - "return_type": "string", - "callee_signature": "string", - "argument_types": "string[]", - "argument_expr": "string[]", - "is_static_call": "boolean", - "is_constructor_call": "boolean", - "is_public": "boolean", - "is_private": "boolean", - "is_protected": "boolean", - "is_unspecified": "boolean", - "start_line": "integer", - "start_column": "integer", - "end_line": "integer", - "end_column": "integer", - "docstring": "string", - "_module": "string" + "end_line": "integer" } }, { @@ -168,6 +121,7 @@ "id": "string", "name": "string", "arguments": "string[]", + "docstring": "string", "_module": "string" } }, @@ -180,73 +134,30 @@ "name": "string", "type": "string", "modifiers": "string[]", - "annotations": "string[]", - "default_value": "string", - "is_var_args": "boolean", - "docstring": "string", - "_module": "string" - } - }, - { - "label": "JInitializationBlock", - "merge_label": "JInitializationBlock", - "key": "id", - "properties": { - "id": "string", - "file_path": "string", - "code": "string", - "annotations": "string[]", - "thrown_exceptions": "string[]", - "referenced_types": "string[]", - "accessed_fields": "string[]", - "is_static": "boolean", - "cyclomatic_complexity": "integer", - "start_line": "integer", - "end_line": "integer", + "is_variadic": "boolean", "docstring": "string", "_module": "string" } }, { - "label": "JCrudOperation", - "merge_label": "JCrudOperation", + "label": "JBodyNode", + "merge_label": "JBodyNode", "key": "id", "properties": { "id": "string", - "line_number": "integer", - "operation_type": "string", - "target_table": "string", - "involved_columns": "string[]", - "condition": "string", - "joined_tables": "string[]", - "_module": "string" - } - }, - { - "label": "JCrudQuery", - "merge_label": "JCrudQuery", - "key": "id", - "properties": { - "id": "string", - "line_number": "integer", - "query_type": "string", - "query_arguments": "string[]", - "_module": "string" - } - }, - { - "label": "JComment", - "merge_label": "JComment", - "key": "id", - "properties": { - "id": "string", - "content": "string", - "is_javadoc": "boolean", + "kind": "string", + "method_name": "string", + "receiver_expr": "string", + "receiver_type": "string", + "return_type": "string", + "accessibility": "string", + "is_constructor_call": "boolean", + "is_static_call": "boolean", + "argument_types": "string[]", + "argument_expr": "string[]", + "_module": "string", "start_line": "integer", - "start_column": "integer", - "end_line": "integer", - "end_column": "integer", - "_module": "string" + "end_line": "integer" } }, { @@ -268,29 +179,21 @@ ], "relationship_types": [ { - "type": "J_HAS_UNIT", + "type": "J_HAS_MODULE", "from": [ "JApplication" ], "to": [ - "JCompilationUnit" + "JModule" ], "properties": {} }, { - "type": "J_DECLARES_TYPE", + "type": "J_DECLARES", "from": [ - "JCompilationUnit" - ], - "to": [ - "JType" - ], - "properties": {} - }, - { - "type": "J_HAS_NESTED_TYPE", - "from": [ - "JType" + "JModule", + "JType", + "JCallable" ], "to": [ "JType" @@ -298,7 +201,7 @@ "properties": {} }, { - "type": "J_HAS_CALLABLE", + "type": "J_HAS_METHOD", "from": [ "JType" ], @@ -318,66 +221,70 @@ "properties": {} }, { - "type": "J_HAS_PARAMETER", + "type": "J_DECLARES_VAR", "from": [ "JCallable" ], "to": [ - "JParameter" + "JVariable" ], "properties": {} }, { - "type": "J_HAS_CALLSITE", + "type": "J_HAS_ENUM_CONSTANT", "from": [ - "JCallable", - "JInitializationBlock" + "JType" ], "to": [ - "JCallSite" + "JEnumConstant" ], "properties": {} }, { - "type": "J_DECLARES_VAR", + "type": "J_HAS_RECORD_COMPONENT", "from": [ - "JCallable", - "JInitializationBlock" + "JType" ], "to": [ - "JVariable" + "JRecordComponent" ], "properties": {} }, { - "type": "J_HAS_ENUM_CONSTANT", + "type": "J_HAS_BODY_NODE", "from": [ - "JType" + "JCallable" ], "to": [ - "JEnumConstant" + "JBodyNode" ], "properties": {} }, { - "type": "J_HAS_RECORD_COMPONENT", + "type": "J_RESOLVES_TO", "from": [ - "JType" + "JBodyNode" ], "to": [ - "JRecordComponent" + "JCallable", + "JExternal" ], "properties": {} }, { - "type": "J_HAS_INIT_BLOCK", + "type": "J_CALLS", "from": [ - "JType" + "JCallable", + "JExternal" ], "to": [ - "JInitializationBlock" + "JCallable", + "JExternal" ], - "properties": {} + "properties": { + "weight": "integer", + "prov": "string[]" + } }, { "type": "J_EXTENDS", @@ -399,6 +306,21 @@ ], "properties": {} }, + { + "type": "J_IMPORTS", + "from": [ + "JModule" + ], + "to": [ + "JModule", + "JPackage" + ], + "properties": { + "spellings": "string[]", + "is_static": "boolean", + "is_wildcard": "boolean" + } + }, { "type": "J_ANNOTATED_BY", "from": [ @@ -409,104 +331,93 @@ "to": [ "JAnnotation" ], - "properties": {} + "properties": { + "arguments": "string[]" + } }, { - "type": "J_IMPORTS", + "type": "J_CFG_NEXT", "from": [ - "JCompilationUnit" + "JBodyNode" ], "to": [ - "JType", - "JPackage" + "JBodyNode" ], "properties": { - "path": "string", - "is_static": "boolean", - "is_wildcard": "boolean" + "kind": "string", + "_k": "string" } }, { - "type": "J_RESOLVES_TO", + "type": "J_CDG", "from": [ - "JCallSite" + "JBodyNode" ], "to": [ - "JCallable" + "JBodyNode" ], "properties": {} }, { - "type": "J_CALLS", + "type": "J_DDG", "from": [ - "JCallable" + "JBodyNode" ], "to": [ - "JCallable" + "JBodyNode" ], "properties": { - "type": "string", - "weight": "integer", - "source_kind": "string", - "destination_kind": "string" + "var": "string", + "prov": "string[]", + "_k": "string" } }, { - "type": "J_HAS_CRUD_OPERATION", + "type": "J_PARAM_IN", "from": [ - "JCallable", - "JCallSite" + "JBodyNode" ], "to": [ - "JCrudOperation" + "JBodyNode" ], - "properties": {} + "properties": { + "var": "string" + } }, { - "type": "J_HAS_CRUD_QUERY", + "type": "J_PARAM_OUT", "from": [ - "JCallable", - "JCallSite" + "JBodyNode" ], "to": [ - "JCrudQuery" + "JBodyNode" ], - "properties": {} + "properties": { + "var": "string" + } }, { - "type": "J_HAS_COMMENT", + "type": "J_SUMMARY", "from": [ - "JCompilationUnit", - "JType", - "JCallable", - "JField", - "JCallSite", - "JVariable", - "JRecordComponent", - "JInitializationBlock" + "JBodyNode" ], "to": [ - "JComment" + "JBodyNode" ], "properties": {} } ], "constraints": [ - "CREATE CONSTRAINT j_symbol_id IF NOT EXISTS FOR (s:JSymbol) REQUIRE s.id IS UNIQUE", - "CREATE CONSTRAINT j_application_name IF NOT EXISTS FOR (a:JApplication) REQUIRE a.name IS UNIQUE", - "CREATE CONSTRAINT j_compilation_unit_key IF NOT EXISTS FOR (c:JCompilationUnit) REQUIRE c.file_key IS UNIQUE", - "CREATE CONSTRAINT j_package_name IF NOT EXISTS FOR (p:JPackage) REQUIRE p.name IS UNIQUE", - "CREATE CONSTRAINT j_annotation_name IF NOT EXISTS FOR (an:JAnnotation) REQUIRE an.name IS UNIQUE", - "CREATE CONSTRAINT j_callsite_id IF NOT EXISTS FOR (cs:JCallSite) REQUIRE cs.id IS UNIQUE", - "CREATE CONSTRAINT j_field_id IF NOT EXISTS FOR (f:JField) REQUIRE f.id IS UNIQUE", - "CREATE CONSTRAINT j_parameter_id IF NOT EXISTS FOR (p:JParameter) REQUIRE p.id IS UNIQUE", - "CREATE CONSTRAINT j_variable_id IF NOT EXISTS FOR (v:JVariable) REQUIRE v.id IS UNIQUE", - "CREATE CONSTRAINT j_enum_constant_id IF NOT EXISTS FOR (e:JEnumConstant) REQUIRE e.id IS UNIQUE", - "CREATE CONSTRAINT j_record_component_id IF NOT EXISTS FOR (r:JRecordComponent) REQUIRE r.id IS UNIQUE", - "CREATE CONSTRAINT j_init_block_id IF NOT EXISTS FOR (ib:JInitializationBlock) REQUIRE ib.id IS UNIQUE", - "CREATE CONSTRAINT j_crud_operation_id IF NOT EXISTS FOR (co:JCrudOperation) REQUIRE co.id IS UNIQUE", - "CREATE CONSTRAINT j_crud_query_id IF NOT EXISTS FOR (cq:JCrudQuery) REQUIRE cq.id IS UNIQUE", - "CREATE CONSTRAINT j_comment_id IF NOT EXISTS FOR (cm:JComment) REQUIRE cm.id IS UNIQUE" + "CREATE CONSTRAINT japplication_name IF NOT EXISTS FOR (x:JApplication) REQUIRE x.name IS UNIQUE", + "CREATE CONSTRAINT jmodule_id IF NOT EXISTS FOR (x:JModule) REQUIRE x.id IS UNIQUE", + "CREATE CONSTRAINT jsymbol_id IF NOT EXISTS FOR (x:JSymbol) REQUIRE x.id IS UNIQUE", + "CREATE CONSTRAINT jfield_id IF NOT EXISTS FOR (x:JField) REQUIRE x.id IS UNIQUE", + "CREATE CONSTRAINT jvariable_id IF NOT EXISTS FOR (x:JVariable) REQUIRE x.id IS UNIQUE", + "CREATE CONSTRAINT jenumconstant_id IF NOT EXISTS FOR (x:JEnumConstant) REQUIRE x.id IS UNIQUE", + "CREATE CONSTRAINT jrecordcomponent_id IF NOT EXISTS FOR (x:JRecordComponent) REQUIRE x.id IS UNIQUE", + "CREATE CONSTRAINT jbodynode_id IF NOT EXISTS FOR (x:JBodyNode) REQUIRE x.id IS UNIQUE", + "CREATE CONSTRAINT jpackage_name IF NOT EXISTS FOR (x:JPackage) REQUIRE x.name IS UNIQUE", + "CREATE CONSTRAINT jannotation_name IF NOT EXISTS FOR (x:JAnnotation) REQUIRE x.name IS UNIQUE" ], "indexes": [ "CREATE INDEX j_callable_name IF NOT EXISTS FOR (c:JCallable) ON (c.name)", diff --git a/src/main/java/com/ibm/cldk/CodeAnalyzer.java b/src/main/java/com/ibm/cldk/CodeAnalyzer.java index 48333040..1ab609c2 100644 --- a/src/main/java/com/ibm/cldk/CodeAnalyzer.java +++ b/src/main/java/com/ibm/cldk/CodeAnalyzer.java @@ -128,11 +128,10 @@ public class CodeAnalyzer implements Runnable { private static String neo4jDatabase; @Option(names = { - "--schema" }, description = "Output schema: v1 (legacy, default) | v2 (canonical CPG). " - + "v2 currently covers analysis levels 1 and 2.") + "--schema" }, description = "Output schema: v2 (canonical CPG, default) | v1 (legacy).") // Deliberately an INSTANCE field: the pre-existing options on this class are static, which leaks // values between CommandLine instances in the same JVM. New flags do not add to that. - private String schema = "v1"; + private String schema = "v2"; @Option(names = {"-c", "--cache-dir" }, description = "Directory holding the incremental analysis cache. When set, " @@ -315,15 +314,8 @@ private void analyze() throws Exception { JsonArray callGraph = combinedJsonObject.has("call_graph") ? combinedJsonObject.getAsJsonArray("call_graph") : null; - // Connection options resolve with precedence: CLI flag > NEO4J_* env var > default. - String uri = firstNonEmpty(neo4jUri, System.getenv("NEO4J_URI")); - BoltConfig bolt = uri == null - ? null - : new BoltConfig(uri, - firstNonEmpty(neo4jUser, System.getenv("NEO4J_USERNAME"), "neo4j"), - firstNonEmpty(neo4jPassword, System.getenv("NEO4J_PASSWORD"), "neo4j"), - firstNonEmpty(neo4jDatabase, System.getenv("NEO4J_DATABASE"))); - Neo4jEmitter.emit(symbolTable, callGraph, appName, input, output, targetFiles != null, bolt); + Neo4jEmitter.emit(symbolTable, callGraph, appName, input, output, targetFiles != null, + boltConfig()); return; } @@ -377,8 +369,20 @@ private void analyzeV2() throws Exception { "error: unknown --l3-engine '" + l3Engine + "'; use ast or wala"); } if ("neo4j".equalsIgnoreCase(emit)) { - throw new ParameterException(spec.commandLine(), - "error: --schema v2 does not support --emit neo4j yet; the graph projection is still v1"); + // The graph is always full-depth (keystone depth rule): depth/section selectors cannot + // be combined with it — error loudly rather than silently project a partial graph. + if (spec.commandLine().getParseResult().hasMatchedOption("--analysis-level")) { + throw new ParameterException(spec.commandLine(), + "error: --analysis-level does not apply to --emit neo4j; " + + "the graph is always projected at full depth"); + } + if (spec.commandLine().getParseResult().hasMatchedOption("--graph-field-depth")) { + throw new ParameterException(spec.commandLine(), + "error: --graph-field-depth does not apply to --emit neo4j; " + + "the graph is always projected at full depth"); + } + analysisLevel = 3; + externalCalls = true; } if (sourceAnalysis != null || targetFiles != null) { throw new ParameterException(spec.commandLine(), @@ -473,6 +477,11 @@ private void analyzeV2() throws Exception { analysis = V2Emitter.emit(application, analysisLevel, modules, version); } + if ("neo4j".equalsIgnoreCase(emit)) { + Neo4jEmitter.emitV2(analysis, appName, input, output, boltConfig()); + return; + } + if (output == null) { // stdout is the data channel: compact JSON only, so the SDK can parse it directly. System.out.println(V2Json.compact().toJson(analysis)); @@ -487,6 +496,17 @@ private void analyzeV2() throws Exception { } } + /** Bolt connection resolution, precedence: CLI flag > NEO4J_* env var > default. Null ⇒ snapshot. */ + private static BoltConfig boltConfig() { + String uri = firstNonEmpty(neo4jUri, System.getenv("NEO4J_URI")); + return uri == null + ? null + : new BoltConfig(uri, + firstNonEmpty(neo4jUser, System.getenv("NEO4J_USERNAME"), "neo4j"), + firstNonEmpty(neo4jPassword, System.getenv("NEO4J_PASSWORD"), "neo4j"), + firstNonEmpty(neo4jDatabase, System.getenv("NEO4J_DATABASE"))); + } + private static String analyzerVersion() { try { String[] versions = new VersionProvider().getVersion(); diff --git a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java index 55322e39..6ea4c900 100644 --- a/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/BoltWriter.java @@ -101,10 +101,12 @@ void run(GraphRows rows, boolean fullRun) { } } - // 2. diff content_hash. + // 2. diff content_hash. Both generations key their per-file node on file_key: v1 + // :JCompilationUnit and v2 :JModule. Map dbHash = new HashMap<>(); try (Session s = session()) { - s.run("MATCH (c:JCompilationUnit) RETURN c.file_key AS k, c.content_hash AS h").list() + s.run("MATCH (c) WHERE c:JCompilationUnit OR c:JModule " + + "RETURN c.file_key AS k, c.content_hash AS h").list() .forEach(rec -> dbHash.put(rec.get("k").asString(null), rec.get("h").asString(null))); } Set changed = new HashSet<>(); @@ -149,14 +151,18 @@ void run(GraphRows rows, boolean fullRun) { } upsertEdges(edges); - // 6. orphan prune — only safe on a full run. + // 6. orphan prune — only safe on a full run. Reaches both generations' per-file nodes + // through the shared application anchor, so a v2 push also prunes a prior v1 graph's + // units (and other applications in the database are never touched). if (fullRun) { List present = new ArrayList<>(byUnit.keySet()); + String app = appNameOf(rows); try (Session s = session()) { - long pruned = s.run("MATCH (c:JCompilationUnit) WHERE NOT c.file_key IN $present " + long pruned = s.run("MATCH (:JApplication {name: $app})-[:J_HAS_UNIT|J_HAS_MODULE]->(c) " + + "WHERE NOT c.file_key IN $present " + "OPTIONAL MATCH (c)-" + CypherWriter.DESCENDANTS + "->(x) " + "DETACH DELETE x, c RETURN count(c) AS pruned", - Values.parameters("present", present)).single().get("pruned").asLong(0); + Values.parameters("present", present, "app", app)).single().get("pruned").asLong(0); Log.info("neo4j(bolt): pruned " + pruned + " vanished unit(s)"); } } else { @@ -193,21 +199,26 @@ private void upsertNodes(List nodes) { private void upsertEdges(List edges) { Map> groups = new LinkedHashMap<>(); for (EdgeRow e : edges) { - String k = e.type + "|" + e.from.label + "." + e.from.keyProp + "|" + e.to.label + "." + e.to.keyProp; + String k = e.type + "|" + e.from.label + "." + e.from.keyProp + "|" + e.to.label + "." + e.to.keyProp + + "|" + (e.key != null); groups.computeIfAbsent(k, x -> new ArrayList<>()).add(e); } for (List group : groups.values()) { EdgeRow head = group.get(0); + boolean keyed = head.key != null; String cypher = "UNWIND $rows AS row " + "MATCH (a:" + head.from.label + " {" + head.from.keyProp + ": row.f}) " + "MATCH (b:" + head.to.label + " {" + head.to.keyProp + ": row.t}) " - + "MERGE (a)-[r:" + head.type + "]->(b) SET r += row.p"; + + "MERGE (a)-[r:" + head.type + (keyed ? " {_k: row.k}" : "") + "]->(b) SET r += row.p"; for (List batch : CypherWriter.chunk(group, BATCH)) { List> payload = new ArrayList<>(); for (EdgeRow e : batch) { Map r = new HashMap<>(); r.put("f", e.from.value); r.put("t", e.to.value); + if (keyed) { + r.put("k", e.key); + } r.put("p", e.props); payload.add(r); } @@ -220,12 +231,25 @@ private void upsertEdges(List edges) { private static String hashOf(List nodes, String fileKey) { for (NodeRow n : nodes) { - if (n.labels.get(0).equals("JCompilationUnit") && n.value.equals(fileKey)) { + String label = n.labels.get(0); + boolean fileNode = label.equals("JCompilationUnit") + || (label.equals("JModule") && fileKey.equals(n.props.get("file_key"))); + if (fileNode && (n.value.equals(fileKey) || fileKey.equals(n.props.get("file_key")))) { Object h = n.props.get("content_hash"); return h instanceof String ? (String) h : null; } } return null; } + + /** The application anchor's name — the one {@code :JApplication} row every projection emits. */ + private static String appNameOf(GraphRows rows) { + for (NodeRow n : rows.nodes) { + if (n.labels.get(0).equals("JApplication")) { + return n.value; + } + } + return "application"; + } } } diff --git a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java index e5cdd2ea..ffec529f 100644 --- a/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java +++ b/src/main/java/com/ibm/cldk/neo4j/CypherWriter.java @@ -31,9 +31,15 @@ public final class CypherWriter { private static final int BATCH = 500; + /** + * Every containment relationship either graph generation emits — v1 (unit-rooted) and v2 + * (module-rooted) together, so a v2 push wipes/prunes a prior v1 graph of the same app and vice + * versa (spec: one app name = one graph, latest push wins). + */ static final String DESCENDANTS = "[:J_DECLARES_TYPE|J_HAS_NESTED_TYPE|J_HAS_CALLABLE|J_HAS_FIELD|J_HAS_PARAMETER" + "|J_HAS_CALLSITE|J_DECLARES_VAR|J_HAS_ENUM_CONSTANT|J_HAS_RECORD_COMPONENT|J_HAS_INIT_BLOCK" - + "|J_HAS_CRUD_OPERATION|J_HAS_CRUD_QUERY|J_HAS_COMMENT*1..]"; + + "|J_HAS_CRUD_OPERATION|J_HAS_CRUD_QUERY|J_HAS_COMMENT" + + "|J_DECLARES|J_HAS_METHOD|J_HAS_BODY_NODE*1..]"; private CypherWriter() {} @@ -65,10 +71,18 @@ public static String renderCypher(GraphRows rows, String appName) { } private static String wipe(String appName) { + // The unit hop is unlabeled and lists both generations' rel types (v1 J_HAS_UNIT → + // :JCompilationUnit, v2 J_HAS_MODULE → :JModule) so either generation's push replaces + // whichever generation the DB currently holds for this app. The second statement sweeps + // fully-isolated :JSymbol nodes the containment traversal cannot reach — v1's + // import-materialized bodyless :JType stubs hang off units via J_IMPORTS only, so the + // DETACH DELETE above orphans them; degree-0 symbols are unreferencable junk in any + // generation, and a symbol another application still uses keeps its edges and survives. return "MATCH (a:JApplication {name: " + cypherValue(appName) + "})\n" - + "OPTIONAL MATCH (a)-[:J_HAS_UNIT]->(c:JCompilationUnit)\n" + + "OPTIONAL MATCH (a)-[:J_HAS_UNIT|J_HAS_MODULE]->(c)\n" + "OPTIONAL MATCH (c)-" + DESCENDANTS + "->(x)\n" - + "DETACH DELETE x, c, a;"; + + "DETACH DELETE x, c, a;\n" + + "MATCH (s:JSymbol) WHERE NOT (s)--() DELETE s;"; } // ---------------------------------------------------------------------------------------------- @@ -108,23 +122,26 @@ private static List nodeStatements(List nodes) { private static List edgeStatements(List edges) { Map> groups = new LinkedHashMap<>(); for (EdgeRow e : edges) { - String k = e.type + "|" + e.from.label + "." + e.from.keyProp + "|" + e.to.label + "." + e.to.keyProp; + String k = e.type + "|" + e.from.label + "." + e.from.keyProp + "|" + e.to.label + "." + e.to.keyProp + + "|" + (e.key != null); groups.computeIfAbsent(k, x -> new ArrayList<>()).add(e); } List blocks = new ArrayList<>(); for (List group : groups.values()) { EdgeRow head = group.get(0); + boolean keyed = head.key != null; for (List batch : chunk(group, BATCH)) { List list = new ArrayList<>(); for (EdgeRow e : batch) { list.add(" {f: " + cypherValue(e.from.value) + ", t: " + cypherValue(e.to.value) + + (keyed ? ", k: " + cypherValue(e.key) : "") + ", p: " + cypherMap(e.props) + "}"); } blocks.add("UNWIND [\n" + String.join(",\n", list) + "\n] AS row\n" + "MATCH (a:" + head.from.label + " {" + head.from.keyProp + ": row.f})\n" + "MATCH (b:" + head.to.label + " {" + head.to.keyProp + ": row.t})\n" - + "MERGE (a)-[r:" + head.type + "]->(b)\n" + + "MERGE (a)-[r:" + head.type + (keyed ? " {_k: row.k}" : "") + "]->(b)\n" + "SET r += row.p;"); } } diff --git a/src/main/java/com/ibm/cldk/neo4j/GraphRows.java b/src/main/java/com/ibm/cldk/neo4j/GraphRows.java index be1f796c..0e2e40c9 100644 --- a/src/main/java/com/ibm/cldk/neo4j/GraphRows.java +++ b/src/main/java/com/ibm/cldk/neo4j/GraphRows.java @@ -70,12 +70,24 @@ public static final class EdgeRow { public final NodeRef from; public final NodeRef to; public final Map props; + /** + * Optional MERGE discriminant. When set, writers MERGE on {@code {_k: key}} so several + * relationships of one type may coexist between one endpoint pair (a conditional's + * true/false CFG edges; one DDG dependence per variable). {@code null} means the plain + * one-relationship-per-(type, from, to) MERGE. + */ + public final String key; public EdgeRow(String type, NodeRef from, NodeRef to, Map props) { + this(type, from, to, props, null); + } + + public EdgeRow(String type, NodeRef from, NodeRef to, Map props, String key) { this.type = type; this.from = from; this.to = to; this.props = props; + this.key = key; } } } diff --git a/src/main/java/com/ibm/cldk/neo4j/Neo4jEmitter.java b/src/main/java/com/ibm/cldk/neo4j/Neo4jEmitter.java index c022baf7..8264681b 100644 --- a/src/main/java/com/ibm/cldk/neo4j/Neo4jEmitter.java +++ b/src/main/java/com/ibm/cldk/neo4j/Neo4jEmitter.java @@ -37,10 +37,13 @@ public final class Neo4jEmitter { private Neo4jEmitter() {} - /** Emit the machine-readable schema contract. {@code output == null} prints to stdout. */ + /** + * Emit the machine-readable schema contract — always the v2 catalog (graph 2.0.0), regardless + * of {@code --schema} (spec decision 5). {@code output == null} prints to stdout. + */ public static void emitSchema(String output) throws IOException { String doc = new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create() - .toJson(SchemaCatalog.buildSchemaDocument()) + "\n"; + .toJson(V2SchemaCatalog.buildSchemaDocument()) + "\n"; if (output == null) { System.out.print(doc); return; @@ -66,8 +69,18 @@ public static void emitSchema(String output) throws IOException { public static void emit(Map symbolTable, JsonArray callGraph, String appName, String input, String output, boolean targetedRun, BoltConfig bolt) throws IOException { String name = appName != null ? appName : deriveAppName(input); - GraphRows rows = GraphProjector.project(symbolTable, callGraph, name); + write(GraphProjector.project(symbolTable, callGraph, name), name, output, targetedRun, bolt); + } + + /** Project + emit the schema v2 graph (graph contract 2.0.0). Always a full-depth, full run. */ + public static void emitV2(com.ibm.cldk.schema.Analysis analysis, String appName, String input, + String output, BoltConfig bolt) throws IOException { + String name = appName != null ? appName : deriveAppName(input); + write(V2GraphProjector.project(analysis, name), name, output, false, bolt); + } + private static void write(GraphRows rows, String name, String output, boolean targetedRun, + BoltConfig bolt) throws IOException { if (bolt != null) { BoltSink sink = loadBoltSink(); if (sink != null) { diff --git a/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java b/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java index 29b30da1..32b17156 100644 --- a/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java +++ b/src/main/java/com/ibm/cldk/neo4j/RowBuilder.java @@ -90,6 +90,11 @@ public void edge(String type, NodeRef from, NodeRef to, Map prop edges.add(new EdgeRow(type, from, to, props)); } + /** As above, with a {@code _k} MERGE discriminant (see {@link EdgeRow#key}). */ + public void keyedEdge(String type, NodeRef from, NodeRef to, Map props, String key) { + edges.add(new EdgeRow(type, from, to, props, key)); + } + public void edge(String type, NodeRef from, NodeRef to) { edges.add(new EdgeRow(type, from, to, RowBuilder.props())); } @@ -119,11 +124,12 @@ public GraphRows finish() { } } // Dedupe edges the way Neo4j's MERGE would: one relationship per - // (type, source, target), last-write-wins on props (mirrors `MERGE (a)-[r]->(b) SET r += p`). + // (type, source, target[, _k]), last-write-wins on props (mirrors `MERGE (a)-[r]->(b) SET r += p`). Map uniqueEdges = new LinkedHashMap<>(); for (EdgeRow e : edges) { uniqueEdges.put(e.type + "|" + e.from.label + ":" + e.from.value - + "|" + e.to.label + ":" + e.to.value, e); + + "|" + e.to.label + ":" + e.to.value + + (e.key == null ? "" : "|" + e.key), e); } List nodeList = new ArrayList<>(nodes.values()); @@ -131,8 +137,9 @@ public GraphRows finish() { (a.labels.get(0) + " " + a.value).compareTo(b.labels.get(0) + " " + b.value)); List edgeList = new ArrayList<>(uniqueEdges.values()); edgeList.sort((a, b) -> - (a.type + " " + a.from.value + " " + a.to.value) - .compareTo(b.type + " " + b.from.value + " " + b.to.value)); + (a.type + " " + a.from.value + " " + a.to.value + " " + (a.key == null ? "" : a.key)) + .compareTo(b.type + " " + b.from.value + " " + b.to.value + " " + + (b.key == null ? "" : b.key))); return new GraphRows(nodeList, edgeList); } } diff --git a/src/main/java/com/ibm/cldk/neo4j/Schema.java b/src/main/java/com/ibm/cldk/neo4j/Schema.java index 5e260ccb..acd43e37 100644 --- a/src/main/java/com/ibm/cldk/neo4j/Schema.java +++ b/src/main/java/com/ibm/cldk/neo4j/Schema.java @@ -39,7 +39,11 @@ private Schema() {} "CREATE CONSTRAINT j_init_block_id IF NOT EXISTS FOR (ib:JInitializationBlock) REQUIRE ib.id IS UNIQUE", "CREATE CONSTRAINT j_crud_operation_id IF NOT EXISTS FOR (co:JCrudOperation) REQUIRE co.id IS UNIQUE", "CREATE CONSTRAINT j_crud_query_id IF NOT EXISTS FOR (cq:JCrudQuery) REQUIRE cq.id IS UNIQUE", - "CREATE CONSTRAINT j_comment_id IF NOT EXISTS FOR (cm:JComment) REQUIRE cm.id IS UNIQUE"); + "CREATE CONSTRAINT j_comment_id IF NOT EXISTS FOR (cm:JComment) REQUIRE cm.id IS UNIQUE", + // Schema v2 (graph 2.0.0) additions — the writers run the union so either generation's + // graph stays constraint-protected in a shared database. + "CREATE CONSTRAINT j_module_id IF NOT EXISTS FOR (m:JModule) REQUIRE m.id IS UNIQUE", + "CREATE CONSTRAINT j_body_node_id IF NOT EXISTS FOR (bn:JBodyNode) REQUIRE bn.id IS UNIQUE"); public static final List INDEXES = Arrays.asList( "CREATE INDEX j_callable_name IF NOT EXISTS FOR (c:JCallable) ON (c.name)", diff --git a/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java b/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java new file mode 100644 index 00000000..d24009b5 --- /dev/null +++ b/src/main/java/com/ibm/cldk/neo4j/V2GraphProjector.java @@ -0,0 +1,492 @@ +/* +Copyright IBM Corporation 2023, 2024 + +Licensed under the Apache Public License 2.0, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.ibm.cldk.neo4j; + +import com.ibm.cldk.neo4j.GraphRows.NodeRef; +import com.ibm.cldk.schema.Analysis; +import com.ibm.cldk.schema.JBodyNode; +import com.ibm.cldk.schema.JCallEdge; +import com.ibm.cldk.schema.JCallable; +import com.ibm.cldk.schema.JCdgEdge; +import com.ibm.cldk.schema.JCfgEdge; +import com.ibm.cldk.schema.JComment; +import com.ibm.cldk.schema.JDdgEdge; +import com.ibm.cldk.schema.JDecorator; +import com.ibm.cldk.schema.JEnumConstant; +import com.ibm.cldk.schema.JExternalSymbol; +import com.ibm.cldk.schema.JField; +import com.ibm.cldk.schema.JImport; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.JRecordComponent; +import com.ibm.cldk.schema.JType; +import com.ibm.cldk.schema.JVariableDeclaration; +import com.ibm.cldk.schema.Span; +import com.ibm.cldk.schema.V2Json; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The schema v2 → Neo4j projection: a pure {@code (Analysis, appName) → GraphRows} function, no + * I/O, no driver. The vocabulary is {@link V2SchemaCatalog} (graph contract 2.0.0), mirroring + * codeanalyzer-python's projection: call sites are {@code :JBodyNode} rows (no call-site nodes), + * parameters flatten to {@code parameters_json}, javadoc collapses to {@code docstring}, and the + * L3 {@code cfg}/{@code cdg}/{@code ddg} overlays become typed relationships between body nodes. + * + *

Body-node identity is the global ordinal: {@code @} for real + * statements ({@code 12:5}), {@code } for synthetic bookends whose local + * key already starts with {@code @} ({@code @entry}, {@code @exit}) — the same rule the JSON + * projection's ids follow, so both projections land on one identity. + */ +public final class V2GraphProjector { + + private V2GraphProjector() {} + + private static final List SYMBOL_TYPE = Arrays.asList("JSymbol", "JType"); + private static final List SYMBOL_CALLABLE = Arrays.asList("JSymbol", "JCallable"); + private static final List SYMBOL_EXTERNAL = Arrays.asList("JSymbol", "JExternal"); + + public static GraphRows project(Analysis analysis, String appName) { + RowBuilder b = new RowBuilder(); + Map symbolTable = analysis.getApplication().getSymbolTable(); + + Map appProps = RowBuilder.props(); + appProps.put("name", appName); + appProps.put("schema_version", V2SchemaCatalog.SCHEMA_VERSION); + if (analysis.getAnalyzer() != null) { + appProps.put("analyzer_name", analysis.getAnalyzer().getName()); + appProps.put("analyzer_version", analysis.getAnalyzer().getVersion()); + } + NodeRef app = b.node(Arrays.asList("JApplication"), "name", appName, RowBuilder.prune(appProps)); + + // First pass: an in-project index from a type's qualified (dotted) name to its node id and + // owning module id, for resolving extends/implements/import spellings to emitted nodes. + Map typeIdByFqn = new LinkedHashMap<>(); + Map moduleIdByFqn = new LinkedHashMap<>(); + for (Map.Entry m : symbolTable.entrySet()) { + indexTypes(m.getValue(), m.getValue().getTypes(), null, typeIdByFqn, moduleIdByFqn); + } + + for (Map.Entry entry : symbolTable.entrySet()) { + String fileKey = entry.getKey(); + JModule module = entry.getValue(); + + Map mp = RowBuilder.props(); + mp.put("id", module.getId()); + mp.put("file_key", fileKey); + mp.put("package", module.getPackageName()); + mp.put("content_hash", module.getContentHash()); + mp.put("_module", fileKey); + NodeRef mod = b.node(Arrays.asList("JModule"), "id", module.getId(), RowBuilder.prune(mp)); + b.edge("J_HAS_MODULE", app, mod); + + projectImports(b, mod, module, typeIdByFqn, moduleIdByFqn); + + for (Map.Entry t : module.getTypes().entrySet()) { + projectType(b, mod, "J_DECLARES", t.getKey(), t.getValue(), module, fileKey, + typeIdByFqn); + } + } + + if (analysis.getApplication().getExternalSymbols() != null) { + for (Map.Entry e + : analysis.getApplication().getExternalSymbols().entrySet()) { + JExternalSymbol ext = e.getValue(); + Map p = RowBuilder.props(); + p.put("id", e.getKey()); + p.put("kind", ext.getKind()); + p.put("signature", ext.getSignature()); + p.put("declaring_type", ext.getDeclaringType()); + b.node(SYMBOL_EXTERNAL, "id", e.getKey(), RowBuilder.prune(p)); + } + } + + if (analysis.getApplication().getCallGraph() != null) { + for (JCallEdge e : analysis.getApplication().getCallGraph()) { + Map p = RowBuilder.props(); + p.put("weight", e.getWeight()); + p.put("prov", e.getProv()); + b.edgeIfBothResolved("J_CALLS", + new NodeRef("JSymbol", "id", e.getSrc()), + new NodeRef("JSymbol", "id", e.getDst()), + RowBuilder.prune(p)); + } + } + + return b.finish(); + } + + // ------------------------------------------------------------------------------------------ + // Types + // ------------------------------------------------------------------------------------------ + + private static void indexTypes(JModule module, Map types, String qualifierFqn, + Map typeIdByFqn, Map moduleIdByFqn) { + for (Map.Entry e : types.entrySet()) { + String fqn = qualifierFqn != null + ? qualifierFqn + "." + e.getKey() + : (module.getPackageName() == null || module.getPackageName().isEmpty() + ? e.getKey() + : module.getPackageName() + "." + e.getKey()); + typeIdByFqn.put(fqn, e.getValue().getId()); + moduleIdByFqn.put(fqn, module.getId()); + indexTypes(module, e.getValue().getTypes(), fqn, typeIdByFqn, moduleIdByFqn); + for (JCallable c : e.getValue().getCallables().values()) { + indexTypes(module, c.getTypes(), fqn, typeIdByFqn, moduleIdByFqn); + } + } + } + + private static void projectType(RowBuilder b, NodeRef parent, String containmentRel, String name, + JType type, JModule module, String fileKey, Map typeIdByFqn) { + List labels = type.isEntrypointClass() + ? Arrays.asList("JSymbol", "JType", "JEntrypoint") + : SYMBOL_TYPE; + Map p = RowBuilder.props(); + p.put("id", type.getId()); + p.put("name", name); + p.put("kind", type.getKind()); + p.put("modifiers", type.getModifiers()); + p.put("base_types", type.getBaseTypes()); + p.put("interfaces", type.getInterfaces()); + p.put("docstring", docstringOf(type.getComments())); + putLines(p, type.getSpan()); + if (type.isEntrypointClass()) { + p.put("is_entrypoint", true); + } + p.put("_module", fileKey); + NodeRef ref = b.node(labels, "id", type.getId(), RowBuilder.prune(p)); + b.edge(containmentRel, parent, ref); + + String pkg = module.getPackageName(); + for (String base : type.getBaseTypes()) { + String target = resolveType(base, pkg, typeIdByFqn); + if (target != null) { + b.edgeToSymbol("J_EXTENDS", ref, target); + } + } + for (String iface : type.getInterfaces()) { + String target = resolveType(iface, pkg, typeIdByFqn); + if (target != null) { + b.edgeToSymbol("J_IMPLEMENTS", ref, target); + } + } + annotate(b, ref, type.getDecorators()); + + for (Map.Entry f : type.getFields().entrySet()) { + projectField(b, ref, f.getValue(), fileKey); + } + for (JEnumConstant ec : type.getEnumConstants()) { + Map ep = RowBuilder.props(); + String id = type.getId() + "#enum#" + ec.getName(); + ep.put("id", id); + ep.put("name", ec.getName()); + ep.put("arguments", ec.getArguments()); + ep.put("docstring", docstringOf(ec.getComments())); + ep.put("_module", fileKey); + NodeRef er = b.node(Arrays.asList("JEnumConstant"), "id", id, RowBuilder.prune(ep)); + b.edge("J_HAS_ENUM_CONSTANT", ref, er); + } + for (JRecordComponent rc : type.getRecordComponents()) { + Map rp = RowBuilder.props(); + String id = type.getId() + "#rec#" + rc.getName(); + rp.put("id", id); + rp.put("name", rc.getName()); + rp.put("type", rc.getType()); + rp.put("modifiers", rc.getModifiers()); + if (rc.isVariadic()) { + rp.put("is_variadic", true); + } + rp.put("docstring", docstringOf(rc.getComments())); + rp.put("_module", fileKey); + NodeRef rr = b.node(Arrays.asList("JRecordComponent"), "id", id, RowBuilder.prune(rp)); + b.edge("J_HAS_RECORD_COMPONENT", ref, rr); + } + for (Map.Entry c : type.getCallables().entrySet()) { + projectCallable(b, ref, c.getKey(), c.getValue(), module, fileKey, typeIdByFqn); + } + for (Map.Entry nested : type.getTypes().entrySet()) { + projectType(b, ref, "J_DECLARES", nested.getKey(), nested.getValue(), module, fileKey, + typeIdByFqn); + } + } + + private static void projectField(RowBuilder b, NodeRef owner, JField field, String fileKey) { + Map p = RowBuilder.props(); + p.put("id", field.getId()); + p.put("name", field.getName()); + p.put("type", field.getType()); + p.put("initializer", field.getInitializer()); + p.put("modifiers", field.getModifiers()); + p.put("docstring", docstringOf(field.getComments())); + putLines(p, field.getSpan()); + p.put("_module", fileKey); + NodeRef ref = b.node(Arrays.asList("JField"), "id", field.getId(), RowBuilder.prune(p)); + b.edge("J_HAS_FIELD", owner, ref); + annotate(b, ref, field.getDecorators()); + } + + // ------------------------------------------------------------------------------------------ + // Callables + the L3 CPG overlay + // ------------------------------------------------------------------------------------------ + + private static void projectCallable(RowBuilder b, NodeRef owner, String signature, JCallable c, + JModule module, String fileKey, Map typeIdByFqn) { + List labels = c.isEntrypoint() + ? Arrays.asList("JSymbol", "JCallable", "JEntrypoint") + : SYMBOL_CALLABLE; + Map p = RowBuilder.props(); + p.put("id", c.getId()); + p.put("name", signature.contains("(") ? signature.substring(0, signature.indexOf('(')) : signature); + p.put("signature", signature); + p.put("kind", c.getKind()); + p.put("declaration", c.getDeclaration()); + p.put("return_type", c.getReturnType()); + if (!c.getParameters().isEmpty()) { + p.put("parameters_json", V2Json.compact().toJson(c.getParameters())); + } + p.put("modifiers", c.getModifiers()); + p.put("error_channel", c.getErrorChannel()); + p.put("code", slice(module.getSource(), c.getSpan())); + p.put("docstring", docstringOf(c.getComments())); + if (c.getMetrics() != null) { + p.put("cyclomatic_complexity", c.getMetrics().getCyclomatic()); + } + if (c.getRefs() != null) { + p.put("referenced_types", c.getRefs().getTypes()); + p.put("accessed_fields", c.getRefs().getFields()); + } + if (c.isImplicit()) { + p.put("is_implicit", true); + } + if (c.isEntrypoint()) { + p.put("is_entrypoint", true); + } + putLines(p, c.getSpan()); + p.put("_module", fileKey); + NodeRef ref = b.node(labels, "id", c.getId(), RowBuilder.prune(p)); + b.edge("J_HAS_METHOD", owner, ref); + annotate(b, ref, c.getDecorators()); + + for (JVariableDeclaration v : c.getLocalVariables()) { + Map vp = RowBuilder.props(); + int line = v.getSpan() != null && v.getSpan().getStart() != null ? v.getSpan().getStart()[0] : 0; + String id = c.getId() + "#" + v.getName() + "@" + line; + vp.put("id", id); + vp.put("name", v.getName()); + vp.put("type", v.getType()); + vp.put("initializer", v.getInitializer()); + putLines(vp, v.getSpan()); + vp.put("_module", fileKey); + NodeRef vr = b.node(Arrays.asList("JVariable"), "id", id, RowBuilder.prune(vp)); + b.edge("J_DECLARES_VAR", ref, vr); + } + + for (Map.Entry e : c.getBody().entrySet()) { + JBodyNode n = e.getValue(); + String id = globalOrdinal(c.getId(), e.getKey()); + Map np = RowBuilder.props(); + np.put("id", id); + np.put("kind", n.getKind()); + np.put("method_name", n.getMethodName()); + np.put("receiver_expr", n.getReceiverExpr()); + np.put("receiver_type", n.getReceiverType()); + np.put("return_type", n.getReturnType()); + np.put("accessibility", n.getAccessibility()); + if (n.isConstructorCall()) { + np.put("is_constructor_call", true); + } + np.put("is_static_call", n.getIsStaticCall()); + np.put("argument_types", n.getArgumentTypes()); + np.put("argument_expr", n.getArgumentExpr()); + putLines(np, n.getSpan()); + np.put("_module", fileKey); + NodeRef nr = b.node(Arrays.asList("JBodyNode"), "id", id, RowBuilder.prune(np)); + b.edge("J_HAS_BODY_NODE", ref, nr); + if (n.getCallee() != null) { + b.edgeToSymbol("J_RESOLVES_TO", nr, n.getCallee()); + } + } + + if (c.getCfg() != null) { + for (JCfgEdge e : c.getCfg()) { + Map ep = RowBuilder.props(); + ep.put("kind", e.getKind()); + b.keyedEdge("J_CFG_NEXT", bodyRef(c, e.getSrc()), bodyRef(c, e.getDst()), + RowBuilder.prune(ep), e.getKind() == null ? "" : e.getKind()); + } + } + if (c.getCdg() != null) { + for (JCdgEdge e : c.getCdg()) { + b.edge("J_CDG", bodyRef(c, e.getSrc()), bodyRef(c, e.getDst())); + } + } + if (c.getDdg() != null) { + for (JDdgEdge e : c.getDdg()) { + Map ep = RowBuilder.props(); + ep.put("var", e.getVar()); + ep.put("prov", e.getProv()); + String k = (e.getVar() == null ? "" : e.getVar()) + "|" + + (e.getProv() == null ? "" : String.join(",", e.getProv())); + b.keyedEdge("J_DDG", bodyRef(c, e.getSrc()), bodyRef(c, e.getDst()), + RowBuilder.prune(ep), k); + } + } + + for (Map.Entry local : c.getTypes().entrySet()) { + projectType(b, ref, "J_DECLARES", local.getKey(), local.getValue(), module, fileKey, + typeIdByFqn); + } + } + + private static NodeRef bodyRef(JCallable c, String localKey) { + return new NodeRef("JBodyNode", "id", globalOrdinal(c.getId(), localKey)); + } + + /** {@code @entry}-style synthetic keys concatenate; real {@code line:col} keys get an {@code @}. */ + private static String globalOrdinal(String callableId, String localKey) { + return localKey.startsWith("@") ? callableId + localKey : callableId + "@" + localKey; + } + + // ------------------------------------------------------------------------------------------ + // Imports, annotations, shared helpers + // ------------------------------------------------------------------------------------------ + + private static void projectImports(RowBuilder b, NodeRef mod, JModule module, + Map typeIdByFqn, Map moduleIdByFqn) { + // Aggregate per target so one edge carries every spelling importing it (python parity). + Map> byTarget = new LinkedHashMap<>(); + Map refByTarget = new LinkedHashMap<>(); + for (JImport imp : module.getImports()) { + String path = imp.getPath(); + if (path == null || path.isEmpty()) { + continue; + } + String targetModule = imp.isWildcard() ? null : moduleIdByFqn.get(path); + String targetKey; + NodeRef target; + if (targetModule != null) { + targetKey = "module:" + targetModule; + target = new NodeRef("JModule", "id", targetModule); + } else { + // Out-of-project (or wildcard): a shared :JPackage keyed by the package prefix. + String pkg = imp.isWildcard() ? path + : (path.contains(".") ? path.substring(0, path.lastIndexOf('.')) : path); + targetKey = "package:" + pkg; + target = b.node(Arrays.asList("JPackage"), "name", pkg, + RowBuilder.prune(mapOf("name", pkg))); + } + byTarget.computeIfAbsent(targetKey, x -> new ArrayList<>()).add(imp); + refByTarget.put(targetKey, target); + } + for (Map.Entry> e : byTarget.entrySet()) { + List spellings = new ArrayList<>(); + boolean anyStatic = false; + boolean anyWildcard = false; + for (JImport imp : e.getValue()) { + spellings.add(imp.getPath()); + anyStatic |= imp.isStatic(); + anyWildcard |= imp.isWildcard(); + } + Map p = RowBuilder.props(); + p.put("spellings", spellings); + if (anyStatic) { + p.put("is_static", true); + } + if (anyWildcard) { + p.put("is_wildcard", true); + } + // In-project targets are :JModule rows this same run emits; the rest are shared + // :JPackage rows created above — either way the endpoint exists, no defer-gating. + b.edge("J_IMPORTS", mod, refByTarget.get(e.getKey()), RowBuilder.prune(p)); + } + } + + private static void annotate(RowBuilder b, NodeRef owner, List decorators) { + for (JDecorator d : decorators) { + if (d.getName() == null || d.getName().isEmpty()) { + continue; + } + NodeRef ann = b.node(Arrays.asList("JAnnotation"), "name", d.getName(), + RowBuilder.prune(mapOf("name", d.getName()))); + Map p = RowBuilder.props(); + p.put("arguments", d.getArgs()); + b.edge("J_ANNOTATED_BY", owner, ann, RowBuilder.prune(p)); + } + } + + /** + * Resolve a type spelling from the tree ({@code com.example.Base} or a bare {@code Base}) to an + * in-project type id, trying the spelling as-is then package-qualified. Unresolved spellings + * stay as the string fallback on the node's {@code base_types}/{@code interfaces} props. + */ + private static String resolveType(String spelling, String pkg, Map typeIdByFqn) { + if (spelling == null || spelling.isEmpty()) { + return null; + } + String direct = typeIdByFqn.get(spelling); + if (direct != null) { + return direct; + } + if (pkg != null && !pkg.isEmpty()) { + return typeIdByFqn.get(pkg + "." + spelling); + } + return null; + } + + /** Join the javadoc comments into one {@code docstring}; non-doc comments are not projected. */ + private static String docstringOf(List comments) { + if (comments == null || comments.isEmpty()) { + return null; + } + List docs = new ArrayList<>(); + for (JComment c : comments) { + if (c.isJavadoc() && c.getContent() != null && !c.getContent().isEmpty()) { + docs.add(c.getContent()); + } + } + return docs.isEmpty() ? null : String.join("\n", docs); + } + + /** UTF-8 byte slice of the module source per the node's {@code span.bytes}. */ + private static String slice(String source, Span span) { + if (source == null || span == null || span.getBytes() == null || span.getBytes().length < 2) { + return null; + } + byte[] bytes = source.getBytes(StandardCharsets.UTF_8); + int start = span.getBytes()[0]; + int end = span.getBytes()[1]; + if (start < 0 || end > bytes.length || start >= end) { + return null; + } + return new String(bytes, start, end - start, StandardCharsets.UTF_8); + } + + private static void putLines(Map p, Span span) { + if (span != null && span.getStart() != null && span.getStart().length > 0) { + p.put("start_line", span.getStart()[0]); + } + if (span != null && span.getEnd() != null && span.getEnd().length > 0) { + p.put("end_line", span.getEnd()[0]); + } + } + + private static Map mapOf(String k, Object v) { + Map m = RowBuilder.props(); + m.put(k, v); + return m; + } +} diff --git a/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java b/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java new file mode 100644 index 00000000..2f420504 --- /dev/null +++ b/src/main/java/com/ibm/cldk/neo4j/V2SchemaCatalog.java @@ -0,0 +1,226 @@ +/* +Copyright IBM Corporation 2023, 2024 + +Licensed under the Apache Public License 2.0, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.ibm.cldk.neo4j; + +import com.ibm.cldk.neo4j.SchemaCatalog.NodeLabel; +import com.ibm.cldk.neo4j.SchemaCatalog.RelType; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * The schema v2 Neo4j graph catalog (graph contract {@code 2.0.0}) — the in-repo source of truth + * for what {@link V2GraphProjector} may emit, serialized by {@code --emit schema} and enforced by + * the v2 conformance test. Mirrors codeanalyzer-python's {@code neo4j/schema.py} vocabulary with + * {@code J}/{@code J_} namespacing; java-only constructs (enum constants, record components, + * {@code J_IMPLEMENTS}) are additive leaves per the cross-language parity clause + * (design spec: docs/design/specs/2026-08-27-v2-neo4j-projection.md). + * + *

Per the python convergence spec there are no call-site / parameter / comment nodes: call sites + * are {@code :JBodyNode} rows with {@code kind == "call"}, parameters flatten to + * {@code JCallable.parameters_json}, and javadoc collapses to a {@code docstring} property. + * + *

{@code SCHEMA_VERSION} bump policy: MAJOR on a breaking change (renamed/removed label, + * relationship or key), MINOR on an additive one. Stamped onto {@code :JApplication} so consumers + * detect producer/consumer mismatch at runtime. + */ +public final class V2SchemaCatalog { + + private V2SchemaCatalog() {} + + public static final String SCHEMA_VERSION = "2.0.0"; + + /** Labels layered onto a node in addition to its merge + specific labels. */ + public static final List MARKER_LABELS = Arrays.asList("JEntrypoint"); + + /** Tiny ordered-map builder for property declarations. */ + private static final class P { + private final Map m = new LinkedHashMap<>(); + + P put(String k, String v) { + m.put(k, v); + return this; + } + + Map done() { + return m; + } + } + + private static Map lines(P p) { + return p.put("start_line", "integer").put("end_line", "integer").done(); + } + + public static final List NODE_LABELS = buildNodeLabels(); + public static final List REL_TYPES = buildRelTypes(); + + private static NodeLabel node(String label, String merge, String key, Map props) { + return new NodeLabel(label, merge, key, props); + } + + private static List buildNodeLabels() { + List n = new ArrayList<>(); + + n.add(node("JApplication", "JApplication", "name", + new P().put("name", "string").put("schema_version", "string") + .put("analyzer_name", "string").put("analyzer_version", "string").done())); + + n.add(node("JModule", "JModule", "id", + new P().put("id", "string").put("file_key", "string").put("package", "string") + .put("content_hash", "string").put("_module", "string").done())); + + n.add(node("JType", "JSymbol", "id", + lines(new P().put("id", "string").put("name", "string").put("kind", "string") + .put("modifiers", "string[]").put("base_types", "string[]") + .put("interfaces", "string[]").put("docstring", "string") + .put("is_entrypoint", "boolean").put("_module", "string")))); + + n.add(node("JCallable", "JSymbol", "id", + lines(new P().put("id", "string").put("name", "string").put("signature", "string") + .put("kind", "string").put("declaration", "string").put("return_type", "string") + .put("parameters_json", "string").put("modifiers", "string[]") + .put("error_channel", "string[]").put("code", "string").put("docstring", "string") + .put("cyclomatic_complexity", "integer") + .put("referenced_types", "string[]").put("accessed_fields", "string[]") + .put("is_implicit", "boolean").put("is_entrypoint", "boolean") + .put("_module", "string")))); + + n.add(node("JExternal", "JSymbol", "id", + new P().put("id", "string").put("kind", "string").put("signature", "string") + .put("declaring_type", "string").done())); + + n.add(node("JField", "JField", "id", + lines(new P().put("id", "string").put("name", "string").put("type", "string") + .put("initializer", "string").put("modifiers", "string[]") + .put("docstring", "string").put("_module", "string")))); + + n.add(node("JVariable", "JVariable", "id", + lines(new P().put("id", "string").put("name", "string").put("type", "string") + .put("initializer", "string").put("_module", "string")))); + + n.add(node("JEnumConstant", "JEnumConstant", "id", + new P().put("id", "string").put("name", "string").put("arguments", "string[]") + .put("docstring", "string").put("_module", "string").done())); + + n.add(node("JRecordComponent", "JRecordComponent", "id", + new P().put("id", "string").put("name", "string").put("type", "string") + .put("modifiers", "string[]").put("is_variadic", "boolean") + .put("docstring", "string").put("_module", "string").done())); + + n.add(node("JBodyNode", "JBodyNode", "id", + lines(new P().put("id", "string").put("kind", "string").put("method_name", "string") + .put("receiver_expr", "string").put("receiver_type", "string") + .put("return_type", "string").put("accessibility", "string") + .put("is_constructor_call", "boolean").put("is_static_call", "boolean") + .put("argument_types", "string[]").put("argument_expr", "string[]") + .put("_module", "string")))); + + n.add(node("JPackage", "JPackage", "name", new P().put("name", "string").done())); + + n.add(node("JAnnotation", "JAnnotation", "name", new P().put("name", "string").done())); + + return n; + } + + private static RelType rel(String type, List from, List to, Map props) { + return new RelType(type, from, to, props); + } + + private static List buildRelTypes() { + List r = new ArrayList<>(); + Map none = new LinkedHashMap<>(); + List symbol = Arrays.asList("JCallable", "JExternal"); + List body = Arrays.asList("JBodyNode"); + + r.add(rel("J_HAS_MODULE", Arrays.asList("JApplication"), Arrays.asList("JModule"), none)); + r.add(rel("J_DECLARES", Arrays.asList("JModule", "JType", "JCallable"), Arrays.asList("JType"), none)); + r.add(rel("J_HAS_METHOD", Arrays.asList("JType"), Arrays.asList("JCallable"), none)); + r.add(rel("J_HAS_FIELD", Arrays.asList("JType"), Arrays.asList("JField"), none)); + r.add(rel("J_DECLARES_VAR", Arrays.asList("JCallable"), Arrays.asList("JVariable"), none)); + r.add(rel("J_HAS_ENUM_CONSTANT", Arrays.asList("JType"), Arrays.asList("JEnumConstant"), none)); + r.add(rel("J_HAS_RECORD_COMPONENT", Arrays.asList("JType"), Arrays.asList("JRecordComponent"), none)); + r.add(rel("J_HAS_BODY_NODE", Arrays.asList("JCallable"), body, none)); + r.add(rel("J_RESOLVES_TO", body, symbol, none)); + r.add(rel("J_CALLS", symbol, symbol, + new P().put("weight", "integer").put("prov", "string[]").done())); + r.add(rel("J_EXTENDS", Arrays.asList("JType"), Arrays.asList("JType"), none)); + r.add(rel("J_IMPLEMENTS", Arrays.asList("JType"), Arrays.asList("JType"), none)); + r.add(rel("J_IMPORTS", Arrays.asList("JModule"), Arrays.asList("JModule", "JPackage"), + new P().put("spellings", "string[]").put("is_static", "boolean") + .put("is_wildcard", "boolean").done())); + r.add(rel("J_ANNOTATED_BY", Arrays.asList("JType", "JCallable", "JField"), + Arrays.asList("JAnnotation"), new P().put("arguments", "string[]").done())); + // L3 CPG overlay. `_k` is the MERGE discriminant (internal, underscore-prefixed): J_CFG_NEXT + // merges per `kind` (a conditional's true/false pair), J_DDG per `(var, prov)`. + r.add(rel("J_CFG_NEXT", body, body, new P().put("kind", "string").put("_k", "string").done())); + r.add(rel("J_CDG", body, body, none)); + r.add(rel("J_DDG", body, body, + new P().put("var", "string").put("prov", "string[]").put("_k", "string").done())); + // L4 SDG — declared so the contract is stable; not emitted until L4 lands. + r.add(rel("J_PARAM_IN", body, body, new P().put("var", "string").done())); + r.add(rel("J_PARAM_OUT", body, body, new P().put("var", "string").done())); + r.add(rel("J_SUMMARY", body, body, none)); + + return r; + } + + /** One uniqueness constraint per distinct {@code (merge_label, key)}, deterministically named. */ + public static List uniquenessConstraints() { + Map seen = new LinkedHashMap<>(); + for (NodeLabel nl : NODE_LABELS) { + seen.putIfAbsent(nl.mergeLabel, nl.key); + } + List out = new ArrayList<>(); + for (Map.Entry e : seen.entrySet()) { + out.add("CREATE CONSTRAINT " + e.getKey().toLowerCase() + "_" + e.getValue() + + " IF NOT EXISTS FOR (x:" + e.getKey() + ") REQUIRE x." + e.getValue() + " IS UNIQUE"); + } + return out; + } + + /** Build the machine-readable schema document emitted by {@code --emit schema}. */ + public static Map buildSchemaDocument() { + Map doc = new LinkedHashMap<>(); + doc.put("schema_version", SCHEMA_VERSION); + doc.put("generator", "codeanalyzer-java"); + doc.put("marker_labels", MARKER_LABELS); + + List> nodeLabels = new ArrayList<>(); + for (NodeLabel nl : NODE_LABELS) { + Map m = new LinkedHashMap<>(); + m.put("label", nl.label); + m.put("merge_label", nl.mergeLabel); + m.put("key", nl.key); + m.put("properties", nl.properties); + nodeLabels.add(m); + } + doc.put("node_labels", nodeLabels); + + List> relTypes = new ArrayList<>(); + for (RelType rt : REL_TYPES) { + Map m = new LinkedHashMap<>(); + m.put("type", rt.type); + m.put("from", rt.from); + m.put("to", rt.to); + m.put("properties", rt.properties); + relTypes.add(m); + } + doc.put("relationship_types", relTypes); + + doc.put("constraints", uniquenessConstraints()); + doc.put("indexes", Schema.INDEXES); + return doc; + } +} diff --git a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java index a82b1333..f3065151 100644 --- a/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java +++ b/src/test/java/com/ibm/cldk/CodeAnalyzerV2CliTest.java @@ -118,17 +118,62 @@ void v2SchemaWritesCanonicalEnvelopeToAnalysisJson(@TempDir Path tmp) throws IOE } @Test - void v2SchemaIsNotTheDefault(@TempDir Path tmp) throws IOException { - // The legacy shape stays the default until the rest of the migration lands, so existing - // consumers are unaffected by this change. + void v2SchemaIsTheDefault(@TempDir Path tmp) throws IOException { + // 3.0.0: the canonical shape is the default; --schema v1 opts into the legacy shape. Path in = project(tmp.resolve("app")); Path out = tmp.resolve("out"); assertEquals(0, run("-i", in.toString(), "-o", out.toString())); JsonObject root = JsonParser.parseString(Files.readString(out.resolve("analysis.json"))).getAsJsonObject(); - assertFalse(root.has("schema_version"), "default output is still the v1 shape"); + assertEquals("2.0.0", root.get("schema_version").getAsString(), "default output is the v2 envelope"); + } + + @Test + void schemaV1StillEmitsTheLegacyShape(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "--schema", "v1")); + JsonObject root = JsonParser.parseString(Files.readString(out.resolve("analysis.json"))).getAsJsonObject(); + assertFalse(root.has("schema_version"), "--schema v1 keeps the legacy shape"); assertTrue(root.has("symbol_table"), "v1 keeps symbol_table at the top level"); } + @Test + void emitNeo4jWritesTheV2GraphAtFullDepth(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + Path out = tmp.resolve("out"); + assertEquals(0, run("-i", in.toString(), "-o", out.toString(), "--emit", "neo4j", "--no-build", + "--app-name", "widgets")); + Path cypher = out.resolve("graph.cypher"); + assertTrue(Files.exists(cypher), "graph.cypher must be written"); + String script = Files.readString(cypher); + assertTrue(script.contains(":JModule"), "v2 graph projects :JModule rows"); + assertTrue(script.contains(":JBodyNode"), "full depth: the L3 body must be projected"); + assertTrue(script.contains("J_CFG_NEXT"), "full depth: cfg overlay edges must be present"); + assertFalse(script.contains(":JCompilationUnit {"), "v1 node rows must not appear in a v2 graph"); + } + + @Test + void emitNeo4jRejectsAnalysisLevelAndFieldDepth(@TempDir Path tmp) throws IOException { + Path in = project(tmp.resolve("app")); + assertNotEquals(0, run("-i", in.toString(), "--emit", "neo4j", "-a", "2"), + "the graph is always full depth; -a with --emit neo4j must error"); + assertNotEquals(0, run("-i", in.toString(), "--emit", "neo4j", "--graph-field-depth", "2"), + "--graph-field-depth with --emit neo4j must error"); + } + + @Test + void emitSchemaAlwaysEmitsTheV2Catalog(@TempDir Path tmp) throws IOException { + Path out = tmp.resolve("out"); + assertEquals(0, run("--emit", "schema", "-o", out.toString())); + JsonObject doc = JsonParser.parseString(Files.readString(out.resolve("schema.neo4j.json"))) + .getAsJsonObject(); + assertEquals("2.0.0", doc.get("schema_version").getAsString()); + assertEquals(0, run("--emit", "schema", "-o", out.toString(), "--schema", "v1"), + "--emit schema ignores --schema"); + assertEquals("2.0.0", JsonParser.parseString(Files.readString(out.resolve("schema.neo4j.json"))) + .getAsJsonObject().get("schema_version").getAsString()); + } + @Test void cacheFileIsWrittenAndReusedOnASecondRun(@TempDir Path tmp) throws IOException { Path in = project(tmp.resolve("app")); @@ -400,10 +445,4 @@ void v2ExternalCallsFlagHomesOutOfProjectTargets(@TempDir Path tmp) throws IOExc "--external-calls homes out-of-project targets (e.g. java.lang.Math)"); } - @Test - void v2WithNeo4jEmitFailsLoudly(@TempDir Path tmp) throws IOException { - Path in = project(tmp.resolve("app")); - assertNotEquals(0, run("-i", in.toString(), "--schema", "v2", "--emit", "neo4j"), - "the graph projection is still v1"); - } } diff --git a/src/test/java/com/ibm/cldk/neo4j/Neo4jSchemaConformanceTest.java b/src/test/java/com/ibm/cldk/neo4j/Neo4jSchemaConformanceTest.java index 268b764e..7e4d1402 100644 --- a/src/test/java/com/ibm/cldk/neo4j/Neo4jSchemaConformanceTest.java +++ b/src/test/java/com/ibm/cldk/neo4j/Neo4jSchemaConformanceTest.java @@ -128,11 +128,12 @@ public void everyEmittedRelationshipIsDeclared() { @Test public void checkedInSchemaMatchesCatalog() throws Exception { + // The checked-in artifact is the v2 contract — `--emit schema` always emits the v2 catalog. Path onDiskPath = Paths.get("schema.neo4j.json"); assertTrue(Files.exists(onDiskPath), "schema.neo4j.json missing — run `--emit schema`"); String onDisk = new String(Files.readAllBytes(onDiskPath), StandardCharsets.UTF_8).trim(); String fresh = new GsonBuilder().setPrettyPrinting().disableHtmlEscaping().create() - .toJson(SchemaCatalog.buildSchemaDocument()).trim(); + .toJson(V2SchemaCatalog.buildSchemaDocument()).trim(); assertEquals(fresh, onDisk, "schema.neo4j.json is stale — regenerate with `--emit schema`"); } } diff --git a/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java b/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java new file mode 100644 index 00000000..f214706a --- /dev/null +++ b/src/test/java/com/ibm/cldk/neo4j/V2Neo4jSchemaConformanceTest.java @@ -0,0 +1,177 @@ +/* +Copyright IBM Corporation 2023, 2024 + +Licensed under the Apache Public License 2.0, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package com.ibm.cldk.neo4j; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.ibm.cldk.neo4j.GraphRows.EdgeRow; +import com.ibm.cldk.neo4j.GraphRows.NodeRow; +import com.ibm.cldk.neo4j.SchemaCatalog.NodeLabel; +import com.ibm.cldk.neo4j.SchemaCatalog.RelType; +import com.ibm.cldk.schema.Analysis; +import com.ibm.cldk.schema.JModule; +import com.ibm.cldk.schema.V2Emitter; +import com.ibm.cldk.syntactic_analysis.L1Extractor; +import com.ibm.cldk.syntactic_analysis.L2CallGraph; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +/** + * Schema v2 graph conformance (no container needed): run the real L1–L3 pipeline over a fixture, + * project with {@link V2GraphProjector}, and assert the projector only ever produces what + * {@link V2SchemaCatalog} declares — the anti-drift guard for the 2.0.0 graph contract. Also pins + * the convergence decisions: body nodes instead of call-site nodes, and the {@code _k}-keyed + * CFG/DDG relationships. + */ +public class V2Neo4jSchemaConformanceTest { + + private static final Path FIXTURE = Paths.get("src/test/resources/test-applications/call-graph-test"); + + private static GraphRows rows; + + private static final Map BY_LABEL = new HashMap<>(); + private static final Map MERGE_OF = new HashMap<>(); + private static final Map REL_BY_TYPE = new HashMap<>(); + private static final Set MARKERS = new HashSet<>(V2SchemaCatalog.MARKER_LABELS); + + @BeforeAll + static void project() throws Exception { + for (NodeLabel nl : V2SchemaCatalog.NODE_LABELS) { + BY_LABEL.put(nl.label, nl); + MERGE_OF.put(nl.label, nl.mergeLabel); + } + for (RelType rt : V2SchemaCatalog.REL_TYPES) { + REL_BY_TYPE.put(rt.type, rt); + } + Map modules = L1Extractor.extractAll( + FIXTURE, "call-graph-test", null, new LinkedHashMap<>(), 3, 3, "ast"); + L2CallGraph.Result l2 = L2CallGraph.build("call-graph-test", modules, null, true); + Analysis analysis = V2Emitter.emit( + "call-graph-test", 3, modules, "test", l2.callGraph(), l2.externalSymbols()); + rows = V2GraphProjector.project(analysis, "call-graph-test"); + } + + private static String specificLabel(List labels) { + String merge = labels.get(0); + if (!merge.equals("JSymbol")) { + return merge; + } + for (String l : labels) { + if (!l.equals("JSymbol") && !MARKERS.contains(l)) { + return l; + } + } + return "JSymbol"; + } + + private static Set mergeLabelsFor(List specifics) { + Set out = new HashSet<>(); + for (String s : specifics) { + out.add(MERGE_OF.get(s)); + } + return out; + } + + @Test + public void everyEmittedNodeLabelAndPropertyIsDeclared() { + assertTrue(rows.nodes.size() > 0, "fixture produced no nodes"); + for (NodeRow node : rows.nodes) { + String specific = specificLabel(node.labels); + NodeLabel decl = BY_LABEL.get(specific); + assertNotNull(decl, "undeclared node label: " + String.join(":", node.labels)); + assertEquals(decl.mergeLabel, node.labels.get(0), "wrong merge label for " + specific); + for (String label : node.labels) { + boolean ok = label.equals(decl.mergeLabel) || label.equals(specific) || MARKERS.contains(label); + assertTrue(ok, "unexpected label '" + label + "' on " + specific); + } + for (String key : node.props.keySet()) { + assertTrue(decl.properties.containsKey(key), "undeclared property '" + specific + "." + key + "'"); + } + } + } + + @Test + public void everyEmittedRelationshipIsDeclared() { + assertTrue(rows.edges.size() > 0, "fixture produced no edges"); + for (EdgeRow edge : rows.edges) { + RelType decl = REL_BY_TYPE.get(edge.type); + assertNotNull(decl, "undeclared relationship type: " + edge.type); + assertTrue(mergeLabelsFor(decl.from).contains(edge.from.label), + "bad source " + edge.from.label + " for " + edge.type); + assertTrue(mergeLabelsFor(decl.to).contains(edge.to.label), + "bad target " + edge.to.label + " for " + edge.type); + for (String key : edge.props.keySet()) { + assertTrue(decl.properties.containsKey(key), "undeclared property on " + edge.type + "." + key); + } + } + } + + @Test + public void convergedNodeModelHasBodyNodesAndNoV1OnlyNodes() { + boolean sawModule = false; + boolean sawBodyNode = false; + for (NodeRow node : rows.nodes) { + String merge = node.labels.get(0); + sawModule |= merge.equals("JModule"); + sawBodyNode |= merge.equals("JBodyNode"); + assertFalse(merge.equals("JCompilationUnit") || merge.equals("JCallSite") + || merge.equals("JParameter") || merge.equals("JComment"), + "v1-only node label leaked into the v2 projection: " + merge); + } + assertTrue(sawModule, "no :JModule rows projected"); + assertTrue(sawBodyNode, "no :JBodyNode rows projected — the L3 body did not project"); + } + + @Test + public void l3OverlayEdgesAreKeyedAndPresent() { + boolean sawCfg = false; + boolean sawHasBody = false; + for (EdgeRow edge : rows.edges) { + if (edge.type.equals("J_CFG_NEXT")) { + sawCfg = true; + assertNotNull(edge.key, "J_CFG_NEXT must carry the _k MERGE discriminant"); + } + if (edge.type.equals("J_DDG")) { + assertNotNull(edge.key, "J_DDG must carry the _k MERGE discriminant"); + } + sawHasBody |= edge.type.equals("J_HAS_BODY_NODE"); + } + assertTrue(sawCfg, "no J_CFG_NEXT edges — the L3 cfg overlay did not project"); + assertTrue(sawHasBody, "no J_HAS_BODY_NODE edges"); + } + + @Test + public void wipeCoversBothGenerationsSoV2ReplacesAPriorV1Graph() { + String cypher = CypherWriter.renderCypher(rows, "call-graph-test"); + assertTrue(cypher.contains("J_HAS_UNIT|J_HAS_MODULE"), + "the wipe must traverse both generations' unit relationship"); + assertTrue(cypher.contains("MATCH (s:JSymbol) WHERE NOT (s)--() DELETE s"), + "the wipe must sweep orphaned symbols (v1 import-materialized type stubs)"); + for (String rel : new String[] {"J_HAS_CALLSITE", "J_HAS_COMMENT", "J_HAS_PARAMETER", + "J_DECLARES", "J_HAS_METHOD", "J_HAS_BODY_NODE"}) { + assertTrue(CypherWriter.DESCENDANTS.contains(rel), + "wipe/prune descendant traversal must include " + rel); + } + } +}