diff --git a/.githooks/pre-commit b/.githooks/pre-commit index defbbc5c..c386bbb8 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -21,7 +21,7 @@ # # It checks the *index*, not the working tree, because the index is what you # are about to commit. Staging pom.xml while leaving the matching -# connector/*/pom.xml edits unstaged is precisely the drift being guarded +# licensed-only/*/pom.xml edits unstaged is precisely the drift being guarded # against, and a working-tree check would call that clean. set -uo pipefail @@ -77,7 +77,7 @@ pre-commit: the parser version disagrees across the POMs you are committing. git add -u && git commit then commit again Never edit a version by hand: it is written in pom.xml and in each of the three -connector/*/pom.xml, which are separate builds with no parent to inherit it. +licensed-only/*/pom.xml, which are separate builds with no parent to inherit it. In practice you should not be editing one at all -- the nightly tests the newest release and opens a pre-verified bump PR. diff --git a/.github/scripts/check-licensed-only-guard.sh b/.github/scripts/check-licensed-only-guard.sh new file mode 100755 index 00000000..0fce7ccb --- /dev/null +++ b/.github/scripts/check-licensed-only-guard.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# +# Prove the licensed-only modules refuse to build with an explanation. +# +# licensed-only/{oracle,snowflake,sqlServer}Connector need a LICENSED parser: +# the public trial artifact this repository resolves does not ship +# gudusoft.gsqlparser.sqlenv.T*SQLDataSource. Before 2026-08-24 they simply +# failed -- javac's "cannot find symbol" for snowflake, and for the other two a +# dependency-resolution error about a JDBC jar nobody had, thrown before any +# plugin could speak. A first-time evaluator read that as "this library does not +# compile" and said so in an evaluation report. +# +# Now each POM stops at validate with a message naming the licensing boundary +# and pointing at the trial-friendly alternative. That message is the whole +# feature, so this asserts on the message, not on the exit status: a build that +# fails for the old confusing reason also exits non-zero. +# +# Usage: +# check-licensed-only-guard.sh + +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../.." + +MODULES=(oracleConnector snowflakeConnector sqlServerConnector) +NEEDLE="needs a LICENSED General SQL Parser" + +failed=0 + +for m in "${MODULES[@]}"; do + pom="licensed-only/$m/pom.xml" + if [ ! -f "$pom" ]; then + echo " FAIL $pom is missing" + failed=1 + continue + fi + + out=$(mvn -B -f "$pom" validate 2>&1 || true) + + if ! grep -qF "$NEEDLE" <<<"$out"; then + echo " FAIL $m built or failed without explaining the licence boundary" + echo "$out" | tail -20 | sed 's/^/ /' + failed=1 + continue + fi + + # It must be the guard that stopped it, not something incidental that + # happened to print the same words. + if ! grep -q "BUILD FAILURE" <<<"$out"; then + echo " FAIL $m printed the message but the build succeeded" + failed=1 + continue + fi + + echo " ok $m stops at validate and says why" +done + +echo + +if [ "$failed" -ne 0 ]; then + echo "::error::a licensed-only module no longer refuses the trial build with an explanation" + exit 1 +fi + +echo "ok: all ${#MODULES[@]} licensed-only modules refuse the trial build with an explanation" diff --git a/.github/scripts/set-parser-version.sh b/.github/scripts/set-parser-version.sh index 8fcdf525..0dde8b99 100755 --- a/.github/scripts/set-parser-version.sh +++ b/.github/scripts/set-parser-version.sh @@ -4,9 +4,10 @@ # it down. # # The version lives in four files. The root build declares it as a -# ${gsp.core.version} property, and the three connector/ modules hardcode the -# version in their gsqlparser dependency, because they are separate builds with -# no parent to inherit a property from. Nothing made them agree, so they could +# ${gsp.core.version} property, and the three licensed-only/ connector modules +# hardcode the version in their gsqlparser dependency, because they are separate +# builds with no parent to inherit a property from. (They lived under +# connector/ until 2026-08-24; see licensed-only/README.md for why they moved.) Nothing made them agree, so they could # drift apart silently -- and a bump meant four hand edits, which is most of why # bumping felt expensive. # @@ -61,8 +62,8 @@ TARGETS = [ ] for mod in ("oracleConnector", "snowflakeConnector", "sqlServerConnector"): TARGETS.append(( - "connector/%s/pom.xml" % mod, - "connector/%s dependency" % mod, + "licensed-only/%s/pom.xml" % mod, + "licensed-only/%s dependency" % mod, # Anchor on the gsqlparser dependency so we never touch the JDBC # driver's sitting a few lines below it. r"(?s)(gsqlparser\s*)([^<]+)()", diff --git a/.github/scripts/smoke-dlineage-jar.sh b/.github/scripts/smoke-dlineage-jar.sh index d9e89963..b71a4fc6 100755 --- a/.github/scripts/smoke-dlineage-jar.sh +++ b/.github/scripts/smoke-dlineage-jar.sh @@ -96,4 +96,43 @@ if not rel: print("ok: XML output, %d relationships" % len(rel)) PY +# --- a whole schema, under the trial cap ----------------------------------- +# samples/dlineage/demo.sql is 366 bytes. Every real schema dump under +# samples/dlineageBasic/ is over the trial parser's 10,000-byte limit, so a +# visitor who tried "lineage on a real schema" got a licence error and no +# lineage. hr_mini exists to be the one that fits, which makes its size the +# thing worth guarding: grow it past the cap and the demo silently stops +# working for every trial user, while the tool goes on exiting 0. +SCHEMA="samples/dlineageBasic/oracle/hr_mini/hr_mini.sql" +CAP=10000 + +[ -f "$SCHEMA" ] || fail "$SCHEMA is missing" + +bytes=$(wc -c <"$SCHEMA") +if [ "$bytes" -ge "$CAP" ]; then + fail "$SCHEMA is $bytes bytes, at or over the trial parser's $CAP-byte limit; it exists precisely so schema-scale lineage is runnable on the trial jar. Shrink it, or put the addition in one of the licensed-only dumps." +fi +echo "ok: $SCHEMA is $bytes bytes, under the $CAP-byte trial limit" + +java -jar "$JAR" /f "$SCHEMA" /t oracle /o "$OUT/schema.json" /json + +[ -s "$OUT/schema.json" ] || fail "schema.json is empty or missing" + +python3 - "$OUT/schema.json" <<'PY' +import json, sys +d = json.load(open(sys.argv[1])) +rel = d.get("relationships") or [] +if not rel: + sys.exit("hr_mini parsed but produced no relationships") +# A licence rejection is not an exception: the tool reports it as an error +# inside an otherwise well-formed document and exits 0, which reads as +# "no lineage found" unless something goes looking for it. +if "trial version can only process" in json.dumps(d): + sys.exit("hr_mini hit the trial size limit; it is no longer trial-evaluable") +if len(rel) < 50: + sys.exit("hr_mini produced only %d relationships; it produced 120 when it " + "was added, so the schema or the analyzer has regressed" % len(rel)) +print("ok: schema-scale lineage, %d relationships from hr_mini" % len(rel)) +PY + echo "ok: the standalone dlineage jar runs and produces lineage in both formats" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 17c84818..911f303d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -73,6 +73,17 @@ jobs: echo .github/scripts/check-stale-docs.sh + # The three licensed-only/ modules cannot compile against the public + # trial parser, and used to say so only in javac's words -- "cannot find + # symbol", or worse, a dependency-resolution error about a JDBC jar + # nobody had. An evaluator read that as a broken library. Each POM now + # stops at validate with a message naming the licensing boundary, and the + # message is the feature, so this asserts on the message: failing for the + # old confusing reason also exits non-zero. + - name: Licensed-only modules refuse the trial build, with a reason + if: matrix.java == '21' + run: .github/scripts/check-licensed-only-guard.sh + - name: Build run: mvn -B package -DskipTests diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5c5b2945..cd54856c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -240,7 +240,7 @@ jobs: --title "Bump the parser from $pinned to $NEW" \ --body "\`$NEW\` is the newest release on sqlparser.com. Tonight's nightly ran it through the same checks the pinned \`$pinned\` gets, and it passed all of them: the full test suite with no failures, every demo starting, and every case in \`.github/scripts/demo-cases.tsv\` producing its expected output. - This touches the four places the version is written -- \`pom.xml\` and the three \`connector/*/pom.xml\` -- via \`.github/scripts/set-parser-version.sh\`, so they cannot drift apart. + This touches the four places the version is written -- \`pom.xml\` and the three \`licensed-only/*/pom.xml\` -- via \`.github/scripts/set-parser-version.sh\`, so they cannot drift apart. Merge to move the demos to \`$NEW\`. Close to stay on \`$pinned\`; the nightly will go on testing \`$NEW\` and will not reopen this until a newer release appears. diff --git a/README.md b/README.md index a391c424..dd64ee59 100644 --- a/README.md +++ b/README.md @@ -146,14 +146,23 @@ any individual demo. `samples/` holds sample `.sql` files to feed them. -### Demos that connect to a database +### Demos that connect to a database — licensed parser only -`connector/{oracleConnector,snowflakeConnector,sqlServerConnector}/` are +`licensed-only/{oracleConnector,snowflakeConnector,sqlServerConnector}/` are separate, independently built Maven modules showing JDBC-connected metadata -extraction. They are **not** part of `mvn package` or `mvn test` at the root; -build each on its own. Each module's `lib/` holds only a readme — you download -the JDBC driver yourself, and the version in that module's `pom.xml` tells you -which. +extraction. **They cannot be built with the trial parser**, which does not ship +`gudusoft.gsqlparser.sqlenv.T*SQLDataSource`; each stops at `validate` with a +message saying so, and `-Plicensed` turns that guard off once you have a +licensed parser. They are not part of `mvn package` or `mvn test` at the root. +See [`licensed-only/README.md`](licensed-only/README.md). + +They sat in `connector/` until 2026-08-24, where they read as part of the +ordinary demo set: a first-time evaluation started there, met `cannot find +symbol`, and concluded the library did not compile. + +**On the trial parser, `columninspect` is the thing to run instead.** It does +the same metadata-aware column resolution from a JSON catalog export rather +than a live connection, and `samples/columninspect/` has a runnable pair. ## Rewriting SQL through the parse tree @@ -217,9 +226,19 @@ version (e.g. `4.1.9`) does not necessarily match the one in the release notes. > **The trial build refuses input larger than 10,000 bytes**, reporting > `trial version can only process query with size of at most 10000 bytes`. +> The limit is on a single parse, not on total throughput. +> > Every demo here works within that except `scriptwriter`, whose built-in query > is ~49 KB on purpose — give it your own smaller file, or use a licensed -> parser. The limit is on a single parse, not on total throughput. +> parser. **16 of the 89 `.sql` files under `samples/` are also over the +> limit**, all of them vendor schema dumps under `samples/dlineageBasic/` +> (10,378 to 99,139 bytes). Those are for licensed evaluation; the rejection +> arrives as an `` inside otherwise-normal output, which reads as "no +> lineage found" rather than as a licence limit. For schema-scale lineage on +> the trial jar use +> [`samples/dlineageBasic/oracle/hr_mini/`](samples/dlineageBasic/oracle/hr_mini/readme.md) +> — 5,212 bytes, 120 relationships, added for that purpose and size-checked in +> CI. ### Published versions are kept; one batch was recalled in July 2026 @@ -265,7 +284,7 @@ One command, never by hand: The version lives in **four** files — the `${gsp.core.version}` property in `pom.xml`, plus a hardcoded `` in each of the three -`connector/*/pom.xml`, which are separate builds with no parent to inherit a +`licensed-only/*/pom.xml`, which are separate builds with no parent to inherit a property from. Both workflows run `--check`, so a missed file is a red build rather than a connector quietly compiling against an older parser. @@ -314,7 +333,7 @@ src/main/java/gudusoft/gsqlparser/demos// the demos, one dir per topic src/main/resources/ classpath resources (one file) src/test/java/gudusoft/gsqlparser/ tests, all exercising demos samples/ sample .sql for the demos -connector/Connector/ separate JDBC-connected modules +licensed-only/Connector/ JDBC modules, licensed parser only lib-repo/ in-project Maven repository setenv/ + per-demo *.bat the Windows route .github/scripts/ CI checks, all runnable locally @@ -408,9 +427,10 @@ than only building them. | Parser version consistency | `set-parser-version.sh --check` across all four POMs | | The pre-commit hook | `test-pre-commit-hook.sh`: a drifting bump is refused in a throwaway clone | | Documentation | `check-stale-docs.sh`: no readme names `pom_dlineage.xml`, `gudusoft.dlineage.jar` or the old `demos` package root; `--self-test` first, so a check that matches nothing cannot pass as a clean repo | +| Licensed-only guard | `check-licensed-only-guard.sh`: each `licensed-only/*` module stops at `validate` **with the licence message**, not with `cannot find symbol` | | Build and test | JDK 8 and 21; 156 tests, and a run that skipped everything fails | | Demo smoke test | `checksyntax` against known SQL | -| Standalone lineage jar | `smoke-dlineage-jar.sh` on JDK 8 and 21 — asserts on **output**, in JSON *and* XML | +| Standalone lineage jar | `smoke-dlineage-jar.sh` on JDK 8 and 21 — asserts on **output**, in JSON *and* XML, and that `hr_mini.sql` stays under the trial parser's 10,000-byte cap | | Windows `.bat` | `windows-latest`: bootstrap, 39 compile scripts, 50 run scripts, 4 driven with real arguments | `.github/workflows/nightly.yml` — at 03:17 UTC, because the parser is the moving @@ -476,7 +496,7 @@ drive with arguments. - **Don't commit jars**; add dependencies by coordinate. - **Don't add a live-JDBC path to a demo** under `src/main/java`. It can't run in CI, and it is what got two demos excluded from the build for years. The - `connector/*` modules are where database connections belong. + `licensed-only/*` modules are where database connections belong. - **Don't add parser tests here**; they belong in `gsp_java_core`. `master` tracks released GSP versions from diff --git a/connector/oracleConnector/lib/readme.md b/connector/oracleConnector/lib/readme.md deleted file mode 100644 index ed6639ef..00000000 --- a/connector/oracleConnector/lib/readme.md +++ /dev/null @@ -1 +0,0 @@ -Please download and put the corresponding jdbc library here before compiling. \ No newline at end of file diff --git a/connector/oracleConnector/pom.xml b/connector/oracleConnector/pom.xml deleted file mode 100644 index 05619202..00000000 --- a/connector/oracleConnector/pom.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - 4.0.0 - gudusoft - gudusoft.gsqlparser.demo.oracleConnector - 1.0-SNAPSHOT - jar - - - UTF-8 - UTF-8 - Gsp Oracle Connector Demo - 1.0 - Gudu software - 1.6 - 1.6 - - - - - gudu-public-releases - https://www.sqlparser.com/maven/ - - - - - - - com.gudusoft - gsqlparser - 4.2.6 - - - com - ojdbc - 1.1.1 - system - ${project.basedir}/lib/ojdbc-1.1.1.jar - - - - - oracleConnector - - - org.apache.maven.plugins - maven-jar-plugin - - target/classes/ - - - demos.connector.OracleDataSourceConnector - false - true - lib/ - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5.1 - - 1.8 - 1.8 - - - - - \ No newline at end of file diff --git a/connector/snowflakeConnector/lib/readme.md b/connector/snowflakeConnector/lib/readme.md deleted file mode 100644 index ed6639ef..00000000 --- a/connector/snowflakeConnector/lib/readme.md +++ /dev/null @@ -1 +0,0 @@ -Please download and put the corresponding jdbc library here before compiling. \ No newline at end of file diff --git a/connector/sqlServerConnector/lib/readme.md b/connector/sqlServerConnector/lib/readme.md deleted file mode 100644 index ed6639ef..00000000 --- a/connector/sqlServerConnector/lib/readme.md +++ /dev/null @@ -1 +0,0 @@ -Please download and put the corresponding jdbc library here before compiling. \ No newline at end of file diff --git a/connector/sqlServerConnector/pom.xml b/connector/sqlServerConnector/pom.xml deleted file mode 100644 index a866f047..00000000 --- a/connector/sqlServerConnector/pom.xml +++ /dev/null @@ -1,74 +0,0 @@ - - - 4.0.0 - gudusoft - gudusoft.gsqlparser.demo.sqlServerConnector - 1.0-SNAPSHOT - jar - - - UTF-8 - UTF-8 - Gsp Sql Server Connector Demo - 1.0 - Gudu software - 1.6 - 1.6 - - - - - gudu-public-releases - https://www.sqlparser.com/maven/ - - - - - - - com.gudusoft - gsqlparser - 4.2.6 - - - com.microsoft.sqlserver - sqljdbc4 - 4.0 - system - ${project.basedir}/lib/sqljdbc4-4.0.jar - - - - - sqlServerConnector - - - org.apache.maven.plugins - maven-jar-plugin - - target/classes/ - - - demos.connector.SqlServerDataSourceConnector - false - true - lib/ - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5.1 - - 1.8 - 1.8 - - - - - \ No newline at end of file diff --git a/licensed-only/README.md b/licensed-only/README.md new file mode 100644 index 00000000..288c5dbe --- /dev/null +++ b/licensed-only/README.md @@ -0,0 +1,72 @@ +# Licensed-only demos + +**Nothing in this directory builds with the trial parser, and that is not a +bug.** These three modules extract table and column metadata from a live +database over JDBC, which needs `gudusoft.gsqlparser.sqlenv.T*SQLDataSource` +and `sqlenv.metadata.DDL` — classes the public trial artifact this repository +resolves does not ship. Confirmed against `gsqlparser-4.2.6.jar`: `TSQLEnv` and +`TJSONSQLEnvParser` are present, `TOracleSQLDataSource`, +`TSnowflakeSQLDataSource`, `TMssqlSQLDataSource` and `metadata.DDL` are not. + +They used to live in `connector/` at the repository root, where they read as +part of the ordinary demo set. A first-time evaluation in August 2026 started +there, hit a wall of `cannot find symbol`, and reasonably concluded the library +did not compile. Moving them here and having them refuse the build with an +explanation is the fix for that; it is not a statement about the code, which is +fine and works on a licensed parser. + +```console +$ mvn -f licensed-only/snowflakeConnector/pom.xml compile +[ERROR] This module needs a LICENSED General SQL Parser, not the public trial +jar this repository resolves by default. ... +``` + +With a licensed parser installed, build normally: + +```bash +mvn -f licensed-only/snowflakeConnector/pom.xml -Plicensed package +``` + +`-Plicensed` deactivates the default `trial-guard` profile. Nothing else about +the modules changes. + +## What to evaluate instead + +Metadata-aware column resolution works on the trial parser — it just takes the +metadata offline rather than over JDBC. `columninspect` reads the same JSON a +catalog export produces, and `samples/columninspect/` has a runnable pair: + +```bash +mvn -q exec:java \ + -Dexec.mainClass=gudusoft.gsqlparser.demos.columninspect.ColumnInspect \ + -Dexec.args="/t mssql /f samples/columninspect/sample.sql \ + /metadata samples/columninspect/metadata.json \ + /db testdb /schema dbo" +``` + +`gettablecolumns/runGetTableColumn` takes the same approach with a `TSQLEnv` +built in code. Both were themselves live-JDBC demos until 2026-07-28 and were +converted rather than deleted, for exactly this reason. + +## The modules + +| module | driver | main class | +|---|---|---| +| `oracleConnector` | `com.oracle.database.jdbc:ojdbc8` | `demos.connector.OracleDataSourceConnector` | +| `snowflakeConnector` | `net.snowflake:snowflake-jdbc` | `demos.connector.SnowflakeDataSourceConnector` | +| `sqlServerConnector` | `com.microsoft.sqlserver:mssql-jdbc` | `demos.connector.SqlServerDataSourceConnector` | + +Each is a **separate Maven build** with no parent — they are not modules of the +root reactor, so `mvn test` and `mvn package` at the root never touch them. +That is also why each POM hardcodes the parser version instead of inheriting +`${gsp.core.version}`; `.github/scripts/set-parser-version.sh` keeps all four +files in step, and both workflows check it. + +All three drivers now resolve from Maven Central. Two of them did not until +2026-08-24: Oracle's was declared as `com:ojdbc:1.1.1` on `system` scope +pointing at a `lib/ojdbc-1.1.1.jar` that has never existed here, and SQL +Server's as `sqljdbc4:4.0` the same way. Both killed the build during +dependency resolution, before any plugin could explain anything — which is why +the guard above could not fire until they were real coordinates. **Do not +reintroduce a `system`-scope dependency or vendor a driver jar**; see the root +README on `lib-repo/`. diff --git a/licensed-only/oracleConnector/lib/readme.md b/licensed-only/oracleConnector/lib/readme.md new file mode 100644 index 00000000..001f164f --- /dev/null +++ b/licensed-only/oracleConnector/lib/readme.md @@ -0,0 +1,11 @@ +Nothing needs to be downloaded here any more. + +The JDBC driver used to be a `system`-scope dependency pointing into this +directory, so building meant fetching a jar by hand first. It is an ordinary +Maven Central coordinate now, declared in the module's `pom.xml`, and Maven +fetches it. This directory only survives because the jar plugin's manifest +still names `lib/` as its runtime classpath prefix, for assembling a runnable +jar by hand. + +See `../../README.md`: these modules need a licensed parser regardless of the +driver. diff --git a/licensed-only/oracleConnector/pom.xml b/licensed-only/oracleConnector/pom.xml new file mode 100644 index 00000000..56e585da --- /dev/null +++ b/licensed-only/oracleConnector/pom.xml @@ -0,0 +1,128 @@ + + + 4.0.0 + gudusoft + gudusoft.gsqlparser.demo.oracleConnector + 1.0-SNAPSHOT + jar + + + UTF-8 + UTF-8 + Gsp Oracle Connector Demo + 1.0 + Gudu software + 1.6 + 1.6 + + + + + gudu-public-releases + https://www.sqlparser.com/maven/ + + + + + + + com.gudusoft + gsqlparser + 4.2.6 + + + + com.oracle.database.jdbc + ojdbc8 + 21.18.0.0 + + + + + oracleConnector + + + org.apache.maven.plugins + maven-jar-plugin + + target/classes/ + + + demos.connector.OracleDataSourceConnector + false + true + lib/ + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + + + + + + + + trial-guard + + true + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.4.1 + + + licensed-parser-required + validate + + enforce + + + + + This module needs a LICENSED General SQL Parser, not the public trial jar this repository resolves by default. The trial artifact does not ship gudusoft.gsqlparser.sqlenv.T*SQLDataSource, so these sources cannot compile against it. See licensed-only/README.md. To evaluate metadata-aware column resolution on the trial parser instead, run the columninspect demo with samples/columninspect/metadata.json. Once you have a licensed parser installed, build with -Plicensed. + + + + + + + + + + + + licensed + + + + diff --git a/connector/oracleConnector/src/main/java/demos/connector/OracleDataSourceConnector.java b/licensed-only/oracleConnector/src/main/java/demos/connector/OracleDataSourceConnector.java similarity index 100% rename from connector/oracleConnector/src/main/java/demos/connector/OracleDataSourceConnector.java rename to licensed-only/oracleConnector/src/main/java/demos/connector/OracleDataSourceConnector.java diff --git a/licensed-only/snowflakeConnector/lib/readme.md b/licensed-only/snowflakeConnector/lib/readme.md new file mode 100644 index 00000000..001f164f --- /dev/null +++ b/licensed-only/snowflakeConnector/lib/readme.md @@ -0,0 +1,11 @@ +Nothing needs to be downloaded here any more. + +The JDBC driver used to be a `system`-scope dependency pointing into this +directory, so building meant fetching a jar by hand first. It is an ordinary +Maven Central coordinate now, declared in the module's `pom.xml`, and Maven +fetches it. This directory only survives because the jar plugin's manifest +still names `lib/` as its runtime classpath prefix, for assembling a runnable +jar by hand. + +See `../../README.md`: these modules need a licensed parser regardless of the +driver. diff --git a/connector/snowflakeConnector/pom.xml b/licensed-only/snowflakeConnector/pom.xml similarity index 59% rename from connector/snowflakeConnector/pom.xml rename to licensed-only/snowflakeConnector/pom.xml index cc402e56..ee88cab4 100644 --- a/connector/snowflakeConnector/pom.xml +++ b/licensed-only/snowflakeConnector/pom.xml @@ -76,4 +76,54 @@ - \ No newline at end of file + + + + + trial-guard + + true + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.4.1 + + + licensed-parser-required + validate + + enforce + + + + + This module needs a LICENSED General SQL Parser, not the public trial jar this repository resolves by default. The trial artifact does not ship gudusoft.gsqlparser.sqlenv.T*SQLDataSource, so these sources cannot compile against it. See licensed-only/README.md. To evaluate metadata-aware column resolution on the trial parser instead, run the columninspect demo with samples/columninspect/metadata.json. Once you have a licensed parser installed, build with -Plicensed. + + + + + + + + + + + + licensed + + + + diff --git a/connector/snowflakeConnector/src/main/java/demos/connector/SnowflakeDataSourceConnector.java b/licensed-only/snowflakeConnector/src/main/java/demos/connector/SnowflakeDataSourceConnector.java similarity index 100% rename from connector/snowflakeConnector/src/main/java/demos/connector/SnowflakeDataSourceConnector.java rename to licensed-only/snowflakeConnector/src/main/java/demos/connector/SnowflakeDataSourceConnector.java diff --git a/licensed-only/sqlServerConnector/lib/readme.md b/licensed-only/sqlServerConnector/lib/readme.md new file mode 100644 index 00000000..001f164f --- /dev/null +++ b/licensed-only/sqlServerConnector/lib/readme.md @@ -0,0 +1,11 @@ +Nothing needs to be downloaded here any more. + +The JDBC driver used to be a `system`-scope dependency pointing into this +directory, so building meant fetching a jar by hand first. It is an ordinary +Maven Central coordinate now, declared in the module's `pom.xml`, and Maven +fetches it. This directory only survives because the jar plugin's manifest +still names `lib/` as its runtime classpath prefix, for assembling a runnable +jar by hand. + +See `../../README.md`: these modules need a licensed parser regardless of the +driver. diff --git a/licensed-only/sqlServerConnector/pom.xml b/licensed-only/sqlServerConnector/pom.xml new file mode 100644 index 00000000..209b4718 --- /dev/null +++ b/licensed-only/sqlServerConnector/pom.xml @@ -0,0 +1,128 @@ + + + 4.0.0 + gudusoft + gudusoft.gsqlparser.demo.sqlServerConnector + 1.0-SNAPSHOT + jar + + + UTF-8 + UTF-8 + Gsp Sql Server Connector Demo + 1.0 + Gudu software + 1.6 + 1.6 + + + + + gudu-public-releases + https://www.sqlparser.com/maven/ + + + + + + + com.gudusoft + gsqlparser + 4.2.6 + + + + com.microsoft.sqlserver + mssql-jdbc + 12.10.0.jre8 + + + + + sqlServerConnector + + + org.apache.maven.plugins + maven-jar-plugin + + target/classes/ + + + demos.connector.SqlServerDataSourceConnector + false + true + lib/ + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.5.1 + + 1.8 + 1.8 + + + + + + + + + trial-guard + + true + + + + + org.apache.maven.plugins + maven-enforcer-plugin + 3.4.1 + + + licensed-parser-required + validate + + enforce + + + + + This module needs a LICENSED General SQL Parser, not the public trial jar this repository resolves by default. The trial artifact does not ship gudusoft.gsqlparser.sqlenv.T*SQLDataSource, so these sources cannot compile against it. See licensed-only/README.md. To evaluate metadata-aware column resolution on the trial parser instead, run the columninspect demo with samples/columninspect/metadata.json. Once you have a licensed parser installed, build with -Plicensed. + + + + + + + + + + + + licensed + + + + diff --git a/connector/sqlServerConnector/src/main/java/demos/connector/SqlServerDataSourceConnector.java b/licensed-only/sqlServerConnector/src/main/java/demos/connector/SqlServerDataSourceConnector.java similarity index 100% rename from connector/sqlServerConnector/src/main/java/demos/connector/SqlServerDataSourceConnector.java rename to licensed-only/sqlServerConnector/src/main/java/demos/connector/SqlServerDataSourceConnector.java diff --git a/samples/dlineageBasic/oracle/hr_mini/hr_mini.sql b/samples/dlineageBasic/oracle/hr_mini/hr_mini.sql new file mode 100644 index 00000000..b1e42e4f --- /dev/null +++ b/samples/dlineageBasic/oracle/hr_mini/hr_mini.sql @@ -0,0 +1,166 @@ +-- hr_mini: a small HR warehouse, small enough for the trial parser. +-- +-- The other schema dumps under samples/dlineageBasic/ are real vendor scripts +-- and every one of them is over the trial parser's 10,000-byte limit, so they +-- return a licence error instead of lineage. This file exists so that "column +-- lineage across a schema" is something you can actually run from a plain +-- clone: DDL, views, a CTE, joins, aggregates and CASE WHEN, all resolvable +-- without a metadata.json. +-- +-- java -jar target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar \ +-- /f samples/dlineageBasic/oracle/hr_mini/hr_mini.sql /t oracle /json + +-- ---------------------------------------------------------------- staging -- + +CREATE TABLE stg_department ( + department_id NUMBER(6), + department_name VARCHAR2(60), + location_city VARCHAR2(40), + country_code CHAR(2) +); + +CREATE TABLE stg_employee ( + employee_id NUMBER(6), + first_name VARCHAR2(40), + last_name VARCHAR2(40), + email VARCHAR2(80), + hire_date DATE, + job_id VARCHAR2(12), + salary NUMBER(10, 2), + commission NUMBER(6, 2), + manager_id NUMBER(6), + department_id NUMBER(6) +); + +CREATE TABLE stg_job_grade ( + job_id VARCHAR2(12), + job_title VARCHAR2(60), + lowest_salary NUMBER(10, 2), + grade_code CHAR(1) +); + +-- ------------------------------------------------------------- warehouse -- + +CREATE TABLE dim_department ( + department_key NUMBER(6), + department_name VARCHAR2(60), + location_city VARCHAR2(40), + country_code CHAR(2), + headcount NUMBER(6) +); + +CREATE TABLE dim_employee ( + employee_key NUMBER(6), + full_name VARCHAR2(90), + email VARCHAR2(80), + job_title VARCHAR2(60), + grade_code CHAR(1), + hire_date DATE, + manager_key NUMBER(6), + department_key NUMBER(6) +); + +CREATE TABLE fact_compensation ( + employee_key NUMBER(6), + department_key NUMBER(6), + base_salary NUMBER(12, 2), + total_pay NUMBER(12, 2), + pay_band VARCHAR2(12), + above_job_floor NUMBER(1) +); + +-- ------------------------------------------------------------------ view -- + +CREATE VIEW v_employee_enriched AS +SELECT e.employee_id, + e.first_name || ' ' || e.last_name AS full_name, + LOWER(e.email) AS email, + e.hire_date, + e.salary, + NVL(e.commission, 0) AS commission, + e.manager_id, + e.department_id, + g.job_title, + g.grade_code, + g.lowest_salary + FROM stg_employee e + LEFT JOIN stg_job_grade g + ON e.job_id = g.job_id; + +-- ------------------------------------------------------------------- ETL -- + +INSERT INTO dim_department (department_key, + department_name, + location_city, + country_code, + headcount) +SELECT d.department_id, + UPPER(d.department_name), + d.location_city, + d.country_code, + COUNT(e.employee_id) + FROM stg_department d + LEFT JOIN stg_employee e + ON e.department_id = d.department_id + GROUP BY d.department_id, + d.department_name, + d.location_city, + d.country_code; + +INSERT INTO dim_employee (employee_key, + full_name, + email, + job_title, + grade_code, + hire_date, + manager_key, + department_key) +SELECT v.employee_id, + v.full_name, + v.email, + NVL(v.job_title, 'UNASSIGNED'), + v.grade_code, + v.hire_date, + v.manager_id, + v.department_id + FROM v_employee_enriched v; + +-- A CTE, an aggregate over it, and a CASE WHEN, so the lineage has both +-- direct (fdd) and indirect (fdr) relations to look at. +INSERT INTO fact_compensation (employee_key, + department_key, + base_salary, + total_pay, + pay_band, + above_job_floor) +WITH paid AS ( + SELECT v.employee_id, + v.department_id, + v.salary, + v.salary + v.commission AS gross_pay, + v.lowest_salary + FROM v_employee_enriched v + WHERE v.hire_date < SYSDATE +) +SELECT p.employee_id, + p.department_id, + p.salary, + SUM(p.gross_pay), + CASE + WHEN SUM(p.gross_pay) >= 15000 THEN 'EXECUTIVE' + WHEN SUM(p.gross_pay) >= 7000 THEN 'SENIOR' + ELSE 'STANDARD' + END, + CASE WHEN p.salary >= p.lowest_salary THEN 1 ELSE 0 END + FROM paid p + GROUP BY p.employee_id, + p.department_id, + p.salary, + p.lowest_salary; + +-- Headcount is recomputed from the dimension it was loaded alongside, so the +-- lineage shows a target column depending on the same table it updates. +UPDATE dim_department dd + SET dd.headcount = (SELECT COUNT(de.employee_key) + FROM dim_employee de + WHERE de.department_key = dd.department_key); diff --git a/samples/dlineageBasic/oracle/hr_mini/readme.md b/samples/dlineageBasic/oracle/hr_mini/readme.md new file mode 100644 index 00000000..a3e3dfec --- /dev/null +++ b/samples/dlineageBasic/oracle/hr_mini/readme.md @@ -0,0 +1,45 @@ +# hr_mini + +A small Oracle HR warehouse — staging tables, a view, three loads and an +update — written so that **column-level lineage across a schema is runnable on +the trial parser**. + +```bash +mvn package -DskipTests +java -jar target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar \ + /f samples/dlineageBasic/oracle/hr_mini/hr_mini.sql /t oracle /json +``` + +120 relationships on parser 4.2.6: 82 direct (`fdd`) and 38 indirect (`fdr`). +`/tableLineage /csv` gives the same flow at table level. + +## Why it exists + +The other directories under `samples/dlineageBasic/` hold real vendor schema +dumps, and **every one of them is over the trial parser's 10,000-byte limit** — +`hr_cre.sql` is 10,481 bytes, `pcus_v3.sql` is 99,139. Point the lineage demo +at one of those and you get a `` document whose only content is an +`` saying the trial build cannot process a query that size. It looks +like "no lineage found". A first-time evaluation in August 2026 hit exactly +that and concluded schema-scale lineage did not work. + +This file is 5,212 bytes, so it fits, and it is written to exercise the parts +that make lineage interesting rather than to be a realistic HR schema: + +| construct | where | +|---|---| +| view over a `LEFT JOIN` | `v_employee_enriched` | +| expression columns (concatenation, `LOWER`, `NVL`) | the same view | +| aggregate into a dimension | `dim_department` load, `COUNT` + `GROUP BY` | +| a CTE feeding an aggregate | `fact_compensation` load | +| `CASE WHEN` over an aggregate, and over a column comparison | `pay_band`, `above_job_floor` | +| correlated subquery in an `UPDATE` | the final `UPDATE dim_department` | + +All the DDL is in the file, so every column resolves without a +`metadata.json`. That is the "put the CREATE TABLE statements in the same +script" answer to ambiguous columns, worked through at schema scale — see +[the lineage demo's readme](../../../../src/main/java/gudusoft/gsqlparser/demos/dlineage/readme.md). + +`.github/scripts/smoke-dlineage-jar.sh` runs this file in CI and fails if it +ever reaches 10,000 bytes, if it stops producing relationships, or if the +output starts carrying the trial-limit error. Keep additions small. diff --git a/src/main/java/gudusoft/gsqlparser/demos/dlineage/DataFlowAnalyzer.java b/src/main/java/gudusoft/gsqlparser/demos/dlineage/DataFlowAnalyzer.java index 9f6e9eb5..a66227c3 100644 --- a/src/main/java/gudusoft/gsqlparser/demos/dlineage/DataFlowAnalyzer.java +++ b/src/main/java/gudusoft/gsqlparser/demos/dlineage/DataFlowAnalyzer.java @@ -83,11 +83,6 @@ public static void main(String[] args) { System.out.println("/showConstant: Optional, show constant table."); System.out.println("/treatArgumentsInCountFunctionAsDirectDataflow: Optional, treat arguments in count function as direct dataflow. Default is false."); System.out.println("/showER: Optional, show entity relationship."); - //add by grq 2022.10.25 issue=I5X3KO - System.out.println("/fromdb: Optional, specifies the database connection parameters."); - System.out.println("/exportonly: Optional, just export metadata.json, no further data analysis."); - System.out.println("/metadataoutput: Optional, specifies the metadata output directory and file name."); - //end by grq System.out.println("/showCaseWhenAsIndirect: Optional, treat CASE WHEN conditions as indirect dataflow. Default is false."); System.out.println("/filterRelationTypes: Optional, specify the relation types to be output, support fdd, fdr, join, call, er, multiple relation types separated by commas"); System.out.println("/lv: Optional, output lineage for visualize"); @@ -125,21 +120,14 @@ public static void main(String[] args) { return; } } - //add by grq 2022.10.25 issue=I5X3KO - else if (argList.indexOf("/fromdb") != -1 && argList.size() > argList.indexOf("/fromdb") + 1){ - String metadataoutput = "metadata.json"; - if(argList.indexOf("/metadataoutput") != -1 && argList.size() > argList.indexOf("/metadataoutput") + 1){ - metadataoutput = args[argList.indexOf("/metadataoutput") + 1]; - } - // if(!SqlflowIngester.export(vendor.name(), args[argList.indexOf("/fromdb") + 1].split("\\s+"), metadataoutput)){ - // return; - // } - if(argList.indexOf("/exportonly") != -1 ){ - return; - } - sqlFiles = new File(metadataoutput ); - } - //end by grq + // /fromdb, /exportonly and /metadataoutput were removed on 2026-08-24. + // They exported a database catalog through SqlflowIngester.export(...), + // a call that had been commented out and whose class was deleted from + // this repository, so the flags parsed and then did nothing: an empty + // and no metadata.json, while the readme taught them in two + // sections with four vendor examples. Live JDBC extraction also needs + // gudusoft.gsqlparser.sqlenv.T*SQLDataSource, which the public trial + // parser does not ship. Pass metadata you exported elsewhere with /env. else { System.out.println("Please specify a sql file path or directory path to analyze dlineage."); return; diff --git a/src/main/java/gudusoft/gsqlparser/demos/dlineage/readme.md b/src/main/java/gudusoft/gsqlparser/demos/dlineage/readme.md index 51a36054..c67c970f 100644 --- a/src/main/java/gudusoft/gsqlparser/demos/dlineage/readme.md +++ b/src/main/java/gudusoft/gsqlparser/demos/dlineage/readme.md @@ -83,28 +83,40 @@ an `` element, which is easy to mistake for "no lineage found". `samples/dlineage/demo.sql` is 366 bytes and works. **16 of the 89 `.sql` files under `samples/` are over the limit, and every one of them is a vendor schema dump under `samples/dlineageBasic/`** — `hr_cre.sql`, `sakila-schema.sql`, -`instawdbdw.sql` and the rest, from 10,378 up to 99,139 bytes. Pointing this -demo at one of those to "try lineage on a real schema" produces the licence -error above, not lineage. Those need a licensed parser. - -## `/fromdb` does not work in this repository - -The `/fromdb`, `/exportonly` and `/metadataoutput` flags are still parsed, and -the option list still describes them, but **the export itself is gone**: the -call to `SqlflowIngester.export(...)` in `DataFlowAnalyzer.java` is commented -out and that class has been deleted from this repository. Run it and you get an -empty document, having written no `metadata.json`: - -```console -$ java -jar ...-dlineage.jar /t oracle /fromdb "-dbVendor dbvoracle -host ..." /o out.xml - - +`instawdbdw.sql` and the rest, from 10,378 up to 99,139 bytes. Those need a +licensed parser. + +For schema-scale lineage on the trial jar, use +[`samples/dlineageBasic/oracle/hr_mini/`](../../../../../../../samples/dlineageBasic/oracle/hr_mini/readme.md), +added for exactly this reason — 5,212 bytes, and 120 relationships: + +```bash +java -jar target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar \ + /f samples/dlineageBasic/oracle/hr_mini/hr_mini.sql /t oracle /json +``` + +It carries staging tables, a view over a `LEFT JOIN`, a CTE, aggregates, `CASE +WHEN` and a correlated subquery, with all its DDL in the file so every column +resolves. CI runs it and fails if it ever crosses the 10,000-byte line. + +## There is no `/fromdb` + +It was removed on 2026-08-24, along with `/exportonly` and `/metadataoutput`. +The flags parsed and then did nothing: the call they existed for, +`SqlflowIngester.export(...)`, had been commented out and that class deleted, so +the tool emitted an empty `` and wrote no `metadata.json` while this +readme taught the feature in two sections with four vendor examples. Passing +`/fromdb` now gets the same answer as passing nothing: + +``` +Please specify a sql file path or directory path to analyze dlineage. ``` -Live JDBC catalog extraction also needs `gudusoft.gsqlparser.sqlenv.T*SQLDataSource`, -which the public trial jar does not ship. To feed real metadata to this demo, -export it elsewhere and pass the JSON with `/env` — see "Resolving ambiguous -columns" below. +Live JDBC catalog extraction needs `gudusoft.gsqlparser.sqlenv.T*SQLDataSource`, +which the public trial parser does not ship — see +[`licensed-only/`](../../../../../../../licensed-only/README.md). To give this +demo real metadata, export it elsewhere and pass the JSON with `/env`; see +"Resolving ambiguous columns" below. ## Options @@ -160,9 +172,6 @@ sybase,teradata,soql,vertica /showConstant: Optional, show constant table. /treatArgumentsInCountFunctionAsDirectDataflow: Optional, treat arguments in count function as direct dataflow. Default is false. /showER: Optional, show entity relationship. -/fromdb: Optional, specifies the database connection parameters. -/exportonly: Optional, just export metadata.json, no further data analysis. -/metadataoutput: Optional, specifies the metadata output directory and file name. /showCaseWhenAsIndirect: Optional, treat CASE WHEN conditions as indirect dataflow. Default is false. /filterRelationTypes: Optional, specify the relation types to be output, support fdd, fdr, join, call, er, multiple relation types separated by commas /lv: Optional, output lineage for visualize @@ -207,10 +216,9 @@ java -jar target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar \ ``` The same metadata JSON also drives the `columninspect` demo, which has a -runnable pair checked in at `samples/columninspect/`. Note that `/fromdb` -cannot produce this file here (see above); export it with a licensed build or -the [sqlflow-ingester](https://github.com/sqlparser/sqlflow_public/releases) -tool. +runnable pair checked in at `samples/columninspect/`. Nothing in this +repository can produce the file — export it with a licensed build or the +[sqlflow-ingester](https://github.com/sqlparser/sqlflow_public/releases) tool. ## How the options map to SQLFlow's settings