diff --git a/.github/scripts/check-stale-docs.sh b/.github/scripts/check-stale-docs.sh new file mode 100755 index 00000000..c6b9a0a1 --- /dev/null +++ b/.github/scripts/check-stale-docs.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +# +# Fail if any documentation still tells the reader to use something this +# repository deleted. +# +# Every string below was, at some point, the documented way to do something +# here, and every one of them outlived the thing it named. The lineage demo's +# own readme told visitors for months that the demo "cannot be built" and gave +# them `mvn -f pom_dlineage.xml package` -- a file removed on 2026-07-28 -- and +# `java -jar gudusoft.dlineage.jar`, a jar this build has never produced. A +# first-time evaluator who opened that folder concluded the product was broken. +# Nothing went red, because no check reads prose. +# +# So this one does. It is deliberately dumb: fixed strings, no cleverness about +# context. A file whose job is to record that these things died is listed in +# ALLOW; anything else naming them is treated as an instruction to the reader. +# +# Run --self-test to prove the check still catches what it claims to. A grep +# that silently matches nothing would pass this repository forever. + +set -euo pipefail + +cd "$(dirname "$0")/../.." + +# Documents whose subject IS the removal. They have to name what was removed. +ALLOW=( + "README.md" + "docs/maintenance-notes.md" +) + +DEAD=( + "pom_dlineage.xml" + "gudusoft.dlineage.jar" + "src/main/java/demos/" + 'src\main\java\demos\' +) + +WHY=( + "merged into pom.xml on 2026-07-28; build with 'mvn package -DskipTests'" + "the standalone jar is target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar" + "second package root removed 2026-07-27; demos are under src/main/java/gudusoft/gsqlparser/demos/" + "same, in Windows path form" +) + +# scan ... -- prints every hit, returns 1 if there was any +scan() { + local failed=0 f i hits line + for f in "$@"; do + for i in "${!DEAD[@]}"; do + if hits=$(grep -Fn -- "${DEAD[$i]}" "$f" 2>/dev/null); then + while IFS= read -r line; do + printf ' %s:%s\n' "$f" "$line" + done <<<"$hits" + printf ' ^ "%s" is gone -- %s\n\n' "${DEAD[$i]}" "${WHY[$i]}" + failed=1 + fi + done + done + return "$failed" +} + +self_test() { + local tmp rc pass=0 fail=0 i + tmp=$(mktemp -d) + trap 'rm -rf "$tmp"' RETURN + + printf 'Build it with `mvn package -DskipTests`.\n' >"$tmp/clean.md" + if scan "$tmp/clean.md" >/dev/null 2>&1; then + echo " ok a clean document passes" + pass=$((pass + 1)) + else + echo " FAIL a clean document was reported as stale" + fail=$((fail + 1)) + fi + + for i in "${!DEAD[@]}"; do + printf 'run %s to build it\n' "${DEAD[$i]}" >"$tmp/stale.md" + rc=0 + scan "$tmp/stale.md" >/dev/null 2>&1 || rc=$? + if [ "$rc" -eq 1 ]; then + echo " ok \"${DEAD[$i]}\" is caught" + pass=$((pass + 1)) + else + echo " FAIL \"${DEAD[$i]}\" slipped through" + fail=$((fail + 1)) + fi + done + + echo + if [ "$fail" -gt 0 ]; then + echo "self-test: $fail of $((pass + fail)) cases FAILED" + return 1 + fi + echo "self-test: all $pass cases pass" +} + +if [ "${1:-}" = "--self-test" ]; then + echo "proving check-stale-docs.sh catches what it claims to" + echo + self_test + exit $? +fi + +files=() +while IFS= read -r f; do + skip=0 + for a in "${ALLOW[@]}"; do + if [ "$f" = "$a" ]; then skip=1; fi + done + if [ "$skip" -eq 0 ]; then files+=("$f"); fi +done < <(git ls-files '*.md') + +if [ "${#files[@]}" -eq 0 ]; then + echo "::error::no markdown files found to check; is this a git checkout?" + exit 1 +fi + +echo "checking ${#files[@]} markdown files for references to deleted things" +echo + +if scan "${files[@]}"; then + echo "ok: no documentation points at anything that has been removed" + exit 0 +fi + +echo "::error::documentation above still instructs the reader to use something that no longer exists" +echo "If a file's purpose is to record the removal, add it to ALLOW in $0." +exit 1 diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c0a51254..17c84818 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,6 +56,23 @@ jobs: if: matrix.java == '21' run: .github/scripts/test-pre-commit-hook.sh + # Prose does not compile, so a readme can name a file that was deleted two + # releases ago and every build stays green. The lineage demo's own readme + # did exactly that: it told visitors the demo "cannot be built", handed + # them `mvn -f pom_dlineage.xml package` for a POM removed on 2026-07-28 + # and `java -jar gudusoft.dlineage.jar` for a jar this build has never + # produced. Someone evaluating GSP for the first time reasonably concluded + # the product was broken. + # + # --self-test runs first and on purpose: a fixed-string check that matches + # nothing looks identical to a clean repository. + - name: Documentation does not point at deleted things + if: matrix.java == '21' + run: | + .github/scripts/check-stale-docs.sh --self-test + echo + .github/scripts/check-stale-docs.sh + - name: Build run: mvn -B package -DskipTests diff --git a/README.md b/README.md index 13542852..a391c424 100644 --- a/README.md +++ b/README.md @@ -407,6 +407,7 @@ than only building them. |---|---| | Parser version consistency | `set-parser-version.sh --check` across all four POMs | | The pre-commit hook | `test-pre-commit-hook.sh`: a drifting bump is refused in a throwaway clone | +| Documentation | `check-stale-docs.sh`: no readme names `pom_dlineage.xml`, `gudusoft.dlineage.jar` or the old `demos` package root; `--self-test` first, so a check that matches nothing cannot pass as a clean repo | | Build and test | JDK 8 and 21; 156 tests, and a run that skipped everything fails | | Demo smoke test | `checksyntax` against known SQL | | Standalone lineage jar | `smoke-dlineage-jar.sh` on JDK 8 and 21 — asserts on **output**, in JSON *and* XML | diff --git a/src/main/java/gudusoft/gsqlparser/demos/dlineage/readme.md b/src/main/java/gudusoft/gsqlparser/demos/dlineage/readme.md index 02cb1ffe..51a36054 100644 --- a/src/main/java/gudusoft/gsqlparser/demos/dlineage/readme.md +++ b/src/main/java/gudusoft/gsqlparser/demos/dlineage/readme.md @@ -1,88 +1,151 @@ - # DataFlowAnalyzer -Collects the end-to-end column-level data lineage in the Data Warehouses environment by connecting to database or -analyzing SQL script especially stored procedure like PL/SQL. - -This tool is built from the scratch, it is the main part of the backend of [the SQLFlow Cloud](https://sqlflow.gudusoft.com). +Collects end-to-end, column-level data lineage from SQL scripts — including +stored procedures such as PL/SQL — by parsing them, with no database +connection. It is the demo form of the engine behind +[Gudu SQLFlow](https://sqlflow.gudusoft.com). -## Building it on its own +## Build and run -`DataFlowAnalyzer` is **excluded from the root build** — it needs a parser -carrying the metadata layer, which the public trial artifact does not ship. The -standalone build for it is `pom_dlineage.xml` in the repository root: +`DataFlowAnalyzer` is part of the ordinary root build. Nothing about it is +excluded, and there is no separate POM: from the repository root, ```bash -mvn -f pom_dlineage.xml package +mvn package -DskipTests ``` -**That build currently fails**, and did before this demo was reorganised: it -pins `lib/gsqlparser-3.1.1.0.jar`, while `DataFlowAnalyzer` has moved on to -`getOption().setTraceTablePosition(...)` and -`ProcessUtility.generateColumnLevelLineageCsvSimple(...)`, neither of which that -jar has. See the root `README.md`. - -> This section used to say `mvn package` against a `pom.xml` in this folder, -> producing `target\dlineage-1.0.jar`, run with a classpath pointing into -> `c:\prg\maven_repo\…\gsqlparser-3.1.0.2.jar` and a main class of -> `gsp.gudusoft.gsqlparser.demos.dlineage.DataFlowAnalyzer`. All three were -> wrong: that POM referenced a parser jar that is not in this repository, the -> Windows path was the original author's, and the class name carried a stray -> `gsp.` prefix. The POM has been removed. +produces `target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar`, an executable uber +jar with every runtime dependency inside it. Analyze the sample script: +```bash +java -jar target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar \ + /f samples/dlineage/demo.sql /o lineage.json /json +``` -## Quick start - -### 1. Analyze data lineage from SQL files +On parser 4.2.6 that writes 24 relationships. Drop `/json` for the XML form. +Run it with no arguments to print the full option list for the parser version +you have; the list under "Options" below is reproduced from that output. + +> **Use the jar, not `mvn exec:java`, for this demo.** Most demos in this +> repository run happily through `exec:java`, but this one marshals its output +> with JAXB, and `exec:java` loads it in a child classloader while +> `javax.xml.datatype` comes from the boot classloader. On JDK 21 that fails +> before any lineage is printed: +> +> ``` +> loader constraint violation: when resolving field "DATETIME" of type +> javax.xml.namespace.QName ... have different Class objects +> ``` +> +> The uber jar has no such split, which is the reason it exists and the reason +> CI exercises it through `.github/scripts/smoke-dlineage-jar.sh`. Any other +> demo class runs from the same jar too: +> `java -cp target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar …` + +> **The jars on [the Releases page](https://github.com/sqlparser/gsp_demo_java/releases) +> are not this build.** The newest, `gudusoft.dlineage-3.0.2.3`, was published +> 2024-11-02 — before the 2026-07 package reorganisation and several parser +> releases. Build from source with the command above rather than downloading +> one. + +### Output formats + +Column-level XML (the default), column-level JSON (`/json`), and table-level +CSV: -Analyze demo.sql under sample directory and save the data lineage outpout in out.xml file. +```bash +java -jar target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar \ + /f samples/dlineage/demo.sql /tableLineage /csv +``` ``` -java -jar gudusoft.dlineage.jar /t oracle /f ../sample/demo.sql /o out.xml +source_db,source_schema,source_table,source_column,target_db,...,target_column,process_type,... +default,default,dept,deptno;dname,default,default,deptsal,dept_no;dept_name,sstinsert,... +default,default,emp,sal;comm,default,default,deptsal,salary,sstinsert,... ``` -### 2. Analyze data lineage from a database -The dlineage tool can connect to the database instance and analyze the metadata to generate the data lineage automatically. +`/s` drops the intermediate result sets and reports only tables and columns, +which is usually what you want when the lineage is going into another tool. -for example, connect to an Oracle database and analzye the data lineage, and save the data lineage in out.xml. +## The trial parser's 10,000-byte limit -a metadata.json file that includes all metadata extracted from Oracle database will be saved to the current directory. +This repository resolves the **trial** parser, which refuses any single script +over 10,000 bytes: ``` -java -jar gudusoft.dlineage.jar /t oracle /fromdb "-dbVendor dbvoracle -host 127.0.0.1 -port 1521 -db orcl -user scott -pwd tiger" /o out.xml +trial version can only process query with size of at most 10000 bytes, +and expired after 90 days after first usage. ``` -### 3.2 Export the meatadata from database only +You do not get a crash — you get a `` document whose only content is +an `` element, which is easy to mistake for "no lineage found". -You can also export the meatadata from database only use this tool and then upload this metadata to [the Gudu SQLFlow Cloud](https://sqlflow.gudusoft.com) -to analyze the lineage. +`samples/dlineage/demo.sql` is 366 bytes and works. **16 of the 89 `.sql` files +under `samples/` are over the limit, and every one of them is a vendor schema +dump under `samples/dlineageBasic/`** — `hr_cre.sql`, `sakila-schema.sql`, +`instawdbdw.sql` and the rest, from 10,378 up to 99,139 bytes. Pointing this +demo at one of those to "try lineage on a real schema" produces the licence +error above, not lineage. Those need a licensed parser. -- Only export the metadta -``` -java -jar gudusoft.dlineage.jar /fromdb "-dbVendor dbvoracle -host 127.0.0.1 -port 1521 -db orcl -user scott -pwd tiger" /exportonly /metadataoutput metadata.json -``` +## `/fromdb` does not work in this repository +The `/fromdb`, `/exportonly` and `/metadataoutput` flags are still parsed, and +the option list still describes them, but **the export itself is gone**: the +call to `SqlflowIngester.export(...)` in `DataFlowAnalyzer.java` is commented +out and that class has been deleted from this repository. Run it and you get an +empty document, having written no `metadata.json`: -## Usage +```console +$ java -jar ...-dlineage.jar /t oracle /fromdb "-dbVendor dbvoracle -host ..." /o out.xml + + ``` -"Usage: java DataFlowAnalyzer [/f ] [/d ] [/stat] [/s [/topselectlist] [/text] ] [/i] [/ic] [/lof] [/j] [/json] [/traceView] [/t ] [/o ] [/version] [/env ] [/tableLineage [/csv]] [/transform [/coor]]"); +Live JDBC catalog extraction also needs `gudusoft.gsqlparser.sqlenv.T*SQLDataSource`, +which the public trial jar does not ship. To feed real metadata to this demo, +export it elsewhere and pass the JSON with `/env` — see "Resolving ambiguous +columns" below. + +## Options + +Reproduced verbatim from the tool's own output on parser **4.2.6** — run it +with no arguments to get the authoritative list for the version you have, +rather than trusting this copy. + +``` +Usage: java DataFlowAnalyzer [/f ] [/d ] [/stat] [/removeResultSetTypes ] [/removeVariable /removeCursor] [/removeUnusedSynonym] [/n] [/s [/topselectlist] [/text] [/withTemporaryTable] [/simpleShowRelationTypes ]] [/i] [/showResultSetTypes ] [/showVariable] [/showCursor] [/showSynonym] [/ic] [/lof] [/j] [/json /graph] [/traceView] [/t ] [/o ] [/version] [/env ] [/tableLineage [/csv [/delimeter ]]] [/csv-simple] [/transform [/coor]] [/showConstant] [/showER] [/treatArgumentsInCountFunctionAsDirectDataflow] [/showCaseWhenAsIndirect] [/filterRelationTypes ] [/lv] [/traceTablePosition] /f: Optional, the full path to SQL file. /d: Optional, the full path to the directory includes the SQL files. /j: Optional, return the result including the join relation. +/n: Optional, normalize output. /s: Optional, simple output, ignore the intermediate results. /topselectlist: Optional, simple output with top select results. -/i: Optional, the same as /s option, but will keep the resultset generated by the SQL function. -/if: Optional, keep all the intermediate resultset, but remove the resultset generated by the SQL function +/simpleShowRelationTypes: Optional, simple output with specified relation types, support fdd, fdr. +/withTemporaryTable: Optional, determine whether to output the temporary tables in simple output, default is false. +/i: Optional, the same as /s option, but will keep the result set generated by the SQL function, this parameter will have the same effect as /s /topselectlist + keep result set generated by the sql function. +/showResultSetTypes: Optional. This option is valid only when /s or /i option is used, and is used to specify the result set types to be output, separate with commas, result set types contains array, struct, result_of, cte, insert_select, update_select, merge_update, merge_insert, output, update_set, + pivot_table, unpivot_table, alias, rs, function, case_when +/showVariable: Optional. This option is valid only when /s or /i option is used, and is used to reserve all the variables and cursors. +/showCursor: Optional. This option is valid only when /s or /i option is used, and is used to reserve all the cursors. +/showSynonym: Optional. This option is valid only when /s or /i option is used, and is used to reserve all the synonyms. +/removeResultSetTypes: Optional. This option is used to remove the specified result set types to be output, separate with commas, result set types contains array, struct, result_of, cte, insert_select, update_select, merge_update, merge_insert, output, update_set, + pivot_table, unpivot_table, alias, rs, function, case_when +/removeVariable: Optional. This option is remove all the variables and cursors. +/removeCursor: Optional. This option is remove all the cursors. +/removeUnusedSynonym: Optional. This option is remove all the unused synonym. +/if: Optional, keep all the intermediate result set, but remove the result set generated by the SQL function /ic: Optional, ignore the coordinates in the output. /lof: Option, link orphan column to the first table. /traceView: Optional, only output the name of source tables and views, ignore all intermediate data. /text: Optional, this option is valid only /s is used, output the column dependency in text mode. /json: Optional, print the json format output. +/graph: Optional, print the json format output with graph information. /stat: Optional, output the analysis statistic information. -/tableLineage [/csv]: Optional, output table level lineage. +/tableLineage [/csv /delimiter]: Optional, output table level lineage. /csv: Optional, output column level lineage in csv format. -/t: Option, set the database type. Support access,bigquery,couchbase,dax,db2,greenplum,hana,hive,impala,informix,mdx,mssql, +/csv-simple: Optional, output column level lineage in a simplified csv format (source schema.table.column, target schema.table.column, relation type), excluding records that reference the synthetic RelationRows column. +/delimiter: Optional, the delimiter of output column level lineage in csv format. +/t: Option, set the database type. Support access,bigquery,couchbase,dax,db2,gaussdb,greenplum,hana,hive,impala,informix,mdx,mssql, sqlserver,mysql,netezza,odbc,openedge,oracle,postgresql,postgres,redshift,snowflake, sybase,teradata,soql,vertica , the default value is oracle @@ -94,121 +157,32 @@ sybase,teradata,soql,vertica /defaultDatabase: Optional, specify the default schema. /defaultSchema: Optional, specify the default schema. /showImplicitSchema: Optional, show implicit schema. +/showConstant: Optional, show constant table. +/treatArgumentsInCountFunctionAsDirectDataflow: Optional, treat arguments in count function as direct dataflow. Default is false. +/showER: Optional, show entity relationship. /fromdb: Optional, specifies the database connection parameters. /exportonly: Optional, just export metadata.json, no further data analysis. /metadataoutput: Optional, specifies the metadata output directory and file name. +/showCaseWhenAsIndirect: Optional, treat CASE WHEN conditions as indirect dataflow. Default is false. /filterRelationTypes: Optional, specify the relation types to be output, support fdd, fdr, join, call, er, multiple relation types separated by commas /lv: Optional, output lineage for visualize +/traceTablePosition: Optional, trace all table positions. Default is false. ``` +## Resolving ambiguous columns -Here is the list of available database after /t option: -``` -access,bigquery,couchbase,dax,db2,greenplum,hana,hive,impala,informix,mdx,mssql, -sqlserver,mysql,netezza,odbc,openedge,oracle,postgresql,postgres,redshift,snowflake, -sybase,teradata,soql,vertica -``` - -## 1. Binary version -https://github.com/sqlparser/gsp_demo_java/releases/ -> update date: 2022/11/01 - -In order to run this utility, please install Oracle JDK1.8 or higher on your computer correctly. - -## 2. Analyze data lineage from SQL files -Please use `/f` parameter to specify a single SQL file, -or use `/d` parameter to specfify a directory that inculdes multiple SQL files. - -``` -java -jar gudusoft.dlineage.jar /t mssql /f path_to_sql_file -``` - -## 3. Analyze data lineage from a database -The dlineage tool can connect to the database instance and analyze the metadata to generate the data lineage automatically. - - -### 3.1 connect and analyze data lineage -Please use `/fromdb` parameter to export metadta from the database. - -`/fromdb` parameter: - --dbVendor: Database type, Use colon to split dbVendor and version if specific version is required. (:, such as dbvmysql:5.7) - --host: Database host name (ip address or domain name) - --port: Port number - --db: Database name - --user: User name - --pwd: User password - --extractedDbsSchemas: Export metadata under the specific schema. Use comma to split if multiple schema required (such as ,). We can use this flag to improve the export performance. - --excludedDbsSchemas: Exclude metadata under the specific schema during the export. Use comma to split if multiple schema required (such as ,). We can use this flag to improve the export performance. - --extractedViews: Export metadata under the specific view. Use comma to split if multiple views required (such as ,). We can use this flag to improve the export performance. - -`/exportonly` just export metadata.json, no further data analysis. - -`/metadataoutput` specifies the metadata output directory and file name. - -for example, connect to an Oracle database and analzye the data lineage. - -- Oracle -``` -java -jar gudusoft.dlineage.jar /t oracle /fromdb "-dbVendor dbvoracle -host 127.0.0.1 -port 1521 -db orcl -user scott -pwd tiger" /o oracle.xml -``` - -- SQL Server -``` -java -jar gudusoft.dlineage.jar /t mssql /fromdb "-dbVendor dbvmssql -host 127.0.0.1 -port 1433 -db AdventureWorksDW2019 -user sa -pwd sa" /o sqlserver.xml -``` - -- MySQL -``` -java -jar gudusoft.dlineage.jar /t mysql /fromdb "-dbVendor dbvmysql -host 127.0.0.1 -port 3306 -db employees -user mysqluser -pwd mysqlpwd" /o mysql.xml -``` - -- PostgreSQL -``` -java -jar gudusoft.dlineage.jar /t postgresql /fromdb "-dbVendor dbvpostgresql -host 127.0.0.1 -port 5432 -db kingland -user pguser -pwd pgpwd" /o pg.xml -``` - - -### 3.2 Export the meatadata only - -You can also export the meatadata from database and analzye the metadata in two steps: - -- Only export the metadta -``` -java -jar gudusoft.dlineage.jar /fromdb "-dbVendor dbvoracle -host 127.0.0.1 -port 1521 -db orcl -user scott -pwd tiger" /exportonly /metadataoutput metadata.json -``` - -the metadata.json exported in this step can also be used with `/env` paramter to resolve the ambiguous columns problem in SQL query. - -- analyze the metadta that generated in the previous step - -``` -java -jar gudusoft.dlineage.jar /t oracle /f metadata.json -``` - - - -## 4. Resolve the ambiguous columns in SQL query ```sql select ename from emp, dept where emp.deptid = dept.id ``` -column `ename` in the first line is not qualified by table name `emp`, so it’s ambiguous to know which table this column belongs to? +`ename` is not qualified, so on its own the analyzer cannot know which table it +belongs to. There are two ways to tell it. -### solution 1, provides create table DDL +### Solution 1 — put the DDL in the same script -Put the following DDL before the above SQL statement in the same SQL file. -the column `ename` will be linked to the table `emp` correctly. +Prepend the `CREATE TABLE` statements and `ename` links to `emp` correctly: ```sql create table emp( @@ -223,53 +197,59 @@ create table dept( ); ``` -### solution 2: provide metadata exported from database -Since dlineage v2.2.0 (2022/7/21), This dlineage tool supports `/env` parameter to accept a metadata json file -which includes the metadata exported from a database. - -By providing metadata.json that includes the metadata, column `ename` should be linked to the table `emp` correctly. +Watch the 10,000-byte trial limit: DDL plus query counts as one script. -You can use `/env` to specify a metadata.json like this: +### Solution 2 — supply metadata with `/env` -``` -java -jar gudusoft.dlineage.jar /t oracle /f path_to_sql_file /env metadata.json +```bash +java -jar target/gsp_demo_java-1.0-SNAPSHOT-dlineage.jar \ + /t oracle /f path_to_sql_file /env metadata.json ``` -You can always extract metadata from the database use the [sqlflow-ingester](https://github.com/sqlparser/sqlflow_public/releases) tool. +The same metadata JSON also drives the `columninspect` demo, which has a +runnable pair checked in at `samples/columninspect/`. Note that `/fromdb` +cannot produce this file here (see above); export it with a licensed build or +the [sqlflow-ingester](https://github.com/sqlparser/sqlflow_public/releases) +tool. + +## How the options map to SQLFlow's settings -## 5. Relationship between this demo and the setting choices in SQLFlow ![sqlflow setting](./sqlflow-settings.png) -### direct dataflow (fdd), indirect dataflow (fdr) -In this demo, there is no corresponding parameter. -You must filter out the relation types that you don't require because this dataflowAnalyzer demo will generate data lineage with all relation types, including fdd, fdr, join, and call. +### Direct dataflow (fdd), indirect dataflow (fdr) -### args in count function -related arg: `/treatArgumentsInCountFunctionAsDirectDataflow` +No corresponding option: this demo emits every relation type — fdd, fdr, join +and call. Use `/filterRelationTypes` to narrow the output. -### show intermediate recordset, show function -The settings for "show intermediate recordset" and "show function" have no corresponding arguments in the demo -But by using those args, you can get the same outcome: +### Arguments in the count function -- don't specify any related args, this will output the same result as if you set `show intermediate recordset = true` and set `show function = true` -- /if, this will output the same result as if you set `show intermediate recordset = true` and set `show function = false` -- /i, this will output the same result as if you set `show intermediate recordset = false` and set `show function = true` -- /topselectlist, this will output the same result as if you set `show intermediate recordset = false` and set `show function = false` +`/treatArgumentsInCountFunctionAsDirectDataflow` -### show constant -/showConstant +### Show intermediate recordset, show function -### show transform -/transform /coor +No single option, but the combinations cover it: +| options | equivalent settings | +|---|---| +| *(none)* | show intermediate recordset = true, show function = true | +| `/if` | show intermediate recordset = true, show function = false | +| `/i` | show intermediate recordset = false, show function = true | +| `/topselectlist` | show intermediate recordset = false, show function = false | -## 6. Links -- [First version, 2017-8](https://github.com/sqlparser/wings/issues/494) +### Show constant + +`/showConstant` + +### Show transform + +`/transform /coor` -## 8、List of Supported dbVendors +## Supported `dbVendor` values -| dbVendor | databases | -|---------------| ---------- | +Used in the `dbVendor` field of a metadata JSON, and by SQLFlow itself: + +| dbVendor | database | +|---------------|------------| | dbvoracle | oracle | | dbvredshift | redshift | | dbvpostgresql | postgresql | @@ -282,4 +262,8 @@ But by using those args, you can get the same outcome: | dbvteradata | teradata | | dbvhive | hive | | dbvimpala | impala | -| dbvdb2 | db2 | +| dbvdb2 | db2 | + +## Links + +- [First version, 2017-8](https://github.com/sqlparser/wings/issues/494) diff --git a/src/main/java/gudusoft/gsqlparser/demos/readme.md b/src/main/java/gudusoft/gsqlparser/demos/readme.md index 72ba12c6..2d670555 100644 --- a/src/main/java/gudusoft/gsqlparser/demos/readme.md +++ b/src/main/java/gudusoft/gsqlparser/demos/readme.md @@ -1,148 +1,58 @@ -## GsqlParser Use Guide +# The demo programs -### 一、Script running mode +One directory per topic — syntax checking, formatting, lineage, AST traversal, +stored-procedure analysis, rewriting, dialect translation, and so on. Most +carry their own `readme.md`. Start from [the root README](../../../../../../README.md) +for the full tour; this page is only about how to run what is in here. -#### 1.Example Modify the java environment directory in setenv/setenv.bat +**The directory path under `src/main/java/` is the package.** Every class here +is `gudusoft.gsqlparser.demos..`, so you can read the +`-Dexec.mainClass` value straight off the file's location without opening it. -```bash -set JAVA_HOME=C:\Program Files\Java\jdk1.8.0_201 -``` - -#### 2.Compile Class -eg：run the script `src/main/java/demos/checksyntax/compile_checksyntax.bat` +## Maven (any platform) -``` -compile_checksyntax.bat -``` +Nothing to edit first. The parser and every other dependency resolve from +Maven, so a fresh clone runs a demo directly: -#### 3.Run Class -eg：run the script `src/main/java/demos/analyzeview/run_analyzeview.bat` +```bash +mvn package -DskipTests -``` -run_checksyntax.bat /f C:\data.sql /t oracle -/f sql file path -/t databse type -/d Output results folder address +mvn -q exec:java \ + -Dexec.mainClass=gudusoft.gsqlparser.demos.checksyntax.OfflineSyntaxCheck \ + -Dexec.args="/f samples/checksyntax/valid-mssql.sql /t mssql" ``` -### 二、Maven running mode +Argument conventions differ per demo — `checksyntax` takes `/f /t `, +`formatsql` takes a bare filename. **Run a demo with no arguments to see its +usage line.** Sample SQL lives in `samples/` at the repository root. -#### 1.modify the pom.xml +The one demo that does not run this way is `dlineage/DataFlowAnalyzer`: it +marshals its output with JAXB, which collides with `exec:java`'s classloader. +Run it from the packaged uber jar instead — see +[its readme](./dlineage/readme.md). -##### 1.1 Comment parent project - -```xml - - - - - -``` +## Windows `.bat` scripts -##### 1.2 Annotated build configuration +The original no-Maven workflow, still maintained and exercised in CI on +`windows-latest`. Each demo directory has a `compile_.bat` and a +`run_.bat` — 39 and 50 of them respectively: -```xml - - - - - - - - - - - - - - - - - - - - - - - - - ``` - -##### 1.3 Change the dependent jar package directory from lib to external_lib - -```xml - - org.simpleframework - simple-xml - 2.6.2 - system - - ${project.basedir}/external_lib/simple-xml-2.6.2.jar - - - - - com.alibaba - fastjson - 1.2.41 - system - ${project.basedir}/external_lib/fastjson-1.2.41.jar - - - - com.github.junrar - junrar - 0.7 - system - ${project.basedir}/external_lib/junrar-0.7.jar - - - - - tk.pratanumandal - expr4j - 0.0.3 - system - ${project.basedir}/external_lib/expr4j.jar - - - - org.jdom - jdom - 1.1 - system - ${project.basedir}/external_lib/jdom.jar - - - - - sqlflow - exporter - 1.0.0 - system - ${project.basedir}/external_lib/sqlflow-exporter.jar - - - sqlflow - library - 1.0.0 - system - ${project.basedir}/external_lib/sqlflow-library.jar - +src\main\java\gudusoft\gsqlparser\demos\checksyntax\compile_checksyntax.bat +src\main\java\gudusoft\gsqlparser\demos\checksyntax\run_checksyntax.bat /f C:\data.sql /t oracle ``` -##### 1.4 Added gsqlparser dependency, which is in the lib path - -```xml - - sqlflow - gsqlparser - 3.0.1.5 - system - ${project.basedir}/lib/gudusoft.gsqlparser-3.0.1.5.jar - -``` +`setenv\setenv.bat` bootstraps itself: it keeps an existing `JAVA_HOME`, and +when `external_lib\` holds no parser it calls `setenv\fetch-parser.bat`, which +runs `mvn dependency:copy-dependencies` to populate it. So Maven is needed once, +to fetch jars; after that the scripts are plain `javac`/`java`. No version is +named in any script — they read `pom.xml`. -##### 2.Run -Execute each demo class directly +> This page used to describe a third route: commenting out a `` block, +> switching a list of `system`-scope dependencies to `external_lib\`, and adding +> a vendored `lib/gudusoft.gsqlparser-3.0.1.5.jar`. None of that applies. The +> private parent POM, the `system` scopes, the vendored parser and the +> `sqlflow-*` jars were all removed during the 2026-07 and 2026-08 cleanups, and +> the `.bat` paths it quoted pointed into a second `demos` package root that no +> longer exists. Dependencies are ordinary Maven coordinates now; there +> is nothing to edit before building.