diff --git a/.github/workflows/loadsim-measure.yml b/.github/workflows/loadsim-measure.yml new file mode 100644 index 0000000..39a16a0 --- /dev/null +++ b/.github/workflows/loadsim-measure.yml @@ -0,0 +1,47 @@ +name: Loadsim measure + +# Throughput and memory per Cloud SQL-like profile and writer count, run by +# hand. Shared runners are noisy: the numbers are for comparison, never a gate. +on: + workflow_dispatch: + inputs: + profiles: + description: Profiles to measure (comma-separated) + default: cloudsql-micro,cloudsql-2vcpu + +permissions: + contents: read + +jobs: + measure: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.25' + cache: true + + - name: Measure + env: + SEEDSTORM_LOADSIM_MEASURE: ${{ github.workspace }}/loadsim-measure.json + SEEDSTORM_LOADSIM_PROFILES: ${{ inputs.profiles || 'cloudsql-micro,cloudsql-2vcpu' }} + run: cd integration && go test -v -tags "integration loadsim" -count=1 -run TestLoadsim_MeasureWriters ./... -timeout 3000s + + - name: Summary + if: always() + run: | + if [ -f loadsim-measure.json ]; then + { echo "### Loadsim measure"; echo '```json'; cat loadsim-measure.json; echo '```'; } >> "$GITHUB_STEP_SUMMARY" + fi + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: loadsim-measure + path: loadsim-measure.json + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 03364e0..89ef585 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -145,6 +145,80 @@ jobs: - name: Run integration tests run: cd integration && go test -race -v -tags integration -count=1 ./... -timeout 1500s + loadsim: + runs-on: ubuntu-latest + timeout-minutes: 40 + name: loadsim (cloud sql profiles) + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_USER: seedstorm + POSTGRES_PASSWORD: seedstorm + POSTGRES_DB: testdb + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + mysql: + image: mysql:8.4 + env: + MYSQL_ROOT_PASSWORD: root + MYSQL_USER: seedstorm + MYSQL_PASSWORD: seedstorm + MYSQL_DATABASE: testdb + ports: + - 3306:3306 + options: >- + --health-cmd "mysqladmin ping -h localhost -u root -proot" + --health-interval 5s + --health-timeout 5s + --health-retries 10 + + steps: + - uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: '1.25' + cache: true + + # Runner size is not assumed: tests skip profiles that do not fit and say so. + - name: Probe runner + run: | + { + echo "### Runner" + echo '```' + echo "cpus: $(nproc)" + free -m + df -h / /var/lib/docker 2>/dev/null || df -h / + echo "cgroup: $(stat -fc %T /sys/fs/cgroup)" + echo "docker data: $(findmnt -no SOURCE,FSTYPE -T /var/lib/docker || true)" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Pull database images + run: docker pull -q postgres:17-alpine && docker pull -q mysql:8.4 && docker pull -q alpine + + - name: Run resource-limited evals + shell: bash # pipefail: tee must not hide a failure + run: cd integration && go test -v -tags "integration loadsim" -count=1 -run TestLoadsim ./... -timeout 1800s 2>&1 | tee loadsim.log + + - name: Summarize skips + if: always() + run: | + { + echo "### Loadsim results" + echo '```' + grep -E -- "^(--- (PASS|FAIL|SKIP)|\s+[a-z_]+_test.go:[0-9]+: (only|skip))" integration/loadsim.log || true + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + e2e: runs-on: ubuntu-latest timeout-minutes: 25 diff --git a/CLAUDE.md b/CLAUDE.md index 0751fdc..bb402ec 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -29,6 +29,9 @@ seedstorm/ │ │ ├── compare.go # compare command │ │ ├── mirror.go # mirror command │ │ ├── snapshot.go # snapshot command (row counts to JSON/YAML) +│ │ ├── tune.go # tune command, --workers auto +│ │ ├── production.go # --production / --allow-production write guard +│ │ ├── relationships.go # --relationships scan flags, --shape-rows, achieved-shape logs │ │ ├── progress.go # --workers flag, progress log lines, profile ignore │ │ ├── profile.go # profile command + shared --profile flag/loading │ │ ├── endpoints.go # --source-*/--target-* flags and connection opening @@ -38,7 +41,12 @@ seedstorm/ │ │ ├── postgres.go # PostgreSQL schema introspection + constraint parsing │ │ ├── mysql.go # MySQL schema introspection + constraint parsing │ │ ├── truncate.go # Truncate helper (FK-safe order) -│ │ ├── counts.go # GetTableRowCounts helper (used by gaps) +│ │ ├── counts.go # CountTables / CountTablesWithin: per-table outcomes, unknown never 0 +│ │ ├── read_scope.go # ReadOnce: read-only tx, statement/lock timeouts, MySQL KILL QUERY on cancel +│ │ ├── relations.go # DegreeHistogram (CASE buckets), LeadingIndexed, EstimateDegrees +│ │ ├── server_info.go # DetectServer (capacity, replica, server id), ConnectionUsage +│ │ ├── explain.go # Explain: disk full / connection lost errors in plain words +│ │ ├── partitions.go # Postgres partitioned tables (bounds, leaves) │ │ ├── stats.go # Table sizes, estimated counts, column lists, DB identity │ │ ├── copy.go # CopyRows: Postgres COPY for a chunk of rows │ │ ├── sequences.go # SyncSequences: move Postgres sequences past inserted ids @@ -56,6 +64,9 @@ seedstorm/ │ │ ├── stream.go # Stream: generation state kept across chunks (NewStream, Generate) │ │ ├── keys.go # keySet: exact map, then scalable Bloom filter │ │ ├── pools.go # PK pool reservoir sampling and capping +│ │ ├── shapes.go # Relationship shapes: degree dealer (Fenwick slots), DeriveShapedRows +│ │ ├── references.go # Pools for FKs to non-key columns ("table.column") +│ │ ├── partitions.go # Partition-key generators, CheckSeedable │ │ └── *_test.go # Unit tests alongside production files │ ├── graph/ │ │ ├── graph.go # Dependency graph (Build, TopologicalSort, RenderPlan) @@ -63,14 +74,21 @@ seedstorm/ │ │ └── graph_test.go # Unit tests │ ├── rules/ # Seed profile rules: model, templates, resolve/validate/compile │ ├── profiles/ # Saved profile store (profiles.yaml) + Resolve(file|name) -│ ├── compare/ # Snapshots, Diff, PlanMirror, snapshot files (Encode/ParseSnapshot), renderers (never writes) +│ ├── compare/ # Snapshots, Diff, PlanMirror, snapshot files v1/v2 (Encode/ParseSnapshot), DiffShapes, renderers (never writes) +│ ├── relations/ # Relationship shapes: Scan (index gate, per-edge timeout, cancel keeps finished), Shape +│ ├── tuning/ # Recommend, ClampWriters, DetectHost (cgroup CPU/memory) +│ ├── safego/ # Run/Recover: a panic becomes an error with an id +│ ├── runerr/ # Located errors: side · phase · table +│ ├── faultinject/ # SEEDSTORM_FAULT points (build tag faultinject only) │ ├── dataio/ # Streaming data documents: writers (yaml/json/sql/csv), ReadTables │ ├── seeder/ # Seed (strict chunked seed/gaps), Fill (resilient mirror inserts), MirrorJob, Preview, │ │ # writer.go (FK-gated concurrent writes), meter.go (rate/ETA), -│ │ # seed.go generateTables (parallel generation) + poolReleases +│ │ # seed.go generateTables (parallel generation) + poolReleases + clampToServer, +│ │ # relationships.go (Endpoint.Shapes, CompareShapes, MeasureShapes), servers.go (RelateServers) │ ├── fsutil/ # WriteFileAtomic for on-disk stores │ ├── tui/ # Bubble Tea flows (seed, gaps, generate, clone, mirror) -│ ├── web/ # serve: handlers_*.go per area, templates/, static/ (page.js + page.css per page) +│ ├── web/ # serve: handlers_*.go per area, templates/, static/ (page.js + page.css per page), +│ │ # jobs.go (list, reattach, eviction), production.go, preflight.go, shapes.go (session shape cache) │ ├── ai/ai.go # Gemini enrichment (prompt building, response parsing) │ ├── schema/schema.go # Schema YAML types and loader │ ├── build/info.go # Version info injected at build time @@ -80,7 +98,9 @@ seedstorm/ │ ├── binary_test.go # Builds the binary; scratch DB helpers per engine │ ├── *_test.go # Scenario evals: reseed, mirror, seeder, keycloak, compare_web, parallel_seed, │ │ # wide_schema (150 generated tables), access, snapshot, clone_objects, -│ │ # web_jobs (+helpers), reproducible, seed_scale (memory bounds) +│ │ # web_jobs (+helpers), reproducible, seed_scale (memory bounds), read_scope, production, +│ │ # reliability (faultinject), relationships, shaped_seed, partitions, tune, servers +│ ├── loadsim_*_test.go # Resource-limited evals in Cloud SQL-shaped containers (tags: integration loadsim) │ ├── fixtures/ # Keycloak schema dumps (real-world 87-table stress schema) │ ├── schema_postgres.sql # 28-table schema for Postgres integration tests │ └── schema_mysql.sql # 28-table schema for MySQL integration tests @@ -88,12 +108,13 @@ seedstorm/ │ ├── playwright.config.ts # chromium, 1 worker; global setup builds the binary and starts serve │ ├── support/ # global.setup/teardown, databases.fixture (ss_e2e_* DBs), wide-schema.fixture, │ │ # db.helpers (SQL assertions), test.fixture, selectors.ts (every data-testid) -│ └── tests/.spec.ts # workspace, seed, clone, compare, access, profiles, mobile +│ └── tests/.spec.ts # workspace, seed, clone, compare, access, profiles, mobile, memory, production, tuning, snapshot, relationships ├── docs/benchmarks.md # Measured throughput, memory and layout numbers behind the defaults ├── README.md # User-facing documentation (keep in sync with code) ├── Makefile # Build, test, lint, dev-up/down targets -├── compose.yaml # Local MySQL + PostgreSQL via Docker Compose -└── .github/workflows/pr.yml # CI: title, review, structlint, unit tests, lint, integration (-race), e2e +├── docker-compose.yaml # Local MySQL + PostgreSQL via Docker Compose +└── .github/workflows/ # pr.yml: title, review, gauntlet (structlint, unit -race), lint, integration, loadsim, e2e; + # loadsim-measure.yml: manual throughput measurement ``` --- @@ -205,12 +226,12 @@ After standard row generation, `topUpEnumCoverage` guarantees every enum value ( | Job | What it checks | |-----|---------------| -| `title` | Conventional Commits format | +| `pr-title` | Conventional Commits format | | `review` | AI code review via reviewforge (Gemini) | -| `validate` | Directory/file structure via structlint | -| `test` | `go test ./...` + `make build` | +| `gauntlet` | structlint, dupehound, unit tests with `-race` | | `lint` | `golangci-lint` | | `integration` | Full suite + scenario evals on Postgres 13/15/17 × MySQL 5.7/8.0/8.4, with the race detector | +| `loadsim` | Resource-limited evals (Cloud SQL-shaped containers); outcomes only, never timings | | `e2e` | Playwright journeys (`e2e/`) against a built binary on Postgres 15 + MySQL 8.0 | The integration job in CI uses `-race -timeout 1500s` (the race detector roughly doubles the ~5 minute suite). Use the same locally. @@ -279,6 +300,24 @@ The integration job in CI uses `-race -timeout 1500s` (the race detector roughly - Don't rely on server defaults (PG < 15 `CREATE` on `public`, MySQL 5.7 CHECK/roles). - Do give scratch databases unique `ss__*` names. +**Shared state a run borrows** +- Do restore a pool setting a run changes (`SetMaxOpenConns`): the web seeds on the pool its pages query. +- Don't judge a replica by MySQL `read_only`: a user with SUPER writes anyway, only `super_read_only` blocks. +- Do buffer every SSE event past the subscriber's channel: a fast job drops events, the handler refills from `job.Events()`. + +**Database truths that break guards** +- Do round datetime keys to the second: MySQL rounds fractional seconds on insert, so adjacent values collide. +- Don't read Postgres row statistics right after inserting: they land asynchronously, a partial count is a valid estimate. + +**Reads and failures** +- Do route new reads through `db.ReadOnce` (read-only, lock timeout, real cancel); never report an unknown count as 0. +- Do wrap goroutines in `safego.Run` and errors in `runerr.At`/`OnSide` so failures name side, phase and table. +- Don't trust the MySQL driver to stop a cancelled query: it keeps running without `KILL QUERY`. + +**Relationship shapes** +- Do keep the unshaped path free of extra random draws (`--seed` hashes must not change). +- Do return dealt slots when a row is retried or dropped; never loop to fit a shape, adjust and warn. + **Web UI** - Do check 320–1920 widths with the interaction active (search, dialogs, runs). - Do test graph changes on `wideSchemaDDL(150)`, not only small schemas. diff --git a/Makefile b/Makefile index 47bcc3a..9727e7f 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,13 @@ test-integration: dev-up done cd integration && go test -race -v -tags integration -count=1 ./... -timeout 1500s +.PHONY: test-loadsim +# Resource-limited evals: throwaway database containers shaped like managed +# Cloud SQL instances (integration/loadsim_*_test.go). Needs Docker; skips when +# the machine has too little free memory. Pass ARGS to filter, e.g. ARGS=-run=TestLoadsim_Read +test-loadsim: + cd integration && go test -race -v -tags "integration loadsim" -count=1 -run TestLoadsim $(ARGS) ./... -timeout 1800s + .PHONY: test-e2e # Playwright journeys against a freshly built `seedstorm serve` and the compose # databases (make dev-up). Pass ARGS to filter, e.g. ARGS=tests/compare.spec.ts diff --git a/README.md b/README.md index 032a00f..506f9cd 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,13 @@ seedstorm mirror --source-dsn "$PROD" --target-dsn "$STAGE" --scale 5 --max-rows No live access to production at mirror time? Save its counts once with `seedstorm snapshot --out prod-counts.yaml` (or **Export counts** in the web UI) and pass `--source-snapshot prod-counts.yaml`. +Row counts are not the whole picture: `--relationships` on `compare` and `snapshot` measures children per parent for every foreign key (min, avg, p95, max, parents with none), read-only and one key at a time on a timeout, and `mirror --shape-like-source` seeds the target with the same skew instead of an even spread: + +```bash +seedstorm compare --source-dsn "$PROD" --target-dsn "$STAGE" --relationships +seedstorm mirror --source-dsn "$PROD" --target-dsn "$STAGE" --shape-like-source +``` + Engines can differ (MySQL volumes onto Postgres works). Only the target is written; the command refuses when source and target are the same database. Rows the database rejects are regenerated, and a table that cannot be filled is reported without @@ -166,13 +173,17 @@ live example values and sample rows. See [docs/profiles.md](docs/profiles.md). - **Mirror volumes** — `mirror` seeds a target to a source's row counts at any scale (top-up or reset), with a reviewable plan, sample rows, FK-aware parents, and resilient inserts that report what could not be filled - **Seed profiles** — value rules (templates with `{{auto}}`/`{{seq}}`/`{{run}}`, fixed values, lists, NULL) applied by column pattern or per column; saved from the web UI and reused with `--profile` in the CLI and TUI - **Parallel writes with live progress** — unrelated tables (and pieces of a large table) write on `--workers` connections while generation streams in memory-bounded chunks; runs report rows written, rate and ETA (174k rows into MySQL: 41s → 14.5s). `--gen-workers` generates tables on several cores (1.79M rows/s on 8) for databases that can keep up — see [benchmarks](docs/benchmarks.md) +- **Relationship shapes** — measure children per parent for every foreign key (exact with an index gate and per-key timeouts, or planner estimates), compare them between databases, save them in counts files, and seed with them (`relationships:` in a profile, `mirror --shape-like-source`): no parent above max, real zero and NULL shares +- **Safe on production** — every read runs in a read-only transaction with a lock timeout and server-side cancel; unknown counts are never 0; connections marked production need an explicit confirmation to write, and relationship scans there read estimates unless confirmed +- **Tuning advice** — `tune` (and **Recommend** in the web UI) picks writers and generators from the database's free connections, vCPU, memory and disk, and checks the rows fit; every run lowers its writers when the server has fewer free connections +- **Failures say what failed** — errors name the side, phase and table and list what was written; a panic in one web job never stops the server or other jobs; exit codes 1 / 70 / 130 - **Reproducible** — `--seed` writes byte-identical data on every run - **Counts snapshots** — `snapshot` saves row counts to JSON/YAML; `compare` and `mirror` accept `--source-snapshot`, and the web UI exports and imports the same files - **Ignored tables** — a profile's `ignore:` globs keep tables out of every seed, fill, generate and mirror run - **Re-seed safely** — seeding a populated table appends: ids, UNIQUE sequences and composite keys continue past existing rows - **Schema clone for test DBs** — copy schema-only structure from one connected Postgres/MySQL database into another matching local target, preserving compatible table metadata before seeding it with safe fake data; `--objects all` adds views, functions/procedures and triggers - **Interactive TUI** — wizard for table selection, global config, self-reference depth, per-table row volumes, and review before seeding -- **Web UI** — `seedstorm serve` exposes an interactive graph workspace with search that zooms to matches, a navigator minimap for large schemas, privilege checks for the connected user, click-to-select tables, self-reference depth, per-table row overrides, truncate-only runs (`Rows = 0` + `truncate`), live SSE job logs with per-table truncate/insert progress, schema clone between connected DBs, and a multi-DB session switcher +- **Web UI** — runs you can leave and come back to (live progress, elapsed time, reattach), remembered settings per connection, relationship badges on the graph; `seedstorm serve` exposes an interactive graph workspace with search that zooms to matches, a navigator minimap for large schemas, privilege checks for the connected user, click-to-select tables, self-reference depth, per-table row overrides, truncate-only runs (`Rows = 0` + `truncate`), live SSE job logs with per-table truncate/insert progress, schema clone between connected DBs, and a multi-DB session switcher - **Saved connections** — connections persist on the machine and survive a restart, with test-before-connect, opt-in password storage, and driver-aware connection parameters (with JDBC-to-Go translation) for both Postgres and MySQL - **Dry-run** — preview the seed plan and INSERT SQL without touching the database - **Export** — generate fake data as YAML, JSON, SQL or CSV without a live connection, streamed in flat memory; SQL files load as-is diff --git a/cmd/seedstorm/main.go b/cmd/seedstorm/main.go index 8e40cda..e2087f2 100644 --- a/cmd/seedstorm/main.go +++ b/cmd/seedstorm/main.go @@ -2,18 +2,61 @@ package main import ( "context" + "errors" "fmt" "os" + "os/signal" + "syscall" + + tea "github.com/charmbracelet/bubbletea" "github.com/AxeForging/seedstorm/internal/app" + "github.com/AxeForging/seedstorm/internal/safego" _ "github.com/go-sql-driver/mysql" _ "github.com/jackc/pgx/v5/stdlib" ) +// Exit codes: 1 a run failed, 70 an internal error (a bug: please report it), +// 130 interrupted. +const ( + exitFailed = 1 + exitInternal = 70 + exitInterrupted = 130 +) + func main() { - if err := app.New().Run(context.Background(), os.Args); err != nil { - fmt.Fprintf(os.Stderr, "error: %v\n", err) - os.Exit(1) + os.Exit(run()) +} + +func run() int { + // The first Ctrl+C cancels the run, so reads stop on the server (MySQL + // queries are killed) and the summary is printed; a second one exits now. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + go func() { + <-ctx.Done() + stop() + again := make(chan os.Signal, 1) + signal.Notify(again, os.Interrupt, syscall.SIGTERM) + <-again + fmt.Fprintln(os.Stderr, "interrupted again: exiting now") + os.Exit(exitInterrupted) + }() + + err := safego.Run("seedstorm", func() error { return app.New().Run(ctx, os.Args) }) + switch { + case err == nil: + return 0 + case ctx.Err() != nil: + fmt.Fprintf(os.Stderr, "interrupted: %v\n", err) + return exitInterrupted + } + var p *safego.PanicError + if errors.As(err, &p) || errors.Is(err, tea.ErrProgramPanic) { + fmt.Fprintf(os.Stderr, "error: %v\nThis is a bug in seedstorm; run with --log-level debug for the stack and please report it.\n", err) + return exitInternal } + fmt.Fprintf(os.Stderr, "error: %v\n", err) + return exitFailed } diff --git a/docs/benchmarks.md b/docs/benchmarks.md index 9217461..fd6879d 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -74,6 +74,49 @@ The integration tests `TestSeed_WideRowsStayUnderAMemoryBound`, process's own `VmHWM`: rusage's max RSS also counts the test process at fork, which under `-race` made every run report 212 MB. +## Relationship shapes + +Shaped seeding keeps the streaming memory bound: a dealer holds a few bytes per +parent (degrees plus a Fenwick tree over the remaining slots) and is freed when +its table ends. Dealing 5M slots over 500,000 parents takes well under a +second (`TestDealDegrees_PoolLimitIsFast`). + +| Run | Peak memory | +|-----|------------:| +| Shaped seed, 100k parents + 225k shaped children (Postgres) | **124 MB** | +| Unshaped seed, 100k + 200k rows (bound held by `TestSeed_LargeRunsStreamWithFlatMemory`) | < 150 MB | + +```bash +cd integration && go test -tags integration -count=1 -run TestShapedSeed_MemoryStaysBounded -v ./... +``` + +With `--seed` and no `relationships:` in the profile, output is byte-identical +to the version before shaping existed (the 6 recorded files below). + +## Tuning and small instances + +Writers for a managed database are limited by its CPU and disk more than by +seedstorm. Local Docker container shaped like Cloud SQL's smallest instance +(`cloudsql-micro`: 1 vCPU, 629MB, 300 write IOPS), 3 tables × 2,000 rows, +one run each (2026-09-17): + +| Engine | 1 writer | 2 writers | 4 writers | 8 writers | +|--------|---------:|----------:|----------:|----------:| +| MySQL 8.4 | 1,672 rows/s | **1,851 rows/s** | 1,496 rows/s | 1,483 rows/s | +| Postgres 17 (COPY) | 50,887 rows/s | 54,240 rows/s | 54,714 rows/s | 57,910 rows/s | + +On MySQL two writers per vCPU is the best of the four and more writers get +slower; that is what `tune` recommends for a small instance. The Postgres run +ends in about 0.1s, too short for the disk limit to matter, so it does not rank +writers. The numbers vary from run to run; read them as ratios. + +Reproduce (numbers go to the JSON report): + +```bash +SEEDSTORM_LOADSIM_MEASURE=/tmp/measure.json SEEDSTORM_LOADSIM_PROFILES=cloudsql-micro \ + make test-loadsim ARGS=-run=TestLoadsim_MeasureWriters +``` + ## Reproducibility `--seed` writes byte-identical data on every run. The generation speedups were diff --git a/docs/commands.md b/docs/commands.md index 9f010ae..a311ff4 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -13,11 +13,16 @@ Every seedstorm command, with all flags and examples. - [`clone-schema`](#clone-schema) — copy schema structure into another DB - [`compare`](#compare) — row counts, sizes and column drift between two DBs - [`mirror`](#mirror) — seed a target so its volumes follow a source -- [`snapshot`](#snapshot) — save row counts to a file for compare/mirror +- [`snapshot`](#snapshot) — save row counts (and relationship shapes) to a file for compare/mirror +- [`tune`](#tune) — recommend writers and generators for a database - [`profile`](#profile) — manage seed profiles (value rules) - [`serve`](#serve) — local web UI for every feature - [`version`](#version) / [`completion`](#completion) +**Exit codes:** `0` success · `1` the command failed (the message names the side, phase and table, e.g. `error: target · write · orders: …`, and a partial run lists what was written) · `70` internal error (a panic, reported with an id; the stack is in `--log-level debug`) · `130` interrupted with Ctrl+C (running queries are cancelled on the server first). + +**Unreachable databases** fail within 10 seconds naming the database (`app@db:5432 did not answer`), instead of waiting for the operating system's TCP timeout. + --- ## `introspect` @@ -46,6 +51,11 @@ SEEDSTORM_DB=postgres SEEDSTORM_DSN="postgres://..." seedstorm introspect | `--db` / `$SEEDSTORM_DB` | `postgres` | Database type: `postgres` or `mysql` | | `--dsn` / `$SEEDSTORM_DSN` | — | Connection string (required) | | `--out` / `-o` | `schema.yaml` | Output file path | +| `--relationships` | — | Also measure every foreign key's shape (read-only, exact) and write it with estimated table counts to this [snapshot](#snapshot) file | +| `--scan-unindexed` | false | With `--relationships`: also scan keys that lead no index (full table scans); otherwise those are estimated | +| `--read-timeout` | `60s` | With `--relationships`: server-side time limit per foreign key; a slower one is reported as `timed out` and the rest continue | + +Introspection logs a line per table on large schemas, so a slow catalog shows progress. --- @@ -159,7 +169,21 @@ The interactive TUI includes a **Volumes** step after global config. Each select | `--workers` | `4` | Connections writing at once. A table writes only after every table it references; self-referencing tables write in order (`1` = one at a time) | | `--gen-workers` | `1` | Tables generated at once on separate cores. Only helps when the database takes rows faster than one core generates them (hundreds of thousands per second, see [benchmarks](benchmarks.md)); ignored with `--seed` so runs stay reproducible | | `--interactive` / `-i` | false | Launch interactive TUI | -| `--profile` / `-p` / `$SEEDSTORM_PROFILE` | — | [Seed profile](profiles.md): rules file or saved profile name; its `ignore:` tables are never written | +| `--profile` / `-p` / `$SEEDSTORM_PROFILE` | — | [Seed profile](profiles.md): rules file or saved profile name; its `ignore:` tables are never written and its `relationships:` shape foreign keys | +| `--shape-rows` | false | Derive the row count of each shaped child table from its parents (parents × share with children × avg); `--table-rows` still wins | +| `--production` / `$SEEDSTORM_PRODUCTION` | false | The database is production: writes are refused unless `--allow-production` is also given (dry runs still work) | +| `--allow-production` | false | Confirm a write to a database marked `--production` | + +`--workers auto` picks writers from the database's free connections (see [`tune`](#tune)). Whatever you ask for, a run never opens more connections than the server has free: it lowers the writers and says so (`Using 3 writers instead of 8: the server has 95 of 100 connections in use`). + +With a profile that has `relationships:`, foreign keys follow those shapes instead of picking parents evenly: each parent gets a number of children drawn from the histogram, no parent exceeds `max` (exact while the parent table fits the 500,000-row key pool), the share of parents without children and of NULL keys is kept, and the run logs the achieved shape next to the target afterwards: + +``` +info Rows derived from relationship shapes rows=14000 table=orders +info Relationship shape (target → table now) avg="4.00 → 4.00" max="25 → 25" relationship=orders.account_id without_children="30% → 30%" +``` + +Shapes that cannot fit the planned rows are adjusted with a warning instead of looping (`14000 rows over 500 parents do not fit max 25: max raised to 28`). Self-references and junction keys are not shaped (reported). A parent table above 500,000 rows is kept as a sample: children are dealt a round at a time over each sample (the run says so), and seeding against a live database rotates the sample so the whole table gets its share. The shape is then close rather than exact — measured on 600,000 parents and 1.6M children: average 3 → 3.4, max 8 → 10, parents without children 10% → 22%. Any `--rows` is safe: rows are generated and written 20,000 at a time, Postgres takes each chunk through `COPY`, and memory stays flat (600k rows on Postgres: 7s, under 100MB). A dry run prints the SQL the same way. @@ -248,6 +272,7 @@ Gap Analysis | `--gen-workers` | `1` | Tables generated at once on separate cores. Only helps when the database takes rows faster than one core generates them (hundreds of thousands per second, see [benchmarks](benchmarks.md)); ignored with `--seed` so runs stay reproducible | | `--interactive` / `-i` | false | Launch interactive TUI | | `--profile` / `-p` / `$SEEDSTORM_PROFILE` | — | [Seed profile](profiles.md): rules file or saved profile name; its `ignore:` tables are never filled | +| `--production` / `--allow-production` | false | As for [`seed`](#seed) | --- @@ -354,6 +379,7 @@ By default only tables, foreign keys, indexes and comments are cloned. `--views` | `--objects` | — | Comma-separated kinds to clone: `views`, `routines`, `triggers`, or `all` | | `--dry-run` / `-n` | false | Print generated DDL, do not execute | | `--interactive` / `-i` | false | Confirm the clone in the terminal UI | +| `--production` / `--allow-production` | false | As for [`seed`](#seed): marks the target as production | Boundaries: `clone-schema` is same-engine only. It does not attempt cross-engine translation, and it does not clone partial/expression indexes, grants, ownership, events, or non-public/non-current schemas. @@ -376,6 +402,9 @@ seedstorm compare --source-dsn "$PROD" --target-dsn "$STAGE" --format json # Source from a counts file instead of a connection (see snapshot) seedstorm compare --source-snapshot prod-counts.yaml --target-dsn "$STAGE" + +# Also compare children per parent for every foreign key +seedstorm compare --source-dsn "$PROD" --target-dsn "$STAGE" --relationships ``` Sample output: @@ -392,6 +421,18 @@ employees 160 80 -80 72.0 KB 64.0 KB rows 3856 → 1888 · size 1.7 MB → 1.3 MB ``` +With `--relationships`: + +``` +Relationships (children per parent: avg / p95 / max · parents without children) +RELATIONSHIP SOURCE TARGET STATUS +orders.user_id → users 5.15 / 15 / 34 · 19% 2.00 / 2 / 2 · 0% differs +reviews.order_id → orders ~1.00 / ? / ? · 70% 0.00 / 0 / 0 · 100% differs +2 relationships · same 0 · differs 2 · source only 0 · target only 0 · unknown 0 +``` + +`~` marks estimates (planner statistics). A relationship that timed out, was locked or failed shows its outcome instead of numbers and counts as `unknown`. + | Flag | Default | Description | |------|---------|-------------| | `--source-db` / `$SEEDSTORM_SOURCE_DB` | `postgres` | Source database type | @@ -401,7 +442,12 @@ rows 3856 → 1888 · size 1.7 MB → 1.3 MB | `--target-dsn` / `$SEEDSTORM_TARGET_DSN` | — | Target connection string (required) | | `--counts` | `exact` | `exact` (COUNT(*)) or `estimate` (Postgres `reltuples`, MySQL `TABLE_ROWS`); tables with no or zero statistics are counted exactly and estimated counts are marked `~` | | `--format` / `-f` | `table` | `table` or `json` | -| `--only-diff` | false | Hide tables whose counts and columns match | +| `--only-diff` | false | Hide tables (and relationships) that match | +| `--relationships` | false | Also compare every foreign key's shape on both sides (a `--source-snapshot` must include relationships) | +| `--scan-unindexed` | false | With `--relationships`: also scan keys that lead no index | +| `--read-timeout` | `60s` | With `--relationships`: server-side time limit per foreign key | + +All reads are safe on a busy database: each runs in a read-only transaction, waits at most 2 seconds for a lock (a table locked by DDL is reported `locked`, not waited on), and is cancelled on the server when you press Ctrl+C. A count that could not be read is `unknown` (`?`), never 0. Both sides are read at the same time; a side that fails is named (`target · count: …`). Statuses: `same`, `differs`, `source_only`, `target_only`. MySQL sizes come from cached statistics and are approximate. @@ -432,6 +478,10 @@ seedstorm mirror --source-db mysql --source-dsn "$MYSQL_PROD" \ # Review, preview samples and confirm in the terminal UI seedstorm mirror --source-dsn "$PROD" --target-dsn "$STAGE" --interactive + +# Give the target production's children per parent, not an even spread +seedstorm mirror --source-dsn "$PROD" --target-dsn "$STAGE" --shape-like-source +seedstorm mirror --source-snapshot prod-with-relationships.yaml --target-dsn "$STAGE" --shape-like-source ``` Sample dry run: @@ -460,6 +510,8 @@ How the plan is built: - Tables missing on the target, or whose source count is unknown, are skipped with a reason. Per-table `rows` in a profile do not apply: volumes come from the source. - The run refuses when source and target are the same database, however the DSNs are spelled. With `--source-snapshot` that check cannot run, and seedstorm warns about it: double-check `--target-dsn`. - Tables a profile lists under `ignore:` are skipped (`ignored by profile`) and never truncated; a table that needs an ignored, empty parent is skipped with the reason. +- A target that refuses every write is refused before anything is planned: a Postgres standby, or MySQL with `super_read_only`. MySQL `read_only` alone only warns, since a user with `SUPER` can still write. When source and target are databases on the same server, the plan says so: reading one and writing the other share its CPU, memory and disk. +- `--shape-like-source` shapes the target's foreign keys like the source's (see [`seed`](#seed)); the shapes come from the snapshot file's relationships, or are measured read-only on the source first. Keys this schema cannot shape (junction keys, key columns, self-references) are named with their reason and spread evenly — on Keycloak's 87 tables, 29 of 67 measured keys are shaped and the other 38 are reported. Large volumes are safe to run: rows are generated and written 20,000 at a time, Postgres takes each chunk through `COPY`, and memory stays flat whatever the table size (a 7.2M-row mirror peaks around 150MB). Existing keys are read once per table; tables with more than 500,000 parents reference a rotating sample of them, so children still spread over the whole parent table. After the run, Postgres sequences behind SERIAL/IDENTITY columns are moved past the inserted ids, so the application's own inserts keep working (`advanced orders.id sequence 0 → 2000000`). @@ -496,6 +548,11 @@ inserted 384 rows · missing 6 | `--yes` / `-y` | false | Skip the `reset` confirmation | | `--seed` | `0` | Random seed for reproducible generation | | `--interactive` / `-i` | false | Review, preview and confirm in the TUI | +| `--shape-like-source` | false | Shape target foreign keys like the source's | +| `--scan-unindexed` / `--read-timeout` | false / `60s` | With `--shape-like-source` on a live source, as for `compare --relationships` | +| `--production` / `--allow-production` | false | As for [`seed`](#seed): marks the target as production | + +`--workers auto` is accepted as for `seed`. --- @@ -508,6 +565,9 @@ seedstorm snapshot --db postgres --dsn "$PROD_DSN" --out prod-counts.yaml seedstorm snapshot --db mysql --dsn "$DSN" --counts estimate --format json > counts.json seedstorm compare --source-snapshot prod-counts.yaml --target-dsn "$STAGE_DSN" seedstorm mirror --source-snapshot prod-counts.yaml --target-dsn "$STAGE_DSN" --scale 0.1 + +# Counts plus every foreign key's shape (writes version 2) +seedstorm snapshot --dsn "$PROD_DSN" --relationships --out prod-with-relationships.yaml ``` ```yaml @@ -535,6 +595,8 @@ tables: orders: 5000 ``` +With `--relationships` the file is `version: 2` and adds a `relationships:` list: per foreign key the parent and child counts, NULL keys, share of parents without children, min / avg / p50 / p95 / max children per parent and a histogram (exact buckets 1–16, then powers of two). Relationship scans run read-only with a 60-second limit per key, two at a time, cheapest tables first; keys that lead no index are estimated unless `--scan-unindexed` is given, and a key that times out is recorded as `timed out` while the others continue. A file without relationships stays `version: 1`, readable by older seedstorm versions. + Tables match the target case-insensitively, so a Postgres snapshot works against MySQL. `-1` means unknown, and mirror skips that table. The web UI's **Compare** page exports either side of a comparison in the same format and imports files or pasted text as a source. | Flag | Default | Description | @@ -544,6 +606,43 @@ Tables match the target case-insensitively, so a Postgres snapshot works against | `--counts` | `exact` | `exact` (COUNT(*)) or `estimate` (planner statistics) | | `--format` / `-f` | `yaml` | `yaml` or `json` | | `--out` / `-o` | stdout | File to write | +| `--relationships` | false | Also measure every foreign key's shape (read-only; exact or estimate per `--counts`) | +| `--scan-unindexed` | false | With `--relationships`: also scan keys that lead no index | +| `--read-timeout` | `60s` | With `--relationships`: server-side time limit per foreign key | + +--- + +## `tune` + +Recommends `--workers` and `--gen-workers` for one database from what it reports (free connections, buffers, used space; read-only) and what SQL cannot tell (vCPU, memory, disk), and checks that the rows fit on the disk. + +```bash +seedstorm tune --dsn "$DSN" --vcpu 1 --memory-mb 629 --storage network-ssd --storage-gb 10 --rows 2000000 --avg-row-bytes 300 +``` + +``` +Host 16 CPUs, 31188MB memory (this machine or container, running seedstorm) +Database postgres 15.18, 6/100 connections in use +Writers 2 (--workers) +Generators 1 (--gen-workers) + +Why: + - writers 2: 1 vCPU on the database, 2 writers each + - generators 1: one generator keeps up with this database + +Disk: About 1.3GB of the 10.0GB free will be used. +``` + +The recommendation is an estimate from benchmarks on one machine: watch the rate during the first minute and adjust. Production, shared or high-availability databases get fewer writers. The web UI's **Recommend** dialog (workspace → Tuning) runs the same rules. + +| Flag | Default | Description | +|------|---------|-------------| +| `--db` / `--dsn` | — | As for `seed` | +| `--vcpu` / `--memory-mb` | — | The database's CPU and memory | +| `--storage` | — | `local-ssd`, `network-ssd` or `hdd` | +| `--storage-gb` / `--iops` | — | Disk size and provisioned IOPS of a network disk | +| `--rows` / `--avg-row-bytes` | — / `256` | Rows the run will write, for the disk check | +| `--shared` / `--ha` / `--production` | false | Other workloads use it / synchronous replica / production: stay conservative | --- @@ -588,6 +687,13 @@ What the UI gives you: - **Multi-session** — hold several DBs open at once and switch from the topbar dropdown; a saved connection that is already live offers **Switch to** instead of a second connect. The workspace can clone schema from the active connection into another matching connected database. - **Compare & mirror** — `/compare` puts any two connections (live or saved, engines may differ) side by side: per-table mirrored gauges of source and target rows, size, delta and column drift, filterable. The mirror panel plans a top-up or reset at any scale for the ticked tables, shows the plan, truncate list, skipped tables and sample rows in a review dialog, and runs only after you confirm (reset needs an extra acknowledgement). Counts refresh when the run ends. The last comparison of each pair is kept in the browser and restored when you come back, marked with its age. **Export counts** downloads or copies either side as a [`snapshot`](#snapshot) file (JSON or YAML); **Import counts** takes a file, a drop or pasted text and uses it as the source for compare and mirror. - **Seed profiles** — `/profiles` builds [value rules](profiles.md) with a generator palette (click or drag tokens into a template), ordered column patterns with live example values and match counts, and a table explorer that shows what every column will get plus sample rows from the active connection. Profiles save to disk, import (file, drop or paste) and export (download or copy) as YAML, and are selectable in the workspace action bar and on Compare. **Ignored tables** lists globs never written, with the tables each one matches; the workspace greys those tables out and lists them in an **Ignored** tab. +- **Runs you can leave** — every page shows a run strip with the phase, progress, elapsed time and whether the server is still answering (`running`, `quiet`, `reconnecting`, `lost`). Leave a page while a seed, compare or mirror runs and come back: the run is reattached with live progress, or its result is shown if it finished. A failure names the side, phase and table and lists what completed; if `serve` restarted, the page says the run was lost instead of spinning. +- **Remembered settings** — rows, tuning, profile and compare options are kept per connection in the browser. Truncate, disable-FK and drop options are never remembered. +- **Fast return to the workspace** — the graph draws from the cached schema first; row counts fill in afterwards (cached per connection, **↻** recounts) and a table that cannot be counted says so. +- **Production connections** — tick *Production database* on a saved connection: writes to it (seed, fill empty, mirror, clone into it) need its label typed back, and relationship scans read planner estimates, one query at a time, unless you confirm an exact scan. +- **Recommend** — the Tuning panel's dialog takes vCPU, memory, storage type and size, and recommends writers and generators with the reasons (`tune` on the command line). +- **Snapshot counts** — saves the active connection's row counts (optionally with its relationships) to YAML or JSON, or hands them to Compare as an imported source; **Calibrate from a file** opens Compare with this connection as target. +- **Analyze relationships** — measures children per parent for every foreign key in the background (exact or estimate, unindexed keys opt-in); each arrow in the graph gets an `avg · max` badge as its key finishes, and the table panel shows the details. On Compare, **Compare relationships** shows the same per key for both sides, the export can include them, and **Shape relationships like the source** makes a mirror follow them. - **Standalone tools** — `/generate`, `/enrich`, `/export` mirror the CLI commands as forms. - **Bounded resources** — text a job returns to the browser is capped at 20MB: a dry run's SQL stops there with a note of the rows left out, while generate and export refuse and point at the CLI `--out` flags, since a cut document would be broken. Job requests are limited to 21MB, and a connection's pooled database connections close after two idle minutes (the next query reopens one). diff --git a/docs/development.md b/docs/development.md index 337c2bb..9c5fd4b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -111,6 +111,15 @@ scratch databases they create and drop, so they test exactly what a user runs: | `TestSeed_ManyTablesDoNotKeepEveryKeyPool` | 100 tables × 30k rows peak 82MB; keeping every key pool peaked at 262MB | | `TestWebJobs_SeedProgressIsTruthful` / `…CancelIsPromptAndLeavesAUsableServer` / `…GapsFillAndMirrorReportProgress` / `…ConcurrentSeedJobs` | In-process server on real DBs: progress is monotonic and ends at done == total == `COUNT(*)`, one `Table written` per table, cancel ends within 15s with no queries left, concurrent jobs with viewers joining and leaving stay correct. Run with `-race` (found two job-manager races) | | `TestCloneSchema_Objects` | `clone-schema --objects all`: view on view, function, procedure and trigger work on the clone; nothing extra without the flags | +| `TestReadScope_*` | Reads run in a read-only transaction the server enforces, time out server-side, stop waiting behind a DDL lock, and a cancelled MySQL query is killed on the server (it keeps running with the driver alone) | +| `TestProduction_CLIRefusesWritesUnlessAllowed` | `--production` refuses seed/gaps/mirror/clone writes without `--allow-production`; nothing is written; dry runs work | +| `TestCLI_ExitCodesSayWhatKindOfFailureItWas` / `TestCLI_InterruptExitsWith130` / `TestServe_PanicInOneJobLeavesTheServerAndOtherJobsRunning` | Built with `-tags faultinject`: a panic exits 70 with the phase and table, an error exits 1, Ctrl+C exits 130; in `serve` a panicking job fails alone while another job completes and the server keeps answering | +| `TestFaultInjection_AbsentFromTheDefaultBinary` | The release binary contains no fault-injection code | +| `TestIntrospect_PostgresConstraintsArePairedAndVisibleToReadOnlyRoles` / `TestSeed_ForeignKeysToNonKeyColumnsInsertValidReferences` | FK columns paired from `pg_constraint` (composite and cross-schema), and FKs to a non-key UNIQUE column reference real values | +| `TestPartitionedTables_*` | Postgres partitioned tables are counted once and seeded inside their partition bounds; an expression partition key is refused before writing | +| `TestDetectServer_ReadsCapacityOnBothEngines` / `TestTune_CLIRecommendsAndSeedUsesAuto` / `TestServers_TwoDatabasesOnOneServerAreShared` | Server capacity reads, `tune` output and `--workers auto`, shared-server detection | +| `TestRelationships_*` | Hand-computed shapes on both engines, the unindexed-key gate, a timed-out key while the others finish, cancel keeping finished keys, `snapshot`/`introspect`/`compare --relationships` through the binary | +| `TestShapedSeed_*` | A shaped profile seeds with no parent above max and the target average and zero share; `mirror --shape-like-source` copies a skewed source; shaped seeding keeps the memory bound (325k rows, ~124MB) | Scratch databases on MySQL are created as `root` (`SEEDSTORM_MYSQL_ROOT_PASSWORD`, default `root`). @@ -145,6 +154,11 @@ SEEDSTORM_E2E_KEEP=1 make test-e2e # keep the scratch databases for debuggin | `access` | SELECT-only MySQL user: read-only badge, banner, warning chips | | `profiles` | Ignore glob with live matches, saved to disk, Ignored tab in the workspace, exported YAML | | `mobile` | 390 and 320 wide with search and a wide preview open: no overlap, no sideways scroll | +| `memory` | Workspace and compare settings survive leaving the page (truncate is never kept); a run started before leaving is reattached | +| `production` | Seeding a connection marked production asks for its label first | +| `tuning` | Recommend dialog for a small managed database applies its writers | +| `snapshot` | Snapshot counts from the workspace, download, compare another database against the file, calibrate from a file | +| `relationships` | Analyze relationships (estimate, then exact), include them in a counts file, compare drift, export with relationships, mirror plan shaped like the source, import into a profile | Specs find elements only through `data-testid` names kept in `e2e/support/selectors.ts`: when you change markup a journey uses, keep or move the test id and update that file. Each journey checks what the user sees and, when it writes, the database via SQL. Static assets are embedded in the binary, so the suite always runs against a fresh build. @@ -169,18 +183,42 @@ All tests run automatically on every PR via GitHub Actions (`.github/workflows/p | Job | What it checks | |-----|---------------| -| `title` | Conventional Commits format | +| `pr-title` | Conventional Commits format | | `review` | AI code review via reviewforge (Gemini) | -| `validate` | Directory/file structure via structlint | -| `test` | `go test ./...` + `make build` | +| `gauntlet` | structlint, dupehound, and unit tests with `-race` (its `gotest` gate) | | `lint` | `golangci-lint` | | `integration` | Full 29-table suite, scenario evals and schema-clone tests with `-race` on each Postgres/MySQL pair | +| `loadsim` | Resource-limited evals in Cloud SQL-shaped containers (below); the job summary prints the runner's size and every skip | | `e2e` | Playwright journeys against a built binary (Postgres 15, MySQL 8.0) | +`loadsim-measure.yml` measures throughput per profile and writer count when started by hand (`workflow_dispatch`); it never gates a PR. + The integration job in CI uses `-race -timeout 1500s` across the database-version matrix. Use the same timeout locally when running both engines back-to-back. Throughput and memory numbers, and how to reproduce them, are in [benchmarks.md](benchmarks.md). +### Resource-limited evals (loadsim) + +`integration/loadsim_*_test.go` (build tags `integration loadsim`) start throwaway database containers shaped like managed instances and run the binary against them, some of it inside a container with its own CPU and memory limits: + +| Profile | Database container | Settings from | +|---------|-------------------|---------------| +| `cloudsql-micro` | 1 vCPU, 629MB, 300 write IOPS | Cloud SQL docs (Postgres); `SHOW VARIABLES` of a MySQL 8.4 1 vCPU / 628.74MB instance | +| `cloudsql-2vcpu` | 2 vCPU, 7.5GB, 3000 write IOPS | Cloud SQL docs; MySQL values estimated from the micro reference | + +They assert outcomes, never speed: seedstorm sees a container's CPU and memory limits (not the host's), few free connections lower the writers and the run completes, a full disk fails naming the table and the cause, a million rows seed under a 256MB container, and the read safeguards hold on the smallest instance. A test skips (visibly) when the machine lacks the memory for its profile or a throttle does not apply. + +```bash +make dev-up +make test-loadsim # needs Docker +make test-loadsim ARGS=-run=TestLoadsim_Read # one test +SEEDSTORM_LOADSIM_MEASURE=/tmp/measure.json SEEDSTORM_LOADSIM_PROFILES=cloudsql-micro make test-loadsim ARGS=-run=TestLoadsim_MeasureWriters +``` + +### Fault injection + +Reliability tests build the binary with `-tags faultinject`; `SEEDSTORM_FAULT=point:table:action` then makes a point fail (`write`, `generate`, `copy`, `job`, `introspect`; action `panic`, `error` or `hang`), e.g. `SEEDSTORM_FAULT=write:orders:panic`. The default build contains none of it. + ### Supported database versions | Postgres | MySQL | Notes | @@ -215,7 +253,10 @@ The engine-specific suites (`TestMySQLIntegration`, `TestMySQLGaps`, `TestMySQLS | `SEEDSTORM_PROFILE` | Default `--profile` for `seed`, `gaps`, `generate`, `mirror` | | `SEEDSTORM_PROFILES` | Path of the saved-profile store (default `~/.config/seedstorm/profiles.yaml`) | | `SEEDSTORM_SOURCE_DSN` / `SEEDSTORM_TARGET_DSN` | Defaults for `compare`, `mirror`, `clone-schema` | +| `SEEDSTORM_PRODUCTION` | Same as `--production` on commands that write | | `SEEDSTORM_PG_HOST` / `SEEDSTORM_PG_PORT` / `SEEDSTORM_MYSQL_HOST` / `SEEDSTORM_MYSQL_PORT` | Integration tests: where the databases listen | +| `SEEDSTORM_FAULT` | Fault-injection builds only: `point:table:panic\|error\|hang` | +| `SEEDSTORM_LOADSIM_MEASURE` / `SEEDSTORM_LOADSIM_PROFILES` / `SEEDSTORM_LOADSIM_ENGINES` | Loadsim measure: report path, profiles and engines | --- @@ -226,6 +267,8 @@ make build Build for current platform → bin/seedstorm make build-all Build for linux/darwin amd64+arm64 → dist/ make test Run unit tests make test-integration Run integration tests (requires make dev-up) +make test-loadsim Run resource-limited evals (requires Docker and make dev-up) +make test-e2e Run Playwright journeys (requires make dev-up) make lint Run golangci-lint make fmt Format with gofumpt make tidy go mod tidy diff --git a/docs/profiles.md b/docs/profiles.md index b3aaa05..2459969 100644 --- a/docs/profiles.md +++ b/docs/profiles.md @@ -42,8 +42,68 @@ tables: # explicit settings per table columns: role: { value: guest } phone: { setNull: true } +relationships: # children per parent for a foreign key (version 2) + orders.user_id: + min: 1 + avg: 5.2 + max: 34 + zeroShare: 0.19 # share of parents with no children + nullShare: 0 # share of NULL keys (nullable columns) + histogram: # optional: parents per degree range + - {min: 1, max: 1, parents: 187} + - {min: 2, max: 3, parents: 340} + - {min: 4, max: 7, parents: 310} + - {min: 8, max: 34, parents: 133} ``` +### Relationships + +`relationships:` makes foreign keys look like real data instead of an even +spread: most users with a few orders, some with many, some with none. Each key +is `table.column` (matched to the database ignoring case). Seed, fill empty, +generate and mirror deal parents so that: + +- no parent gets more than `max` children, and each parent with children gets at least `min` (exact while the parent table fits the key pool of 500,000 — see below); +- the share of parents without children is `zeroShare`, and exactly `nullShare` of the rows have a NULL key; +- degrees follow the `histogram` when given (bucket by its parent weight, then a value inside it), otherwise they spread around `avg`. + +Every foreign key of a table can be shaped at once; the keys are dealt +independently, so they are not correlated (the users with many orders are not +necessarily the ones with many reviews). + +When the planned rows cannot fit the shape, seedstorm adjusts in one bounded +pass and warns instead of retrying: `max raised`, `share of parents without +children lowered`, or fewer parents with children. More rows than planned +(enum coverage) pick parents evenly and are reported. After a run that writes, +the achieved shape is measured and logged next to the target. + +Not shaped (every run names them with the reason, and the profile validation +warns too): self-references, junction tables whose key is made of foreign keys, +and foreign keys that are part of the primary key. Real schemas hit this often: +on Keycloak's 87 tables, 29 of 67 measured keys are shaped. A parent table above 500,000 rows is held as a +sample: children are then dealt a round at a time over each sample, and the run +says so. Seeding against a live database rotates the sample, so the whole +parent table gets its share, and the shape lands close rather than exact — a +parent caught in two samples can pass `max`, and parents never sampled raise +the share without children (measured on 600,000 parents and 1.6M children: +average 3 → 3.4, max 8 → 10, without children 10% → 22%). `generate` (no +connection) can only reach the parents in the sample. + +Where shapes come from: + +- **Analyze relationships** in the workspace, `seedstorm snapshot --relationships` + or `introspect --relationships` measure them from a database; **Import from a + counts file** on the Profiles page turns such a file into `relationships:`. +- `seed --shape-rows` derives the row count of each shaped child table from its + parents (parents × (1 − zeroShare) × avg ÷ (1 − nullShare)); `--table-rows` + still wins. +- `mirror --shape-like-source` (or **Shape relationships like the source** on + Compare) uses the source's shapes directly, without a profile. + +A profile with relationships is written as `version: 2`; older seedstorm +versions refuse it rather than seeding it unshaped. Without relationships a +profile stays `version: 1`. + ### Ignored tables `ignore:` lists table-name globs that seedstorm never writes: seed, gaps, generate and mirror skip them (no inserts, no truncation). Globs match the whole name and ignore case, so `flyway_*` also matches MySQL's `FLYWAY_SCHEMA_HISTORY`; `*` matches any run of characters and `?` exactly one. @@ -109,7 +169,8 @@ Validation (web UI issues panel, `seedstorm profile validate --dsn …`) reports | Level | Examples | |-------|----------| | error | no action / two actions, unknown generator or token, bad glob, explicit rule on a key, `setNull` on `NOT NULL`, a value that cannot fit the column type | -| warning | table or column not in this database (profiles stay portable), a rule that matches nothing or is shadowed by an earlier rule, a UNIQUE column (or a column inside a multi-column UNIQUE) given a fixed value or short list, a UNIQUE template relying on `{{seq}}` alone | +| warning | table or column not in this database (profiles stay portable), a rule that matches nothing or is shadowed by an earlier rule, a UNIQUE column (or a column inside a multi-column UNIQUE) given a fixed value or short list, a UNIQUE template relying on `{{seq}}` alone, a relationship on a key that cannot be shaped | +| error (relationships) | a key not written as `table.column`, max below min, avg outside min … max, shares outside 0 … 1, a bucket with min above max | ## CLI diff --git a/docs/specs/001-connect-flow-overhaul.md b/docs/specs/001-connect-flow-overhaul.md new file mode 100644 index 0000000..c61b248 --- /dev/null +++ b/docs/specs/001-connect-flow-overhaul.md @@ -0,0 +1,375 @@ +# 001 — Connect flow overhaul: extra params, test connection, saved connections + +> Working document. **Not committed** — this file stays untracked and out of the PR. + +## Context + +`seedstorm serve` opens on `/connect`, a single form that is the only door into the +whole web UI. Three problems make that door harder to walk through than it should be. + +**1. No way to pass driver params from the structured form.** `buildDSN` +(`internal/web/dsn.go:15`) hardcodes the query string per driver — `sslmode` for +Postgres, `parseTime=true&multiStatements=true` for MySQL — with no seam for anything +else. Any connection that needs an extra driver parameter is unreachable through the +structured fields. Real drivers demand these routinely: `go-sql-driver/mysql` refuses +to authenticate against a `caching_sha2_password` account over a non-TLS socket unless +`allowPublicKeyRetrieval=1` is set, and against a cleartext-plugin account unless +`allowCleartextPasswords=1`. The failure mode today is a red banner quoting the driver +("please add 'allowCleartextPasswords=1' to your DSN") next to a form that has no field +to add it to. The only workaround is abandoning the structured fields entirely and +hand-assembling a raw DSN in the **Connection string** box — which also means +hand-escaping the password. + +**2. Validating a connection costs a full page navigation.** The only way to find out +whether a connection works is to submit the form. `handleConnect` +(`internal/web/handlers_pages.go:22`) either opens a session and 303s to the workspace, +or re-renders the whole connect page with `Error:` set. Iterating on one wrong parameter +means a full round trip through a re-rendered page each time, and every re-render drops +the password field. + +**3. Saved connections are browser-local, half-wired, and never the starting point.** +The server already supports several live connections at once — `SessionRegistry` +(`internal/web/session.go:52`) holds many, `/api/connections`, `/switch` and +`/disconnect`+`Pick` all exist, and the header menu lists them. But *saved* connections +are `localStorage` only (`PRESET_KEY = "seedstorm.connections.v1"`, +`internal/web/static/app.js:6`): they are invisible to a second browser, lost on a cache +clear, and — because sessions live only in server memory — every `serve` restart drops +all live connections while the saved list is stranded client-side. Worse, the header +menu disables any preset saved without a password (`btn.disabled = !p.password && !p.dsn`, +`app.js:202`), so a security-conscious save produces an entry that cannot be opened. +And on `/connect` itself, saved connections appear only as a bare ` + + `; + document.body.appendChild(dlg); + const input = dlg.querySelector("input"); + const ok = dlg.querySelector('[value="ok"]'); + input.addEventListener("input", () => { ok.disabled = input.value !== label; }); + dlg.addEventListener("close", () => { + const typed = dlg.returnValue === "ok" && input.value === label ? input.value : null; + dlg.remove(); + resolve(typed); + }); + dlg.showModal(); + input.focus(); + }); + } + + // postRun starts a job. A refusal to write to a production connection asks + // for its label and retries; a request that never reaches the server, or a + // non-JSON answer, rejects with a readable message instead of hanging. + async function postRun(endpoint, body) { + let res; + try { + res = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }); + } catch (err) { + throw new Error(`seedstorm did not answer (${err.message || err}). Is the server still running?`); + } + const j = await res.json().catch(() => ({ error: `unexpected answer (${res.status} ${res.statusText})` })); + if (res.status === 409 && j.code === "production_confirm") { + const typed = await confirmProduction(j.label, j.error); + if (typed == null) { + const err = new Error(`Not written: ${j.label} is a production database and the write was not confirmed.`); + err.cancelled = true; + throw err; + } + return postRun(endpoint, { ...body, confirmProduction: typed }); + } + if (!res.ok) throw new Error(j.error || res.statusText); + return j; + } + + // ── browser storage (per-viewer conveniences only) ───────────────────── + // Storage can be unavailable (private windows, blocked site data): every + // read and write falls back to nothing and the page keeps working. + function readStore(key, fallback) { + try { return JSON.parse(localStorage.getItem(key) || "null") ?? fallback; } catch (_) { return fallback; } + } + function writeStore(key, value) { + try { localStorage.setItem(key, JSON.stringify(value)); return true; } catch (_) { return false; } + } + + // ── run strip: what the current job is doing, next to the action ──────── + // Every page with jobs has one or more [data-run-strip] elements. The strip + // shows the job, its phase and progress, elapsed time, and the connection + // state: running (events arriving), quiet (no update for a while but the + // server still pings), reconnecting, or lost. A failure shows where it + // happened and what to do next. + const QUIET_AFTER_MS = 30000; + const LOST_AFTER_MS = 45000; + const run = { + jobId: "", name: "", status: "idle", phase: "", done: 0, total: 0, label: "", + startedAt: 0, endedAt: 0, lastEventAt: 0, lastPingAt: 0, conn: "running", + failure: null, nextStep: "", message: "", timer: null, + }; + function stripElements() { + return document.querySelectorAll("[data-run-strip]"); + } + function formatElapsed(ms) { + const s = Math.max(0, ms) / 1000; + if (s < 60) return s.toFixed(1) + "s"; + const m = Math.floor(s / 60); + return m + "m " + String(Math.floor(s % 60)).padStart(2, "0") + "s"; + } + function failureText(f, fallback) { + if (!f) return fallback || ""; + const where = [f.side, f.phase, f.table].filter(Boolean).join(" · "); + return where ? `${where}: ${fallback || f.message || ""}` : (fallback || f.message || ""); + } + function renderRunStrip() { + const strips = stripElements(); + if (!strips.length || !run.jobId) return; + const now = Date.now(); + const active = run.status === "running" || run.status === "pending"; + const elapsed = formatElapsed((run.endedAt || now) - run.startedAt); + const pct = run.total > 0 ? Math.min(100, (run.done / run.total) * 100) : null; + let state = run.status; + if (active) state = run.conn; + let detail = run.label || ""; + if (active && run.conn === "quiet") { + detail = `No update for ${formatElapsed(now - run.lastEventAt)} — last: ${[run.phase, run.label].filter(Boolean).join(" · ") || "starting"}. Still connected.`; + } else if (active && run.conn === "reconnecting") { + detail = "Connection to seedstorm dropped; reconnecting…"; + } else if (run.conn === "lost") { + detail = run.message || "Lost contact with seedstorm. Is the server still running?"; + } + const failed = run.status === "failed" || run.status === "canceled"; + for (const el of strips) { + el.hidden = false; + el.dataset.state = state; + el.innerHTML = ` +
+ + ${escapeHTML(stateLabel(state))} + ${escapeHTML(run.name)}${run.phase ? " · " + escapeHTML(run.phase) : ""} + ${elapsed} +
+ ${pct != null && !failed ? ` + ${run.done.toLocaleString()} / ${run.total.toLocaleString()} · ${pct.toFixed(pct < 10 ? 1 : 0)}%` : ""} + ${failed ? `` : ""} + ${failed && run.nextStep ? `

${escapeHTML(run.nextStep)}

` : ""} + ${!failed && detail ? `

${escapeHTML(detail)}

` : ""}`; + } + } + function stateLabel(state) { + return { + running: "Running", pending: "Starting", quiet: "Still working", reconnecting: "Reconnecting", + lost: "Connection lost", done: "Done", failed: "Failed", canceled: "Cancelled", + }[state] || state; + } + function tickRunStrip() { + const active = run.status === "running" || run.status === "pending"; + if (active) { + const now = Date.now(); + if (run.conn !== "reconnecting" && run.conn !== "lost") { + run.conn = now - run.lastEventAt > QUIET_AFTER_MS ? "quiet" : "running"; + } + if (now - Math.max(run.lastPingAt, run.lastEventAt) > LOST_AFTER_MS && run.conn !== "lost") { + run.conn = "lost"; + checkServerAfterLoss(); + } + } + renderRunStrip(); + if (!active && run.timer) { clearInterval(run.timer); run.timer = null; } + } + async function checkServerAfterLoss() { + const stored = readStore(LAST_JOB_KEY, null); + try { + const list = await fetchJSON("/api/jobs", { timeoutMs: 5000 }); + if (stored && list.bootId !== stored.bootId) { + run.status = "failed"; + run.message = `seedstorm restarted: the ${run.name} job was lost. Nothing more will be written by it.`; + } else { + run.conn = "reconnecting"; + } + } catch (_) { + run.message = "Lost contact with seedstorm. Is the server still running?"; + } + renderRunStrip(); + } + function beginRun(jobId, name, bootId) { + Object.assign(run, { + jobId, name, status: "running", phase: "", done: 0, total: 0, label: "", failure: null, nextStep: "", message: "", + startedAt: Date.now(), endedAt: 0, lastEventAt: Date.now(), lastPingAt: Date.now(), conn: "running", + }); + if (bootId) writeStore(LAST_JOB_KEY, { bootId, jobId, name, page: document.body.dataset.active || "" }); + if (!run.timer) run.timer = setInterval(tickRunStrip, 1000); + renderRunStrip(); + } + function endRun(job) { + const stored = readStore(LAST_JOB_KEY, null); + if (stored && stored.jobId === job.id) writeStore(LAST_JOB_KEY, { ...stored, seen: true }); + run.status = job.status || "failed"; + run.endedAt = job.end ? Date.parse(job.end) || Date.now() : Date.now(); + const result = job.result || {}; + run.failure = result.failure || null; + run.nextStep = result.nextStep || ""; + run.message = job.error || ""; + if (run.conn !== "lost") run.conn = "running"; + renderRunStrip(); + } + const LAST_JOB_KEY = "seedstorm.lastJob.v1"; + + // fetchJSON is the one way page scripts read the API: it times out, reads a + // non-JSON answer as an error, and rejects with a message a person can act on. + async function fetchJSON(url, opts = {}) { + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), opts.timeoutMs || 30000); + let res; + try { + res = await fetch(url, { cache: "no-store", ...opts, signal: ctrl.signal }); + } catch (err) { + throw new Error(err.name === "AbortError" ? `seedstorm did not answer ${url} in time` : `seedstorm did not answer (${err.message || err})`); + } finally { + clearTimeout(timer); + } + const body = await res.json().catch(() => null); + if (!res.ok) throw new Error((body && body.error) || `${res.status} ${res.statusText}`); + if (body == null) throw new Error(`unexpected answer from ${url}`); + return body; + } + + // resumeRun reattaches to this session's running job after the page was + // left and opened again, or says the server restarted and the job is gone. + async function resumeRun(names, hooks) { + if (!stripElements().length) return; + const stored = readStore(LAST_JOB_KEY, null); + let list; + try { + list = await fetchJSON("/api/jobs", { timeoutMs: 5000 }); + } catch (_) { return; } + const running = (list.jobs || []).find((j) => (j.status === "running" || j.status === "pending") && names.includes(j.name)); + if (running) { + streamJob(running.id, running.name, hooks, list.bootId); + return; + } + // The job this page started ended while the user was away: show how. + const finished = stored && stored.bootId === list.bootId && (list.jobs || []).find((j) => j.id === stored.jobId && names.includes(j.name)); + if (finished && !stored.seen) { + let job = finished; + try { job = await fetchJSON(`/api/jobs/${finished.id}`, { timeoutMs: 5000 }); } catch (_) { /* the summary is enough */ } + Object.assign(run, { jobId: job.id, name: job.name, phase: "", done: 0, total: 0, label: "", + startedAt: Date.parse(job.start) || Date.now(), conn: "running" }); + endRun(job); + writeStore(LAST_JOB_KEY, { ...stored, seen: true }); + hooks?.onEnd?.(job); + return; + } + if (stored && stored.bootId && stored.bootId !== list.bootId && names.includes(stored.name)) { + Object.assign(run, { jobId: stored.jobId, name: stored.name, status: "failed", startedAt: Date.now(), endedAt: Date.now(), conn: "lost", + message: `seedstorm restarted since this page started the ${stored.name} job: it was stopped and will write nothing more.` }); + writeStore(LAST_JOB_KEY, null); + renderRunStrip(); + } + } + + // A page error must never be silent: running jobs are not affected, and the + // user is told a reload is safe. + function setupGlobalErrorNotice() { + let shown = false; + const show = (detail) => { + console.error("seedstorm page error:", detail); + if (shown) return; + shown = true; + const box = document.createElement("div"); + box.className = "page-error-notice"; + box.setAttribute("role", "alert"); + box.setAttribute("data-testid", "page-error-notice"); + box.innerHTML = `A page error happened; running jobs are not affected. Reload is safe. + `; + box.querySelector("button").addEventListener("click", () => { box.remove(); shown = false; }); + document.body.appendChild(box); + }; + window.addEventListener("error", (ev) => show(ev.error || ev.message)); + window.addEventListener("unhandledrejection", (ev) => show(ev.reason)); + } + // ── shared job streaming ────────────────────────────────────────────── let elapsedTimer = null; function startElapsed() { @@ -665,13 +928,15 @@ const m = /^\[(\d+)\]\s?(.*)$/.exec(s); return m ? m[2] : s; } - function streamJob(jobId, jobName, hooks) { + function streamJob(jobId, jobName, hooks, bootId) { const cancel = document.getElementById("job-cancel"); setStatus("running"); resetPhases(); + beginRun(jobId, jobName, bootId); if (cancel) { cancel.disabled = false; - cancel.onclick = () => fetch(`/api/jobs/${jobId}/cancel`, { method: "POST" }); + cancel.onclick = () => fetch(`/api/jobs/${jobId}/cancel`, { method: "POST" }) + .catch((err) => appendLog("ERROR: could not cancel: " + (err.message || err))); } const expandAll = document.getElementById("job-expand-all"); if (expandAll) { @@ -681,36 +946,72 @@ expandAll.dataset.open = (!open).toString(); }; } + // The browser reconnects on its own after a dropped connection and sends + // Last-Event-ID, so the server resumes after the last event we saw. const es = new EventSource(`/api/jobs/${jobId}/stream`); + let settled = false; + const settle = (job) => { + if (settled) return; + settled = true; + es.close(); + if (cancel) cancel.disabled = true; + endRun(job); + setStatus(job.status); + finalizeLastPhase(job.status); + hooks?.onEnd?.(job); + }; + const settleFromServer = () => + fetch(`/api/jobs/${jobId}`, { cache: "no-store" }) + .then((r) => (r.ok ? r.json() : Promise.reject(new Error(`job lookup failed (${r.status})`)))) + .then(settle) + .catch((err) => settle({ id: jobId, name: jobName, status: "failed", error: `Lost track of the job: ${err.message || err}` })); + const seen = () => { run.lastEventAt = Date.now(); run.lastPingAt = run.lastEventAt; if (run.conn !== "lost") run.conn = "running"; }; + es.addEventListener("ping", () => { run.lastPingAt = Date.now(); if (run.conn === "reconnecting") run.conn = "running"; }); es.addEventListener("log", (e) => { + seen(); const text = stripSeq(e.data); appendLog(text); hooks?.onLog?.(text); }); es.addEventListener("phase", (e) => { + seen(); const text = stripSeq(e.data); + run.phase = text; startPhase(text); }); es.addEventListener("progress", (e) => { // payload: `[seq] done/total label` const m = /^\[\d+\]\s?(\d+)\/(\d+)\s?(.*)$/.exec(e.data); if (!m) return; + seen(); + run.done = Number(m[1]); run.total = Number(m[2]); run.label = m[3]; setProgress(Number(m[1]), Number(m[2]), m[3]); }); es.addEventListener("status", (e) => setStatus(e.data)); - es.addEventListener("error", (e) => { + // The server names a job's failure "failure": a named "error" event would + // trigger onerror below and used to close the stream before "end". + es.addEventListener("failure", (e) => { if (e.data) appendLog("ERROR: " + e.data); }); es.addEventListener("end", () => { es.close(); - if (cancel) cancel.disabled = true; - fetch(`/api/jobs/${jobId}`).then(r => r.json()).then((j) => { - setStatus(j.status); - finalizeLastPhase(j.status); - hooks?.onEnd?.(j); - }); - }); - es.onerror = () => { es.close(); }; + settleFromServer(); + }); + es.onerror = () => { + if (settled) return; + if (es.readyState === EventSource.CONNECTING) { run.conn = "reconnecting"; renderRunStrip(); } + // Connection trouble: if the job is over, finish with its real state; + // if it is still running, let the browser reconnect and resume. + fetch(`/api/jobs/${jobId}`, { cache: "no-store" }) + .then((r) => (r.ok ? r.json() : Promise.reject(new Error(`job lookup failed (${r.status})`)))) + .then((job) => { + if (job.status !== "running" && job.status !== "pending") settle(job); + else if (es.readyState === EventSource.CLOSED) streamJob(jobId, jobName, hooks); + }) + .catch(() => { + if (es.readyState === EventSource.CLOSED) settle({ id: jobId, name: jobName, status: "failed", error: "Connection to seedstorm lost" }); + }); + }; } // ── simple run-form (used by /generate, /enrich, /export pages) ─────── @@ -736,14 +1037,15 @@ form.querySelectorAll('input[type="checkbox"]').forEach((el) => { if (!(el.name in payload)) payload[el.name] = false; }); - const res = await fetch(endpoint, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(payload), - }); - const j = await res.json(); document.getElementById("job-panel").hidden = false; - if (!res.ok) { resetPhases(); appendLog("ERROR: " + (j.error || res.statusText)); return; } + let j; + try { + j = await postRun(endpoint, payload); + } catch (err) { + resetPhases(); + appendLog("ERROR: " + (err.message || err)); + return; + } streamJob(j.id, j.name, { onEnd: (job) => { const r = job.result || {}; @@ -751,7 +1053,7 @@ if (!out) return; renderJobResult(out, r, job.name || j.name || "run"); }, - }); + }, j.bootId); }); } @@ -1158,8 +1460,10 @@ b.addEventListener("click", () => { document.querySelectorAll(".ws-mode-pill").forEach(x => x.classList.remove("active")); b.classList.add("active"); + const wasClone = ws.mode === "clone"; ws.mode = b.dataset.mode; updateCloneControls(); + if (ws.mode === "clone" && !wasClone) loadCloneTargetAccess(); recomputeAuto(); refreshSelectionUI(); }); @@ -1169,7 +1473,7 @@ document.querySelector('[data-act="none"]').addEventListener("click", () => clearSelection()); document.querySelector('[data-act="empty"]').addEventListener("click", () => selectEmpty()); document.querySelector('[data-act="invert"]').addEventListener("click", () => invertSelection()); - document.querySelector('[data-act="refresh"]').addEventListener("click", () => refreshCounts()); + document.querySelector('[data-act="refresh"]').addEventListener("click", () => refreshCounts(true)); document.getElementById("ws-search")?.addEventListener("input", (ev) => applySearch(ev.target.value)); document.getElementById("ws-search")?.addEventListener("keydown", (ev) => { if (ev.key === "Enter") { @@ -1195,7 +1499,16 @@ if (ws.onlyMatches) { exitOnlyMatches(); fitGraph(); } else enterOnlyMatches(); }); setupMinimap(); + setupWorkspaceFormMemory(); syncTuningSummary(); + document.getElementById("ws-recommend")?.addEventListener("click", openRecommendDialog); + document.getElementById("ws-snapshot")?.addEventListener("click", takeSnapshot); + document.getElementById("ws-relationships")?.addEventListener("click", openShapesDialog); + fetchConnections().then((conns) => { + const active = (conns || []).find((c) => c.active); + const link = document.getElementById("ws-calibrate"); + if (active && link) link.href = "/compare?import=1&target=" + encodeURIComponent("id:" + active.id); + }).catch(() => {}); document.getElementById("ws-fit")?.addEventListener("click", () => fitGraph()); document.getElementById("ws-zoom-in")?.addEventListener("click", () => zoomGraph(1.18)); document.getElementById("ws-zoom-out")?.addEventListener("click", () => zoomGraph(0.84)); @@ -1217,11 +1530,340 @@ document.getElementById("ws-run").addEventListener("click", runMode); loadCloneTargets(); - loadProfileOptions(); + loadProfileOptions().then(() => { + restoreWorkspaceForm(true); + const id = document.getElementById("cfg-profile")?.value; + if (id) loadProfileInsights(id); + }); loadGraph(); + resumeRun(["seed", "gaps", "generate", "clone-schema"], { onLog: (line) => onLogPulse(line), onEnd: (job) => onJobEnd(job) }); loadWorkspaceAccess(false); } + // ── remembered workspace form (per connection) ───────────────────────── + // Volume, tuning, profile and dry-run survive leaving the page. Destructive + // toggles (truncate, disable FK checks, drop existing on clone) always start + // off: they are never remembered. + const WS_FORM_KEY = "seedstorm.workspaceForm.v1"; + const WS_FORM_FIELDS = [ + ["cfg-rows", "value"], ["cfg-workers", "value"], ["cfg-gen-workers", "value"], ["cfg-batch", "value"], + ["cfg-enum", "value"], ["cfg-selfref-depth", "value"], ["cfg-dryrun", "checked"], ["cfg-profile", "value"], + ["cfg-clone-dryrun", "checked"], ["ws-clone-views", "checked"], ["ws-clone-routines", "checked"], ["ws-clone-triggers", "checked"], + ]; + function formConnKey() { + return document.body.dataset.connKey || "default"; + } + function saveWorkspaceForm() { + const all = readStore(WS_FORM_KEY, {}); + const values = {}; + for (const [id, prop] of WS_FORM_FIELDS) { + const el = document.getElementById(id); + if (el) values[id] = el[prop]; + } + all[formConnKey()] = { ...values, savedAt: Date.now() }; + // Keep the ten most recent connections. + const keys = Object.keys(all).sort((a, b) => (all[b].savedAt || 0) - (all[a].savedAt || 0)); + for (const k of keys.slice(10)) delete all[k]; + writeStore(WS_FORM_KEY, all); + } + // restoreWorkspaceForm sets remembered values; the profile select only once + // its options exist (they load asynchronously). + function restoreWorkspaceForm(onlyProfile) { + const values = readStore(WS_FORM_KEY, {})[formConnKey()]; + if (!values) return; + for (const [id, prop] of WS_FORM_FIELDS) { + if ((id === "cfg-profile") !== !!onlyProfile) continue; + const el = document.getElementById(id); + if (!el || !(id in values)) continue; + if (id === "cfg-profile") { + const known = [...el.options].some((o) => o.value === values[id]); + if (!known) continue; // the profile was deleted since + } + el[prop] = values[id]; + } + } + function setupWorkspaceFormMemory() { + for (const [id] of WS_FORM_FIELDS) { + const el = document.getElementById(id); + if (!el) continue; + el.addEventListener("input", saveWorkspaceForm); + el.addEventListener("change", saveWorkspaceForm); + } + restoreWorkspaceForm(false); + } + + // ── tuning recommendation ───────────────────────────────────────────── + // The dialog asks what SQL cannot see (size, disk), remembered per + // connection, and shows writers and generators with a reason each. + const SERVER_SHAPE_KEY = "seedstorm.serverShape.v1"; + function openRecommendDialog() { + const shape = readStore(SERVER_SHAPE_KEY, {})[formConnKey()] || {}; + const dlg = document.createElement("dialog"); + dlg.className = "prod-dialog tune-dialog"; + dlg.setAttribute("data-testid", "tune-dialog"); + const field = (id, label, input) => ``; + dlg.innerHTML = ` +
+

Recommend writers and generators

+

Connections, buffers and used space are read from the database. Enter what it cannot report.

+
+ ${field("vcpu", "Database vCPU", ``)} + ${field("memoryMB", "Database memory (MB)", ``)} + ${field("storage", "Storage", ``)} + ${field("storageGB", "Storage size (GB)", ``)} + ${field("iops", "Provisioned IOPS", ``)} +
+ + +

Checking the database…

+ +
`; + document.body.appendChild(dlg); + const form = dlg.querySelector("form"); + if (shape.storage) form.storage.value = shape.storage; + let rec = null; + const refresh = async () => { + const params = new URLSearchParams(); + const values = {}; + for (const name of ["vcpu", "memoryMB", "storage", "storageGB", "iops"]) { + const v = form[name].value.trim(); + values[name] = v; + if (v) params.set(name, v); + } + values.shared = form.shared.checked; + values.ha = form.ha.checked; + if (values.shared) params.set("shared", "1"); + if (values.ha) params.set("ha", "1"); + const all = readStore(SERVER_SHAPE_KEY, {}); + all[formConnKey()] = values; + writeStore(SERVER_SHAPE_KEY, all); + const scope = ws.selected.size + ws.auto.size || ws.nodes.length; + params.set("rows", document.getElementById("cfg-rows")?.value || "0"); + params.set("tables", String(scope)); + const box = dlg.querySelector(".tune-result"); + try { + const data = await fetchJSON("/api/tuning?" + params.toString(), { timeoutMs: 15000 }); + rec = data.recommendation; + const growth = rec.growth || {}; + box.innerHTML = ` +

${rec.writers} writers · + ${rec.generators} generators

+ +

${escapeHTML(growth.message || "")}

+ ${data.production ? '

Production database: kept conservative.

' : ""} + ${data.detectError ? `

Could not read the server's limits: ${escapeHTML(data.detectError)}

` : `

Server: ${escapeHTML(data.server.engine)} ${escapeHTML(data.server.version)} · ${data.server.usedConnections}/${data.server.maxConnections} connections${data.server.replica ? " · replica" : ""}

`} +

${escapeHTML(data.caveat)}

`; + dlg.querySelector('[value="apply"]').disabled = false; + } catch (err) { + box.innerHTML = `

Could not recommend: ${escapeHTML(err.message || String(err))}

`; + } + }; + let timer; + form.addEventListener("input", () => { clearTimeout(timer); timer = setTimeout(refresh, 350); }); + dlg.addEventListener("close", () => { + if (dlg.returnValue === "apply" && rec) { + const workers = document.getElementById("cfg-workers"); + const gens = document.getElementById("cfg-gen-workers"); + if (workers) workers.value = String(rec.writers); + if (gens) gens.value = String(rec.generators); + syncTuningSummary(); + saveWorkspaceForm(); + } + dlg.remove(); + }); + dlg.showModal(); + refresh(); + } + + // ── relationship shapes of the active connection ────────────────────── + // Shapes are measured by a job; the server keeps each one as it finishes, so + // the graph polls while a scan runs and draws a badge per foreign key. + ws.shapes = {}; + ws.shapesProduction = false; + let shapesPoll = null; + const shapeMeasured = (sh) => sh.outcome === "ok" || sh.outcome === "estimated" || sh.outcome === "skipped: unindexed"; + function shapeBadge(sh) { + if (!shapeMeasured(sh)) return sh.outcome; + const n = (v) => (v == null || v < 0 ? "?" : String(v)); + return `${sh.outcome === "ok" ? "" : "~"}avg ${Number(sh.avg || 0).toFixed(1)} · max ${n(sh.max)}`; + } + function shapeTitle(sh) { + if (!shapeMeasured(sh)) return `${sh.child}.${sh.column}: ${sh.outcome}${sh.detail ? " — " + sh.detail : ""}`; + const n = (v) => (v == null || v < 0 ? "?" : Number(v).toLocaleString()); + return `${sh.child}.${sh.column} → ${sh.parent}: children per parent min ${n(sh.min)} · avg ${Number(sh.avg || 0).toFixed(2)} · p50 ${n(sh.p50)} · p95 ${n(sh.p95)} · max ${n(sh.max)} · ${Math.round((sh.zeroShare || 0) * 100)}% of parents have none` + + (sh.nullShare ? ` · ${Math.round(sh.nullShare * 100)}% NULL keys` : "") + (sh.outcome === "ok" ? "" : ` · ${sh.detail || "estimated"}`); + } + + async function loadShapes() { + let view; + try { + view = await fetchJSON("/api/relationships", { cache: "no-store" }); + } catch (_) { return; } + ws.shapesProduction = !!view.production; + ws.shapes = Object.fromEntries((view.shapes || []).map((sh) => [sh.child + "." + sh.column, sh])); + applyShapes(); + const status = document.getElementById("ws-shapes-status"); + const shapes = view.shapes || []; + if (status) { + status.hidden = !shapes.length && !view.running; + const unknown = shapes.filter((sh) => !shapeMeasured(sh)).length; + const est = shapes.filter((sh) => sh.outcome !== "ok" && shapeMeasured(sh)).length; + status.dataset.state = view.running ? "loading" : "ready"; + status.textContent = view.running + ? `Measuring relationships… ${shapes.length}${view.total ? "/" + view.total : ""}` + : `${shapes.length} ${shapes.length === 1 ? "relationship" : "relationships"} measured${est ? ` · ${est} estimated (~)` : ""}${unknown ? ` · ${unknown} not measured` : ""}`; + } + if (view.running && !shapesPoll) shapesPoll = setInterval(loadShapes, 1500); + if (!view.running && shapesPoll) { clearInterval(shapesPoll); shapesPoll = null; } + } + + function applyShapes() { + if (!ws.cy) return; + ws.cy.batch(() => { + for (const e of ws.edges) { + const edge = ws.cy.getElementById(e.id); + const sh = ws.shapes[e.target + "." + e.column]; + if (!edge || edge.empty()) continue; + if (sh) { + edge.data("shapeLabel", shapeBadge(sh)); + edge.data("shapeTitle", shapeTitle(sh)); + edge.toggleClass("shape-unknown", !shapeMeasured(sh)); + } else { + edge.removeData("shapeLabel shapeTitle"); + edge.removeClass("shape-unknown"); + } + } + }); + } + + function openShapesDialog() { + const dlg = document.createElement("dialog"); + dlg.className = "prod-dialog shapes-dialog"; + dlg.setAttribute("data-testid", "shapes-dialog"); + const prod = ws.shapesProduction; + dlg.innerHTML = ` +
+

Analyze relationships

+

For every foreign key: how many children each parent has (min, avg, p95, max) and how many parents have none. Read-only, two queries at a time, 60s limit per key; cancel keeps what finished.

+
+ + +
+

Exact aggregates each key in the database. Estimate reads planner statistics: instant, averages only where the database keeps them.

+ + ${prod ? `` : ""} + +
`; + document.body.appendChild(dlg); + dlg.querySelector("[data-start]").addEventListener("click", async () => { + const form = dlg.querySelector("form"); + const body = { + counts: form.querySelector('input[name="counts"]:checked').value, + scanUnindexed: form.querySelector('input[name="unindexed"]').checked, + confirmExact: !!form.querySelector('input[name="confirmExact"]')?.checked, + }; + dlg.close(); + let j; + try { + j = await postRun("/api/relationships", body); + } catch (err) { + appendLog("ERROR: " + (err.message || err)); + return; + } + activateTab("logs"); + streamJob(j.id, j.name, { onEnd: () => loadShapes() }, j.bootId); + setTimeout(loadShapes, 300); + }); + dlg.addEventListener("close", () => dlg.remove()); + dlg.showModal(); + } + + // ── counts snapshot of the active connection ────────────────────────── + const IMPORTED_COUNTS_KEY = "seedstorm.importedCounts.v1"; + async function takeSnapshot() { + let j; + try { + j = await postRun("/api/snapshot", { counts: "exact" }); + } catch (err) { + appendLog("ERROR: " + (err.message || err)); + return; + } + activateTab("logs"); + streamJob(j.id, j.name, { + onEnd: (job) => { + if (job.status === "done" && job.result?.snapshot) openSnapshotDialog(job.result.snapshot, job.result.unknown || 0); + }, + }, j.bootId); + } + + function openSnapshotDialog(snapshot, unknown) { + const dlg = document.createElement("dialog"); + dlg.className = "prod-dialog snapshot-dialog"; + dlg.setAttribute("data-testid", "snapshot-dialog"); + const tables = Object.keys(snapshot.tables || {}).length; + dlg.innerHTML = ` +
+

Row counts of ${escapeHTML(snapshot.label || "this database")}

+

${tables} tables${unknown ? ` · ${unknown} could not be counted (written as -1)` : ""}. Import it on Compare to fill another database to match, or use it with seedstorm mirror --source-snapshot.

+
+ + +
+ + 
Rendering…
+ +
`; + document.body.appendChild(dlg); + const preview = dlg.querySelector("pre"); + const link = dlg.querySelector("[data-download]"); + let content = ""; + const render = async () => { + const format = dlg.querySelector('input[name="format"]:checked').value; + const withShapes = dlg.querySelector('input[name="relationships"]').checked; + const payload = withShapes ? { ...snapshot, relationships: Object.values(ws.shapes) } : snapshot; + try { + const out = await fetchJSON("/api/snapshots/encode", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ snapshot: payload, format, relationships: withShapes }) }); + content = out.content; + preview.textContent = content; + if (link.dataset.url) URL.revokeObjectURL(link.dataset.url); + link.dataset.url = URL.createObjectURL(new Blob([content], { type: format === "json" ? "application/json" : "text/yaml" })); + link.href = link.dataset.url; + link.download = out.filename; + } catch (err) { + preview.textContent = "Could not render: " + (err.message || err); + } + }; + dlg.querySelectorAll('input[name="format"], input[name="relationships"]').forEach((r) => r.addEventListener("change", render)); + dlg.querySelector("[data-copy]").addEventListener("click", () => copyText(content)); + dlg.querySelector("[data-compare]").addEventListener("click", () => { + // Hand the snapshot to Compare as an imported source. + const id = Date.now().toString(36) + Math.random().toString(36).slice(2, 6); + const rows = Object.values(snapshot.tables || {}).reduce((n, t) => n + Math.max(0, t.rows || 0), 0); + const imported = readStore(IMPORTED_COUNTS_KEY, []); + const entry = { id, name: snapshot.label || "snapshot", importedAt: Date.now(), tables: Object.keys(snapshot.tables || {}).length, rows, snapshot }; + writeStore(IMPORTED_COUNTS_KEY, [entry, ...imported.filter((it) => it.name !== entry.name)].slice(0, 8)); + window.location.href = "/compare?source=" + encodeURIComponent("snap:" + id); + }); + dlg.addEventListener("close", () => { if (link.dataset.url) URL.revokeObjectURL(link.dataset.url); dlg.remove(); }); + dlg.showModal(); + render(); + } + function syncTuningSummary() { const n = Number(document.getElementById("cfg-workers")?.value || 0); const g = Number(document.getElementById("cfg-gen-workers")?.value || 0); @@ -1253,6 +1895,14 @@ const data = await res.json(); if (!res.ok) throw new Error(data.error || res.statusText); initGraph(data); + if (data.countsTakenAt) { + ws.countsTakenAt = Date.parse(data.countsTakenAt); + const status = document.getElementById("ws-counts-status"); + if (status) { status.dataset.state = "ready"; status.textContent = countsAgeText(); } + } else { + refreshCounts(false); + } + loadShapes(); } catch (err) { setGraphLoading("Graph failed", err.message || String(err), true); } @@ -1382,6 +2032,8 @@ }); ws.cy.on("tap", "node", (ev) => toggleSelect(ev.target.id())); + // A measured relationship's numbers live on its child table's columns. + ws.cy.on("tap", "edge[shapeLabel]", (ev) => showDetail(ev.target.data("target"))); ws.cy.on("cxttap", "node", (ev) => { ev.preventDefault?.(); showDetail(ev.target.id()); @@ -1643,6 +2295,24 @@ "control-point-step-size": 42, }, }, + { + selector: "edge[shapeLabel]", + style: { + "label": "data(shapeLabel)", + "font-size": 9, + "color": "#c9d4cc", + "text-background-color": "#11161a", + "text-background-opacity": 0.85, + "text-background-padding": "2px", + "text-background-shape": "roundrectangle", + "text-rotation": "autorotate", + "min-zoomed-font-size": 7, + }, + }, + { + selector: "edge.shape-unknown", + style: { "color": "#d8b56f" }, + }, { selector: "edge[?nullable]", style: { "line-style": "dashed", "line-color": "#4a5169", "target-arrow-color": "#4a5169" }, @@ -1934,10 +2604,11 @@ target.hidden = false; target.innerHTML = '

Loading rows...

'; const q = new URLSearchParams({ table: tableName, limit: "5", offset: "0", _: String(Date.now()) }); - const res = await fetch("/api/table?" + q.toString(), { cache: "no-store" }); - const data = await res.json(); - if (!res.ok) { - target.innerHTML = `

Preview failed: ${escapeHTML(data.error || res.statusText)}

`; + let data; + try { + data = await fetchJSON("/api/table?" + q.toString()); + } catch (err) { + target.innerHTML = `

Preview failed: ${escapeHTML(err.message || String(err))}

`; return; } if (!data.rows || data.rows.length === 0) { @@ -2373,10 +3044,16 @@ if (legend) legend.hidden = noInsert.size === 0; } + // Checking a saved target opens (and pings) that connection on the server, + // so it only happens while clone mode is active. async function loadCloneTargetAccess() { const target = document.getElementById("cfg-clone-target"); const selected = target?.selectedOptions?.[0]; ws.cloneAccess = null; + if (ws.mode !== "clone") { + updateAccessWarnings(); + return; + } if (selected && selected.value && selected.dataset.kind !== "empty") { const key = selected.dataset.kind === "saved" ? "savedId" : "id"; try { @@ -2482,7 +3159,7 @@ ws.preview.offset = 0; const target = document.getElementById("ws-detail"); target.innerHTML = "

loading...

"; - fetch("/api/schema").then(r => r.json()).then((sc) => { + fetchJSON("/api/schema").then((sc) => { const t = (sc.tables && sc.tables[tableName]) || (sc.Tables && sc.Tables[tableName]); if (!t) { target.innerHTML = "

not in schema

"; return; } const entries = Object.entries(t.columns || t.Columns); @@ -2497,6 +3174,8 @@ const flags = []; if (c.pk || c.PK) flags.push('PK'); if (c.fk || c.FK) flags.push(`FK -> ${escapeHTML(c.fk || c.FK)}`); + const sh = ws.shapes[tableName + "." + col]; + if (sh) flags.push(`${escapeHTML(shapeBadge(sh))}`); if (c.nullable || c.Nullable) flags.push('nullable'); return `${escapeHTML(col)} ${flags.join(" ")}${escapeHTML(c.type || c.Type || "")}`; }).join(""); @@ -2560,6 +3239,8 @@ loadPreview(tableName); }); loadPreview(tableName); + }).catch((err) => { + target.innerHTML = `

Could not load ${escapeHTML(tableName)}: ${escapeHTML(err.message || String(err))}

`; }); } @@ -2603,7 +3284,12 @@ async function ensureSchemaColumns(tableName) { if (ws.schemaColumns[tableName]) return; - const sc = await fetch("/api/schema").then(r => r.json()); + let sc; + try { + sc = await fetchJSON("/api/schema"); + } catch (_) { + return; // column hints are optional: the preview still renders without them + } const t = (sc.tables && sc.tables[tableName]) || (sc.Tables && sc.Tables[tableName]); if (!t) return; const entries = Object.entries(t.columns || t.Columns); @@ -2630,10 +3316,11 @@ offset: String(ws.modal.offset), _: String(Date.now()), }); - const res = await fetch("/api/table?" + q.toString(), { cache: "no-store" }); - const data = await res.json(); - if (!res.ok) { - box.innerHTML = `

Preview failed: ${escapeHTML(data.error || res.statusText)}

`; + let data; + try { + data = await fetchJSON("/api/table?" + q.toString()); + } catch (err) { + box.innerHTML = `

Preview failed: ${escapeHTML(err.message || String(err))}

`; return; } const start = data.total === 0 ? 0 : data.offset + 1; @@ -2659,10 +3346,11 @@ offset: String(ws.preview.offset), _: String(Date.now()), }); - const res = await fetch("/api/table?" + q.toString(), { cache: "no-store" }); - const data = await res.json(); - if (!res.ok) { - box.innerHTML = `

Preview failed: ${escapeHTML(data.error || res.statusText)}

`; + let data; + try { + data = await fetchJSON("/api/table?" + q.toString()); + } catch (err) { + box.innerHTML = `

Preview failed: ${escapeHTML(err.message || String(err))}

`; return; } const start = data.total === 0 ? 0 : data.offset + 1; @@ -2754,20 +3442,17 @@ document.getElementById("job-result").innerHTML = ""; if (ws.cy) ws.cy.nodes().removeClass("seeding done failed"); - const res = await fetch(endpoint, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(cfg), - }); - const j = await res.json(); - if (!res.ok) { - appendLog("ERROR: " + (j.error || res.statusText)); + let j; + try { + j = await postRun(endpoint, cfg); + } catch (err) { + appendLog("ERROR: " + (err.message || err)); return; } streamJob(j.id, j.name, { onLog: (line) => onLogPulse(line), onEnd: (job) => onJobEnd(job), - }); + }, j.bootId); } async function runCloneSchema() { @@ -2790,14 +3475,11 @@ activateTab("logs"); resetPhases(); document.getElementById("job-result").innerHTML = ""; - const res = await fetch("/api/clone-schema", { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(cfg), - }); - const j = await res.json(); - if (!res.ok) { - appendLog("ERROR: " + (j.error || res.statusText)); + let j; + try { + j = await postRun("/api/clone-schema", cfg); + } catch (err) { + appendLog("ERROR: " + (err.message || err)); return; } streamJob(j.id, j.name, { @@ -2815,7 +3497,7 @@ out.querySelector(".result-shell")?.appendChild(box); } }, - }); + }, j.bootId); } function tableRowPayload() { @@ -2847,37 +3529,71 @@ const out = document.getElementById("job-result"); if (out) renderJobResult(out, job.result || {}, job.name || ws.mode); refreshCounts(); + loadShapes(); } - function refreshCounts() { + // refreshCounts fills node counts without blocking the graph: the page is + // usable while tables are counted. force recounts instead of reusing the + // counts the server cached for this connection. + let countsRequest = 0; + function refreshCounts(force) { if (!ws.cy) return; - setGraphLoading("Refreshing counts", "Reading row counts for every table."); - fetch("/api/counts").then(r => r.json()).then((counts) => { - ws.cy.batch(() => { - ws.cy.nodes().forEach((n) => { - const id = n.id(); - if (id in counts) { - n.data("count", counts[id]); - n.data("counted", true); - n.data("countLabel", formatCount(counts[id])); - } - }); + const status = document.getElementById("ws-counts-status"); + const setState = (state, text) => { + if (!status) return; + status.dataset.state = state; + status.textContent = text; + }; + const seq = ++countsRequest; + setState("loading", `Counting rows in ${ws.nodes.length} tables…`); + fetch("/api/counts" + (force ? "?refresh=1" : ""), { cache: "no-store" }) + .then(async (r) => { + const body = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(body.error || r.statusText); + return { counts: body, takenAt: r.headers.get("X-Counts-Taken-At") }; + }) + .then(({ counts, takenAt }) => { + if (seq !== countsRequest) return; + applyCounts(counts); + const missing = ws.nodes.filter((n) => !(n.id in counts)).length; + ws.countsTakenAt = takenAt ? Date.parse(takenAt) : Date.now(); + setState("ready", countsAgeText() + (missing ? ` · ${missing} could not be counted` : "")); + }) + .catch((err) => { + if (seq !== countsRequest) return; + setState("failed", "Row counts unavailable: " + (err.message || err)); }); - // Keep the JS-side mirror in sync so isPopulated() sees fresh counts. - for (const n of ws.nodes) { - if (n.id in counts) { - n.count = counts[n.id]; - n.counted = true; + } + + function applyCounts(counts) { + ws.cy.batch(() => { + ws.cy.nodes().forEach((n) => { + const id = n.id(); + if (id in counts) { + n.data("count", counts[id]); + n.data("counted", true); + n.data("countLabel", formatCount(counts[id])); } - } - updateStats(); - recomputeAuto(); - refreshSelectionUI(); - renderIgnoredTab(document.getElementById("ws-ignored-profile")?.textContent); - clearGraphLoading(); - }).catch((err) => { - setGraphLoading("Count refresh failed", err.message || String(err), true); + }); }); + // Keep the JS-side mirror in sync so isPopulated() sees fresh counts. + for (const n of ws.nodes) { + if (n.id in counts) { + n.count = counts[n.id]; + n.counted = true; + } + } + updateStats(); + recomputeAuto(); + refreshSelectionUI(); + renderIgnoredTab(document.getElementById("ws-ignored-profile")?.textContent); + } + + function countsAgeText() { + if (!ws.countsTakenAt) return ""; + const s = Math.max(0, Math.round((Date.now() - ws.countsTakenAt) / 1000)); + const age = s < 45 ? "just now" : s < 3600 ? `${Math.round(s / 60)} min ago` : `${Math.round(s / 3600)} h ago`; + return `Row counts from ${age} · ↻ to recount`; } // Narrow screens collapse the top navigation into a drawer. @@ -2928,8 +3644,19 @@ } } + function setupProductionToggle() { + const box = document.getElementById("conn-production"); + const row = document.getElementById("conn-confirm-label-row"); + if (!box || !row) return; + const sync = () => { row.hidden = !(box.dataset.wasProduction && !box.checked); }; + box.addEventListener("change", sync); + sync(); + } + document.addEventListener("DOMContentLoaded", () => { + setupGlobalErrorNotice(); setupNavDrawer(); + setupProductionToggle(); setupAccessBadge(); setupConnectForm(); setupConnectionDialog(); @@ -2947,7 +3674,8 @@ window.seedstorm = { // Shared helpers for page scripts (compare.js, profiles.js). ui: { - streamJob, resetPhases, appendLog, escapeHTML, formatCount, copyText, + streamJob, postRun, confirmProduction, resumeRun, fetchJSON, readStore, writeStore, + resetPhases, appendLog, escapeHTML, formatCount, copyText, fetchConnections, fetchSavedConnections, connectionLabel, connectionKey, fetchAccess, }, state: ws, diff --git a/internal/web/static/compare.css b/internal/web/static/compare.css index 9387fc4..00c5321 100644 --- a/internal/web/static/compare.css +++ b/internal/web/static/compare.css @@ -86,6 +86,39 @@ .cmp-meter { display: block; height: 4px; border-radius: 2px; background: rgba(216,181,111,0.18); overflow: hidden; } .cmp-meter i { display: block; height: 100%; background: var(--tgt); transition: width 500ms var(--ease-out); } +/* ── relationship shapes ── */ +.cmp-shapes { + border: 1px solid var(--line); + border-radius: var(--radius-lg); + background: var(--panel); + overflow: hidden; +} +.cmp-shapes-head { + display: flex; flex-wrap: wrap; align-items: center; justify-content: space-between; gap: 10px; + padding: 10px 12px; +} +.cmp-shapes-head h2 { margin: 0; font-size: 15px; } +.cmp-shapes-head p { margin: 2px 0 0; max-width: 72ch; } +.cmp-shapes-actions { display: flex; flex-wrap: wrap; align-items: center; gap: 10px 14px; } +.cmp-shapes-body { border-top: 1px solid var(--line); max-height: 52vh; overflow: auto; } +.cmp-shapes-summary { padding: 8px 12px; color: var(--muted); font-size: 12px; border-bottom: 1px solid var(--line); } +.cmp-shape-row { + display: grid; + grid-template-columns: minmax(160px, 1.2fr) minmax(0, 1fr) minmax(0, 1fr) 96px; + align-items: center; gap: 10px; + padding: 7px 12px; + border-bottom: 1px solid rgba(231,236,232,0.05); + font-size: 12px; +} +.cmp-shape-row.head { color: var(--muted); font-family: var(--mono); font-size: 11px; text-transform: uppercase; letter-spacing: 0.06em; background: rgba(11,15,18,0.35); } +.cmp-shape-row code { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.cmp-shape-src { color: var(--src); text-align: right; font-variant-numeric: tabular-nums; } +.cmp-shape-tgt { color: var(--tgt); font-variant-numeric: tabular-nums; } +.cmp-shape-status { font-size: 11px; color: var(--muted); text-align: right; } +.cmp-shape-status.differs { color: var(--warn); } +.cmp-shape-row .cell-unknown { color: var(--muted); } +.cmp-export-shapes { align-self: end; } + /* ── table gauges ── */ .cmp-grid { display: grid; grid-template-columns: minmax(0, 1fr) 340px; gap: 14px; align-items: start; } .cmp-table-panel { @@ -285,6 +318,12 @@ .cmp-empty { flex-direction: column; align-items: flex-start; } .cmp-search, .cmp-search input { width: 100%; } .cmp-gauge-head { display: none; } + .cmp-shape-row.head { display: none; } + .cmp-shape-row { grid-template-columns: minmax(0, 1fr) auto; grid-template-areas: "name status" "src src" "tgt tgt"; row-gap: 3px; } + .cmp-shape-row > code { grid-area: name; } + .cmp-shape-src { grid-area: src; text-align: left; } + .cmp-shape-tgt { grid-area: tgt; } + .cmp-shape-status { grid-area: status; } .cmp-gauge { grid-template-columns: 24px minmax(0, 1fr) auto; grid-template-areas: "check name delta" "check src src" "check tgt tgt"; diff --git a/internal/web/static/compare.js b/internal/web/static/compare.js index 448ae50..f50507e 100644 --- a/internal/web/static/compare.js +++ b/internal/web/static/compare.js @@ -9,18 +9,21 @@ const $ = (id) => document.getElementById(id); const IMPORTED_KEY = "seedstorm.importedCounts.v1"; + // Picks and mirror settings survive leaving the page. Reset mode is never + // remembered: it truncates the target. + const COMPARE_FORM_KEY = "seedstorm.compareForm.v1"; + const COMPARE_FORM_FIELDS = [ + ["cmp-scale", "value"], ["cmp-max-rows", "value"], ["cmp-parent-rows", "value"], ["cmp-profile", "value"], + ["cmp-batch", "value"], ["cmp-selfref", "value"], ["cmp-workers", "value"], ["cmp-stop", "checked"], + ]; const REPORT_KEY = "seedstorm.compareReport.v1"; const MAX_IMPORTED = 8; const MAX_SAVED_REPORTS = 6; - // Browser storage can be unavailable (private windows, blocked site data): - // every read and write falls back to nothing. - function readStore(key, fallback) { - try { return JSON.parse(localStorage.getItem(key) || "null") ?? fallback; } catch (_) { return fallback; } - } - function writeStore(key, value) { - try { localStorage.setItem(key, JSON.stringify(value)); return true; } catch (_) { return false; } - } + // Browser storage helpers are shared (app.js): unavailable storage falls + // back to nothing. + const readStore = (key, fallback) => window.seedstorm.ui.readStore(key, fallback); + const writeStore = (key, value) => window.seedstorm.ui.writeStore(key, value); const state = { report: null, @@ -42,14 +45,14 @@ const liveKeys = new Set(); for (const c of live) { liveKeys.add(ui().connectionKey(c.info)); - options.push({ value: "id:" + c.id, label: ui().connectionLabel(c.info), group: "Connected", active: c.active, dbType: c.info.dbType }); + options.push({ value: "id:" + c.id, label: ui().connectionLabel(c.info) + (c.production ? " · production" : ""), group: "Connected", active: c.active, dbType: c.info.dbType }); } for (const c of saved) { if (liveKeys.has(ui().connectionKey(c))) continue; const locked = !c.hasPassword && !c.dsn; options.push({ value: "saved:" + c.id, - label: ui().connectionLabel(c) + (locked ? " — connect once to store its password" : ""), + label: ui().connectionLabel(c) + (c.production ? " · production" : "") + (locked ? " — connect once to store its password" : ""), group: "Saved", disabled: locked, dbType: c.dbType, }); } @@ -60,11 +63,14 @@ })); fillSelect($("cmp-source"), [...options, ...importedOptions]); fillSelect($("cmp-target"), options); + // URL (a link from another page) > remembered picks > active and next connection. const params = new URLSearchParams(location.search); + const remembered = readStore(COMPARE_FORM_KEY, {}); const active = options.find((o) => o.active); const other = options.find((o) => !o.active && !o.disabled); - setSelect($("cmp-source"), params.get("source") || active?.value); - setSelect($("cmp-target"), params.get("target") || other?.value); + const known = (v) => v && [...$("cmp-source").options].some((o) => o.value === v && !o.disabled); + setSelect($("cmp-source"), params.get("source") || (known(remembered.source) ? remembered.source : active?.value)); + setSelect($("cmp-target"), params.get("target") || ([...$("cmp-target").options].some((o) => o.value === remembered.target && !o.disabled) ? remembered.target : other?.value)); const usable = options.filter((o) => !o.disabled).length; const hint = $("cmp-hint"); @@ -158,14 +164,12 @@ } // ── jobs ──────────────────────────────────────────────────────────── - function runJob(endpoint, body) { - return new Promise(async (resolve, reject) => { - const res = await fetch(endpoint, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(body) }); - const j = await res.json().catch(() => ({})); - if (!res.ok) return reject(new Error(j.error || res.statusText)); - $("job-name").textContent = j.name; - ui().streamJob(j.id, j.name, { onEnd: (job) => resolve(job) }); - }); + // runJob starts a job and settles when it ends; a request that never reaches + // the server rejects instead of leaving the caller waiting. + async function runJob(endpoint, body) { + const j = await ui().postRun(endpoint, body); + $("job-name").textContent = j.name; + return new Promise((resolve) => ui().streamJob(j.id, j.name, { onEnd: resolve }, j.bootId)); } function setBusy(busy, label) { @@ -286,6 +290,7 @@ } app.querySelectorAll("[data-count]").forEach((b) => { b.textContent = counts[b.dataset.count]; }); renderGauges(); + renderShapes(); $("cmp-writes").innerHTML = `Writes only to ${ui().escapeHTML(targetLabel())}. The source is read.`; $("cmp-plan").disabled = false; } @@ -297,12 +302,12 @@ function renderStats(r) { const t = r.totals; const pct = t.sourceRows > 0 ? Math.round((t.targetRows / t.sourceRows) * 100) : null; - const attention = t.sourceOnly + t.targetOnly + t.columnDrift; + const attention = t.sourceOnly + t.targetOnly + t.columnDrift + (t.unknown || 0); const tiles = [ - { label: "tables matched", value: fmt(t.same + t.differs), note: `${fmt(t.same)} same · ${fmt(t.differs)} differ` }, + { label: "tables matched", value: fmt(t.same + t.differs + (t.unknown || 0)), note: `${fmt(t.same)} same · ${fmt(t.differs)} differ` + (t.unknown ? ` · ${fmt(t.unknown)} count unknown` : "") }, { label: "rows", value: `${fmt(t.sourceRows)} → ${fmt(t.targetRows)}`, note: (pct == null ? "source is empty" : `target holds ${pct}% of source`) + (hasEstimates(r) ? " · ~ = estimated" : ""), meter: pct }, { label: "size", value: `${bytes(t.sourceBytes)} → ${bytes(t.targetBytes)}`, note: r.source.dbType === "mysql" || r.target.dbType === "mysql" ? "MySQL sizes are cached estimates" : "data + indexes" }, - { label: "needs a look", value: fmt(attention), note: `${t.sourceOnly} source-only · ${t.targetOnly} target-only · ${t.columnDrift} column drift`, warn: attention > 0 }, + { label: "needs a look", value: fmt(attention), note: `${t.sourceOnly} source-only · ${t.targetOnly} target-only · ${t.columnDrift} column drift` + (t.unknown ? ` · ${t.unknown} count unknown` : ""), warn: attention > 0 }, ]; $("cmp-stats").innerHTML = tiles.map((tile, i) => `
@@ -329,7 +334,7 @@ const mirrorable = row.status === "same" || row.status === "differs"; const checked = mirrorable && !state.excluded.has(row.table); const delta = row.delta || 0; - const statusText = { same: "same", differs: "differs", source_only: "not on target", target_only: "not on source" }[row.status]; + const statusText = { same: "same", differs: "differs", source_only: "not on target", target_only: "not on source", unknown: "count unknown" }[row.status]; const driftBadge = drift(row) ? `columns ≠` : ""; @@ -394,6 +399,7 @@ selfRefDepth: Number($("cmp-selfref").value || 0), stopOnError: $("cmp-stop").checked, previewRows: 3, + sourceShapes: $("cmp-shape-source").checked ? sourceShapes() : undefined, dryRun, }; } @@ -406,7 +412,7 @@ const job = await runJob("/api/mirror", mirrorRequest(true)); if (job.status !== "done") throw new Error(job.error || "plan " + job.status); state.plan = job.result; - state.report = job.result.report; // counts are fresh from the plan run + state.report = { ...job.result.report, relationships: state.report?.relationships }; // counts are fresh from the plan run state.pairKey = pairKey(); saveReport(); render(); @@ -433,6 +439,10 @@ : ""; const issues = ((result.issues || []).length ? `
Profile notes${result.issues.map((i) => esc(`${i.path}: ${i.message}`)).join("
")}
` + : "") + ((result.serverNotices || []).length + ? `
Servers${result.serverNotices.map(esc).join("
")}
` + : "") + (result.shapedKeys + ? `
Relationships shaped${fmt(result.shapedKeys)} foreign ${result.shapedKeys === 1 ? "key gets" : "keys get"} the source's children per parent; the job log shows the result next to the target after the run.${(result.shapesSkipped || []).length ? ` ${result.shapesSkipped.length} cannot be shaped in this schema (junction keys, key columns) and are spread evenly — see the log.` : ""}
` : "") + (result.sameDatabaseUnchecked ? `
Source is an imported counts fileseedstorm cannot check that the target is a different database. Make sure ${esc(result.target)} is the one you mean to write.
` : ""); @@ -615,6 +625,84 @@ compare(); } + // ── relationship shapes ───────────────────────────────────────────── + // A side's cell: avg / p95 / max children per parent and parents without any. + function shapeCell(sh) { + if (!sh) return `—`; + const measured = sh.outcome === "ok" || sh.outcome === "estimated" || sh.outcome === "skipped: unindexed"; + if (!measured) return `${ui().escapeHTML(sh.outcome)}`; + const est = sh.outcome !== "ok" ? "~" : ""; + const n = (v) => (v == null || v < 0 ? "?" : Number(v).toLocaleString()); + const title = `${sh.parents >= 0 ? n(sh.parents) + " parents · " : ""}${n(sh.children)} children · min ${n(sh.min)} · p50 ${n(sh.p50)}${sh.nullShare ? ` · ${Math.round(sh.nullShare * 100)}% NULL keys` : ""}${est ? " · estimated" : ""}`; + return `${est}${Number(sh.avg || 0).toFixed(2)} / ${n(sh.p95)} / ${n(sh.max)} · ${Math.round((sh.zeroShare || 0) * 100)}%`; + } + + const shapeStatusText = { same: "same", differs: "differs", source_only: "source only", target_only: "target only", unknown: "unknown" }; + + function renderShapes() { + const body = $("cmp-shapes-body"); + const drift = state.report?.relationships; + const exportBox = $("cmp-export-relationships"); + exportBox.disabled = !drift?.length; + if (exportBox.disabled) exportBox.checked = false; + $("cmp-export-relationships-note").textContent = drift?.length ? `(${drift.length})` : "(compare them first)"; + const usable = sourceShapes().length; + const shapeBox = $("cmp-shape-source"); + shapeBox.disabled = !usable; + if (!usable) shapeBox.checked = false; + $("cmp-shape-source-note").textContent = usable ? `(${usable} ${usable === 1 ? "key" : "keys"})` : "(compare relationships first)"; + if (!drift) { body.hidden = true; body.innerHTML = ""; return; } + body.hidden = false; + if (!drift.length) { + body.innerHTML = `

Neither side has foreign keys to compare.

`; + return; + } + const counts = {}; + for (const d of drift) counts[d.status] = (counts[d.status] || 0) + 1; + const rows = drift.map((d) => { + const parent = (d.source || d.target)?.parent; + return `
+ ${ui().escapeHTML(d.child + "." + d.column)}${parent ? ` → ${ui().escapeHTML(parent)}` : ""} + ${shapeCell(d.source)} + ${shapeCell(d.target)} + ${shapeStatusText[d.status] || d.status} +
`; + }).join(""); + body.innerHTML = `

${drift.length} ${drift.length === 1 ? "relationship" : "relationships"} · ${counts.same || 0} same · ${counts.differs || 0} differ · ${(counts.source_only || 0) + (counts.target_only || 0)} on one side · ${counts.unknown || 0} unknown · children per parent: avg / p95 / max · parents without children · ~ = estimated

+ ${rows}`; + } + + // sourceShapes are the measured source side of the compared relationships. + function sourceShapes() { + return (state.report?.relationships || []).map((d) => d.source).filter((s) => s && s.max > 0 && ["ok", "estimated", "skipped: unindexed"].includes(s.outcome)); + } + + async function compareShapes() { + if (state.busy || !state.report) return; + setBusy(true); + const button = $("cmp-shapes-run"); + button.disabled = true; + button.textContent = "Measuring…"; + const counts = app.querySelector('input[name="counts"]:checked').value; + try { + const job = await runJob("/api/compare/relationships", { + source: refOf($("cmp-source")), sourceSnapshot: sourceSnapshot(), target: refOf($("cmp-target")), counts, + scanUnindexed: $("cmp-shapes-unindexed").checked, confirmExact: $("cmp-shapes-exact-prod").checked, + }); + if (job.status !== "done") throw new Error(job.error || "relationships " + job.status); + state.report.relationships = job.result.relationships || []; + saveReport(); + renderShapes(); + } catch (err) { + showOutcome("error", "Relationships could not be compared", err.message); + $("cmp-logs").open = true; + } finally { + button.disabled = false; + button.textContent = "Compare relationships"; + setBusy(false); + } + } + // ── counts export ─────────────────────────────────────────────────── let exportSeq = 0; async function renderExport() { @@ -625,7 +713,7 @@ $("cmp-export-status").textContent = "Rendering…"; let out; try { - const res = await fetch("/api/snapshots/encode", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ report: state.report, side, format }) }); + const res = await fetch("/api/snapshots/encode", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ report: state.report, side, format, relationships: $("cmp-export-relationships").checked }) }); out = await res.json(); if (!res.ok) throw new Error(out.error || res.statusText); } catch (err) { @@ -637,7 +725,7 @@ } if (seq !== exportSeq) return; $("cmp-export-preview").textContent = out.content; - $("cmp-export-status").textContent = `${out.tables} tables · ${out.filename}`; + $("cmp-export-status").textContent = `${out.tables} tables${out.relationships ? ` · ${out.relationships} relationships` : ""} · ${out.filename}`; const link = $("cmp-export-download"); const type = format === "json" ? "application/json" : "text/yaml"; if (link.dataset.url) URL.revokeObjectURL(link.dataset.url); @@ -677,7 +765,34 @@ } catch (_) { /* optional */ } } + // A compare or mirror started before leaving the page keeps running on the + // server: reattach to it and show its outcome when it ends. + function resumeRunningJob() { + ui().resumeRun(["compare", "mirror", "compare relationships"], { + onEnd: (job) => { + if (job.name === "compare relationships" && job.status === "done" && state.report) { + state.report.relationships = job.result?.relationships || []; + saveReport(); + renderShapes(); + } else if (job.name === "compare" && job.status === "done" && job.result?.report) { + state.report = job.result.report; + state.pairKey = pairKey(); + saveReport(); + render(); + } else if (job.status !== "done") { + showOutcome("error", `${job.name === "mirror" ? "Mirror" : "Compare"} ${job.status}`, job.error || ""); + $("cmp-results").hidden = false; + } else if (job.name === "mirror") { + const run = job.result?.run || {}; + showOutcome("ok", `Inserted ${fmt(run.inserted || 0)} rows`, "The mirror that was running when you left finished."); + $("cmp-results").hidden = false; + } + }, + }); + } + document.addEventListener("DOMContentLoaded", () => { + resumeRunningJob(); $("cmp-form").addEventListener("submit", (ev) => { ev.preventDefault(); compare(); }); $("cmp-source").addEventListener("change", syncPickers); $("cmp-target").addEventListener("change", syncPickers); @@ -736,7 +851,8 @@ readImportFile(ev.dataTransfer?.files?.[0]); }); $("cmp-export").addEventListener("click", openExport); - app.querySelectorAll('input[name="export-side"], input[name="export-format"]').forEach((r) => r.addEventListener("change", renderExport)); + app.querySelectorAll('input[name="export-side"], input[name="export-format"], #cmp-export-relationships').forEach((r) => r.addEventListener("change", renderExport)); + $("cmp-shapes-run").addEventListener("click", compareShapes); $("cmp-export-copy").addEventListener("click", async () => { await ui().copyText($("cmp-export-preview").textContent); $("cmp-export-copy").textContent = "Copied"; @@ -748,7 +864,44 @@ app.querySelectorAll("[data-close]").forEach((el) => el.addEventListener("click", closeModal)); app.querySelectorAll(".cmp-tab").forEach((b) => b.addEventListener("click", () => activateTab(b.dataset.tab))); document.addEventListener("keydown", (ev) => { if (ev.key === "Escape" && !$("cmp-modal").hidden) closeModal(); }); - loadPickers().then(() => restoreReport()); - loadProfiles(); + loadPickers().then(() => { + restoreReport(); + // "Calibrate from a file" on the workspace opens the import directly. + if (new URLSearchParams(location.search).get("import") === "1") openImport(); + }); + loadProfiles().then(() => restoreCompareForm(true)); + restoreCompareForm(false); + for (const [id] of COMPARE_FORM_FIELDS) { + $(id)?.addEventListener("input", saveCompareForm); + $(id)?.addEventListener("change", saveCompareForm); + } + ["cmp-source", "cmp-target"].forEach((id) => $(id).addEventListener("change", saveCompareForm)); + app.querySelectorAll('input[name="counts"]').forEach((r) => r.addEventListener("change", saveCompareForm)); }); + + function saveCompareForm() { + const values = { source: $("cmp-source").value, target: $("cmp-target").value, counts: app.querySelector('input[name="counts"]:checked')?.value }; + for (const [id, prop] of COMPARE_FORM_FIELDS) { + if ($(id)) values[id] = $(id)[prop]; + } + writeStore(COMPARE_FORM_KEY, values); + } + + // restoreCompareForm sets remembered settings; the profile only once its + // options have loaded, and only if it still exists. + function restoreCompareForm(onlyProfile) { + const values = readStore(COMPARE_FORM_KEY, null); + if (!values) return; + if (!onlyProfile && values.counts) { + const radio = app.querySelector(`input[name="counts"][value="${values.counts}"]`); + if (radio) radio.checked = true; + } + for (const [id, prop] of COMPARE_FORM_FIELDS) { + const el = $(id); + if (!el || !(id in values) || (id === "cmp-profile") !== !!onlyProfile) continue; + if (id === "cmp-profile" && ![...el.options].some((o) => o.value === values[id])) continue; + el[prop] = values[id]; + } + if (!onlyProfile && values["cmp-scale"]) setScale(values["cmp-scale"]); + } })(); diff --git a/internal/web/static/profiles.css b/internal/web/static/profiles.css index 8de9dfa..3bf29b8 100644 --- a/internal/web/static/profiles.css +++ b/internal/web/static/profiles.css @@ -278,6 +278,17 @@ .pf-warn-text { color: var(--warn); } .pf-ignore-toggle { align-self: end; min-height: 38px; align-items: center; } .pf-ignored-note { margin: 0 0 8px; color: var(--warn); } +.pf-shapes-import { display: flex; flex-wrap: wrap; align-items: center; gap: 8px; } +.pf-file-btn { cursor: pointer; } +.pf-shape-item { align-items: start; } +.pf-shape-fields { display: flex; flex-wrap: wrap; gap: 6px 10px; align-items: end; min-width: 0; } +.pf-shape-field { display: grid; gap: 2px; font-size: 11px; color: var(--muted); } +.pf-shape-field input { width: 76px; min-height: 28px; padding: 3px 6px; font-size: 12px; font-family: var(--mono); } +@media (max-width: 640px) { + .pf-shape-item { grid-template-columns: minmax(0, 1fr) auto; } + .pf-shape-fields { grid-column: 1 / -1; grid-row: 2; } + .pf-shape-field input { font-size: 16px; } +} /* ── profile file import ── */ .pf-yaml-file { diff --git a/internal/web/static/profiles.js b/internal/web/static/profiles.js index d92cac5..c7045fc 100644 --- a/internal/web/static/profiles.js +++ b/internal/web/static/profiles.js @@ -8,6 +8,7 @@ const app = document.getElementById("profiles-app"); if (!app) return; const $ = (id) => document.getElementById(id); + const ui = () => window.seedstorm.ui; const esc = (v) => window.seedstorm.ui.escapeHTML(v == null ? "" : v); const KINDS = [ @@ -187,6 +188,7 @@ $("pf-description").value = state.doc.description || ""; renderRules(); renderIgnore(); + renderShapes(); renderStatus(); renderSelect(); scheduleExplain(0); @@ -235,6 +237,42 @@ note.textContent = ignored ? `Ignored by ${pattern}: runs never write ${state.table}, so its column rules below never apply.` : ""; } + // ── relationships ─────────────────────────────────────────────────── + const SHAPE_FIELDS = [["min", 1, 1], ["avg", 0.01, 0], ["max", 1, 1], ["zeroShare", 0.01, 0], ["nullShare", 0.01, 0]]; + function renderShapes() { + const rels = state.doc.relationships || {}; + const keys = Object.keys(rels).sort(); + $("pf-shapes-empty").hidden = keys.length > 0; + $("pf-shapes-list").innerHTML = keys.map((key) => { + const r = rels[key]; + const inputs = SHAPE_FIELDS.map(([f, step, minv]) => ``).join(""); + return `
  • + ${esc(key)} + ${inputs}${r.histogram?.length ? `${r.histogram.length} buckets` : ""} + +
    • `; + }).join(""); + } + + async function importShapes(file) { + if (!file) return; + const status = $("pf-shapes-status"); + try { + const data = await file.text(); + const res = await fetch("/api/profiles/relationships", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ data }) }); + const out = await res.json(); + if (!res.ok) throw new Error(out.error || res.statusText); + state.doc.relationships = { ...(state.doc.relationships || {}), ...out.relationships }; + const n = Object.keys(out.relationships).length; + status.textContent = `Imported ${n} ${n === 1 ? "relationship" : "relationships"}` + (out.skipped.length ? ` · skipped ${out.skipped.length} without a measured max: ${out.skipped.join(", ")}` : ""); + renderShapes(); + markDirty(); + scheduleExplain(0); + } catch (err) { + status.textContent = "Could not import: " + (err.message || err); + } + } + function addIgnore(glob) { const value = String(glob || "").trim(); if (!value) return; @@ -726,12 +764,13 @@ setStatusNote("Give the profile a name first; the CLI uses it with --profile."); return; } - const res = await fetch("/api/profiles", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ id: state.id, rules: state.doc }) }); - const data = await res.json().catch(() => ({})); - if (!res.ok) { + let data; + try { + data = await ui().fetchJSON("/api/profiles", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ id: state.id, rules: state.doc }) }); + } catch (err) { const el = $("pf-status"); el.className = "pf-status small error"; - el.textContent = data.error || res.statusText; + el.textContent = "Not saved: " + (err.message || err); return; } await loadProfiles(); @@ -743,7 +782,14 @@ async function remove() { const p = state.profiles.find((x) => x.id === state.id); if (!p || !window.confirm(`Delete profile “${p.rules.name}”? Runs that reference it by name will stop working.`)) return; - await fetch("/api/profiles?id=" + encodeURIComponent(p.id), { method: "DELETE" }); + try { + await ui().fetchJSON("/api/profiles?id=" + encodeURIComponent(p.id), { method: "DELETE" }); + } catch (err) { + const el = $("pf-status"); + el.className = "pf-status small error"; + el.textContent = "Not deleted: " + (err.message || err); + return; + } await loadProfiles(); openProfile(state.profiles[0]?.id || ""); } @@ -754,8 +800,14 @@ $("pf-yaml-error").hidden = true; dialog.dataset.mode = mode; if (mode === "export") { - const res = await fetch("/api/profiles/yaml", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ rules: state.doc }) }); - const data = await res.json(); + let data; + try { + data = await ui().fetchJSON("/api/profiles/yaml", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ rules: state.doc }) }); + } catch (err) { + data = {}; + $("pf-yaml-error").hidden = false; + $("pf-yaml-error").textContent = "Could not render the YAML: " + (err.message || err); + } text.value = data.yaml || ""; text.readOnly = true; $("pf-yaml-eyebrow").textContent = "export"; @@ -807,12 +859,13 @@ } async function loadYAML() { - const res = await fetch("/api/profiles/yaml", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ yaml: $("pf-yaml-text").value }) }); - const data = await res.json().catch(() => ({})); const err = $("pf-yaml-error"); - if (!res.ok) { + let data; + try { + data = await ui().fetchJSON("/api/profiles/yaml", { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ yaml: $("pf-yaml-text").value }) }); + } catch (e) { err.hidden = false; - err.textContent = data.error || res.statusText; + err.textContent = e.message || String(e); return; } const existing = state.profiles.find((p) => p.rules.name && p.rules.name.toLowerCase() === String(data.rules.name || "").toLowerCase()); @@ -910,6 +963,25 @@ const btn = ev.target.closest("[data-ignore-index]"); if (btn) removeIgnore(Number(btn.dataset.ignoreIndex)); }); + $("pf-shapes-file").addEventListener("change", (ev) => { importShapes(ev.target.files?.[0]); ev.target.value = ""; }); + $("pf-shapes-list").addEventListener("click", (ev) => { + const btn = ev.target.closest("[data-shape-remove]"); + if (!btn) return; + delete state.doc.relationships[btn.dataset.shapeRemove]; + if (!Object.keys(state.doc.relationships).length) delete state.doc.relationships; + renderShapes(); + markDirty(); + }); + $("pf-shapes-list").addEventListener("change", (ev) => { + const input = ev.target.closest("[data-shape-field]"); + if (!input) return; + const rel = state.doc.relationships?.[input.dataset.shapeKey]; + if (!rel) return; + const n = Number(input.value); + if (input.value === "" || !Number.isFinite(n)) delete rel[input.dataset.shapeField]; + else rel[input.dataset.shapeField] = input.dataset.shapeField === "min" || input.dataset.shapeField === "max" ? Math.round(n) : n; + markDirty(); + }); $("pf-table-ignore").addEventListener("change", (ev) => { if (!state.table) return; if (ev.target.checked) { diff --git a/internal/web/static/style.css b/internal/web/static/style.css index 0ecae57..cbee123 100644 --- a/internal/web/static/style.css +++ b/internal/web/static/style.css @@ -1842,6 +1842,15 @@ textarea { font-family: var(--mono); font-size: 12px; } text-transform: lowercase; white-space: nowrap; } .pill.ok { background: rgba(121,216,179,0.14); border-color: rgba(121,216,179,0.4); color: var(--ok); } +.pill.prod { background: rgba(240,120,110,0.14); border-color: rgba(240,120,110,0.45); color: var(--danger, #f0786e); font-weight: 600; } +.prod-dialog { max-width: min(460px, calc(100vw - 32px)); border: 1px solid var(--line, #333); border-radius: 14px; padding: 0; background: var(--panel, #16181c); color: inherit; } +.prod-dialog::backdrop { background: rgba(0,0,0,0.55); } +.prod-dialog form { display: grid; gap: 12px; padding: 18px; } +.prod-dialog h2 { margin: 0; font-size: 1.05rem; } +.prod-dialog input { width: 100%; box-sizing: border-box; } +.prod-dialog input[type="checkbox"], .prod-dialog input[type="radio"] { width: auto; } +.badge.shape { background: rgba(125, 180, 150, 0.14); color: #b9d8c4; } +.prod-dialog footer { display: flex; justify-content: flex-end; gap: 8px; flex-wrap: wrap; } .pill.param { background: rgba(216,181,111,0.12); border-color: rgba(216,181,111,0.32); color: var(--accent-2); } /* ── Connect: driver parameters ────────────────────────────────────────── */ @@ -2382,3 +2391,41 @@ textarea { font-family: var(--mono); font-size: 12px; } .ws-rail, .ws-tab-body, #ws-detail { min-width: 0; } .preview-table-wrap { max-width: 100%; overflow-x: auto; } + +/* ── run strip: the current job next to its action ─────────────────────── */ +.run-strip { display: grid; gap: 4px; min-width: 0; max-width: 100%; padding: 8px 10px; border: 1px solid var(--line, #2a2d33); border-radius: 10px; background: var(--panel, #16181c); font-size: 0.86rem; } +.run-strip-head { display: flex; flex-wrap: wrap; align-items: baseline; gap: 6px 8px; min-width: 0; } +.run-strip-name { min-width: 0; overflow-wrap: anywhere; } +.run-strip-elapsed { margin-left: auto; font-variant-numeric: tabular-nums; } +.run-strip-dot { width: 8px; height: 8px; border-radius: 50%; background: var(--accent, #79d8b3); flex: none; align-self: center; } +.run-strip[data-state="running"] .run-strip-dot, .run-strip[data-state="pending"] .run-strip-dot { animation: run-strip-pulse 1.2s ease-in-out infinite; } +.run-strip[data-state="quiet"] .run-strip-dot { background: #d8b56f; } +.run-strip[data-state="reconnecting"] .run-strip-dot { background: #d8b56f; animation: run-strip-pulse 0.6s ease-in-out infinite; } +.run-strip[data-state="lost"] .run-strip-dot, .run-strip[data-state="failed"] .run-strip-dot { background: var(--danger, #f0786e); } +.run-strip[data-state="canceled"] .run-strip-dot { background: #9aa0a6; } +.run-strip[data-state="failed"], .run-strip[data-state="lost"] { border-color: rgba(240,120,110,0.45); } +.run-strip-bar { width: 100%; height: 6px; } +.run-strip-count { font-variant-numeric: tabular-nums; } +.run-strip p { margin: 0; overflow-wrap: anywhere; } +.run-strip-failure { color: var(--danger, #f0786e); } +.cmp-run-strip { grid-column: 1 / -1; } +@keyframes run-strip-pulse { 50% { opacity: 0.35; } } +@media (prefers-reduced-motion: reduce) { .run-strip-dot { animation: none !important; } } +.page-error-notice { position: fixed; right: 16px; bottom: 16px; left: 16px; max-width: 520px; margin-left: auto; display: flex; gap: 10px; align-items: center; justify-content: space-between; padding: 10px 12px; border-radius: 10px; border: 1px solid rgba(240,120,110,0.45); background: var(--panel, #16181c); z-index: 1000; } + +/* ── tuning recommendation dialog ──────────────────────────────────────── */ +.tune-dialog { max-width: min(560px, calc(100vw - 32px)); } +.tune-grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(min(180px, 100%), 1fr)); gap: 10px; } +.tune-grid .field { min-width: 0; } +.tune-result { min-width: 0; padding: 10px; border-radius: 10px; background: rgba(121,216,179,0.06); border: 1px solid var(--line, #2a2d33); } +.tune-result p, .tune-result li { overflow-wrap: anywhere; } +.tune-values { font-size: 1.05rem; margin: 0 0 6px; } +.tune-reasons { margin: 0 0 6px; padding-left: 18px; } +.tune-growth-warn { color: #d8b56f; } +.tune-growth-refuse, .tune-prod { color: var(--danger, #f0786e); } +.ws-recommend-btn { justify-self: start; } + +.ws-counts-actions { display: flex; flex-wrap: wrap; gap: 6px; min-width: 0; } +.ws-counts-actions .btn-ghost { min-height: 30px; padding: 4px 9px; font-size: 12px; } +.snapshot-dialog { max-width: min(640px, calc(100vw - 32px)); } +.snapshot-preview { max-height: 40vh; overflow: auto; } diff --git a/internal/web/store.go b/internal/web/store.go index c1bcaa5..7f083c7 100644 --- a/internal/web/store.go +++ b/internal/web/store.go @@ -40,6 +40,10 @@ type SavedConnection struct { Password string `json:"-" yaml:"password,omitempty"` HasPassword bool `json:"hasPassword" yaml:"-"` + // Production marks a database seedstorm must not write to without the + // user typing its label back. + Production bool `json:"production" yaml:"production,omitempty"` + CreatedAt time.Time `json:"createdAt,omitempty" yaml:"createdAt,omitempty"` UsedAt time.Time `json:"usedAt,omitempty" yaml:"usedAt,omitempty"` } diff --git a/internal/web/templates/compare.html.tmpl b/internal/web/templates/compare.html.tmpl index 12fb900..792d7f7 100644 --- a/internal/web/templates/compare.html.tmpl +++ b/internal/web/templates/compare.html.tmpl @@ -31,6 +31,7 @@ +
    @@ -117,6 +118,8 @@ + +

    All tables present on both sides.

    @@ -145,6 +148,21 @@
    +
    +
    +
    +

    Relationships

    +

    Children per parent for every foreign key, on both sides. Read-only; keys without an index are estimated unless you include them. Counts mode applies: estimate reads statistics only.

    +
    +
    + + + +
    +
    + +
    +
    Job log idle
    @@ -217,6 +235,7 @@
    +