From 88a0db9c1d54fbf5baf092b59c7a696b1ac35916 Mon Sep 17 00:00:00 2001 From: "Gustavo Bertoi (WSL Windows 29/06/2026)" Date: Fri, 10 Jul 2026 17:09:33 -0300 Subject: [PATCH 1/2] feat(expose): standard host ports + MySQL/MariaDB/MongoDB/Cassandra/ArangoDB engines Publish shared engines on their WELL-KNOWN host ports instead of a 5xxxx offset band, so `defaultPort` (in-network) and the exposed host port match and a GUI client's defaults just work: postgres 5432, mysql/mariadb 3306, mongodb 27017, cassandra 9042, arangodb 8529, redis 6379, minio 9000/9001. The standard port becomes the FreeHostPort search base. This is safe because AllocatePort skips every already-allocated port across all owners, so a lone engine lands on the standard port, a host-native server holding it makes the allocator fall back to the next free port, and protocol twins (mysql+mariadb on 3306, localstack+ministack on 4566) deconflict to base/base+1. Kafka keeps its fixed advertised 49092. Add five image-based shared database engines (built-in templates), each reachable at shared-, exposable, and with a client-ready connection URL: mysql, mariadb, mongodb, cassandra, arangodb. Wire the devdock-import engine map and the templates README. Determinism, vet, gofmt, and the race detector on orchestrate/generate pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/cli/template_new_test.go | 10 +++--- internal/migrate/migrate.go | 5 +++ internal/orchestrate/expose.go | 56 ++++++++++++++++++++--------- internal/orchestrate/expose_test.go | 41 +++++++++++++-------- templates/README.md | 11 ++++-- templates/arangodb/template.yaml | 37 +++++++++++++++++++ templates/cassandra/template.yaml | 35 ++++++++++++++++++ templates/embed.go | 2 +- templates/mariadb/template.yaml | 40 +++++++++++++++++++++ templates/mongodb/template.yaml | 36 +++++++++++++++++++ templates/mysql/template.yaml | 40 +++++++++++++++++++++ 11 files changed, 273 insertions(+), 40 deletions(-) create mode 100644 templates/arangodb/template.yaml create mode 100644 templates/cassandra/template.yaml create mode 100644 templates/mariadb/template.yaml create mode 100644 templates/mongodb/template.yaml create mode 100644 templates/mysql/template.yaml diff --git a/internal/cli/template_new_test.go b/internal/cli/template_new_test.go index 5a0edaa..80eb6ad 100644 --- a/internal/cli/template_new_test.go +++ b/internal/cli/template_new_test.go @@ -123,15 +123,15 @@ func TestTemplateNewEngineHasNoBuildTree(t *testing.T) { t.Setenv("DEVSTACK_HOME", t.TempDir()) dir := t.TempDir() if out, err := runCmd(t, "template", "new", "--no-input", "--dir", dir, - "--kind", "engine", "--name", "mariadb", "--base-image", "mariadb:11", - "--provides", "mariadb", "--exports", "host,port,user", "--port", "3306"); err != nil { + "--kind", "engine", "--name", "couchdb", "--base-image", "couchdb:3", + "--provides", "couchdb", "--exports", "host,port,user", "--port", "5984"); err != nil { t.Fatalf("author engine: %v\n%s", err, out) } - if _, err := os.Stat(filepath.Join(dir, "mariadb", "build")); !os.IsNotExist(err) { + if _, err := os.Stat(filepath.Join(dir, "couchdb", "build")); !os.IsNotExist(err) { t.Errorf("engine template must have no build/ tree, stat err = %v", err) } - manifest, _ := os.ReadFile(filepath.Join(dir, "mariadb", "template.yaml")) - for _, want := range []string{"image: mariadb:11", "provides: mariadb"} { + manifest, _ := os.ReadFile(filepath.Join(dir, "couchdb", "template.yaml")) + for _, want := range []string{"image: couchdb:3", "provides: couchdb"} { if !strings.Contains(string(manifest), want) { t.Errorf("engine template.yaml missing %q:\n%s", want, manifest) } diff --git a/internal/migrate/migrate.go b/internal/migrate/migrate.go index 40a3428..dbfbd0f 100644 --- a/internal/migrate/migrate.go +++ b/internal/migrate/migrate.go @@ -30,6 +30,11 @@ const APIVersion = "devstack/v1" // knownEngines maps a devdock template/image keyword to a devstack shared engine. var knownEngines = map[string]string{ "postgres": "postgres", "postgresql": "postgres", "postgis": "postgres", + "mysql": "mysql", "percona": "mysql", + "mariadb": "mariadb", + "mongo": "mongodb", "mongodb": "mongodb", + "cassandra": "cassandra", + "arango": "arangodb", "arangodb": "arangodb", "redis": "redis", "valkey": "redis", "minio": "minio", } diff --git a/internal/orchestrate/expose.go b/internal/orchestrate/expose.go index a4508fe..1236c6e 100644 --- a/internal/orchestrate/expose.go +++ b/internal/orchestrate/expose.go @@ -19,11 +19,16 @@ import ( // is an explicit opt-in that, like provisioning, is an UP-TIME compose overlay — // it never touches the deterministic, golden-asserted generated compose. // -// Exposure uses its OWN host-port range (55xxx/58xxx…), distinct from the -// provisioning range (45xxx), so the expose overlay and the provision overlay -// never publish the same host port and can both be applied without a duplicate -// binding. Ports are ledger-allocated (FreeHostPort), so the same engine keeps -// the same host port across runs and two terminals never collide. +// Exposure publishes each engine on its OWN WELL-KNOWN host port — the same port +// the template advertises in-network (postgres→5432, mysql→3306, redis→6379, …) — +// so a GUI client's default connection settings just work and there is no gap +// between what the template's `defaultPort` says and what the host sees. That +// deliberately differs from the provisioning range (45xxx): the two overlays map +// different host ports onto the same container port, so both can be applied +// without a duplicate binding. Ports remain ledger-allocated (FreeHostPort) with +// the standard port as the search base, so the same engine keeps the same host +// port across runs, and if a host-native server already holds the standard port +// the allocator transparently falls back to the next free one in the band. const exposeFile = "compose.expose.yaml" @@ -37,19 +42,26 @@ type exposePort struct { } // exposeEngines maps a shared engine (template name) to the ports `shared expose` -// publishes on 127.0.0.1. Bases sit in the 5xxxx range so they never collide with -// the 4xxxx provisioning overlay. Kafka is the exception: host clients MUST reach -// the broker on 127.0.0.1:49092 (the fixed advertised external listener from the -// template), so it reuses the kafka provision port rather than a 5xxxx one. +// publishes on 127.0.0.1. The search base is the engine's WELL-KNOWN port (equal +// to the in-container port), so clients connect on the port they already expect +// and the allocator only drifts off it when a host-native server already holds it. +// Kafka is the exception: host clients MUST reach the broker on 127.0.0.1:49092 +// (the fixed advertised external listener from the template), so it keeps that +// base and reuses the kafka provision port rather than the broker's 19092. var exposeEngines = map[string][]exposePort{ - "postgres": {{5432, "postgres", "pg-expose", 55432, true}}, - "redis": {{6379, "redis", "redis-expose", 56379, true}}, - "minio": {{9000, "s3", "minio-expose", 59000, true}, {9001, "console", "minio-console-expose", 59001, false}}, - "localstack": {{4566, "aws", "localstack-expose", 54566, true}}, - "ministack": {{4566, "aws", "ministack-expose", 54567, true}}, - "nats": {{4222, "nats", "nats-expose", 54222, true}, {8222, "monitor", "nats-monitor-expose", 58222, false}}, + "postgres": {{5432, "postgres", "pg-expose", 5432, true}}, + "mysql": {{3306, "mysql", "mysql-expose", 3306, true}}, + "mariadb": {{3306, "mariadb", "mariadb-expose", 3306, true}}, + "mongodb": {{27017, "mongodb", "mongodb-expose", 27017, true}}, + "cassandra": {{9042, "cassandra", "cassandra-expose", 9042, true}}, + "arangodb": {{8529, "arangodb", "arangodb-expose", 8529, true}}, + "redis": {{6379, "redis", "redis-expose", 6379, true}}, + "minio": {{9000, "s3", "minio-expose", 9000, true}, {9001, "console", "minio-console-expose", 9001, false}}, + "localstack": {{4566, "aws", "localstack-expose", 4566, true}}, + "ministack": {{4566, "aws", "ministack-expose", 4566, true}}, + "nats": {{4222, "nats", "nats-expose", 4222, true}, {8222, "monitor", "nats-monitor-expose", 8222, false}}, "kafka": {{19092, "kafka", "kafka-provision", 49092, true}}, - "rabbitmq": {{5672, "amqp", "rmq-expose", 55672, true}, {15672, "management", "rmq-mgmt-expose", 55673, false}}, + "rabbitmq": {{5672, "amqp", "rmq-expose", 5672, true}, {15672, "management", "rmq-mgmt-expose", 15672, false}}, } // ExposableEngine reports whether an engine has a defined host-expose port set. @@ -284,6 +296,18 @@ func connectionURL(engine string, ep exposePort, params map[string]any, port int user := paramString(params, "rootUser", "devstack") pass := paramString(params, "rootPassword", "devstack") return fmt.Sprintf("postgres://%s:%s@%s/postgres?sslmode=disable", user, pass, host) + case "mysql", "mariadb": + user := paramString(params, "rootUser", "devstack") + pass := paramString(params, "rootPassword", "devstack") + return fmt.Sprintf("mysql://%s:%s@%s/%s", user, pass, host, user) + case "mongodb": + user := paramString(params, "rootUser", "devstack") + pass := paramString(params, "rootPassword", "devstack") + return fmt.Sprintf("mongodb://%s:%s@%s/?authSource=admin", user, pass, host) + case "cassandra": + return host // contact point host:9042 (CQL native transport) + case "arangodb": + return "http://" + host // HTTP API + web UI (root / rootPassword) case "redis": return "redis://" + host case "minio": diff --git a/internal/orchestrate/expose_test.go b/internal/orchestrate/expose_test.go index 8364179..79bbc37 100644 --- a/internal/orchestrate/expose_test.go +++ b/internal/orchestrate/expose_test.go @@ -93,16 +93,21 @@ func TestConnectionURL(t *testing.T) { port int want string }{ - {"postgres", exposePort{5432, "postgres", "", 0, true}, pgParams, 55432, "postgres://admin:s3cret@127.0.0.1:55432/postgres?sslmode=disable"}, - {"postgres", exposePort{5432, "postgres", "", 0, true}, nil, 55432, "postgres://devstack:devstack@127.0.0.1:55432/postgres?sslmode=disable"}, - {"redis", exposePort{6379, "redis", "", 0, true}, nil, 56379, "redis://127.0.0.1:56379"}, - {"minio", exposePort{9000, "s3", "", 0, true}, nil, 59000, "http://127.0.0.1:59000"}, - {"localstack", exposePort{4566, "aws", "", 0, true}, nil, 54566, "http://127.0.0.1:54566"}, - {"nats", exposePort{8222, "monitor", "", 0, false}, nil, 58222, "http://127.0.0.1:58222"}, - {"nats", exposePort{4222, "nats", "", 0, true}, nil, 54222, "nats://127.0.0.1:54222"}, + {"postgres", exposePort{5432, "postgres", "", 0, true}, pgParams, 5432, "postgres://admin:s3cret@127.0.0.1:5432/postgres?sslmode=disable"}, + {"postgres", exposePort{5432, "postgres", "", 0, true}, nil, 5432, "postgres://devstack:devstack@127.0.0.1:5432/postgres?sslmode=disable"}, + {"mysql", exposePort{3306, "mysql", "", 0, true}, nil, 3306, "mysql://devstack:devstack@127.0.0.1:3306/devstack"}, + {"mariadb", exposePort{3306, "mariadb", "", 0, true}, nil, 3306, "mysql://devstack:devstack@127.0.0.1:3306/devstack"}, + {"mongodb", exposePort{27017, "mongodb", "", 0, true}, nil, 27017, "mongodb://devstack:devstack@127.0.0.1:27017/?authSource=admin"}, + {"cassandra", exposePort{9042, "cassandra", "", 0, true}, nil, 9042, "127.0.0.1:9042"}, + {"arangodb", exposePort{8529, "arangodb", "", 0, true}, nil, 8529, "http://127.0.0.1:8529"}, + {"redis", exposePort{6379, "redis", "", 0, true}, nil, 6379, "redis://127.0.0.1:6379"}, + {"minio", exposePort{9000, "s3", "", 0, true}, nil, 9000, "http://127.0.0.1:9000"}, + {"localstack", exposePort{4566, "aws", "", 0, true}, nil, 4566, "http://127.0.0.1:4566"}, + {"nats", exposePort{8222, "monitor", "", 0, false}, nil, 8222, "http://127.0.0.1:8222"}, + {"nats", exposePort{4222, "nats", "", 0, true}, nil, 4222, "nats://127.0.0.1:4222"}, {"kafka", exposePort{19092, "kafka", "", 0, true}, nil, 49092, "127.0.0.1:49092"}, - {"rabbitmq", exposePort{15672, "management", "", 0, false}, nil, 55673, "http://127.0.0.1:55673"}, - {"rabbitmq", exposePort{5672, "amqp", "", 0, true}, nil, 55672, "amqp://devstack@127.0.0.1:55672"}, + {"rabbitmq", exposePort{15672, "management", "", 0, false}, nil, 15672, "http://127.0.0.1:15672"}, + {"rabbitmq", exposePort{5672, "amqp", "", 0, true}, nil, 5672, "amqp://devstack@127.0.0.1:5672"}, } for _, tc := range cases { if got := connectionURL(tc.engine, tc.ep, tc.params, tc.port); got != tc.want { @@ -128,17 +133,23 @@ func TestExposePortsNeverCollideWithProvision(t *testing.T) { } } } - // Every expose base must be unique across all engines/ports (no two services - // fight for the same host port at allocation time either). - seen := map[int]string{} - for _, ports := range exposeEngines { + // Within a SINGLE engine, its ports must not share a base (else a two-port + // engine like minio/nats/rabbitmq would self-collide on the same host port). + for engine, ports := range exposeEngines { + seen := map[int]string{} for _, ep := range ports { - if prev, ok := seen[ep.base]; ok && prev != ep.purpose { - t.Errorf("expose base %d reused across purposes %q and %q", ep.base, prev, ep.purpose) + if prev, ok := seen[ep.base]; ok { + t.Errorf("engine %q reuses expose base %d across purposes %q and %q", engine, ep.base, prev, ep.purpose) } seen[ep.base] = ep.purpose } } + // Across DIFFERENT engines the base MAY repeat on purpose: two engines that + // speak the same wire protocol want the same well-known port (mysql/mariadb on + // 3306, localstack/ministack on 4566). That is safe because the ledger's + // AllocatePort skips every already-allocated port (AllocatedPorts spans all + // owners), so a lone engine lands on the standard port and, when both are + // exposed, the second transparently deconflicts to base+1. } func TestFileExists(t *testing.T) { diff --git a/templates/README.md b/templates/README.md index 232ce13..630450e 100644 --- a/templates/README.md +++ b/templates/README.md @@ -8,9 +8,14 @@ Built-in service templates, compiled into the binary via `go:embed` | Template | Kind | Notes | |---|---|---| -| `postgres` | shared engine | `provides: postgres`; version-aware PGDATA mount (PG18+ moved it — DECISIONS D8) | -| `redis` | shared engine | `provides: redis` | -| `minio` | shared engine | `provides: minio` | +| `postgres` | shared engine | `provides: postgres`; version-aware PGDATA mount (PG18+ moved it — DECISIONS D8); exposes on `5432` | +| `mysql` | shared engine | `provides: mysql`; exposes on `3306` | +| `mariadb` | shared engine | `provides: mariadb` (MySQL-compatible); exposes on `3306` | +| `mongodb` | shared engine | `provides: mongodb`; exposes on `27017` | +| `cassandra` | shared engine | `provides: cassandra` (CQL); exposes on `9042` | +| `arangodb` | shared engine | `provides: arangodb` (multi-model + web UI); exposes on `8529` | +| `redis` | shared engine | `provides: redis`; exposes on `6379` | +| `minio` | shared engine | `provides: minio`; exposes on `9000`/`9001` | | `php.nginx` | project base | PHP-FPM build (`build/Dockerfile`); parent template | | `php.laravel.nginx` | project | `extends: php.nginx`; adds Laravel env + entrypoint | | `node.vite` | project | Node + Vite dev server build | diff --git a/templates/arangodb/template.yaml b/templates/arangodb/template.yaml new file mode 100644 index 0000000..62a37b9 --- /dev/null +++ b/templates/arangodb/template.yaml @@ -0,0 +1,37 @@ +schemaVersion: 1 +description: "Shared ArangoDB engine (multi-model + web UI), reached over the shared network at shared-arangodb." +provides: arangodb +exports: [host, port, user, password] +defaultPort: 8529 +params: + version: + type: string + default: "3.12" + description: "ArangoDB version image tag." + rootPassword: + type: string + default: devstack + description: "Password for the built-in root account (local development)." + +service: + image: "arangodb:[[ .params.version ]]" + restart: unless-stopped + environment: + ARANGO_ROOT_PASSWORD: "[[ .params.rootPassword ]]" + volumes: + - "arangodata:/var/lib/arangodb3" + - "arangoapps:/var/lib/arangodb3-apps" + # The dev password is inlined (loopback-only threat model) so the check needs no + # shell env expansion; arangosh exits non-zero if the server is not yet serving. + healthcheck: + test: + - CMD-SHELL + - arangosh --server.endpoint tcp://127.0.0.1:8529 --server.password '[[ .params.rootPassword ]]' --javascript.execute-string 'db._version()' || exit 1 + interval: 15s + timeout: 10s + retries: 10 + start_period: 30s + +volumes: + arangodata: {} + arangoapps: {} diff --git a/templates/cassandra/template.yaml b/templates/cassandra/template.yaml new file mode 100644 index 0000000..ac329dd --- /dev/null +++ b/templates/cassandra/template.yaml @@ -0,0 +1,35 @@ +schemaVersion: 1 +description: "Shared Apache Cassandra engine, reached over the shared network at shared-cassandra." +provides: cassandra +exports: [host, port] +defaultPort: 9042 +params: + version: + type: string + default: "5" + description: "Cassandra major version image tag." + clusterName: + type: string + default: devstack + description: "Cluster name advertised by the node." + +service: + image: "cassandra:[[ .params.version ]]" + restart: unless-stopped + environment: + CASSANDRA_CLUSTER_NAME: "[[ .params.clusterName ]]" + # Single-node dev cluster: skip the gossip/token bootstrap wait. + CASSANDRA_ENDPOINT_SNITCH: SimpleSnitch + volumes: + - "cassandradata:/var/lib/cassandra" + # Cassandra is slow to accept CQL; the long start_period keeps the up saga from + # aborting before the native transport is listening. + healthcheck: + test: ["CMD-SHELL", "cqlsh -e 'describe keyspaces' 127.0.0.1 9042 || exit 1"] + interval: 30s + timeout: 10s + retries: 10 + start_period: 90s + +volumes: + cassandradata: {} diff --git a/templates/embed.go b/templates/embed.go index cea3bfa..6dde91a 100644 --- a/templates/embed.go +++ b/templates/embed.go @@ -7,7 +7,7 @@ package templates import "embed" -//go:embed all:postgres all:redis all:minio all:php.nginx all:php.laravel.nginx all:node.vite all:localstack all:ministack all:nats all:kafka all:rabbitmq all:node.express all:node.nestjs all:node.next all:react.vite all:bun.app all:turborepo +//go:embed all:postgres all:mysql all:mariadb all:mongodb all:cassandra all:arangodb all:redis all:minio all:php.nginx all:php.laravel.nginx all:node.vite all:localstack all:ministack all:nats all:kafka all:rabbitmq all:node.express all:node.nestjs all:node.next all:react.vite all:bun.app all:turborepo var builtinFS embed.FS // FS is the embedded built-in templates root: template-name directories at the diff --git a/templates/mariadb/template.yaml b/templates/mariadb/template.yaml new file mode 100644 index 0000000..5b3168e --- /dev/null +++ b/templates/mariadb/template.yaml @@ -0,0 +1,40 @@ +schemaVersion: 1 +description: "Shared MariaDB engine (MySQL-compatible), reached over the shared network at shared-mariadb." +provides: mariadb +exports: [host, port, user, password, database] +defaultPort: 3306 +params: + version: + type: string + default: "11" + description: "MariaDB major version image tag." + rootUser: + type: string + default: devstack + description: "Application user created on first boot (per-project users are provisioned on top in M2)." + rootPassword: + type: string + default: devstack + description: "Password for both the MariaDB root account and the application user (local development)." + +service: + image: "mariadb:[[ .params.version ]]" + restart: unless-stopped + environment: + MARIADB_ROOT_PASSWORD: "[[ .params.rootPassword ]]" + MARIADB_DATABASE: "[[ .params.rootUser ]]" + MARIADB_USER: "[[ .params.rootUser ]]" + MARIADB_PASSWORD: "[[ .params.rootPassword ]]" + volumes: + - "mariadbdata:/var/lib/mysql" + # The mariadb image ships /usr/local/bin/healthcheck.sh; --connect proves the + # socket accepts connections and --innodb_initialized proves it is done booting. + healthcheck: + test: ["CMD", "healthcheck.sh", "--connect", "--innodb_initialized"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + +volumes: + mariadbdata: {} diff --git a/templates/mongodb/template.yaml b/templates/mongodb/template.yaml new file mode 100644 index 0000000..5bf43b2 --- /dev/null +++ b/templates/mongodb/template.yaml @@ -0,0 +1,36 @@ +schemaVersion: 1 +description: "Shared MongoDB engine, reached over the shared network at shared-mongodb." +provides: mongodb +exports: [host, port, user, password, database] +defaultPort: 27017 +params: + version: + type: string + default: "7" + description: "MongoDB major version image tag." + rootUser: + type: string + default: devstack + description: "Root username created on first boot (authSource=admin)." + rootPassword: + type: string + default: devstack + description: "Root password for local development." + +service: + image: "mongo:[[ .params.version ]]" + restart: unless-stopped + environment: + MONGO_INITDB_ROOT_USERNAME: "[[ .params.rootUser ]]" + MONGO_INITDB_ROOT_PASSWORD: "[[ .params.rootPassword ]]" + volumes: + - "mongodata:/data/db" + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--eval", "db.adminCommand('ping').ok"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 20s + +volumes: + mongodata: {} diff --git a/templates/mysql/template.yaml b/templates/mysql/template.yaml new file mode 100644 index 0000000..539eb3d --- /dev/null +++ b/templates/mysql/template.yaml @@ -0,0 +1,40 @@ +schemaVersion: 1 +description: "Shared MySQL engine, reached over the shared network at shared-mysql." +provides: mysql +exports: [host, port, user, password, database] +defaultPort: 3306 +params: + version: + type: string + default: "8" + description: "MySQL major version image tag." + rootUser: + type: string + default: devstack + description: "Application user created on first boot (per-project users are provisioned on top in M2)." + rootPassword: + type: string + default: devstack + description: "Password for both the MySQL root account and the application user (local development)." + +service: + image: "mysql:[[ .params.version ]]" + restart: unless-stopped + # MYSQL_USER must not be "root" (the image refuses it); the app user is created + # alongside the always-present root account, both sharing the dev password. + environment: + MYSQL_ROOT_PASSWORD: "[[ .params.rootPassword ]]" + MYSQL_DATABASE: "[[ .params.rootUser ]]" + MYSQL_USER: "[[ .params.rootUser ]]" + MYSQL_PASSWORD: "[[ .params.rootPassword ]]" + volumes: + - "mysqldata:/var/lib/mysql" + healthcheck: + test: ["CMD-SHELL", "mysqladmin ping -h 127.0.0.1 -uroot -p'[[ .params.rootPassword ]]' --silent"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 30s + +volumes: + mysqldata: {} From b2d2870abcef12f72303ba64043db6671507ab6b Mon Sep 17 00:00:00 2001 From: "Gustavo Bertoi (WSL Windows 29/06/2026)" Date: Fri, 10 Jul 2026 17:44:19 -0300 Subject: [PATCH 2/2] feat(up): unify host ports to the standard port, expose by default, fix up/down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There were TWO host-port overlays fighting over the same shared containers' `ports:` — the expose overlay (standard ports) and a separate provisioning band (45xxx via writeProvisionOverlay/engineTarget). Every provision, `db reset`, snapshot, resource op and re-up applied a DIFFERENT subset, so compose kept recreating the shared container with a different port set. That churn is the root of "port already allocated", the connection-reset races, and re-up flakiness. Collapse the two into ONE unified overlay published on the engine's STANDARD 127.0.0.1 port: - ensureExposed / primaryExposePort (expose.go): the single host-reachability primitive. Provisioning, reset, snapshot and resource ops all resolve their admin endpoint from it, so there is exactly one host port per engine — the well-known one — never a separate band. Idempotent: same ports + same overlay bytes → no container recreate. - engineTarget, the resources phase, and the provision phase now go through ensureExposed. Deleted engineOverlays/perEngineOverlay, writeProvisionOverlay, and the provisionPortBase/provisionPurpose (pg-provision → pg-expose) band. - Expose-by-default: `up` auto-publishes every exposable shared engine on its standard 127.0.0.1 port (loopback-only), so a GUI client's defaults just work and provisioning has one port to dial. Opt out with `up --no-expose`; skipped on a remote backend. - `down` now actually brings things DOWN: after dropping a project's refs it stops every shared engine that fell to zero refs (Manager.GC stop=true), respecting cross-workspace ref-counting, and CLEARS the saga phase records for what it stopped (compose-up per project; shared/provision/resources when the shared stack is stopped) so the next `up` restarts them instead of skipping on a stale fingerprint — the "re-up after down is a no-op" bug. Verified end-to-end against a real Docker daemon (up → provision on 127.0.0.1:5432 → down stops shared → re-up restarts and re-publishes): standard ports throughout, no collisions across the cycle. Unit suite + race + determinism pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/cli/up.go | 44 ++++++++++- internal/orchestrate/expose.go | 89 +++++++++++++++++++++++ internal/orchestrate/expose_test.go | 32 +++++--- internal/orchestrate/minio_ops_test.go | 4 +- internal/orchestrate/provision.go | 56 +++----------- internal/orchestrate/reset_test.go | 2 +- internal/orchestrate/resource_ops.go | 38 +++------- internal/orchestrate/resource_ops_test.go | 2 +- internal/orchestrate/resources.go | 53 +++----------- internal/orchestrate/snapshot.go | 8 +- internal/orchestrate/snapshot_test.go | 4 +- internal/orchestrate/up.go | 32 +++----- internal/orchestrate/up_test.go | 4 +- 13 files changed, 207 insertions(+), 161 deletions(-) diff --git a/internal/cli/up.go b/internal/cli/up.go index a308810..ad93c09 100644 --- a/internal/cli/up.go +++ b/internal/cli/up.go @@ -33,6 +33,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command { noHooks bool noPreflight bool noProvision bool + noExpose bool profiles []string healthTimeout time.Duration ) @@ -56,6 +57,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command { d.NoHooks = noHooks d.NoPreflight = noPreflight d.NoProvision = noProvision + d.NoExpose = noExpose d.Profiles = profiles d.HealthTimeout = healthTimeout @@ -112,6 +114,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command { cmd.Flags().BoolVar(&noHooks, "no-hooks", false, "skip lifecycle hooks") cmd.Flags().BoolVar(&noPreflight, "no-preflight", false, "skip the preflight checks") cmd.Flags().BoolVar(&noProvision, "no-provision", false, "skip per-project Postgres role/db provisioning") + cmd.Flags().BoolVar(&noExpose, "no-expose", false, "do not auto-publish shared engines on their standard 127.0.0.1 ports") cmd.Flags().StringArrayVarP(&profiles, "profile", "p", nil, "service slice(s) to start — repeatable & comma-separated (spec 12); empty → defaultProfile or all") return cmd @@ -174,8 +177,39 @@ func newDownCmd(g *GlobalOpts) *cobra.Command { fmt.Fprintf(w, "[ok] down %s\n", p) } } + // Tear down anything now left running: after the project refs are dropped, + // stop every shared service that fell to zero refs (`shared gc --stop`). + // This is what makes `down` actually bring the workspace DOWN instead of + // leaving warm engines behind, while still respecting cross-workspace + // sharing — an engine another workspace still references keeps running. + var stopped []string + if gc, err := d.Manager.GC(ctx, true); err != nil { + if firstErr == nil { + firstErr = err + } + } else { + stopped = gc.Stopped + // When shared services were actually stopped, their saga phases are no + // longer satisfied — clear them so the next `up` restarts the shared + // stack (and re-publishes its ports) instead of skipping. + if len(stopped) > 0 { + _ = lock.WithLock(ctx, d.LockPath, func() error { + for _, ph := range []string{"shared", "provision", "resources"} { + if e := d.DB.ClearPhase(d.Model.Workspace.Name, "", ph); e != nil { + return e + } + } + return nil + }) + } + if !g.JSON && !g.Quiet { + for _, s := range stopped { + fmt.Fprintf(w, "[ok] stopped shared %s (0 refs)\n", s) + } + } + } if g.JSON { - if err := writeJSON(cmd, map[string]any{"down": results}); err != nil { + if err := writeJSON(cmd, map[string]any{"down": results, "sharedStopped": stopped}); err != nil { return err } } @@ -216,7 +250,13 @@ func downProject(ctx context.Context, d orchestrate.UpDeps, project string) erro if _, err := d.Manager.RegisterDown(ctx, project); err != nil { return err } - return nil + // The compose-up phase is no longer satisfied — its containers were just + // removed. Clear the saga record so the NEXT `up` re-runs it instead of + // skipping on a stale fingerprint (the "re-up after down is a no-op" bug). + // firstRun/hooks are intentionally NOT cleared (they keep run-once semantics). + return lock.WithLock(ctx, d.LockPath, func() error { + return d.DB.ClearPhase(d.Model.Workspace.Name, project, "compose-up") + }) } // buildUpDeps assembles the up/down dependencies from the current directory. It diff --git a/internal/orchestrate/expose.go b/internal/orchestrate/expose.go index 1236c6e..13b16fa 100644 --- a/internal/orchestrate/expose.go +++ b/internal/orchestrate/expose.go @@ -70,6 +70,95 @@ func ExposableEngine(engine string) bool { return ok } +// primaryExposePort returns an engine's PRIMARY host-published port — the one a +// client (and devstack's own host-side provisioning) connects the engine's main +// protocol on. This is the single source of truth for "the host port of engine +// X": provisioning, reset, snapshot and resource ops all resolve their admin +// endpoint from it, so there is exactly ONE host port per engine (the standard +// one), never a separate provisioning band. +func primaryExposePort(engine string) (exposePort, bool) { + for _, ep := range exposeEngines[engine] { + if ep.primary { + return ep, true + } + } + return exposePort{}, false +} + +// exposableUnion returns the shared instances to publish: the requested set +// unioned with any already-exposed instance (so writing the overlay never drops +// another instance's ports), filtered to engines that support exposure. Sorted +// for a byte-stable overlay. +func exposableUnion(d UpDeps, want []string) []string { + set := map[string]bool{} + for _, i := range want { + set[i] = true + } + for _, i := range exposedInstances(d.Model.Root) { + set[i] = true + } + var insts []string + for i := range set { + if s, ok := d.Model.Workspace.Shared[i]; ok && ExposableEngine(s.Template) { + insts = append(insts, i) + } + } + sort.Strings(insts) + return insts +} + +// exposeOverlayFor allocates the standard host ports for the exposable instances +// among want (unioned with the currently-exposed set) and WRITES the single +// expose overlay, returning its path ("" when there is nothing to expose). It does +// NOT run compose — the caller (the shared phase) folds the returned path into its +// own `compose up` so ports are published as the services come up. +func exposeOverlayFor(ctx context.Context, d UpDeps, want []string) (string, error) { + insts := exposableUnion(d, want) + if len(insts) == 0 { + return "", nil + } + _, pub, err := allocateExposePorts(ctx, d, insts) + if err != nil { + return "", err + } + return writeExposeOverlay(d.Model.Root, pub) +} + +// ensureExposed is the unified host-reachability primitive for callers that need +// the ports published NOW (provisioning, reset, snapshot, resource ops): it writes +// the single expose overlay for the exposable instances among want (unioned with +// the already-exposed set, so it never drops another instance's ports) and applies +// it via `compose up`. It is idempotent — the ledger returns the same standard +// ports and the overlay bytes are unchanged, so compose does not recreate the +// container on repeat calls. Returns instance→primary host port. Because both +// auto-expose and every host-side admin op go through this one overlay, they can +// never fight over a container's `ports:`. +func ensureExposed(ctx context.Context, d UpDeps, want []string) (map[string]int, error) { + insts := exposableUnion(d, want) + if len(insts) == 0 { + return map[string]int{}, nil + } + out, pub, err := allocateExposePorts(ctx, d, insts) + if err != nil { + return nil, err + } + overlay, err := writeExposeOverlay(d.Model.Root, pub) + if err != nil { + return nil, err + } + outDir := filepath.Join(d.Model.Root, generate.GenDir, "shared") + if err := composeUpShared(ctx, d, outDir, []string{overlay}, insts); err != nil { + return nil, fmt.Errorf("apply host-port overlay: %w", err) + } + ports := map[string]int{} + for _, ep := range out { + if ep.Primary { + ports[ep.Instance] = ep.Port + } + } + return ports, nil +} + // ExposedPort is one host-published shared-service port with a client-ready // connection hint (the `--json` schema + the plain-table source). type ExposedPort struct { diff --git a/internal/orchestrate/expose_test.go b/internal/orchestrate/expose_test.go index 79bbc37..0d9e685 100644 --- a/internal/orchestrate/expose_test.go +++ b/internal/orchestrate/expose_test.go @@ -116,23 +116,31 @@ func TestConnectionURL(t *testing.T) { } } -// TestExposePortsNeverCollideWithProvision is the load-bearing invariant: the -// expose overlay and the provision overlay must never publish the SAME host port -// (base) for the SAME engine, or applying both recreates the container with a -// duplicate binding. Kafka is the deliberate exception — its host clients MUST -// use the fixed advertised 49092, so it reuses the provision port. -func TestExposePortsNeverCollideWithProvision(t *testing.T) { - provBase := map[string]int{} - for engine, ov := range engineOverlays { - provBase[engine] = ov.portBase - } +// TestExposeUsesStandardPorts locks in the unification: there is exactly ONE host +// port per engine — the well-known one — and it equals the in-container port. That +// is what lets provisioning/reset/snapshot and `expose` share a single overlay +// instead of two fighting bands. Kafka is the deliberate exception: its broker +// advertises a fixed 127.0.0.1:49092 external listener, so its host base is 49092 +// while the container port is 19092. +func TestExposeUsesStandardPorts(t *testing.T) { for engine, ports := range exposeEngines { for _, ep := range ports { - if pb, ok := provBase[engine]; ok && ep.base == pb && engine != "kafka" { - t.Errorf("engine %q expose base %d collides with provision base %d", engine, ep.base, pb) + if engine == "kafka" { + continue + } + if ep.base != ep.container { + t.Errorf("engine %q port %q: host base %d must equal container port %d (standard-port unification)", + engine, ep.label, ep.base, ep.container) } } } + // Every provisionable engine must have a PRIMARY expose port, since provisioning + // now resolves its host-reachable admin endpoint from that single overlay. + for _, engine := range []string{"postgres", "redis", "minio", "nats", "kafka", "localstack"} { + if _, ok := primaryExposePort(engine); !ok { + t.Errorf("engine %q has no primary expose port — provisioning cannot reach it", engine) + } + } // Within a SINGLE engine, its ports must not share a base (else a two-port // engine like minio/nats/rabbitmq would self-collide on the same host port). for engine, ports := range exposeEngines { diff --git a/internal/orchestrate/minio_ops_test.go b/internal/orchestrate/minio_ops_test.go index 60e1532..ecaddbb 100644 --- a/internal/orchestrate/minio_ops_test.go +++ b/internal/orchestrate/minio_ops_test.go @@ -135,7 +135,7 @@ func TestCreateBucketImperative(t *testing.T) { t.Errorf("bucket ownership row not recorded: %v", rows) } // The minio loopback overlay was applied publishing :9000 (not 5432). - overlay := filepath.Join(d.Model.Root, generate.GenDir, "shared", "compose.provision.yaml") + overlay := filepath.Join(d.Model.Root, generate.GenDir, "shared", "compose.expose.yaml") body, err := os.ReadFile(overlay) if err != nil { t.Fatalf("overlay not written: %v", err) @@ -143,7 +143,7 @@ func TestCreateBucketImperative(t *testing.T) { if !strings.Contains(string(body), ":9000") { t.Errorf("minio overlay must publish container port 9000, got:\n%s", body) } - if !fr.saw("-p "+generate.SharedStackName, "compose.provision.yaml") { + if !fr.saw("-p "+generate.SharedStackName, "compose.expose.yaml") { t.Errorf("overlay not applied via compose up: %v", fr.cmds) } } diff --git a/internal/orchestrate/provision.go b/internal/orchestrate/provision.go index 64c7620..fa07958 100644 --- a/internal/orchestrate/provision.go +++ b/internal/orchestrate/provision.go @@ -3,10 +3,7 @@ package orchestrate import ( "context" "fmt" - "os" - "path/filepath" "sort" - "strings" "github.com/open-source-cloud/devstack/internal/config" "github.com/open-source-cloud/devstack/internal/generate" @@ -25,12 +22,7 @@ import ( // is generated or stored, and an app opts in via the documented DSN // `postgres://:@shared-postgres:5432/`. -const ( - provisionPurpose = "pg-provision" // ledger port_alloc purpose - provisionPortBase = 45432 // host port search base for shared Postgres - provisionFile = "compose.provision.yaml" - pgTemplate = "postgres" // shared engine template that this phase provisions -) +const pgTemplate = "postgres" // shared engine template that this phase provisions // PgConnector opens an admin connection to a Postgres DSN. Injectable so the // provision phase is unit-testable without a live server (the default wraps @@ -99,33 +91,6 @@ func provInstanceList(targets []provTarget) []string { return sortedStringSlice(keysOf(set)) } -// writeProvisionOverlay writes the up-time compose overlay that publishes each -// provisioned instance on 127.0.0.1::. Returns the overlay -// path. Loopback-only so nothing is exposed beyond the host (spec 03 / no host -// ports). containerPort is the engine's in-container port (5432 postgres / 9000 -// minio); every instance in ports shares one engine, so one container port covers all. -func writeProvisionOverlay(root string, ports map[string]int, containerPort int) (string, error) { - var b strings.Builder - b.WriteString("services:\n") - insts := make([]string, 0, len(ports)) - for inst := range ports { - insts = append(insts, inst) - } - sort.Strings(insts) - for _, inst := range insts { - fmt.Fprintf(&b, " %s:\n ports:\n - \"127.0.0.1:%d:%d\"\n", inst, ports[inst], containerPort) - } - dir := filepath.Join(root, generate.GenDir, "shared") - if err := os.MkdirAll(dir, 0o755); err != nil { - return "", err - } - path := filepath.Join(dir, provisionFile) - if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil { - return "", err - } - return path, nil -} - // provisionPhase creates each project's role+database on its shared Postgres, // idempotently, holding the flock for the SQL mutations (DECISIONS D7/D8). It // re-derives the host port from the ledger (the same one sharedPhase published), @@ -156,20 +121,19 @@ func provisionPhase(d UpDeps, targets []provTarget) Phase { byInst[t.instance] = append(byInst[t.instance], t.project) } - // Resolve each instance's published host port (FreeHostPort self-locks - // and is idempotent — returns the port sharedPhase already allocated). - ports := map[string]int{} - for _, inst := range sortedStringSlice(keysOf(byInst)) { - p, err := d.Manager.FreeHostPort(ctx, generate.SharedAlias(inst), provisionPurpose, provisionPortBase) - if err != nil { - return nil, fmt.Errorf("resolve provision port for %s: %w", inst, err) - } - ports[inst] = p + // Publish each shared Postgres on its stable STANDARD 127.0.0.1 port and + // resolve the port host-side pgx dials. This is the SAME unified overlay as + // `expose`/auto-expose (idempotent — no container recreate), and it also + // guarantees the port exists even under --no-expose, since host-side + // provisioning fundamentally needs one host port to reach. + ports, err := ensureExposed(ctx, d, sortedStringSlice(keysOf(byInst))) + if err != nil { + return nil, fmt.Errorf("publish shared postgres host port: %w", err) } provisioned := []map[string]any{} // Hold the flock for the role/db mutations (provision pkg contract). - err := lock.WithLock(ctx, d.LockPath, func() error { + err = lock.WithLock(ctx, d.LockPath, func() error { for _, inst := range sortedStringSlice(keysOf(byInst)) { params := d.Model.Workspace.Shared[inst].Params user := paramString(params, "rootUser", "devstack") diff --git a/internal/orchestrate/reset_test.go b/internal/orchestrate/reset_test.go index 623dcba..0f90573 100644 --- a/internal/orchestrate/reset_test.go +++ b/internal/orchestrate/reset_test.go @@ -87,7 +87,7 @@ func TestResetDropsAndReprovisions(t *testing.T) { // The loopback overlay was applied via compose up on the shared stack (same // host-reachability path as the provision phase). - if !fr.saw("-p "+"devstack-shared", "compose.provision.yaml") { + if !fr.saw("-p "+"devstack-shared", "compose.expose.yaml") { t.Errorf("reset did not apply the provision overlay via compose up: %v", fr.cmds) } } diff --git a/internal/orchestrate/resource_ops.go b/internal/orchestrate/resource_ops.go index 18680e8..b8e98e7 100644 --- a/internal/orchestrate/resource_ops.go +++ b/internal/orchestrate/resource_ops.go @@ -3,10 +3,8 @@ package orchestrate import ( "context" "fmt" - "path/filepath" "github.com/open-source-cloud/devstack/internal/config" - "github.com/open-source-cloud/devstack/internal/docker" "github.com/open-source-cloud/devstack/internal/generate" "github.com/open-source-cloud/devstack/internal/lock" "github.com/open-source-cloud/devstack/internal/resource" @@ -79,35 +77,23 @@ func engineDefaultAdmin(engine string) string { } } -// engineTarget resolves the host-reachable admin endpoint for an instance: it -// allocates/looks up the ledger port, writes+applies the per-engine 127.0.0.1 -// overlay via `compose up -d ` (idempotent, no recreate), and returns the -// Target with the instance's admin creds. Postgres + MinIO overlays are wired. +// engineTarget resolves the host-reachable admin endpoint for an instance. It +// publishes the instance on its stable STANDARD 127.0.0.1 port through the single +// unified expose overlay (ensureExposed — idempotent, no recreate) and returns the +// Target with the instance's admin creds. There is exactly one host port per +// engine (the well-known one), shared with `expose`, so this can never fight the +// expose overlay over the container's `ports:`. func engineTarget(ctx context.Context, d UpDeps, engine, instance string) (resource.Target, error) { - ov, ok := engineOverlays[engine] - if !ok { - return resource.Target{}, fmt.Errorf("engine %q has no host-reachability overlay (postgres/minio in this milestone)", engine) - } - port, err := d.Manager.FreeHostPort(ctx, generate.SharedAlias(instance), ov.purpose, ov.portBase) - if err != nil { - return resource.Target{}, fmt.Errorf("allocate host port for %s: %w", instance, err) + if _, ok := primaryExposePort(engine); !ok { + return resource.Target{}, fmt.Errorf("engine %q has no host-reachability port defined", engine) } - overlay, err := writeProvisionOverlay(d.Model.Root, map[string]int{instance: port}, ov.containerPort) + ports, err := ensureExposed(ctx, d, []string{instance}) if err != nil { return resource.Target{}, err } - outDir := filepath.Join(d.Model.Root, generate.GenDir, "shared") - runner := d.Runner - if runner == nil { - runner = docker.ExecRunner{} - } - cp := docker.Compose{ - Project: generate.SharedStackName, - File: filepath.Join(outDir, generate.ComposeFile), - Dir: outDir, Runner: runner, Overrides: []string{overlay}, - } - if err := cp.Up(ctx, instance); err != nil { - return resource.Target{}, fmt.Errorf("apply host overlay for %s: %w", instance, err) + port, ok := ports[instance] + if !ok { + return resource.Target{}, fmt.Errorf("no host port resolved for %s (engine %q)", instance, engine) } params := d.Model.Workspace.Shared[instance].Params def := engineDefaultAdmin(engine) diff --git a/internal/orchestrate/resource_ops_test.go b/internal/orchestrate/resource_ops_test.go index 51af7c6..1a344c3 100644 --- a/internal/orchestrate/resource_ops_test.go +++ b/internal/orchestrate/resource_ops_test.go @@ -95,7 +95,7 @@ func TestCreateResourceImperative(t *testing.T) { t.Errorf("provisioned rows = %v, want database:reports + role:reports", kinds) } // The shared instance's loopback overlay was applied (compose up on shared stack). - if !fr.saw("-p "+"devstack-shared", "compose.provision.yaml") { + if !fr.saw("-p "+"devstack-shared", "compose.expose.yaml") { t.Errorf("overlay not applied via compose up: %v", fr.cmds) } // The create ran the guarded DDL on loopback. diff --git a/internal/orchestrate/resources.go b/internal/orchestrate/resources.go index 0ab71d8..0530f2c 100644 --- a/internal/orchestrate/resources.go +++ b/internal/orchestrate/resources.go @@ -27,32 +27,6 @@ import ( // ledger INSERT OR IGNORE). Non-postgres engines without a live provisioner in // this milestone are skipped with a note (their provisioners land in Full scope). -// perEngineOverlay is the per-engine host-reachability registry (spec 27 -// §"host-port overlay"): each engine publishes 127.0.0.1:: -// under its own (purpose, portBase). Postgres reuses the provision phase's values -// so one overlay/port covers both phases. -type perEngineOverlay struct { - purpose string - portBase int - containerPort int -} - -var engineOverlays = map[string]perEngineOverlay{ - "postgres": {provisionPurpose, provisionPortBase, 5432}, - "redis": {"redis-provision", 46379, 6379}, - "minio": {"minio-provision", 49000, 9000}, - "nats": {"nats-provision", 44222, 4222}, - // Kafka (Redpanda) advertises its EXTERNAL listener at a fixed 127.0.0.1:49092 - // (template), so host clients must reach the broker on exactly that port — the - // overlay publishes the in-container external listener (19092) there. The port - // base is 49092 to match the advertised address (a mismatch breaks the Kafka - // bootstrap→redirect handshake, the #1 local-Kafka footgun). - "kafka": {"kafka-provision", 49092, 19092}, - // LocalStack's edge port (4566) serves every AWS service (SQS/SNS/S3/…); keyed by - // the template name "localstack" (its `provides: aws` is reached on this port). - "localstack": {"localstack-provision", 44566, 4566}, -} - // declaredKind reports whether a ledger kind is one the declarative resources // phase manages (so drift detection ignores the implicitly-provisioned // role/database/redis_index kinds and never false-flags them). @@ -161,27 +135,22 @@ func resourcesPhase(d UpDeps, decls []resDecl) Phase { Run: func(ctx context.Context) (any, error) { reg := buildRegistry(d) - // Resolve each instance's published host port (idempotent — returns the - // port the shared/provision phase already allocated). - ports := map[string]int{} + // Resolve each instance's published host port through the unified expose + // overlay (idempotent — returns the standard port the shared phase already + // published, no container recreate). Non-exposable engines simply don't + // appear in the map and are reported as skipped below. + want := make([]string, 0, len(decls)) for _, r := range decls { - if _, done := ports[r.instance]; done { - continue - } - ov, ok := engineOverlays[r.engine] - if !ok { - continue // no host-reachability overlay for this engine yet - } - p, err := d.Manager.FreeHostPort(ctx, generate.SharedAlias(r.instance), ov.purpose, ov.portBase) - if err != nil { - return nil, fmt.Errorf("resolve resource port for %s: %w", r.instance, err) - } - ports[r.instance] = p + want = append(want, r.instance) + } + ports, err := ensureExposed(ctx, d, want) + if err != nil { + return nil, fmt.Errorf("publish host ports for resources: %w", err) } provisioned := []map[string]any{} skipped := []map[string]any{} - err := lock.WithLock(ctx, d.LockPath, func() error { + err = lock.WithLock(ctx, d.LockPath, func() error { for _, r := range decls { prov, ok := reg.For(r.engine) if !ok { diff --git a/internal/orchestrate/snapshot.go b/internal/orchestrate/snapshot.go index 8c4b1f9..0d5026f 100644 --- a/internal/orchestrate/snapshot.go +++ b/internal/orchestrate/snapshot.go @@ -22,10 +22,10 @@ import ( // This file is the imperative side of spec 15 (thin v2 scope): Postgres-only // `db snapshot` / `db restore` / `db snapshot ls` against a project's per-project // tenant database on the SHARED Postgres. It reuses the provision phase's exact -// host-reachability pattern (engineTarget → FreeHostPort + writeProvisionOverlay -// + `compose up -d ` on the shared stack, DECISIONS D8) so the dump/restore -// client tooling reaches the warm server over a ledger-allocated 127.0.0.1 host -// port WITHOUT publishing a permanent one. +// host-reachability pattern (engineTarget → ensureExposed: the single unified +// standard-port overlay + `compose up -d ` on the shared stack, DECISIONS +// D8) so the dump/restore client tooling reaches the warm server over the same +// stable 127.0.0.1 host port `expose`/auto-expose publishes. // // Lock discipline (spec 15): the streaming dump/restore PROCESS runs OUTSIDE the // flock (it is long — holding the lock for a multi-GB pg_restore would serialize diff --git a/internal/orchestrate/snapshot_test.go b/internal/orchestrate/snapshot_test.go index f3f98f2..03a5a2c 100644 --- a/internal/orchestrate/snapshot_test.go +++ b/internal/orchestrate/snapshot_test.go @@ -100,11 +100,11 @@ func TestSnapshotRestoreRoundTrip(t *testing.T) { } // The host-port overlay was allocated in the ledger and applied via compose up. - port, ok, _ := ledger.PortFor("shared-postgres", "pg-provision") + port, ok, _ := ledger.PortFor("shared-postgres", "pg-expose") if !ok || port == 0 { t.Errorf("host port not allocated for the snapshot overlay: port=%d ok=%v", port, ok) } - if !fr.saw("-p devstack-shared", "compose.provision.yaml") { + if !fr.saw("-p devstack-shared", "compose.expose.yaml") { t.Errorf("loopback overlay not applied via compose up: %v", fr.cmds) } diff --git a/internal/orchestrate/up.go b/internal/orchestrate/up.go index 3c4350c..619cc90 100644 --- a/internal/orchestrate/up.go +++ b/internal/orchestrate/up.go @@ -92,6 +92,7 @@ type UpDeps struct { NoHooks bool // skip the hooks phase NoPreflight bool // skip the preflight phase (fast inner loops) NoProvision bool // skip the per-project Postgres provision phase + NoExpose bool // skip auto-publishing shared engines on their standard host ports HealthTimeout time.Duration // per-shared-service gate cap (0 → health.Compile default) } @@ -424,31 +425,20 @@ func sharedPhase(d UpDeps, projects, names, provInstances []string) Phase { "(spec 21 follow-up). Re-run with --no-provision, or provision from the remote host", d.Backend.String()) } - // Publish each provisioned Postgres on 127.0.0.1: via an - // up-time overlay so host-side pgx (the provision phase) can reach it, - // without touching the deterministic generated compose. + // Expose-by-default: publish every exposable shared engine on its stable + // STANDARD 127.0.0.1 port through the single unified overlay, so a GUI + // client's defaults just work AND host-side pgx (the provision phase) has + // one port to dial — there is no separate provisioning band. Opt out with + // --no-expose; skipped on a remote backend (bridge is not host-routable, so + // nothing host-published would be reachable anyway). var overrides []string - if len(prov) > 0 { - ports := map[string]int{} - for _, inst := range prov { - port, err := d.Manager.FreeHostPort(ctx, generate.SharedAlias(inst), provisionPurpose, provisionPortBase) - if err != nil { - return nil, fmt.Errorf("allocate provision port for %s: %w", inst, err) - } - ports[inst] = port - } - overlay, err := writeProvisionOverlay(d.Model.Root, ports, 5432) + if !d.NoExpose && d.Backend.Reachability() != docker.ViaProxy { + pub, err := exposeOverlayFor(ctx, d, names) if err != nil { return nil, err } - overrides = append(overrides, overlay) - } - // Re-apply a prior `shared expose` so GUI-client host ports persist - // across up/down (its 5xxxx range never collides with provisioning's - // 4xxxx). Skipped on a remote backend (bridge is not host-routable). - if d.Backend.Reachability() != docker.ViaProxy { - if p := exposeOverlayPath(d.Model.Root); fileExists(p) { - overrides = append(overrides, p) + if pub != "" { + overrides = append(overrides, pub) } } cp.Overrides = overrides diff --git a/internal/orchestrate/up_test.go b/internal/orchestrate/up_test.go index afafdd8..510af56 100644 --- a/internal/orchestrate/up_test.go +++ b/internal/orchestrate/up_test.go @@ -756,11 +756,11 @@ func TestBuildUpProvisionsPerProjectDB(t *testing.T) { t.Errorf("provisioned rows = %v, want role:app + database:app", kinds) } // The shared stack was brought up WITH the loopback port overlay. - if !fr.saw("-p "+generate.SharedStackName, "-f", "compose.provision.yaml") { + if !fr.saw("-p "+generate.SharedStackName, "-f", "compose.expose.yaml") { t.Errorf("shared up did not include the provision overlay: %v", fr.cmds) } // The overlay file was written, loopback-bound. - overlay := filepath.Join(d.Model.Root, generate.GenDir, "shared", "compose.provision.yaml") + overlay := filepath.Join(d.Model.Root, generate.GenDir, "shared", "compose.expose.yaml") body, err := os.ReadFile(overlay) if err != nil { t.Fatalf("overlay not written: %v", err)