Skip to content

feat: standard DB host ports, expose-by-default, unified provisioning + up/down fixes #123

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
gustavobertoi merged 2 commits into from
Jul 10, 2026
Merged

feat: standard DB host ports, expose-by-default, unified provisioning + up/down fixes #123

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
88a0db9
feat(expose): standard host ports + MySQL/MariaDB/MongoDB/Cassandra/A…
gustavobertoi Jul 10, 2026
b2d2870
feat(up): unify host ports to the standard port, expose by default, f…
gustavobertoi Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions internal/cli/template_new_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,15 +123,15 @@ func TestTemplateNewEngineHasNoBuildTree(t *testing.T) {
t.Setenv("DEVSTACK_HOME", t.TempDir())
dir := t.TempDir()
if out, err := runCmd(t, "template", "new", "--no-input", "--dir", dir,
"--kind", "engine", "--name", "mariadb", "--base-image", "mariadb:11",
"--provides", "mariadb", "--exports", "host,port,user", "--port", "3306"); err != nil {
"--kind", "engine", "--name", "couchdb", "--base-image", "couchdb:3",
"--provides", "couchdb", "--exports", "host,port,user", "--port", "5984"); err != nil {
t.Fatalf("author engine: %v\n%s", err, out)
}
if _, err := os.Stat(filepath.Join(dir, "mariadb", "build")); !os.IsNotExist(err) {
if _, err := os.Stat(filepath.Join(dir, "couchdb", "build")); !os.IsNotExist(err) {
t.Errorf("engine template must have no build/ tree, stat err = %v", err)
}
manifest, _ := os.ReadFile(filepath.Join(dir, "mariadb", "template.yaml"))
for _, want := range []string{"image: mariadb:11", "provides: mariadb"} {
manifest, _ := os.ReadFile(filepath.Join(dir, "couchdb", "template.yaml"))
for _, want := range []string{"image: couchdb:3", "provides: couchdb"} {
if !strings.Contains(string(manifest), want) {
t.Errorf("engine template.yaml missing %q:\n%s", want, manifest)
}
Expand Down
44 changes: 42 additions & 2 deletions internal/cli/up.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command {
noHooks bool
noPreflight bool
noProvision bool
noExpose bool
profiles []string
healthTimeout time.Duration
)
Expand All @@ -56,6 +57,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command {
d.NoHooks = noHooks
d.NoPreflight = noPreflight
d.NoProvision = noProvision
d.NoExpose = noExpose
d.Profiles = profiles
d.HealthTimeout = healthTimeout

Expand Down Expand Up @@ -112,6 +114,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command {
cmd.Flags().BoolVar(&noHooks, "no-hooks", false, "skip lifecycle hooks")
cmd.Flags().BoolVar(&noPreflight, "no-preflight", false, "skip the preflight checks")
cmd.Flags().BoolVar(&noProvision, "no-provision", false, "skip per-project Postgres role/db provisioning")
cmd.Flags().BoolVar(&noExpose, "no-expose", false, "do not auto-publish shared engines on their standard 127.0.0.1 ports")
cmd.Flags().StringArrayVarP(&profiles, "profile", "p", nil,
"service slice(s) to start — repeatable & comma-separated (spec 12); empty → defaultProfile or all")
return cmd
Expand Down Expand Up @@ -174,8 +177,39 @@ func newDownCmd(g *GlobalOpts) *cobra.Command {
fmt.Fprintf(w, "[ok] down %s\n", p)
}
}
// Tear down anything now left running: after the project refs are dropped,
// stop every shared service that fell to zero refs (`shared gc --stop`).
// This is what makes `down` actually bring the workspace DOWN instead of
// leaving warm engines behind, while still respecting cross-workspace
// sharing — an engine another workspace still references keeps running.
var stopped []string
if gc, err := d.Manager.GC(ctx, true); err != nil {
if firstErr == nil {
firstErr = err
}
} else {
stopped = gc.Stopped
// When shared services were actually stopped, their saga phases are no
// longer satisfied — clear them so the next `up` restarts the shared
// stack (and re-publishes its ports) instead of skipping.
if len(stopped) > 0 {
_ = lock.WithLock(ctx, d.LockPath, func() error {
for _, ph := range []string{"shared", "provision", "resources"} {
if e := d.DB.ClearPhase(d.Model.Workspace.Name, "", ph); e != nil {
return e
}
}
return nil
})
}
if !g.JSON && !g.Quiet {
for _, s := range stopped {
fmt.Fprintf(w, "[ok] stopped shared %s (0 refs)\n", s)
}
}
}
if g.JSON {
if err := writeJSON(cmd, map[string]any{"down": results}); err != nil {
if err := writeJSON(cmd, map[string]any{"down": results, "sharedStopped": stopped}); err != nil {
return err
}
}
Expand Down Expand Up @@ -216,7 +250,13 @@ func downProject(ctx context.Context, d orchestrate.UpDeps, project string) erro
if _, err := d.Manager.RegisterDown(ctx, project); err != nil {
return err
}
return nil
// The compose-up phase is no longer satisfied — its containers were just
// removed. Clear the saga record so the NEXT `up` re-runs it instead of
// skipping on a stale fingerprint (the "re-up after down is a no-op" bug).
// firstRun/hooks are intentionally NOT cleared (they keep run-once semantics).
return lock.WithLock(ctx, d.LockPath, func() error {
return d.DB.ClearPhase(d.Model.Workspace.Name, project, "compose-up")
})
}

// buildUpDeps assembles the up/down dependencies from the current directory. It
Expand Down
5 changes: 5 additions & 0 deletions internal/migrate/migrate.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ const APIVersion = "devstack/v1"
// knownEngines maps a devdock template/image keyword to a devstack shared engine.
var knownEngines = map[string]string{
"postgres": "postgres", "postgresql": "postgres", "postgis": "postgres",
"mysql": "mysql", "percona": "mysql",
"mariadb": "mariadb",
"mongo": "mongodb", "mongodb": "mongodb",
"cassandra": "cassandra",
"arango": "arangodb", "arangodb": "arangodb",
"redis": "redis", "valkey": "redis",
"minio": "minio",
}
Expand Down
145 changes: 129 additions & 16 deletions internal/orchestrate/expose.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,16 @@ import (
// is an explicit opt-in that, like provisioning, is an UP-TIME compose overlay —
// it never touches the deterministic, golden-asserted generated compose.
//
// Exposure uses its OWN host-port range (55xxx/58xxx…), distinct from the
// provisioning range (45xxx), so the expose overlay and the provision overlay
// never publish the same host port and can both be applied without a duplicate
// binding. Ports are ledger-allocated (FreeHostPort), so the same engine keeps
// the same host port across runs and two terminals never collide.
// Exposure publishes each engine on its OWN WELL-KNOWN host port — the same port
// the template advertises in-network (postgres→5432, mysql→3306, redis→6379, …) —
// so a GUI client's default connection settings just work and there is no gap
// between what the template's `defaultPort` says and what the host sees. That
// deliberately differs from the provisioning range (45xxx): the two overlays map
// different host ports onto the same container port, so both can be applied
// without a duplicate binding. Ports remain ledger-allocated (FreeHostPort) with
// the standard port as the search base, so the same engine keeps the same host
// port across runs, and if a host-native server already holds the standard port
// the allocator transparently falls back to the next free one in the band.

const exposeFile = "compose.expose.yaml"

Expand All @@ -37,19 +42,26 @@ type exposePort struct {
}

// exposeEngines maps a shared engine (template name) to the ports `shared expose`
// publishes on 127.0.0.1. Bases sit in the 5xxxx range so they never collide with
// the 4xxxx provisioning overlay. Kafka is the exception: host clients MUST reach
// the broker on 127.0.0.1:49092 (the fixed advertised external listener from the
// template), so it reuses the kafka provision port rather than a 5xxxx one.
// publishes on 127.0.0.1. The search base is the engine's WELL-KNOWN port (equal
// to the in-container port), so clients connect on the port they already expect
// and the allocator only drifts off it when a host-native server already holds it.
// Kafka is the exception: host clients MUST reach the broker on 127.0.0.1:49092
// (the fixed advertised external listener from the template), so it keeps that
// base and reuses the kafka provision port rather than the broker's 19092.
var exposeEngines = map[string][]exposePort{
"postgres": {{5432, "postgres", "pg-expose", 55432, true}},
"redis": {{6379, "redis", "redis-expose", 56379, true}},
"minio": {{9000, "s3", "minio-expose", 59000, true}, {9001, "console", "minio-console-expose", 59001, false}},
"localstack": {{4566, "aws", "localstack-expose", 54566, true}},
"ministack": {{4566, "aws", "ministack-expose", 54567, true}},
"nats": {{4222, "nats", "nats-expose", 54222, true}, {8222, "monitor", "nats-monitor-expose", 58222, false}},
"postgres": {{5432, "postgres", "pg-expose", 5432, true}},
"mysql": {{3306, "mysql", "mysql-expose", 3306, true}},
"mariadb": {{3306, "mariadb", "mariadb-expose", 3306, true}},
"mongodb": {{27017, "mongodb", "mongodb-expose", 27017, true}},
"cassandra": {{9042, "cassandra", "cassandra-expose", 9042, true}},
"arangodb": {{8529, "arangodb", "arangodb-expose", 8529, true}},
"redis": {{6379, "redis", "redis-expose", 6379, true}},
"minio": {{9000, "s3", "minio-expose", 9000, true}, {9001, "console", "minio-console-expose", 9001, false}},
"localstack": {{4566, "aws", "localstack-expose", 4566, true}},
"ministack": {{4566, "aws", "ministack-expose", 4566, true}},
"nats": {{4222, "nats", "nats-expose", 4222, true}, {8222, "monitor", "nats-monitor-expose", 8222, false}},
"kafka": {{19092, "kafka", "kafka-provision", 49092, true}},
"rabbitmq": {{5672, "amqp", "rmq-expose", 55672, true}, {15672, "management", "rmq-mgmt-expose", 55673, false}},
"rabbitmq": {{5672, "amqp", "rmq-expose", 5672, true}, {15672, "management", "rmq-mgmt-expose", 15672, false}},
}

// ExposableEngine reports whether an engine has a defined host-expose port set.
Expand All @@ -58,6 +70,95 @@ func ExposableEngine(engine string) bool {
return ok
}

// primaryExposePort returns an engine's PRIMARY host-published port — the one a
// client (and devstack's own host-side provisioning) connects the engine's main
// protocol on. This is the single source of truth for "the host port of engine
// X": provisioning, reset, snapshot and resource ops all resolve their admin
// endpoint from it, so there is exactly ONE host port per engine (the standard
// one), never a separate provisioning band.
func primaryExposePort(engine string) (exposePort, bool) {
for _, ep := range exposeEngines[engine] {
if ep.primary {
return ep, true
}
}
return exposePort{}, false
}

// exposableUnion returns the shared instances to publish: the requested set
// unioned with any already-exposed instance (so writing the overlay never drops
// another instance's ports), filtered to engines that support exposure. Sorted
// for a byte-stable overlay.
func exposableUnion(d UpDeps, want []string) []string {
set := map[string]bool{}
for _, i := range want {
set[i] = true
}
for _, i := range exposedInstances(d.Model.Root) {
set[i] = true
}
var insts []string
for i := range set {
if s, ok := d.Model.Workspace.Shared[i]; ok && ExposableEngine(s.Template) {
insts = append(insts, i)
}
}
sort.Strings(insts)
return insts
}

// exposeOverlayFor allocates the standard host ports for the exposable instances
// among want (unioned with the currently-exposed set) and WRITES the single
// expose overlay, returning its path ("" when there is nothing to expose). It does
// NOT run compose — the caller (the shared phase) folds the returned path into its
// own `compose up` so ports are published as the services come up.
func exposeOverlayFor(ctx context.Context, d UpDeps, want []string) (string, error) {
insts := exposableUnion(d, want)
if len(insts) == 0 {
return "", nil
}
_, pub, err := allocateExposePorts(ctx, d, insts)
if err != nil {
return "", err
}
return writeExposeOverlay(d.Model.Root, pub)
}

// ensureExposed is the unified host-reachability primitive for callers that need
// the ports published NOW (provisioning, reset, snapshot, resource ops): it writes
// the single expose overlay for the exposable instances among want (unioned with
// the already-exposed set, so it never drops another instance's ports) and applies
// it via `compose up`. It is idempotent — the ledger returns the same standard
// ports and the overlay bytes are unchanged, so compose does not recreate the
// container on repeat calls. Returns instance→primary host port. Because both
// auto-expose and every host-side admin op go through this one overlay, they can
// never fight over a container's `ports:`.
func ensureExposed(ctx context.Context, d UpDeps, want []string) (map[string]int, error) {
insts := exposableUnion(d, want)
if len(insts) == 0 {
return map[string]int{}, nil
}
out, pub, err := allocateExposePorts(ctx, d, insts)
if err != nil {
return nil, err
}
overlay, err := writeExposeOverlay(d.Model.Root, pub)
if err != nil {
return nil, err
}
outDir := filepath.Join(d.Model.Root, generate.GenDir, "shared")
if err := composeUpShared(ctx, d, outDir, []string{overlay}, insts); err != nil {
return nil, fmt.Errorf("apply host-port overlay: %w", err)
}
ports := map[string]int{}
for _, ep := range out {
if ep.Primary {
ports[ep.Instance] = ep.Port
}
}
return ports, nil
}

// ExposedPort is one host-published shared-service port with a client-ready
// connection hint (the `--json` schema + the plain-table source).
type ExposedPort struct {
Expand Down Expand Up @@ -284,6 +385,18 @@ func connectionURL(engine string, ep exposePort, params map[string]any, port int
user := paramString(params, "rootUser", "devstack")
pass := paramString(params, "rootPassword", "devstack")
return fmt.Sprintf("postgres://%s:%s@%s/postgres?sslmode=disable", user, pass, host)
case "mysql", "mariadb":
user := paramString(params, "rootUser", "devstack")
pass := paramString(params, "rootPassword", "devstack")
return fmt.Sprintf("mysql://%s:%s@%s/%s", user, pass, host, user)
case "mongodb":
user := paramString(params, "rootUser", "devstack")
pass := paramString(params, "rootPassword", "devstack")
return fmt.Sprintf("mongodb://%s:%s@%s/?authSource=admin", user, pass, host)
case "cassandra":
return host // contact point host:9042 (CQL native transport)
case "arangodb":
return "http://" + host // HTTP API + web UI (root / rootPassword)
case "redis":
return "redis://" + host
case "minio":
Expand Down
73 changes: 46 additions & 27 deletions internal/orchestrate/expose_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,16 +93,21 @@ func TestConnectionURL(t *testing.T) {
port int
want string
}{
{"postgres", exposePort{5432, "postgres", "", 0, true}, pgParams, 55432, "postgres://admin:s3cret@127.0.0.1:55432/postgres?sslmode=disable"},
{"postgres", exposePort{5432, "postgres", "", 0, true}, nil, 55432, "postgres://devstack:devstack@127.0.0.1:55432/postgres?sslmode=disable"},
{"redis", exposePort{6379, "redis", "", 0, true}, nil, 56379, "redis://127.0.0.1:56379"},
{"minio", exposePort{9000, "s3", "", 0, true}, nil, 59000, "http://127.0.0.1:59000"},
{"localstack", exposePort{4566, "aws", "", 0, true}, nil, 54566, "http://127.0.0.1:54566"},
{"nats", exposePort{8222, "monitor", "", 0, false}, nil, 58222, "http://127.0.0.1:58222"},
{"nats", exposePort{4222, "nats", "", 0, true}, nil, 54222, "nats://127.0.0.1:54222"},
{"postgres", exposePort{5432, "postgres", "", 0, true}, pgParams, 5432, "postgres://admin:s3cret@127.0.0.1:5432/postgres?sslmode=disable"},
{"postgres", exposePort{5432, "postgres", "", 0, true}, nil, 5432, "postgres://devstack:devstack@127.0.0.1:5432/postgres?sslmode=disable"},
{"mysql", exposePort{3306, "mysql", "", 0, true}, nil, 3306, "mysql://devstack:devstack@127.0.0.1:3306/devstack"},
{"mariadb", exposePort{3306, "mariadb", "", 0, true}, nil, 3306, "mysql://devstack:devstack@127.0.0.1:3306/devstack"},
{"mongodb", exposePort{27017, "mongodb", "", 0, true}, nil, 27017, "mongodb://devstack:devstack@127.0.0.1:27017/?authSource=admin"},
{"cassandra", exposePort{9042, "cassandra", "", 0, true}, nil, 9042, "127.0.0.1:9042"},
{"arangodb", exposePort{8529, "arangodb", "", 0, true}, nil, 8529, "http://127.0.0.1:8529"},
{"redis", exposePort{6379, "redis", "", 0, true}, nil, 6379, "redis://127.0.0.1:6379"},
{"minio", exposePort{9000, "s3", "", 0, true}, nil, 9000, "http://127.0.0.1:9000"},
{"localstack", exposePort{4566, "aws", "", 0, true}, nil, 4566, "http://127.0.0.1:4566"},
{"nats", exposePort{8222, "monitor", "", 0, false}, nil, 8222, "http://127.0.0.1:8222"},
{"nats", exposePort{4222, "nats", "", 0, true}, nil, 4222, "nats://127.0.0.1:4222"},
{"kafka", exposePort{19092, "kafka", "", 0, true}, nil, 49092, "127.0.0.1:49092"},
{"rabbitmq", exposePort{15672, "management", "", 0, false}, nil, 55673, "http://127.0.0.1:55673"},
{"rabbitmq", exposePort{5672, "amqp", "", 0, true}, nil, 55672, "amqp://devstack@127.0.0.1:55672"},
{"rabbitmq", exposePort{15672, "management", "", 0, false}, nil, 15672, "http://127.0.0.1:15672"},
{"rabbitmq", exposePort{5672, "amqp", "", 0, true}, nil, 5672, "amqp://devstack@127.0.0.1:5672"},
}
for _, tc := range cases {
if got := connectionURL(tc.engine, tc.ep, tc.params, tc.port); got != tc.want {
Expand All @@ -111,34 +116,48 @@ func TestConnectionURL(t *testing.T) {
}
}

// TestExposePortsNeverCollideWithProvision is the load-bearing invariant: the
// expose overlay and the provision overlay must never publish the SAME host port
// (base) for the SAME engine, or applying both recreates the container with a
// duplicate binding. Kafka is the deliberate exception — its host clients MUST
// use the fixed advertised 49092, so it reuses the provision port.
func TestExposePortsNeverCollideWithProvision(t *testing.T) {
provBase := map[string]int{}
for engine, ov := range engineOverlays {
provBase[engine] = ov.portBase
}
// TestExposeUsesStandardPorts locks in the unification: there is exactly ONE host
// port per engine — the well-known one — and it equals the in-container port. That
// is what lets provisioning/reset/snapshot and `expose` share a single overlay
// instead of two fighting bands. Kafka is the deliberate exception: its broker
// advertises a fixed 127.0.0.1:49092 external listener, so its host base is 49092
// while the container port is 19092.
func TestExposeUsesStandardPorts(t *testing.T) {
for engine, ports := range exposeEngines {
for _, ep := range ports {
if pb, ok := provBase[engine]; ok && ep.base == pb && engine != "kafka" {
t.Errorf("engine %q expose base %d collides with provision base %d", engine, ep.base, pb)
if engine == "kafka" {
continue
}
if ep.base != ep.container {
t.Errorf("engine %q port %q: host base %d must equal container port %d (standard-port unification)",
engine, ep.label, ep.base, ep.container)
}
}
}
// Every expose base must be unique across all engines/ports (no two services
// fight for the same host port at allocation time either).
seen := map[int]string{}
for _, ports := range exposeEngines {
// Every provisionable engine must have a PRIMARY expose port, since provisioning
// now resolves its host-reachable admin endpoint from that single overlay.
for _, engine := range []string{"postgres", "redis", "minio", "nats", "kafka", "localstack"} {
if _, ok := primaryExposePort(engine); !ok {
t.Errorf("engine %q has no primary expose port — provisioning cannot reach it", engine)
}
}
// Within a SINGLE engine, its ports must not share a base (else a two-port
// engine like minio/nats/rabbitmq would self-collide on the same host port).
for engine, ports := range exposeEngines {
seen := map[int]string{}
for _, ep := range ports {
if prev, ok := seen[ep.base]; ok && prev != ep.purpose {
t.Errorf("expose base %d reused across purposes %q and %q", ep.base, prev, ep.purpose)
if prev, ok := seen[ep.base]; ok {
t.Errorf("engine %q reuses expose base %d across purposes %q and %q", engine, ep.base, prev, ep.purpose)
}
seen[ep.base] = ep.purpose
}
}
// Across DIFFERENT engines the base MAY repeat on purpose: two engines that
// speak the same wire protocol want the same well-known port (mysql/mariadb on
// 3306, localstack/ministack on 4566). That is safe because the ledger's
// AllocatePort skips every already-allocated port (AllocatedPorts spans all
// owners), so a lone engine lands on the standard port and, when both are
// exposed, the second transparently deconflicts to base+1.
}

func TestFileExists(t *testing.T) {
Expand Down
Loading
Loading