From 81208ae9f447959350ed5361319f1ea9233903bf Mon Sep 17 00:00:00 2001 From: Gustavo Bertoi Date: Mon, 29 Jun 2026 17:04:21 -0300 Subject: [PATCH] =?UTF-8?q?feat(orchestrate):=20provision=20saga=20phase?= =?UTF-8?q?=20=E2=80=94=20per-project=20Postgres=20role+db=20(D8)=20?= =?UTF-8?q?=E2=80=94=20M2=20capstone?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the long-deferred provision phase into the up saga. Per-project data isolation on the shared Postgres (spec 03, DECISIONS D8): each active project that `uses: workspace.shared.postgres` gets its own login role + owned database, created idempotently via the existing `internal/provision` (existence-guarded pgx SQL). Determinism-safe host port: provisioning runs pgx FROM THE HOST, so the shared Postgres needs a reachable port. Rather than publish it in the deterministic, golden-asserted generated compose (which would also flip the "no host ports by default" posture for everyone), the shared phase writes an UP-TIME overlay (`.devstack/shared/compose.provision.yaml`) mapping `127.0.0.1::5432` and brings the shared stack up with `-f compose.yaml -f compose.provision.yaml`. `generate` output is untouched (CI determinism still byte-identical) and the port is loopback-only. `docker.Compose` gained `Overrides` for the extra `-f`. Flow: shared (postgres healthy, port published) → provision (pgx-connect to 127.0.0.1: as the template admin, EnsureProject per project, record role+db ownership in the ledger under the flock) → per-project compose-up. The host port is ledger-allocated (`Manager.FreeHostPort`, purpose `pg-provision`) and re-derived idempotently by the provision phase, so it survives re-runs/crashes; the shared fingerprint folds in the provisioned set so adding a consumer re-publishes. Dev-credential default: the per-project password is the project name — a predictable credential for a loopback-only, network-isolated dev DB (THREAT-MODEL: container isolation is a non-goal), so nothing secret is generated/stored and an app opts in via `postgres://:@shared-postgres:5432/` (never auto-overriding the app's own DB config). `--no-provision` opts out. Tested daemon-free via an injectable `PgConnect` seam: the full saga provisions app's role+db (CREATE ROLE/DATABASE on the create path), connects on loopback, records ownership, brings shared up WITH the overlay (loopback-bound to 5432), and `--no-provision` skips the phase entirely. `make ci` + `make determinism` green. Unblocks X3 firstRun (provision scope_key now exists). Co-Authored-By: Claude Opus 4.8 (1M context) --- internal/cli/up.go | 3 + internal/docker/compose.go | 19 ++- internal/orchestrate/provision.go | 230 ++++++++++++++++++++++++++++++ internal/orchestrate/up.go | 60 +++++++- internal/orchestrate/up_test.go | 129 ++++++++++++++++- 5 files changed, 429 insertions(+), 12 deletions(-) create mode 100644 internal/orchestrate/provision.go diff --git a/internal/cli/up.go b/internal/cli/up.go index 5cf8cae..1785998 100644 --- a/internal/cli/up.go +++ b/internal/cli/up.go @@ -29,6 +29,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command { build bool noHooks bool noPreflight bool + noProvision bool profiles []string ) cmd := &cobra.Command{ @@ -48,6 +49,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command { d.Build = build d.NoHooks = noHooks d.NoPreflight = noPreflight + d.NoProvision = noProvision d.Profiles = profiles // Memory-budget warning (spec 12 §budget): opt-in — only when the @@ -90,6 +92,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command { cmd.Flags().BoolVar(&build, "build", false, "build images before starting (compose build)") cmd.Flags().BoolVar(&noHooks, "no-hooks", false, "skip lifecycle hooks") cmd.Flags().BoolVar(&noPreflight, "no-preflight", false, "skip the preflight checks") + cmd.Flags().BoolVar(&noProvision, "no-provision", false, "skip per-project Postgres role/db provisioning") cmd.Flags().StringArrayVarP(&profiles, "profile", "p", nil, "service slice(s) to start — repeatable & comma-separated (spec 12); empty → defaultProfile or all") return cmd diff --git a/internal/docker/compose.go b/internal/docker/compose.go index 846653b..572fd16 100644 --- a/internal/docker/compose.go +++ b/internal/docker/compose.go @@ -71,11 +71,12 @@ func (e *CmdError) Unwrap() error { return e.Err } // project name and compose file (DECISIONS D5). Lifecycle verbs run here; // container enumeration stays on the read-only SDK Client. type Compose struct { - Project string // -p - File string // -f - Dir string // working dir (build contexts resolve relative to it) - Env []string // extra env (resolved secrets), appended to os.Environ - Runner Runner + Project string // -p + File string // -f + Overrides []string // additional -f overlays, applied in order after File (up-time only) + Dir string // working dir (build contexts resolve relative to it) + Env []string // extra env (resolved secrets), appended to os.Environ + Runner Runner } // NewCompose builds a Compose driver using the real exec runner. @@ -89,7 +90,13 @@ func (c *Compose) base() []string { if c.File == "" { return []string{"compose", "-p", c.Project} } - return []string{"compose", "-p", c.Project, "-f", c.File} + args := []string{"compose", "-p", c.Project, "-f", c.File} + // Overlays (e.g. the up-time provision port mapping) are applied after the base + // file so their values win; later files override earlier ones (compose merge). + for _, ov := range c.Overrides { + args = append(args, "-f", ov) + } + return args } // Up brings the stack (or the named subset of services) up detached. With no diff --git a/internal/orchestrate/provision.go b/internal/orchestrate/provision.go new file mode 100644 index 0000000..e954bca --- /dev/null +++ b/internal/orchestrate/provision.go @@ -0,0 +1,230 @@ +package orchestrate + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/open-source-cloud/devstack/internal/config" + "github.com/open-source-cloud/devstack/internal/generate" + "github.com/open-source-cloud/devstack/internal/lock" + "github.com/open-source-cloud/devstack/internal/provision" +) + +// This file is the provision saga phase (M2 capstone, DECISIONS D8): per-project +// Postgres role+database isolation on the shared engine. Because provisioning runs +// pgx FROM THE HOST, the shared Postgres needs a reachable port — published as a +// 127.0.0.1-only mapping via an UP-TIME compose overlay (so the deterministic, +// golden-asserted `generate` output is untouched and the "no host ports by +// default" posture holds for every other service). The per-project password is the +// project name: a predictable dev credential for a loopback-only, network-isolated +// dev database (THREAT-MODEL: container isolation is a non-goal), so nothing secret +// is generated or stored, and an app opts in via the documented DSN +// `postgres://:@shared-postgres:5432/`. + +const ( + provisionPurpose = "pg-provision" // ledger port_alloc purpose + provisionPortBase = 45432 // host port search base for shared Postgres + provisionFile = "compose.provision.yaml" + pgTemplate = "postgres" // shared engine template that this phase provisions +) + +// PgConnector opens an admin connection to a Postgres DSN. Injectable so the +// provision phase is unit-testable without a live server (the default wraps +// provision.Connect / pgx). +type PgConnector func(ctx context.Context, dsn string) (provision.Conn, func() error, error) + +func defaultPgConnect(ctx context.Context, dsn string) (provision.Conn, func() error, error) { + c, closeFn, err := provision.Connect(ctx, dsn) + if err != nil { + return nil, nil, err + } + return c, closeFn, nil +} + +// provTarget is one (project, shared-Postgres-instance) pair to provision. +type provTarget struct { + project string + instance string +} + +// pgInstances returns the set of shared services that are Postgres engines. +func pgInstances(m *config.Model) map[string]bool { + out := map[string]bool{} + for name, s := range m.Workspace.Shared { + if s.Template == pgTemplate { + out[name] = true + } + } + return out +} + +// provTargets returns the (project, instance) pairs to provision: every active +// project that `uses` a shared Postgres instance. Sorted and de-duplicated. +func provTargets(m *config.Model, activeServices map[string][]string, pg map[string]bool) []provTarget { + seen := map[string]bool{} + var out []provTarget + for _, project := range sortedStringSlice(keysOf(activeServices)) { + p, ok := m.Projects[project] + if !ok { + continue + } + for _, sname := range activeServices[project] { + for _, u := range p.Services[sname].Uses { + ref, ok := config.ParseRef(u) + if !ok || ref.Kind != config.RefShared || !pg[ref.Name] { + continue + } + key := project + "\x00" + ref.Name + if seen[key] { + continue + } + seen[key] = true + out = append(out, provTarget{project: project, instance: ref.Name}) + } + } + } + return out +} + +// provInstanceList returns the sorted distinct instances across targets. +func provInstanceList(targets []provTarget) []string { + set := map[string]bool{} + for _, t := range targets { + set[t.instance] = true + } + return sortedStringSlice(keysOf(set)) +} + +// writeProvisionOverlay writes the up-time compose overlay that publishes each +// provisioned Postgres instance on 127.0.0.1:. Returns the overlay path. +// Loopback-only so nothing is exposed beyond the host (spec 03 / no host ports). +func writeProvisionOverlay(root string, ports map[string]int) (string, error) { + var b strings.Builder + b.WriteString("services:\n") + insts := make([]string, 0, len(ports)) + for inst := range ports { + insts = append(insts, inst) + } + sort.Strings(insts) + for _, inst := range insts { + fmt.Fprintf(&b, " %s:\n ports:\n - \"127.0.0.1:%d:5432\"\n", inst, ports[inst]) + } + dir := filepath.Join(root, generate.GenDir, "shared") + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", err + } + path := filepath.Join(dir, provisionFile) + if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil { + return "", err + } + return path, nil +} + +// provisionPhase creates each project's role+database on its shared Postgres, +// idempotently, holding the flock for the SQL mutations (DECISIONS D7/D8). It +// re-derives the host port from the ledger (the same one sharedPhase published), +// so it is safe across re-runs and crashes. Compensation is intentionally empty: +// provisioned roles/dbs are data and survive a failed `up`. +func provisionPhase(d UpDeps, targets []provTarget) Phase { + return Phase{ + Name: "provision", + Mutating: true, + Fingerprint: func(context.Context) (string, error) { + keys := make([]string, 0, len(targets)) + for _, t := range targets { + keys = append(keys, t.project+"@"+t.instance) + } + return Fingerprint(append([]string{"provision"}, keys...)...), nil + }, + Run: func(ctx context.Context) (any, error) { + connect := d.PgConnect + if connect == nil { + connect = defaultPgConnect + } + byInst := map[string][]string{} + for _, t := range targets { + byInst[t.instance] = append(byInst[t.instance], t.project) + } + + // Resolve each instance's published host port (FreeHostPort self-locks + // and is idempotent — returns the port sharedPhase already allocated). + ports := map[string]int{} + for _, inst := range sortedStringSlice(keysOf(byInst)) { + p, err := d.Manager.FreeHostPort(ctx, generate.SharedAlias(inst), provisionPurpose, provisionPortBase) + if err != nil { + return nil, fmt.Errorf("resolve provision port for %s: %w", inst, err) + } + ports[inst] = p + } + + provisioned := []map[string]any{} + // Hold the flock for the role/db mutations (provision pkg contract). + err := lock.WithLock(ctx, d.LockPath, func() error { + for _, inst := range sortedStringSlice(keysOf(byInst)) { + params := d.Model.Workspace.Shared[inst].Params + user := paramString(params, "rootUser", "devstack") + pass := paramString(params, "rootPassword", "devstack") + dsn := provision.DSN("127.0.0.1", ports[inst], user, pass, user) + conn, closeConn, err := connect(ctx, dsn) + if err != nil { + return fmt.Errorf("connect to shared %s on 127.0.0.1:%d: %w", inst, ports[inst], err) + } + for _, project := range byInst[inst] { + creds, err := provision.Postgres{}.EnsureProject(ctx, conn, project, project) + if err != nil { + _ = closeConn() + return fmt.Errorf("provision %s on %s: %w", project, inst, err) + } + if err := d.DB.RecordProvisioned(project, "role", creds.Role); err != nil { + _ = closeConn() + return err + } + if err := d.DB.RecordProvisioned(project, "database", creds.Database); err != nil { + _ = closeConn() + return err + } + d.DB.LogEvent("provision", project, "role+db on "+generate.SharedAlias(inst)) + provisioned = append(provisioned, map[string]any{ + "project": project, "instance": inst, "role": creds.Role, "database": creds.Database, + }) + } + if err := closeConn(); err != nil { + return err + } + } + return nil + }) + if err != nil { + return nil, err + } + return map[string]any{"provisioned": provisioned}, nil + }, + } +} + +// paramString reads a string param with a default. +func paramString(params map[string]any, key, def string) string { + if v, ok := params[key]; ok { + if s, ok := v.(string); ok && s != "" { + return s + } + } + return def +} + +func keysOf[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} + +func sortedStringSlice(s []string) []string { + sort.Strings(s) + return s +} diff --git a/internal/orchestrate/up.go b/internal/orchestrate/up.go index 5532e8e..766930b 100644 --- a/internal/orchestrate/up.go +++ b/internal/orchestrate/up.go @@ -56,13 +56,32 @@ type UpDeps struct { // Trust installs the local CA when network.proxy.httpsLocal; nil → trust.New(). // Injected for tests (the trust phase is fenced — failure never aborts up). Trust *trust.Trust + // PgConnect opens an admin Postgres connection for the provision phase; nil → + // the pgx-backed default. Injected for tests (so provisioning runs daemon-free). + PgConnect PgConnector Build bool // compose up --build NoHooks bool // skip the hooks phase NoPreflight bool // skip the preflight phase (fast inner loops) + NoProvision bool // skip the per-project Postgres provision phase HealthTimeout time.Duration // per-shared-service gate cap (0 → health.Compile default) } +// intersect returns the elements of a that are also in b, preserving a's order. +func intersect(a, b []string) []string { + set := make(map[string]bool, len(b)) + for _, x := range b { + set[x] = true + } + var out []string + for _, x := range a { + if set[x] { + out = append(out, x) + } + } + return out +} + // BuildUp assembles the ordered up-saga phases for the requested projects. func BuildUp(d UpDeps) ([]Phase, error) { if d.Runner == nil { @@ -105,13 +124,26 @@ func BuildUp(d UpDeps) ([]Phase, error) { if !d.NoPreflight { phases = append(phases, preflightPhase(d)) } + // Provision targets: active projects that `uses` a shared Postgres get a + // per-project role+db (DECISIONS D8). The instances they need are published on + // 127.0.0.1 by the shared phase so host-side pgx can reach them. + var targets []provTarget + var provInstances []string + if !d.NoProvision { + targets = provTargets(d.Model, active.Services, pgInstances(d.Model)) + provInstances = provInstanceList(targets) + } + phases = append(phases, networkPhase(d), generatePhase(d, gen), secretsPhase(d, projects, secretEnv), trustPhase(d), - sharedPhase(d, projects, active.Shared), + sharedPhase(d, projects, active.Shared, provInstances), ) + if len(targets) > 0 { + phases = append(phases, provisionPhase(d, targets)) + } // Hook ordering (spec 11): workspace preUp → per-project (preUp → compose-up → // postUp) → workspace postUp. if !d.NoHooks { @@ -293,12 +325,16 @@ func generatePhase(d UpDeps, gen *generate.Generator) Phase { // shared — register ref rows, bring up only the shared services the requested // projects use, then health-gate them. Compensation drops the ref rows. -func sharedPhase(d UpDeps, projects, names []string) Phase { +func sharedPhase(d UpDeps, projects, names, provInstances []string) Phase { + // Only provision instances that are actually being brought up this run. + prov := intersect(provInstances, names) return Phase{ Name: "shared", Mutating: true, Fingerprint: func(context.Context) (string, error) { - return Fingerprint(append([]string{"shared"}, names...)...), nil + // Fold the provisioned set in: newly publishing a port (a consumer was + // added) must re-run shared-up rather than skip on a stale fingerprint. + return Fingerprint(append(append([]string{"shared"}, names...), append([]string{"prov"}, prov...)...)...), nil }, Run: func(ctx context.Context) (any, error) { if len(names) == 0 { @@ -315,6 +351,24 @@ func sharedPhase(d UpDeps, projects, names []string) Phase { File: filepath.Join(outDir, generate.ComposeFile), Dir: outDir, Runner: d.Runner, } + // Publish each provisioned Postgres on 127.0.0.1: via an + // up-time overlay so host-side pgx (the provision phase) can reach it, + // without touching the deterministic generated compose. + if len(prov) > 0 { + ports := map[string]int{} + for _, inst := range prov { + port, err := d.Manager.FreeHostPort(ctx, generate.SharedAlias(inst), provisionPurpose, provisionPortBase) + if err != nil { + return nil, fmt.Errorf("allocate provision port for %s: %w", inst, err) + } + ports[inst] = port + } + overlay, err := writeProvisionOverlay(d.Model.Root, ports) + if err != nil { + return nil, err + } + cp.Overrides = []string{overlay} + } if err := cp.Up(ctx, names...); err != nil { return nil, fmt.Errorf("compose up shared: %w", err) } diff --git a/internal/orchestrate/up_test.go b/internal/orchestrate/up_test.go index ce2d569..7134dac 100644 --- a/internal/orchestrate/up_test.go +++ b/internal/orchestrate/up_test.go @@ -12,6 +12,7 @@ import ( "github.com/open-source-cloud/devstack/internal/config" "github.com/open-source-cloud/devstack/internal/docker" "github.com/open-source-cloud/devstack/internal/generate" + "github.com/open-source-cloud/devstack/internal/provision" "github.com/open-source-cloud/devstack/internal/secrets" "github.com/open-source-cloud/devstack/internal/state" "github.com/open-source-cloud/devstack/internal/template" @@ -81,6 +82,24 @@ func (f *fakeRunner) upServices(project string) []string { return nil } +// saw reports whether any single recorded command contains all the needles. +func (f *fakeRunner) saw(needles ...string) bool { + for _, c := range f.cmds { + joined := strings.Join(c, " ") + all := true + for _, n := range needles { + if !strings.Contains(joined, n) { + all = false + break + } + } + if all { + return true + } + } + return false +} + func (f *fakeRunner) sawDown(project string) bool { for _, c := range f.cmds { joined := strings.Join(c, " ") @@ -141,7 +160,7 @@ hooks: fr := &fakeRunner{} d := UpDeps{ Model: m, DB: db, Docker: mc, Manager: mgr, Source: src, - LockPath: lockPath, Runner: fr, Env: map[string]string{}, + LockPath: lockPath, Runner: fr, Env: map[string]string{}, PgConnect: okPgConnect, } return d, fr, db } @@ -312,7 +331,7 @@ func TestBuildUpInjectsSecretEnv(t *testing.T) { d := UpDeps{ Model: m, DB: db, Docker: mc, Manager: &workspace.Manager{Model: m, DB: db, Docker: mc, Source: src, LockPath: lockPath}, - Source: src, LockPath: lockPath, Runner: fr, Env: map[string]string{}, + Source: src, LockPath: lockPath, Runner: fr, Env: map[string]string{}, PgConnect: okPgConnect, Secrets: reg, } phases, err := BuildUp(d) @@ -475,7 +494,7 @@ services: fr := &fakeRunner{} d := UpDeps{ Model: m, DB: db, Docker: mc, Manager: mgr, Source: src, - LockPath: lockPath, Runner: fr, Env: map[string]string{}, NoHooks: true, + LockPath: lockPath, Runner: fr, Env: map[string]string{}, NoHooks: true, PgConnect: okPgConnect, } return d, fr, db } @@ -538,3 +557,107 @@ func mustPhases(t *testing.T, d UpDeps) []Phase { } return phases } + +// --- provision phase (D8) --------------------------------------------------- + +// fakePgConn records the SQL the provisioner runs; Exists=false so EnsureProject +// takes the create path. Implements provision.Conn. +type fakePgConn struct{ execs []string } + +func (c *fakePgConn) Exec(_ context.Context, sql string, _ ...any) error { + c.execs = append(c.execs, sql) + return nil +} +func (c *fakePgConn) Exists(_ context.Context, _ string, _ ...any) (bool, error) { + return false, nil +} + +// okPgConnect is the no-op connector used by the shared fixtures so the full saga +// (including provision) runs daemon-free. +func okPgConnect(context.Context, string) (provision.Conn, func() error, error) { + return &fakePgConn{}, func() error { return nil }, nil +} + +// recordingPg captures every connection + DSN for assertions. +type recordingPg struct { + conns []*fakePgConn + dsns []string +} + +func (r *recordingPg) connect(_ context.Context, dsn string) (provision.Conn, func() error, error) { + c := &fakePgConn{} + r.conns = append(r.conns, c) + r.dsns = append(r.dsns, dsn) + return c, func() error { return nil }, nil +} + +func TestBuildUpProvisionsPerProjectDB(t *testing.T) { + d, fr, db := upFixture(t) // project app, service web, uses workspace.shared.postgres + rp := &recordingPg{} + d.PgConnect = rp.connect + + recs, err := (&Saga{Workspace: d.Model.Workspace.Name, DB: db, LockPath: d.LockPath}). + Run(context.Background(), mustPhases(t, d)) + if err != nil || AnyFailed(recs) { + t.Fatalf("saga: %v\n%+v", err, recs) + } + + // The provision phase ran. + got := map[string]string{} + for _, r := range recs { + got[r.Phase+scopeSuffix(r.Scope)] = r.Status + } + if got["provision"] != StatusOK { + t.Fatalf("provision phase = %q, want ok (all: %+v)", got["provision"], got) + } + // Connected once to the shared Postgres on loopback. + if len(rp.dsns) != 1 || !strings.Contains(rp.dsns[0], "127.0.0.1") { + t.Fatalf("provision DSNs = %v, want one loopback DSN", rp.dsns) + } + // EnsureProject ran the create path (CREATE ROLE + CREATE DATABASE). + joined := strings.Join(rp.conns[0].execs, " | ") + if !strings.Contains(joined, "CREATE ROLE") || !strings.Contains(joined, "CREATE DATABASE") { + t.Errorf("provisioning SQL missing role/db creation: %s", joined) + } + // Ownership recorded in the ledger. + rows, _ := db.ProvisionedFor("app") + var kinds []string + for _, r := range rows { + kinds = append(kinds, r.Kind+":"+r.Name) + } + if !slices.Contains(kinds, "role:app") || !slices.Contains(kinds, "database:app") { + t.Errorf("provisioned rows = %v, want role:app + database:app", kinds) + } + // The shared stack was brought up WITH the loopback port overlay. + if !fr.saw("-p "+generate.SharedStackName, "-f", "compose.provision.yaml") { + t.Errorf("shared up did not include the provision overlay: %v", fr.cmds) + } + // The overlay file was written, loopback-bound. + overlay := filepath.Join(d.Model.Root, generate.GenDir, "shared", "compose.provision.yaml") + body, err := os.ReadFile(overlay) + if err != nil { + t.Fatalf("overlay not written: %v", err) + } + if !strings.Contains(string(body), "127.0.0.1:") || !strings.Contains(string(body), ":5432") { + t.Errorf("overlay not loopback-bound to 5432:\n%s", body) + } +} + +func TestBuildUpNoProvisionSkips(t *testing.T) { + d, _, db := upFixture(t) + d.NoProvision = true + d.PgConnect = func(context.Context, string) (provision.Conn, func() error, error) { + t.Fatal("PgConnect must not be called when NoProvision is set") + return nil, nil, nil + } + recs, err := (&Saga{Workspace: d.Model.Workspace.Name, DB: db, LockPath: d.LockPath}). + Run(context.Background(), mustPhases(t, d)) + if err != nil || AnyFailed(recs) { + t.Fatalf("saga: %v\n%+v", err, recs) + } + for _, r := range recs { + if r.Phase == "provision" { + t.Error("provision phase present despite NoProvision") + } + } +}