feat(orchestrate): provision saga phase — per-project Postgres role+db (D8) — M2 capstone

internal/cli/up.go

@@ -29,6 +29,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command {
 		build       bool
 		noHooks     bool
 		noPreflight bool
+		noProvision bool
 		profiles    []string
 	)
 	cmd := &cobra.Command{
@@ -48,6 +49,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command {
 			d.Build = build
 			d.NoHooks = noHooks
 			d.NoPreflight = noPreflight
+			d.NoProvision = noProvision
 			d.Profiles = profiles
 
 			// Memory-budget warning (spec 12 §budget): opt-in — only when the
@@ -90,6 +92,7 @@ func newUpCmd(g *GlobalOpts) *cobra.Command {
 	cmd.Flags().BoolVar(&build, "build", false, "build images before starting (compose build)")
 	cmd.Flags().BoolVar(&noHooks, "no-hooks", false, "skip lifecycle hooks")
 	cmd.Flags().BoolVar(&noPreflight, "no-preflight", false, "skip the preflight checks")
+	cmd.Flags().BoolVar(&noProvision, "no-provision", false, "skip per-project Postgres role/db provisioning")
 	cmd.Flags().StringArrayVarP(&profiles, "profile", "p", nil,
 		"service slice(s) to start — repeatable & comma-separated (spec 12); empty → defaultProfile or all")
 	return cmd

internal/docker/compose.go

@@ -71,11 +71,12 @@ func (e *CmdError) Unwrap() error { return e.Err }
 // project name and compose file (DECISIONS D5). Lifecycle verbs run here;
 // container enumeration stays on the read-only SDK Client.
 type Compose struct {
-	Project string   // -p <project>
-	File    string   // -f <compose file>
-	Dir     string   // working dir (build contexts resolve relative to it)
-	Env     []string // extra env (resolved secrets), appended to os.Environ
-	Runner  Runner
+	Project   string   // -p <project>
+	File      string   // -f <compose file>
+	Overrides []string // additional -f overlays, applied in order after File (up-time only)
+	Dir       string   // working dir (build contexts resolve relative to it)
+	Env       []string // extra env (resolved secrets), appended to os.Environ
+	Runner    Runner
 }
 
 // NewCompose builds a Compose driver using the real exec runner.
@@ -89,7 +90,13 @@ func (c *Compose) base() []string {
 	if c.File == "" {
 		return []string{"compose", "-p", c.Project}
 	}
-	return []string{"compose", "-p", c.Project, "-f", c.File}
+	args := []string{"compose", "-p", c.Project, "-f", c.File}
+	// Overlays (e.g. the up-time provision port mapping) are applied after the base
+	// file so their values win; later files override earlier ones (compose merge).
+	for _, ov := range c.Overrides {
+		args = append(args, "-f", ov)
+	}
+	return args
 }
 
 // Up brings the stack (or the named subset of services) up detached. With no

internal/orchestrate/provision.go

@@ -0,0 +1,230 @@
+package orchestrate
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/open-source-cloud/devstack/internal/config"
+	"github.com/open-source-cloud/devstack/internal/generate"
+	"github.com/open-source-cloud/devstack/internal/lock"
+	"github.com/open-source-cloud/devstack/internal/provision"
+)
+
+// This file is the provision saga phase (M2 capstone, DECISIONS D8): per-project
+// Postgres role+database isolation on the shared engine. Because provisioning runs
+// pgx FROM THE HOST, the shared Postgres needs a reachable port — published as a
+// 127.0.0.1-only mapping via an UP-TIME compose overlay (so the deterministic,
+// golden-asserted `generate` output is untouched and the "no host ports by
+// default" posture holds for every other service). The per-project password is the
+// project name: a predictable dev credential for a loopback-only, network-isolated
+// dev database (THREAT-MODEL: container isolation is a non-goal), so nothing secret
+// is generated or stored, and an app opts in via the documented DSN
+// `postgres://<project>:<project>@shared-postgres:5432/<project>`.
+
+const (
+	provisionPurpose  = "pg-provision" // ledger port_alloc purpose
+	provisionPortBase = 45432          // host port search base for shared Postgres
+	provisionFile     = "compose.provision.yaml"
+	pgTemplate        = "postgres" // shared engine template that this phase provisions
+)
+
+// PgConnector opens an admin connection to a Postgres DSN. Injectable so the
+// provision phase is unit-testable without a live server (the default wraps
+// provision.Connect / pgx).
+type PgConnector func(ctx context.Context, dsn string) (provision.Conn, func() error, error)
+
+func defaultPgConnect(ctx context.Context, dsn string) (provision.Conn, func() error, error) {
+	c, closeFn, err := provision.Connect(ctx, dsn)
+	if err != nil {
+		return nil, nil, err
+	}
+	return c, closeFn, nil
+}
+
+// provTarget is one (project, shared-Postgres-instance) pair to provision.
+type provTarget struct {
+	project  string
+	instance string
+}
+
+// pgInstances returns the set of shared services that are Postgres engines.
+func pgInstances(m *config.Model) map[string]bool {
+	out := map[string]bool{}
+	for name, s := range m.Workspace.Shared {
+		if s.Template == pgTemplate {
+			out[name] = true
+		}
+	}
+	return out
+}
+
+// provTargets returns the (project, instance) pairs to provision: every active
+// project that `uses` a shared Postgres instance. Sorted and de-duplicated.
+func provTargets(m *config.Model, activeServices map[string][]string, pg map[string]bool) []provTarget {
+	seen := map[string]bool{}
+	var out []provTarget
+	for _, project := range sortedStringSlice(keysOf(activeServices)) {
+		p, ok := m.Projects[project]
+		if !ok {
+			continue
+		}
+		for _, sname := range activeServices[project] {
+			for _, u := range p.Services[sname].Uses {
+				ref, ok := config.ParseRef(u)
+				if !ok || ref.Kind != config.RefShared || !pg[ref.Name] {
+					continue
+				}
+				key := project + "\x00" + ref.Name
+				if seen[key] {
+					continue
+				}
+				seen[key] = true
+				out = append(out, provTarget{project: project, instance: ref.Name})
+			}
+		}
+	}
+	return out
+}
+
+// provInstanceList returns the sorted distinct instances across targets.
+func provInstanceList(targets []provTarget) []string {
+	set := map[string]bool{}
+	for _, t := range targets {
+		set[t.instance] = true
+	}
+	return sortedStringSlice(keysOf(set))
+}
+
+// writeProvisionOverlay writes the up-time compose overlay that publishes each
+// provisioned Postgres instance on 127.0.0.1:<hostPort>. Returns the overlay path.
+// Loopback-only so nothing is exposed beyond the host (spec 03 / no host ports).
+func writeProvisionOverlay(root string, ports map[string]int) (string, error) {
+	var b strings.Builder
+	b.WriteString("services:\n")
+	insts := make([]string, 0, len(ports))
+	for inst := range ports {
+		insts = append(insts, inst)
+	}
+	sort.Strings(insts)
+	for _, inst := range insts {
+		fmt.Fprintf(&b, "  %s:\n    ports:\n      - \"127.0.0.1:%d:5432\"\n", inst, ports[inst])
+	}
+	dir := filepath.Join(root, generate.GenDir, "shared")
+	if err := os.MkdirAll(dir, 0o755); err != nil {
+		return "", err
+	}
+	path := filepath.Join(dir, provisionFile)
+	if err := os.WriteFile(path, []byte(b.String()), 0o644); err != nil {
+		return "", err
+	}
+	return path, nil
+}
+
+// provisionPhase creates each project's role+database on its shared Postgres,
+// idempotently, holding the flock for the SQL mutations (DECISIONS D7/D8). It
+// re-derives the host port from the ledger (the same one sharedPhase published),
+// so it is safe across re-runs and crashes. Compensation is intentionally empty:
+// provisioned roles/dbs are data and survive a failed `up`.
+func provisionPhase(d UpDeps, targets []provTarget) Phase {
+	return Phase{
+		Name:     "provision",
+		Mutating: true,
+		Fingerprint: func(context.Context) (string, error) {
+			keys := make([]string, 0, len(targets))
+			for _, t := range targets {
+				keys = append(keys, t.project+"@"+t.instance)
+			}
+			return Fingerprint(append([]string{"provision"}, keys...)...), nil
+		},
+		Run: func(ctx context.Context) (any, error) {
+			connect := d.PgConnect
+			if connect == nil {
+				connect = defaultPgConnect
+			}
+			byInst := map[string][]string{}
+			for _, t := range targets {
+				byInst[t.instance] = append(byInst[t.instance], t.project)
+			}
+
+			// Resolve each instance's published host port (FreeHostPort self-locks
+			// and is idempotent — returns the port sharedPhase already allocated).
+			ports := map[string]int{}
+			for _, inst := range sortedStringSlice(keysOf(byInst)) {
+				p, err := d.Manager.FreeHostPort(ctx, generate.SharedAlias(inst), provisionPurpose, provisionPortBase)
+				if err != nil {
+					return nil, fmt.Errorf("resolve provision port for %s: %w", inst, err)
+				}
+				ports[inst] = p
+			}
+
+			provisioned := []map[string]any{}
+			// Hold the flock for the role/db mutations (provision pkg contract).
+			err := lock.WithLock(ctx, d.LockPath, func() error {
+				for _, inst := range sortedStringSlice(keysOf(byInst)) {
+					params := d.Model.Workspace.Shared[inst].Params
+					user := paramString(params, "rootUser", "devstack")
+					pass := paramString(params, "rootPassword", "devstack")
+					dsn := provision.DSN("127.0.0.1", ports[inst], user, pass, user)
+					conn, closeConn, err := connect(ctx, dsn)
+					if err != nil {
+						return fmt.Errorf("connect to shared %s on 127.0.0.1:%d: %w", inst, ports[inst], err)
+					}
+					for _, project := range byInst[inst] {
+						creds, err := provision.Postgres{}.EnsureProject(ctx, conn, project, project)
+						if err != nil {
+							_ = closeConn()
+							return fmt.Errorf("provision %s on %s: %w", project, inst, err)
+						}
+						if err := d.DB.RecordProvisioned(project, "role", creds.Role); err != nil {
+							_ = closeConn()
+							return err
+						}
+						if err := d.DB.RecordProvisioned(project, "database", creds.Database); err != nil {
+							_ = closeConn()
+							return err
+						}
+						d.DB.LogEvent("provision", project, "role+db on "+generate.SharedAlias(inst))
+						provisioned = append(provisioned, map[string]any{
+							"project": project, "instance": inst, "role": creds.Role, "database": creds.Database,
+						})
+					}
+					if err := closeConn(); err != nil {
+						return err
+					}
+				}
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+			return map[string]any{"provisioned": provisioned}, nil
+		},
+	}
+}
+
+// paramString reads a string param with a default.
+func paramString(params map[string]any, key, def string) string {
+	if v, ok := params[key]; ok {
+		if s, ok := v.(string); ok && s != "" {
+			return s
+		}
+	}
+	return def
+}
+
+func keysOf[V any](m map[string]V) []string {
+	out := make([]string, 0, len(m))
+	for k := range m {
+		out = append(out, k)
+	}
+	return out
+}
+
+func sortedStringSlice(s []string) []string {
+	sort.Strings(s)
+	return s
+}

internal/orchestrate/up.go

@@ -56,13 +56,32 @@ type UpDeps struct {
 	// Trust installs the local CA when network.proxy.httpsLocal; nil → trust.New().
 	// Injected for tests (the trust phase is fenced — failure never aborts up).
 	Trust *trust.Trust
+	// PgConnect opens an admin Postgres connection for the provision phase; nil →
+	// the pgx-backed default. Injected for tests (so provisioning runs daemon-free).
+	PgConnect PgConnector
 
 	Build         bool          // compose up --build
 	NoHooks       bool          // skip the hooks phase
 	NoPreflight   bool          // skip the preflight phase (fast inner loops)
+	NoProvision   bool          // skip the per-project Postgres provision phase
 	HealthTimeout time.Duration // per-shared-service gate cap (0 → health.Compile default)
 }
 
+// intersect returns the elements of a that are also in b, preserving a's order.
+func intersect(a, b []string) []string {
+	set := make(map[string]bool, len(b))
+	for _, x := range b {
+		set[x] = true
+	}
+	var out []string
+	for _, x := range a {
+		if set[x] {
+			out = append(out, x)
+		}
+	}
+	return out
+}
+
 // BuildUp assembles the ordered up-saga phases for the requested projects.
 func BuildUp(d UpDeps) ([]Phase, error) {
 	if d.Runner == nil {
@@ -105,13 +124,26 @@ func BuildUp(d UpDeps) ([]Phase, error) {
 	if !d.NoPreflight {
 		phases = append(phases, preflightPhase(d))
 	}
+	// Provision targets: active projects that `uses` a shared Postgres get a
+	// per-project role+db (DECISIONS D8). The instances they need are published on
+	// 127.0.0.1 by the shared phase so host-side pgx can reach them.
+	var targets []provTarget
+	var provInstances []string
+	if !d.NoProvision {
+		targets = provTargets(d.Model, active.Services, pgInstances(d.Model))
+		provInstances = provInstanceList(targets)
+	}
+
 	phases = append(phases,
 		networkPhase(d),
 		generatePhase(d, gen),
 		secretsPhase(d, projects, secretEnv),
 		trustPhase(d),
-		sharedPhase(d, projects, active.Shared),
+		sharedPhase(d, projects, active.Shared, provInstances),
 	)
+	if len(targets) > 0 {
+		phases = append(phases, provisionPhase(d, targets))
+	}
 	// Hook ordering (spec 11): workspace preUp → per-project (preUp → compose-up →
 	// postUp) → workspace postUp.
 	if !d.NoHooks {
@@ -293,12 +325,16 @@ func generatePhase(d UpDeps, gen *generate.Generator) Phase {
 
 // shared — register ref rows, bring up only the shared services the requested
 // projects use, then health-gate them. Compensation drops the ref rows.
-func sharedPhase(d UpDeps, projects, names []string) Phase {
+func sharedPhase(d UpDeps, projects, names, provInstances []string) Phase {
+	// Only provision instances that are actually being brought up this run.
+	prov := intersect(provInstances, names)
 	return Phase{
 		Name:     "shared",
 		Mutating: true,
 		Fingerprint: func(context.Context) (string, error) {
-			return Fingerprint(append([]string{"shared"}, names...)...), nil
+			// Fold the provisioned set in: newly publishing a port (a consumer was
+			// added) must re-run shared-up rather than skip on a stale fingerprint.
+			return Fingerprint(append(append([]string{"shared"}, names...), append([]string{"prov"}, prov...)...)...), nil
 		},
 		Run: func(ctx context.Context) (any, error) {
 			if len(names) == 0 {
@@ -315,6 +351,24 @@ func sharedPhase(d UpDeps, projects, names []string) Phase {
 				File:    filepath.Join(outDir, generate.ComposeFile),
 				Dir:     outDir, Runner: d.Runner,
 			}
+			// Publish each provisioned Postgres on 127.0.0.1:<ledger port> via an
+			// up-time overlay so host-side pgx (the provision phase) can reach it,
+			// without touching the deterministic generated compose.
+			if len(prov) > 0 {
+				ports := map[string]int{}
+				for _, inst := range prov {
+					port, err := d.Manager.FreeHostPort(ctx, generate.SharedAlias(inst), provisionPurpose, provisionPortBase)
+					if err != nil {
+						return nil, fmt.Errorf("allocate provision port for %s: %w", inst, err)
+					}
+					ports[inst] = port
+				}
+				overlay, err := writeProvisionOverlay(d.Model.Root, ports)
+				if err != nil {
+					return nil, err
+				}
+				cp.Overrides = []string{overlay}
+			}
 			if err := cp.Up(ctx, names...); err != nil {
 				return nil, fmt.Errorf("compose up shared: %w", err)
 			}