diff --git a/internal/generate/secrets.go b/internal/generate/secrets.go new file mode 100644 index 0000000..2939c90 --- /dev/null +++ b/internal/generate/secrets.go @@ -0,0 +1,34 @@ +package generate + +import ( + "github.com/open-source-cloud/devstack/internal/config" + "github.com/open-source-cloud/devstack/internal/secrets" +) + +// SecretRefs returns the compose env-key → secret:// reference for every +// secret-valued env entry across a project's services. The KEY here is byte-for- +// byte the valueless key projectEnv emits (env.raw verbatim; env.prefixed as +// _), so the up saga can inject `KEY=` through the +// compose-up process env and have Compose substitute the valueless key (§7.5) — +// no secret value ever in a generated file. +func SecretRefs(m *config.Model, project string) map[string]string { + out := map[string]string{} + p, ok := m.Projects[project] + if !ok { + return out + } + for _, sname := range sortedKeys(p.Services) { + svc := p.Services[sname] + for k, v := range svc.Env.Raw { + if secrets.IsRef(v) { + out[k] = v + } + } + for k, v := range svc.Env.Prefixed { + if secrets.IsRef(v) { + out[envPrefix(sname)+"_"+k] = v + } + } + } + return out +} diff --git a/internal/orchestrate/up.go b/internal/orchestrate/up.go index 172068c..1648a48 100644 --- a/internal/orchestrate/up.go +++ b/internal/orchestrate/up.go @@ -14,6 +14,7 @@ import ( "github.com/open-source-cloud/devstack/internal/health" "github.com/open-source-cloud/devstack/internal/hooks" "github.com/open-source-cloud/devstack/internal/lock" + "github.com/open-source-cloud/devstack/internal/secrets" "github.com/open-source-cloud/devstack/internal/state" "github.com/open-source-cloud/devstack/internal/template" "github.com/open-source-cloud/devstack/internal/workspace" @@ -46,6 +47,9 @@ type UpDeps struct { Env map[string]string // generate env (nil → process env, via generate default) Profile string Projects []string // explicit subset; empty → every project in the workspace + // Secrets resolves secret:// refs; nil → built from workspace.secrets.providers + // with the built-in factories (SOPS+age). Injected for tests. + Secrets *secrets.Registry Build bool // compose up --build NoHooks bool // skip the hooks phase @@ -73,6 +77,11 @@ func BuildUp(d UpDeps) ([]Phase, error) { return nil, err } + // Resolved secret env per project, shared between the secrets phase (writes) + // and each compose-up phase (reads). Values live only here + in the child + // process env — never on disk. + secretEnv := map[string][]string{} + var phases []Phase if !d.NoPreflight { phases = append(phases, preflightPhase(d)) @@ -80,10 +89,11 @@ func BuildUp(d UpDeps) ([]Phase, error) { phases = append(phases, networkPhase(d), generatePhase(d, gen), + secretsPhase(d, projects, secretEnv), sharedPhase(d, projects), ) for _, p := range projects { - phases = append(phases, composeUpPhase(d, p)) + phases = append(phases, composeUpPhase(d, p, secretEnv)) } if !d.NoHooks { for _, p := range projects { @@ -93,6 +103,76 @@ func BuildUp(d UpDeps) ([]Phase, error) { return phases, nil } +// secretsPhase resolves every secret:// ref the requested projects reference and +// stashes the resolved KEY=VALUE env per project (spec 04 §6). It ALWAYS runs +// (never cached — values stay in memory) and mutates nothing global, so it has no +// compensation. A nil Secrets registry with no secret refs is a no-op; refs with +// no registry is a clear error. +func secretsPhase(d UpDeps, projects []string, out map[string][]string) Phase { + return Phase{ + Name: "secrets", + AlwaysRun: true, + Run: func(ctx context.Context) (any, error) { + total := 0 + for _, p := range projects { + keyRefs := generate.SecretRefs(d.Model, p) + if len(keyRefs) == 0 { + continue + } + reg, err := d.secretRegistry() + if err != nil { + return nil, err + } + raws := make([]string, 0, len(keyRefs)) + for _, raw := range keyRefs { + raws = append(raws, raw) + } + refs, err := secrets.Collect(raws...) + if err != nil { + return nil, err + } + resolved, err := secrets.Resolve(ctx, reg, refs) + if err != nil { + return nil, fmt.Errorf("resolve secrets for %s: %w", p, err) + } + env := make([]string, 0, len(keyRefs)) + for _, key := range sortedStringKeys(keyRefs) { + env = append(env, key+"="+resolved[keyRefs[key]]) + } + out[p] = env + total += len(env) + } + return map[string]any{"resolved": total}, nil + }, + } +} + +// secretRegistry returns the injected registry, or builds one from the +// workspace's declared providers with the built-in factories (SOPS+age). +func (d UpDeps) secretRegistry() (*secrets.Registry, error) { + if d.Secrets != nil { + return d.Secrets, nil + } + reg := secrets.NewRegistry() + secrets.RegisterBuiltins(reg) + for _, pr := range d.Model.Workspace.Secrets.Providers { + reg.Configure(secrets.ProviderConfig{ + Name: pr.Name, Kind: pr.Kind, Env: pr.Env, + ProjectID: pr.ProjectID, Region: pr.Region, + }) + } + return reg, nil +} + +func sortedStringKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + // preflight — daemon reachable (critical). The full doctor matrix is X6. func preflightPhase(d UpDeps) Phase { return Phase{ @@ -243,13 +323,17 @@ func gateShared(ctx context.Context, d UpDeps, names []string) ([]map[string]any // composeUpPhase brings one project stack up. Compensation tears it back down // (idempotent) — refs are owned by the shared phase, not unwound here. -func composeUpPhase(d UpDeps, project string) Phase { +func composeUpPhase(d UpDeps, project string, secretEnv map[string][]string) Phase { outDir := filepath.Join(d.Model.ProjectDir(project), generate.GenDir) cp := func() docker.Compose { return docker.Compose{ Project: "devstack-" + project, File: filepath.Join(outDir, generate.ComposeFile), Dir: outDir, Runner: d.Runner, + // Resolved secret values reach the containers ONLY here, via the + // compose-up process env (Compose substitutes the valueless keys); they + // are never written to a file (§7.5). + Env: secretEnv[project], } } return Phase{ diff --git a/internal/orchestrate/up_test.go b/internal/orchestrate/up_test.go index 68ed2db..01701f5 100644 --- a/internal/orchestrate/up_test.go +++ b/internal/orchestrate/up_test.go @@ -11,30 +11,45 @@ import ( "github.com/open-source-cloud/devstack/internal/config" "github.com/open-source-cloud/devstack/internal/docker" "github.com/open-source-cloud/devstack/internal/generate" + "github.com/open-source-cloud/devstack/internal/secrets" "github.com/open-source-cloud/devstack/internal/state" "github.com/open-source-cloud/devstack/internal/template" "github.com/open-source-cloud/devstack/internal/workspace" "github.com/open-source-cloud/devstack/templates" ) -// fakeRunner records compose invocations and can fail selectively. +// fakeRunner records compose invocations (with their injected env) and can fail +// selectively. type fakeRunner struct { cmds [][]string + envs [][]string // env passed alongside each cmd (same index) fail func(args []string) bool } -func (f *fakeRunner) record(name string, args []string) error { +func (f *fakeRunner) record(env []string, name string, args []string) error { f.cmds = append(f.cmds, append([]string{name}, args...)) + f.envs = append(f.envs, env) if f.fail != nil && f.fail(args) { return errors.New("compose failed") } return nil } -func (f *fakeRunner) Run(_ context.Context, _ []string, _, name string, args ...string) error { - return f.record(name, args) +func (f *fakeRunner) Run(_ context.Context, env []string, _, name string, args ...string) error { + return f.record(env, name, args) } -func (f *fakeRunner) Output(_ context.Context, _ []string, _, name string, args ...string) ([]byte, error) { - return nil, f.record(name, args) +func (f *fakeRunner) Output(_ context.Context, env []string, _, name string, args ...string) ([]byte, error) { + return nil, f.record(env, name, args) +} + +// envForUp returns the env injected for a project's compose up (or nil). +func (f *fakeRunner) envForUp(project string) []string { + for i, c := range f.cmds { + joined := strings.Join(c, " ") + if strings.Contains(joined, "-p "+project) && strings.Contains(joined, " up ") { + return f.envs[i] + } + } + return nil } func (f *fakeRunner) sawUp(project string) bool { for _, c := range f.cmds { @@ -163,7 +178,7 @@ func TestBuildUpHappyPath(t *testing.T) { } for _, r := range recs2 { switch r.Phase { - case "preflight", "hooks": + case "preflight", "secrets", "hooks": // AlwaysRun phases if r.Status != StatusOK { t.Errorf("%s should re-run ok, got %q", r.Phase, r.Status) } @@ -225,3 +240,77 @@ func scopeSuffix(scope string) string { } return "@" + scope } + +// fakeSecretProvider resolves any ref to a fixed value (S6 injection test). +type fakeSecretProvider struct{ val string } + +func (fakeSecretProvider) Name() string { return "fake" } +func (p fakeSecretProvider) Resolve(_ context.Context, refs []secrets.Ref) (map[string]string, error) { + out := map[string]string{} + for _, r := range refs { + out[r.Raw] = p.val + } + return out, nil +} + +func TestBuildUpInjectsSecretEnv(t *testing.T) { + root := t.TempDir() + write := func(rel, body string) { + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("workspace.yaml", "apiVersion: devstack/v1\nkind: Workspace\nname: demo\nprojects:\n - { name: app, path: app }\n") + write("app/devstack.yaml", "apiVersion: devstack/v1\nkind: Project\nname: app\nservices:\n web:\n template: node.vite\n env:\n raw: { DB_PASSWORD: \"secret://vault/f#k\" }\n") + + m, err := config.LoadAt(root) + if err != nil { + t.Fatalf("load: %v", err) + } + db, err := state.Open(context.Background(), filepath.Join(root, "state"), "ctx") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { db.Close() }) + mc := &docker.MockClient{} + src := template.NewFSSource(templates.FS) + lockPath := filepath.Join(root, "lock") + fr := &fakeRunner{} + + // Inject a registry whose provider resolves the ref to a known value. + reg := secrets.NewRegistry() + reg.RegisterFactory("fake", func(secrets.ProviderConfig) (secrets.Provider, error) { + return fakeSecretProvider{val: "resolved-pw"}, nil + }) + reg.Configure(secrets.ProviderConfig{Name: "vault", Kind: "fake"}) + + d := UpDeps{ + Model: m, DB: db, Docker: mc, + Manager: &workspace.Manager{Model: m, DB: db, Docker: mc, Source: src, LockPath: lockPath}, + Source: src, LockPath: lockPath, Runner: fr, Env: map[string]string{}, + Secrets: reg, + } + phases, err := BuildUp(d) + if err != nil { + t.Fatal(err) + } + saga := &Saga{Workspace: m.Workspace.Name, DB: db, LockPath: lockPath} + if _, err := saga.Run(context.Background(), phases); err != nil { + t.Fatalf("saga: %v", err) + } + // The resolved secret reaches compose up via its process env — and only there. + env := fr.envForUp("devstack-app") + found := false + for _, kv := range env { + if kv == "DB_PASSWORD=resolved-pw" { + found = true + } + } + if !found { + t.Errorf("compose-up env should carry DB_PASSWORD=resolved-pw, got %v", env) + } +}