diff --git a/go.mod b/go.mod index 695e5f1..0ad14ef 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,8 @@ require ( github.com/moby/moby/api v1.54.2 github.com/moby/moby/client v0.4.1 github.com/nats-io/nats.go v1.52.0 + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.1.1 github.com/spf13/cobra v1.10.2 github.com/twmb/franz-go v1.21.4 github.com/twmb/franz-go/pkg/kadm v1.18.0 @@ -35,6 +37,7 @@ require ( golang.org/x/sync v0.20.0 golang.org/x/term v0.44.0 modernc.org/sqlite v1.52.0 + oras.land/oras-go/v2 v2.6.1 ) require ( @@ -95,8 +98,6 @@ require ( github.com/nats-io/nkeys v0.4.15 // indirect github.com/nats-io/nuid v1.0.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/pierrec/lz4/v4 v4.1.26 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect diff --git a/go.sum b/go.sum index e0f5360..5958e65 100644 --- a/go.sum +++ b/go.sum @@ -298,5 +298,7 @@ modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +oras.land/oras-go/v2 v2.6.1 h1:bonOEkjLfp8tt6qXWRRWP6p1F+9octchOf2EqnWB4Zs= +oras.land/oras-go/v2 v2.6.1/go.mod h1:dhtFrFOuZuDtAVeZ9FUnaa5zfzplG3ZnFX9/uH1J/Yk= pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/internal/cli/backend.go b/internal/cli/backend.go new file mode 100644 index 0000000..b1ac161 --- /dev/null +++ b/internal/cli/backend.go @@ -0,0 +1,28 @@ +package cli + +import ( + "github.com/open-source-cloud/devstack/internal/config" + "github.com/open-source-cloud/devstack/internal/docker" + "github.com/open-source-cloud/devstack/internal/store" +) + +// backendFor resolves the Docker backend (spec 21) for a loaded workspace: WHERE +// the shared stack (and, in the all-remote topology, the project stacks) run. The +// precedence is workspace.yaml `backend:` → the machine-global store default → +// local. A local backend is the default and reproduces today's behavior verbatim. +// +// The read-only docker client and the compose CLI are then bound to the returned +// backend (Backend.NewClient / Compose.ContextEnv), and the state ledger is keyed +// by the backend's context so remote rows never bleed into local counts. +func backendFor(m *config.Model) docker.Backend { + if m != nil && m.Workspace.Backend.IsRemote() { + b := m.Workspace.Backend + return docker.Backend{Context: b.Context, Host: b.Host} + } + // Fall back to the machine-global default (best-effort: a missing/broken store + // simply yields the local backend). + if cfg, ok, err := store.Load(); err == nil && ok && cfg.Backend.IsRemote() { + return docker.Backend{Context: cfg.Backend.Context, Host: cfg.Backend.Host} + } + return docker.Backend{} +} diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index e894248..682c56b 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -171,6 +171,7 @@ type doctorSession struct { model *config.Model // nil when cwd is not a workspace ctxName string lockPath string + backend docker.Backend // the resolved Docker backend (spec 21); zero = local } // openDoctorSession opens the docker client and state ledger once (best-effort) @@ -187,7 +188,11 @@ func openDoctorSession(cmd *cobra.Command) (*doctorSession, func()) { if m, err := config.Load(cwd); err == nil { s.model = m } - if c, err := docker.NewClient(ctx); err == nil { + // Resolve the Docker backend (spec 21) so doctor probes the endpoint the rest + // of the tool will actually target — a remote context/DOCKER_HOST when the + // workspace (or the store default) selects one, else the local daemon. + s.backend = backendFor(s.model) + if c, err := s.backend.NewClient(ctx); err == nil { s.client = c s.ctxName = c.ContextName() } else { @@ -208,6 +213,21 @@ func openDoctorSession(cmd *cobra.Command) (*doctorSession, func()) { } } +// remoteBackendProbe returns the backend.remote reachability check (spec 21), +// or ok=false for the local backend (nothing remote to probe — the docker-daemon +// preflight already covers the local endpoint). Extracted so it is testable +// without the full probe matrix (which needs a live ledger handle). +func (s *doctorSession) remoteBackendProbe(ctx context.Context) (probe, bool) { + if !s.backend.IsRemote() { + return probe{}, false + } + c := s.backend.RemoteReachable(ctx, s.client) + if c.Name == "" { + return probe{}, false + } + return plain(withCategory(c, catCritical)), true +} + // manager builds a workspace.Manager over the session's shared handles. Reconcile // (the state.refs fix) uses only DB/Docker/LockPath, so a nil Model is fine. func (s *doctorSession) manager() *workspace.Manager { @@ -275,6 +295,13 @@ func (s *doctorSession) probes(ctx context.Context) []probe { } } + // backend.remote — the configured remote backend is reachable (spec 21). Only + // emitted when a remote context/DOCKER_HOST is selected; the local backend adds + // nothing here (the docker-daemon preflight above already covers it). + if p, ok := s.remoteBackendProbe(ctx); ok { + probes = append(probes, p) + } + // state.ledger — the ledger opens/migrates cleanly (critical). if s.dbErr != nil { probes = append(probes, plain(docker.Check{ diff --git a/internal/cli/doctor_backend_test.go b/internal/cli/doctor_backend_test.go new file mode 100644 index 0000000..c6d4cd0 --- /dev/null +++ b/internal/cli/doctor_backend_test.go @@ -0,0 +1,57 @@ +package cli + +import ( + "context" + "errors" + "testing" + + "github.com/open-source-cloud/devstack/internal/docker" +) + +// TestDoctorRemoteBackendProbeReachable: with a remote backend selected and a +// reachable client, doctor emits a green backend.remote check reporting the +// engine version. +func TestDoctorRemoteBackendProbeReachable(t *testing.T) { + s := &doctorSession{ + backend: docker.Backend{Context: "prod"}, + client: &docker.MockClient{Context: "prod", Server: "27.1.1"}, + } + p, ok := s.remoteBackendProbe(context.Background()) + if !ok { + t.Fatal("no backend.remote probe emitted for a remote backend") + } + if p.check.ID != "backend.remote" || p.check.Status != docker.StatusOK || p.check.Detail != "Engine v27.1.1" { + t.Fatalf("backend.remote = %+v, want ok / Engine v27.1.1", p.check) + } + if p.check.Category != catCritical { + t.Errorf("category = %q, want %q", p.check.Category, catCritical) + } +} + +// TestDoctorRemoteBackendProbeUnreachable: an unreachable remote endpoint yields +// a failing backend.remote check with an actionable remediation. +func TestDoctorRemoteBackendProbeUnreachable(t *testing.T) { + s := &doctorSession{ + backend: docker.Backend{Host: "ssh://dev@box"}, + client: &docker.MockClient{PingErr: errors.New("dial ssh: no route to host")}, + } + p, ok := s.remoteBackendProbe(context.Background()) + if !ok { + t.Fatal("no backend.remote probe emitted for a remote backend") + } + if p.check.Status != docker.StatusFail || p.check.Remediation == "" { + t.Fatalf("backend.remote = %+v, want fail with remediation", p.check) + } +} + +// TestDoctorLocalBackendNoRemoteProbe: the default local backend adds no +// backend.remote probe (the docker-daemon preflight already covers it). +func TestDoctorLocalBackendNoRemoteProbe(t *testing.T) { + s := &doctorSession{ + backend: docker.LocalBackend(), + client: &docker.MockClient{Context: "default"}, + } + if _, ok := s.remoteBackendProbe(context.Background()); ok { + t.Fatal("local backend emitted a backend.remote probe") + } +} diff --git a/internal/cli/generate.go b/internal/cli/generate.go index 926ccc1..1a3c74c 100644 --- a/internal/cli/generate.go +++ b/internal/cli/generate.go @@ -13,14 +13,26 @@ import ( ) // builtinSource is the template source used by generation and the template -// tooling: custom templates in the store (~/.devstack/templates) override the -// embedded built-ins by name; the embedded set is always the fallback. +// tooling. Resolution priority is embedded < store < remote (first match wins in +// the chain, so the highest-priority source is listed first): a digest-pinned +// REMOTE template (spec 19) overrides a store template, which overrides an +// embedded built-in of the same name. A cold/missing remote cache contributes +// nothing, keeping generation offline-first and deterministic (with no remote +// templates registered the chain is byte-identical to the pre-spec-19 behavior). func builtinSource() template.TemplateSource { embedded := template.NewFSSource(templates.FS) + var chain []template.TemplateSource + if remote := remoteTemplateSource(); remote != nil { + chain = append(chain, remote) + } if dir := userTemplatesDir(); dir != "" { - return template.NewChainSource(template.NewFSSource(os.DirFS(dir)), embedded) + chain = append(chain, template.NewFSSource(os.DirFS(dir))) + } + if len(chain) == 0 { + return embedded } - return embedded + chain = append(chain, embedded) + return template.NewChainSource(chain...) } // newGenerateCmd wires `devstack generate` — the M1 deterministic pipeline entry diff --git a/internal/cli/template.go b/internal/cli/template.go index 64656cf..2467182 100644 --- a/internal/cli/template.go +++ b/internal/cli/template.go @@ -27,13 +27,13 @@ func newTemplateCmd(g *GlobalOpts) *cobra.Command { newTemplateTestCmd(g), newTemplateInitCmd(g), newTemplateNewCmd(g), - // Reserved remote-registry verbs (spec 19, v2) — tree-only stubs so - // help/completions stay consistent (spec 26 / spec 07). - stub("push", "Publish a template to a remote registry", "v2 (spec 19)"), - stub("add", "Add a remote template source", "v2 (spec 19)"), - stub("update", "Update cached remote templates", "v2 (spec 19)"), - stub("diff", "Diff a local template against its remote", "v2 (spec 19)"), - stub("verify", "Verify a remote template's signature", "v2 (spec 19)"), + // Versioned OCI template registry (spec 19). + newTemplatePushCmd(g), + newTemplateAddCmd(g), + newTemplateUpdateCmd(g), + newTemplateDiffCmd(g), + newTemplateVerifyCmd(g), + newTemplateLsCmd(g), ) return cmd } diff --git a/internal/cli/template_registry.go b/internal/cli/template_registry.go new file mode 100644 index 0000000..52914f2 --- /dev/null +++ b/internal/cli/template_registry.go @@ -0,0 +1,561 @@ +package cli + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + + "github.com/spf13/cobra" + + "github.com/open-source-cloud/devstack/internal/lock" + "github.com/open-source-cloud/devstack/internal/registry" + "github.com/open-source-cloud/devstack/internal/store" + "github.com/open-source-cloud/devstack/internal/template" +) + +// maxTemplateSchemaVersion is the newest template-bundle schemaVersion this binary +// understands. A pinned bundle declaring a NEWER schema is refused on `add` with an +// "upgrade devstack" message rather than a silent partial render (spec 19 AC). +const maxTemplateSchemaVersion = 1 + +// newRegistryClient is the registry-client seam. Production builds an oras client +// that talks to real registries with docker credentials; tests swap in a client +// backed by an in-memory store so the whole add/update/diff flow round-trips with +// no network. +var newRegistryClient = func() (*registry.Client, error) { return registry.New() } + +// remoteTemplateSource returns a TemplateSource over the digest-pinned cache for +// every registered remote template whose content is present, or nil when none are +// registered/cached. It is chained AHEAD of the store + embedded sources so a +// registered remote name wins (embedded < store < remote), while a cold or missing +// cache entry simply contributes nothing (generation stays offline-first). +func remoteTemplateSource() template.TemplateSource { + cfg, ok, err := store.Load() + if err != nil || !ok || len(cfg.Templates) == 0 { + return nil + } + var sources []template.TemplateSource + for _, rt := range cfg.Templates { + dir, err := store.TemplateCacheDir(rt.Digest) + if err != nil { + continue + } + if fi, err := os.Stat(filepath.Join(dir, rt.Name, template.TemplateFile)); err != nil || fi.IsDir() { + continue // cold cache — `template update` will populate it + } + sources = append(sources, template.NewFSSource(os.DirFS(dir))) + } + if len(sources) == 0 { + return nil + } + return template.NewChainSource(sources...) +} + +// newTemplatePushCmd graduates `template push ` (spec 19). +func newTemplatePushCmd(g *GlobalOpts) *cobra.Command { + var ( + sign bool + keyPath string + ) + cmd := &cobra.Command{ + Use: "push ", + Short: "Package a template directory and push it as an OCI artifact", + Long: "push packages a template bundle (template.yaml + optional build/ tree + golden.yaml)\n" + + "as a DETERMINISTIC OCI artifact, pushes it to (e.g. ghcr.io/OWNER/NAME:TAG), and\n" + + "prints the resolved sha256 manifest digest. Auth rides ~/.docker/config.json (or a\n" + + "GITHUB_TOKEN for ghcr.io); with --sign the artifact is cosign-signed.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + dir, refStr := args[0], args[1] + ref, err := registry.ParseReference(refStr) + if err != nil { + return err + } + if ref.Tag == "" { + return fmt.Errorf("push requires a tag, e.g. %s:1.0.0", ref.Name()) + } + client, err := newRegistryClient() + if err != nil { + return err + } + ctx := context.Background() + desc, err := client.Push(ctx, ref, dir) + if err != nil { + return err + } + if sign { + if err := registry.SignArtifact(ctx, nil, desc, keyPath); err != nil { + return err + } + } + if g.JSON { + return writeJSON(cmd, desc) + } + if !g.Quiet { + w := cmd.OutOrStdout() + fmt.Fprintf(w, "pushed %s\n", desc.Ref) + fmt.Fprintf(w, " digest: %s\n", desc.Digest) + if sign { + fmt.Fprintln(w, " signed: cosign") + } + } + return nil + }, + } + cmd.Flags().BoolVar(&sign, "sign", false, "cosign-sign the pushed artifact (requires the cosign binary)") + cmd.Flags().StringVar(&keyPath, "key", "", "cosign private key for keyed signing (default: keyless)") + return cmd +} + +// newTemplateAddCmd graduates `template add ` (spec 19). +func newTemplateAddCmd(g *GlobalOpts) *cobra.Command { + var ( + allowFloating bool + policy registry.VerifyPolicy + ) + cmd := &cobra.Command{ + Use: "add ", + Short: "Register a remote template, pinning its resolved digest", + Long: "add resolves 's tag to a manifest digest, verifies it (digest always;\n" + + "cosign signature when --identity/--issuer or --key is given), caches the bundle\n" + + "under the digest-keyed template cache, and records a pinned `templates:` entry in\n" + + "the store config so the template resolves by name in generation and lint/test.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + ref, err := registry.ParseReference(args[0]) + if err != nil { + return err + } + if ref.IsFloatingTag() && !allowFloating { + return fmt.Errorf("refusing to pin a floating tag %q (breaks determinism); pass --allow-floating to override", ref.Tag) + } + return addOrUpdate(cmd, g, ref, policy) + }, + } + cmd.Flags().BoolVar(&allowFloating, "allow-floating", false, "allow pinning a floating tag like :latest") + registerVerifyFlags(cmd, &policy) + return cmd +} + +// newTemplateUpdateCmd graduates `template update [name]` (spec 19). +func newTemplateUpdateCmd(g *GlobalOpts) *cobra.Command { + var ( + toTag string + dryRun bool + ) + cmd := &cobra.Command{ + Use: "update [name]", + Short: "Re-resolve pinned templates to a new digest (opt --to )", + Long: "update re-resolves each registered template's tag (or --to ) to a fresh\n" + + "digest, refreshes the digest-keyed cache, and rewrites the lockfile. With\n" + + "--dry-run nothing is written — it only reports which pins would move.", + Args: cobra.RangeArgs(0, 1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, ok, err := store.Load() + if err != nil { + return err + } + if !ok || len(cfg.Templates) == 0 { + return fmt.Errorf("no remote templates registered — add one with `%s template add `", rootName(cmd)) + } + var targets []store.RemoteTemplate + if len(args) == 1 { + rt, found := cfg.Template(args[0]) + if !found { + return fmt.Errorf("template %q is not registered", args[0]) + } + targets = []store.RemoteTemplate{rt} + } else { + targets = cfg.Templates + } + + type change struct { + Name string `json:"name"` + OldDigest string `json:"oldDigest"` + NewDigest string `json:"newDigest"` + Version string `json:"version"` + Changed bool `json:"changed"` + } + var changes []change + for _, rt := range targets { + ref, err := registry.ParseRepository(rt.Source) + if err != nil { + return fmt.Errorf("template %q: bad source %q: %w", rt.Name, rt.Source, err) + } + ref.Tag = rt.Version + if toTag != "" { + ref.Tag = toTag + } + ref.Digest = "" // re-resolve from the tag + ch := change{Name: rt.Name, OldDigest: rt.Digest, Version: ref.Tag} + if dryRun { + client, err := newRegistryClient() + if err != nil { + return err + } + desc, err := client.ResolveDigest(context.Background(), ref) + if err != nil { + return err + } + ch.NewDigest = desc.Digest + ch.Changed = desc.Digest != rt.Digest + changes = append(changes, ch) + continue + } + // A real update goes through the same pin+cache+lockfile path as add. + desc, err := doPin(context.Background(), ref, registry.VerifyPolicy{}) + if err != nil { + return err + } + ch.NewDigest = desc.Digest + ch.Changed = desc.Digest != rt.Digest + changes = append(changes, ch) + } + + if g.JSON { + return writeJSON(cmd, map[string]any{"dryRun": dryRun, "templates": changes}) + } + if !g.Quiet { + w := cmd.OutOrStdout() + for _, c := range changes { + switch { + case !c.Changed: + fmt.Fprintf(w, "%-16s up to date (%s)\n", c.Name, shortManifestDigest(c.NewDigest)) + case dryRun: + fmt.Fprintf(w, "%-16s %s → %s (would update)\n", c.Name, shortManifestDigest(c.OldDigest), shortManifestDigest(c.NewDigest)) + default: + fmt.Fprintf(w, "%-16s %s → %s (updated)\n", c.Name, shortManifestDigest(c.OldDigest), shortManifestDigest(c.NewDigest)) + } + } + } + return nil + }, + } + cmd.Flags().StringVar(&toTag, "to", "", "re-resolve to this tag instead of the recorded version") + cmd.Flags().BoolVar(&dryRun, "dry-run", false, "report pin changes without writing") + return cmd +} + +// newTemplateDiffCmd graduates `template diff ` (spec 19). +func newTemplateDiffCmd(g *GlobalOpts) *cobra.Command { + var against string + cmd := &cobra.Command{ + Use: "diff ", + Short: "Render-diff a pinned template against its latest remote tag", + Long: "diff renders the pinned (cached) template and the current remote tag (or --against\n" + + ") and reports whether the resolved digests — and the rendered compose — differ.\n" + + "It writes nothing.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + name := args[0] + cfg, ok, err := store.Load() + if err != nil { + return err + } + var rt store.RemoteTemplate + if ok { + rt, ok = cfg.Template(name) + } + if !ok { + return fmt.Errorf("template %q is not registered", name) + } + ref, err := registry.ParseRepository(rt.Source) + if err != nil { + return err + } + ref.Tag = rt.Version + if against != "" { + ref.Tag = against + } + ref.Digest = "" + + client, err := newRegistryClient() + if err != nil { + return err + } + latest, err := client.ResolveDigest(context.Background(), ref) + if err != nil { + return err + } + digestDrift := latest.Digest != rt.Digest + + // Render the pinned (cached) template and, on drift, the remote one, and + // compare the resolved compose so the diff is meaningful, not just a hash. + oldRender, oldErr := renderPinned(name, rt.Digest) + var newRender []byte + var renderDrift bool + if digestDrift { + pulled, perr := client.Pull(context.Background(), refWithTagDigest(ref, latest.Digest)) + if perr != nil { + return perr + } + newRender, err = renderPulled(name, pulled) + if err != nil { + return err + } + renderDrift = string(oldRender) != string(newRender) + } + + if g.JSON { + return writeJSON(cmd, map[string]any{ + "name": name, "pinnedDigest": rt.Digest, "remoteDigest": latest.Digest, + "remoteTag": ref.Tag, "digestDrift": digestDrift, "renderDrift": renderDrift, + }) + } + w := cmd.OutOrStdout() + if oldErr != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: pinned template not cached (run `%s template update`): %v\n", rootName(cmd), oldErr) + } + if !digestDrift { + fmt.Fprintf(w, "%s: up to date at %s (%s)\n", name, ref.Tag, shortManifestDigest(rt.Digest)) + return nil + } + fmt.Fprintf(w, "%s: DRIFT %s → %s (tag %s)\n", name, shortManifestDigest(rt.Digest), shortManifestDigest(latest.Digest), ref.Tag) + if renderDrift { + fmt.Fprintf(w, " rendered compose differs — review before `%s template update %s`\n", rootName(cmd), name) + } else { + fmt.Fprintln(w, " digest moved but the rendered compose is identical") + } + return nil + }, + } + cmd.Flags().StringVar(&against, "against", "", "compare against this remote tag (default: the recorded version)") + return cmd +} + +// newTemplateVerifyCmd graduates `template verify [name]` (spec 19). +func newTemplateVerifyCmd(g *GlobalOpts) *cobra.Command { + var policy registry.VerifyPolicy + cmd := &cobra.Command{ + Use: "verify [name]", + Short: "Re-pull pinned templates and re-check digest (and signature)", + Long: "verify re-pulls each registered template at its pinned digest and asserts no\n" + + "drift; with --identity/--issuer (keyless) or --key (keyed) it also re-checks the\n" + + "cosign signature. A mismatch or bad signature is a hard error.", + Args: cobra.RangeArgs(0, 1), + RunE: func(cmd *cobra.Command, args []string) error { + cfg, ok, err := store.Load() + if err != nil { + return err + } + if !ok || len(cfg.Templates) == 0 { + return fmt.Errorf("no remote templates registered") + } + var targets []store.RemoteTemplate + if len(args) == 1 { + rt, found := cfg.Template(args[0]) + if !found { + return fmt.Errorf("template %q is not registered", args[0]) + } + targets = []store.RemoteTemplate{rt} + } else { + targets = cfg.Templates + } + client, err := newRegistryClient() + if err != nil { + return err + } + ctx := context.Background() + type result struct { + Name string `json:"name"` + Digest string `json:"digest"` + Verified bool `json:"verified"` + Signed bool `json:"signed"` + } + var results []result + for _, rt := range targets { + ref, err := registry.ParseRepository(rt.Source) + if err != nil { + return err + } + ref.Tag = rt.Version + ref.Digest = rt.Digest + pulled, err := client.Pull(ctx, ref) // enforces digest == pin + if err != nil { + return fmt.Errorf("template %q: %w", rt.Name, err) + } + if err := registry.VerifySignature(ctx, nil, pulled.Descriptor, policy); err != nil { + return fmt.Errorf("template %q: %w", rt.Name, err) + } + results = append(results, result{Name: rt.Name, Digest: rt.Digest, Verified: true, Signed: !policy.IsZero()}) + } + if g.JSON { + return writeJSON(cmd, map[string]any{"ok": true, "templates": results}) + } + if !g.Quiet { + w := cmd.OutOrStdout() + for _, r := range results { + sig := "" + if r.Signed { + sig = " + signature" + } + fmt.Fprintf(w, "%-16s verified %s%s\n", r.Name, shortManifestDigest(r.Digest), sig) + } + } + return nil + }, + } + registerVerifyFlags(cmd, &policy) + return cmd +} + +// newTemplateLsCmd lists registered remote templates + digests + cache state. +func newTemplateLsCmd(g *GlobalOpts) *cobra.Command { + return &cobra.Command{ + Use: "ls", + Short: "List pinned remote templates and their cache state", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + cfg, ok, _ := store.Load() + type row struct { + Name string `json:"name"` + Source string `json:"source"` + Version string `json:"version"` + Digest string `json:"digest"` + Cached bool `json:"cached"` + } + var rows []row + if ok { + for _, rt := range cfg.Templates { + rows = append(rows, row{rt.Name, rt.Source, rt.Version, rt.Digest, templateCached(rt)}) + } + } + sort.Slice(rows, func(i, j int) bool { return rows[i].Name < rows[j].Name }) + if g.JSON { + return writeJSON(cmd, map[string]any{"templates": rows}) + } + w := cmd.OutOrStdout() + if len(rows) == 0 { + if !g.Quiet { + fmt.Fprintln(w, "no remote templates registered") + } + return nil + } + for _, r := range rows { + state := "cached" + if !r.Cached { + state = "cold" + } + fmt.Fprintf(w, "%-16s %s@%s [%s]\n", r.Name, r.Source, r.Version, state) + fmt.Fprintf(w, " %s (%s)\n", r.Digest, state) + } + return nil + }, + } +} + +// addOrUpdate pins a ref, verifies + caches the bundle, and records the lockfile +// entry — the shared body of `add`. +func addOrUpdate(cmd *cobra.Command, g *GlobalOpts, ref registry.Reference, policy registry.VerifyPolicy) error { + desc, err := doPin(context.Background(), ref, policy) + if err != nil { + return err + } + if g.JSON { + return writeJSON(cmd, desc) + } + if !g.Quiet { + w := cmd.OutOrStdout() + fmt.Fprintf(w, "added %s as %q\n", desc.Ref, desc.Name) + fmt.Fprintf(w, " digest: %s\n", desc.Digest) + if policy.IsZero() { + fmt.Fprintln(w, " note: templates are digest-pinned but UNSIGNED — pass --identity/--issuer or --key to require a cosign signature") + } else { + fmt.Fprintln(w, " signature: verified") + } + } + return nil +} + +// doPin performs the lock-guarded pin: resolve+pull, schema-gate, verify signature, +// atomically populate the digest cache, and upsert the store lockfile entry. It is +// the single writer of both the cache and the lockfile (spec 19 §"add/update/verify +// take the flock around the cache write"). +func doPin(ctx context.Context, ref registry.Reference, policy registry.VerifyPolicy) (registry.Descriptor, error) { + client, err := newRegistryClient() + if err != nil { + return registry.Descriptor{}, err + } + pulled, err := client.Pull(ctx, ref) + if err != nil { + return registry.Descriptor{}, err + } + desc := pulled.Descriptor + + // Signature gate before anything is trusted / unpacked. + if err := registry.VerifySignature(ctx, nil, desc, policy); err != nil { + return registry.Descriptor{}, err + } + + var result registry.Descriptor + lockErr := lock.WithLock(ctx, lockPath(), func() error { + cacheDir, err := store.TemplateCacheDir(desc.Digest) + if err != nil { + return err + } + name, err := materializeBundle(pulled.Tar, cacheDir) + if err != nil { + return err + } + // Schema-version gate: refuse a bundle newer than this binary understands. + sv, err := bundleSchemaVersion(cacheDir, name) + if err != nil { + return err + } + if sv > maxTemplateSchemaVersion { + return fmt.Errorf("template %q declares schemaVersion %d but this devstack understands up to %d — upgrade devstack to use this template", name, sv, maxTemplateSchemaVersion) + } + + cfg, ok, err := store.Load() + if err != nil { + return err + } + if !ok { + c := store.DefaultConfig() + cfg = &c + } + cfg.UpsertTemplate(store.RemoteTemplate{ + Name: name, + Source: registry.Scheme + ref.Registry + "/" + ref.Repository, + Version: ref.Tag, + Digest: desc.Digest, + SchemaVersion: sv, + }) + if err := cfg.Save(); err != nil { + return err + } + result = registry.Descriptor{ + Ref: registry.Scheme + ref.Registry + "/" + ref.Repository + "@" + desc.Digest, + Repository: ref.Name(), + Tag: ref.Tag, + Digest: desc.Digest, + SchemaVersion: sv, + Name: name, + Size: desc.Size, + } + return nil + }) + if lockErr != nil { + return registry.Descriptor{}, lockErr + } + return result, nil +} + +// registerVerifyFlags wires the cosign policy flags onto a command. Enabled is +// derived in a PreRunE: any of the three flags being set turns verification on +// (an explicit signature policy), otherwise the digest pin stands alone. +func registerVerifyFlags(cmd *cobra.Command, p *registry.VerifyPolicy) { + cmd.Flags().StringVar(&p.IdentityRegexp, "identity", "", "require a keyless cosign signature whose identity matches this regexp") + cmd.Flags().StringVar(&p.OIDCIssuer, "issuer", "", "require this keyless cosign OIDC issuer") + cmd.Flags().StringVar(&p.KeyPath, "key", "", "require a keyed cosign signature verifiable with this public key") + orig := cmd.PreRunE + cmd.PreRunE = func(c *cobra.Command, args []string) error { + p.Enabled = p.IdentityRegexp != "" || p.OIDCIssuer != "" || p.KeyPath != "" + if orig != nil { + return orig(c, args) + } + return nil + } +} diff --git a/internal/cli/template_registry_helpers.go b/internal/cli/template_registry_helpers.go new file mode 100644 index 0000000..ebb3684 --- /dev/null +++ b/internal/cli/template_registry_helpers.go @@ -0,0 +1,146 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/goccy/go-yaml" + + "github.com/open-source-cloud/devstack/internal/generate" + "github.com/open-source-cloud/devstack/internal/registry" + "github.com/open-source-cloud/devstack/internal/store" + "github.com/open-source-cloud/devstack/internal/template" +) + +// materializeBundle unpacks a bundle tar into the digest-keyed cache dir, atomically +// and idempotently. When cacheDir already holds the bundle (a prior pull of the same +// digest), it is reused as-is — the digest is the content address, so identical +// digest ⇒ identical content. Otherwise the tar is unpacked into a sibling temp dir +// and renamed into place, so a kill mid-unpack never leaves a half-populated, +// wrongly-trusted cache entry (spec 19 §"unpack atomically"). Returns the bundle's +// template name (its single top-level directory). +func materializeBundle(tarData []byte, cacheDir string) (string, error) { + // Fast path: already cached — find the single top-level template dir. + if name, ok := cachedBundleName(cacheDir); ok { + return name, nil + } + if err := os.MkdirAll(filepath.Dir(cacheDir), 0o755); err != nil { + return "", err + } + tmp, err := os.MkdirTemp(filepath.Dir(cacheDir), ".tmp-tmpl-*") + if err != nil { + return "", err + } + defer os.RemoveAll(tmp) + + name, err := registry.UnpackBundle(tarData, tmp) + if err != nil { + return "", err + } + // Rename the populated temp dir into the final cache dir. If a concurrent pin + // won the race (dir now exists), reuse it. + if err := os.Rename(tmp, cacheDir); err != nil { + if _, statErr := os.Stat(cacheDir); statErr == nil { + return name, nil + } + return "", err + } + return name, nil +} + +// cachedBundleName returns the single template-name subdir of a populated cache +// dir, or ("", false) when the dir is absent/empty. +func cachedBundleName(cacheDir string) (string, bool) { + entries, err := os.ReadDir(cacheDir) + if err != nil { + return "", false + } + for _, e := range entries { + if !e.IsDir() { + continue + } + if _, err := os.Stat(filepath.Join(cacheDir, e.Name(), template.TemplateFile)); err == nil { + return e.Name(), true + } + } + return "", false +} + +// bundleSchemaVersion reads the cached template's declared schemaVersion. +func bundleSchemaVersion(cacheDir, name string) (int, error) { + raw, err := os.ReadFile(filepath.Join(cacheDir, name, template.TemplateFile)) + if err != nil { + return 0, err + } + var m struct { + SchemaVersion int `yaml:"schemaVersion"` + } + if err := yaml.Unmarshal(raw, &m); err != nil { + return 0, fmt.Errorf("template %q: %s: %w", name, template.TemplateFile, err) + } + return m.SchemaVersion, nil +} + +// renderPinned renders the pinned (cached) template's single-service compose, +// resolving its extends chain through the built-in source (a base may be a +// built-in). Used by `template diff` to compare against the remote render. +func renderPinned(name, digest string) ([]byte, error) { + dir, err := store.TemplateCacheDir(digest) + if err != nil { + return nil, err + } + src := template.NewChainSource(template.NewFSSource(os.DirFS(dir)), builtinSource()) + return renderVia(src, name) +} + +// renderPulled renders a freshly-pulled bundle by unpacking it to a temp dir and +// resolving through the built-in source. Used by `template diff`. +func renderPulled(name string, pulled registry.Pulled) ([]byte, error) { + tmp, err := os.MkdirTemp("", "devstack-diff-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(tmp) + top, err := registry.UnpackBundle(pulled.Tar, tmp) + if err != nil { + return nil, err + } + src := template.NewChainSource(template.NewFSSource(os.DirFS(tmp)), builtinSource()) + return renderVia(src, top) +} + +// renderVia resolves name through src and renders its single-service compose. +func renderVia(src template.TemplateSource, name string) ([]byte, error) { + res, err := template.Resolve(src, name, nil) + if err != nil { + return nil, err + } + return generate.LintResolved(name, res) +} + +// refWithTagDigest returns a copy of ref pinned to digest (tag preserved). +func refWithTagDigest(ref registry.Reference, digest string) registry.Reference { + ref.Digest = digest + return ref +} + +// templateCached reports whether a registered template's content is present in the +// digest cache. +func templateCached(rt store.RemoteTemplate) bool { + dir, err := store.TemplateCacheDir(rt.Digest) + if err != nil { + return false + } + _, ok := cachedBundleName(dir) + return ok +} + +// shortDigest renders the first 12 hex chars of a sha256:… digest for display. +func shortManifestDigest(d string) string { + const prefix = "sha256:" + if len(d) > len(prefix)+12 { + return d[len(prefix) : len(prefix)+12] + } + return d +} diff --git a/internal/cli/template_registry_test.go b/internal/cli/template_registry_test.go new file mode 100644 index 0000000..3d77daf --- /dev/null +++ b/internal/cli/template_registry_test.go @@ -0,0 +1,240 @@ +package cli + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "oras.land/oras-go/v2/content/memory" + + "github.com/open-source-cloud/devstack/internal/registry" + "github.com/open-source-cloud/devstack/internal/store" +) + +// registryHarness isolates the store home, digest cache, and lock dir, and points +// the registry-client seam at a single shared in-memory oras store so the whole +// push/add/update/diff/verify flow round-trips with no network. +func registryHarness(t *testing.T) { + t.Helper() + tmp := t.TempDir() + t.Setenv("DEVSTACK_HOME", filepath.Join(tmp, "home")) + t.Setenv("XDG_CACHE_HOME", filepath.Join(tmp, "cache")) + t.Setenv("XDG_RUNTIME_DIR", filepath.Join(tmp, "run")) + + mem := memory.New() + resolver := func(context.Context, registry.Reference) (registry.Target, error) { return mem, nil } + orig := newRegistryClient + newRegistryClient = func() (*registry.Client, error) { return registry.NewWithResolver(resolver), nil } + t.Cleanup(func() { newRegistryClient = orig }) +} + +// writeCLIBundle scaffolds a template bundle dir named `name` with the given image +// tag baked into template.yaml, and returns its path. +func writeCLIBundle(t *testing.T, name, image string, schemaVersion int) string { + t.Helper() + dir := filepath.Join(t.TempDir(), name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + manifest := "schemaVersion: " + itoa(schemaVersion) + "\ndescription: \"t\"\nservice:\n image: " + image + "\n" + if err := os.WriteFile(filepath.Join(dir, "template.yaml"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + return dir +} + +func itoa(i int) string { + if i == 0 { + return "0" + } + digits := "" + for i > 0 { + digits = string(rune('0'+i%10)) + digits + i /= 10 + } + return digits +} + +func TestTemplateAddPinsDigest(t *testing.T) { + registryHarness(t) + dir := writeCLIBundle(t, "acmepg", "postgres:16", 1) + + // push then add. + if out, err := runCmd(t, "template", "push", dir, "oci://ghcr.io/acme/acmepg:1.0.0"); err != nil { + t.Fatalf("push: %v\n%s", err, out) + } + out, err := runCmd(t, "template", "add", "oci://ghcr.io/acme/acmepg:1.0.0", "--json") + if err != nil { + t.Fatalf("add: %v\n%s", err, out) + } + var desc registry.Descriptor + if err := json.Unmarshal([]byte(out), &desc); err != nil { + t.Fatalf("parse add json: %v\n%s", err, out) + } + if desc.Name != "acmepg" || !strings.HasPrefix(desc.Digest, "sha256:") { + t.Fatalf("unexpected add descriptor: %+v", desc) + } + + // The store config carries a pinned entry: the human version + the digest. + cfg, ok, err := store.Load() + if err != nil || !ok { + t.Fatalf("store load: ok=%v err=%v", ok, err) + } + rt, found := cfg.Template("acmepg") + if !found { + t.Fatal("acmepg not registered in the store config") + } + if rt.Version != "1.0.0" { + t.Errorf("recorded version = %q, want 1.0.0 (provenance)", rt.Version) + } + if rt.Digest != desc.Digest { + t.Errorf("lockfile digest %q != resolved %q — generation must fetch by digest", rt.Digest, desc.Digest) + } + + // The bundle content is cached under the digest key and resolves by name. + cacheDir, _ := store.TemplateCacheDir(rt.Digest) + if _, err := os.Stat(filepath.Join(cacheDir, "acmepg", "template.yaml")); err != nil { + t.Errorf("bundle not cached under digest key: %v", err) + } + if src := remoteTemplateSource(); src == nil || !src.Has("acmepg") { + t.Error("registered remote template does not resolve in the source chain") + } +} + +func TestTemplateAddRejectsNewerSchema(t *testing.T) { + registryHarness(t) + dir := writeCLIBundle(t, "future", "redis:7", maxTemplateSchemaVersion+1) + if out, err := runCmd(t, "template", "push", dir, "oci://ghcr.io/acme/future:1.0.0"); err != nil { + t.Fatalf("push: %v\n%s", err, out) + } + _, err := runCmd(t, "template", "add", "oci://ghcr.io/acme/future:1.0.0") + if err == nil { + t.Fatal("adding a bundle with a newer schemaVersion must fail with an upgrade message") + } + if !strings.Contains(err.Error(), "upgrade devstack") { + t.Errorf("error should tell the user to upgrade, got: %v", err) + } +} + +func TestTemplateAddRejectsFloatingTag(t *testing.T) { + registryHarness(t) + if _, err := runCmd(t, "template", "add", "oci://ghcr.io/acme/x:latest"); err == nil { + t.Fatal("add must refuse a floating :latest tag without --allow-floating") + } +} + +func TestTemplateUpdateAndDiffDetectDrift(t *testing.T) { + registryHarness(t) + + pgV1 := writeCLIBundle(t, "acmepg", "postgres:16", 1) + if _, err := runCmd(t, "template", "push", pgV1, "oci://ghcr.io/acme/acmepg:1.0.0"); err != nil { + t.Fatal(err) + } + if _, err := runCmd(t, "template", "add", "oci://ghcr.io/acme/acmepg:1.0.0"); err != nil { + t.Fatal(err) + } + cfg, _, _ := store.Load() + rt, _ := cfg.Template("acmepg") + oldDigest := rt.Digest + + // Re-push DIFFERENT content to the SAME tag: the pinned workspace is unaffected + // until update, but diff detects the drift. + pgV2 := writeCLIBundle(t, "acmepg", "postgres:17", 1) + if _, err := runCmd(t, "template", "push", pgV2, "oci://ghcr.io/acme/acmepg:1.0.0"); err != nil { + t.Fatal(err) + } + + diffOut, err := runCmd(t, "template", "diff", "acmepg", "--json") + if err != nil { + t.Fatalf("diff: %v\n%s", err, diffOut) + } + var d struct { + DigestDrift bool `json:"digestDrift"` + RenderDrift bool `json:"renderDrift"` + } + if err := json.Unmarshal([]byte(diffOut), &d); err != nil { + t.Fatalf("parse diff json: %v\n%s", err, diffOut) + } + if !d.DigestDrift || !d.RenderDrift { + t.Fatalf("diff should report digest + render drift, got %+v", d) + } + + // dry-run update reports the move but does NOT rewrite the lockfile. + if _, err := runCmd(t, "template", "update", "acmepg", "--dry-run"); err != nil { + t.Fatal(err) + } + cfg, _, _ = store.Load() + rt, _ = cfg.Template("acmepg") + if rt.Digest != oldDigest { + t.Error("--dry-run must not rewrite the pinned digest") + } + + // A real update rewrites the pin to the new digest. + if _, err := runCmd(t, "template", "update", "acmepg"); err != nil { + t.Fatal(err) + } + cfg, _, _ = store.Load() + rt, _ = cfg.Template("acmepg") + if rt.Digest == oldDigest { + t.Error("update must move the pinned digest to the re-pushed content") + } + + // After update, diff is clean again. + if out, _ := runCmd(t, "template", "diff", "acmepg"); !strings.Contains(out, "up to date") { + t.Errorf("diff after update should be up to date, got: %s", out) + } +} + +func TestTemplateVerifySignaturePolicy(t *testing.T) { + registryHarness(t) + dir := writeCLIBundle(t, "acmepg", "postgres:16", 1) + if _, err := runCmd(t, "template", "push", dir, "oci://ghcr.io/acme/acmepg:1.0.0"); err != nil { + t.Fatal(err) + } + if _, err := runCmd(t, "template", "add", "oci://ghcr.io/acme/acmepg:1.0.0"); err != nil { + t.Fatal(err) + } + + // Inject a fake verifier accepting the signature: verify with a keyless policy passes. + origV := registry.DefaultVerifier + registry.DefaultVerifier = &acceptVerifier{} + t.Cleanup(func() { registry.DefaultVerifier = origV }) + if out, err := runCmd(t, "template", "verify", "acmepg", "--identity", "^https://github.com/acme/.+$", "--issuer", "https://token.actions.githubusercontent.com"); err != nil { + t.Fatalf("verify with good signature should pass: %v\n%s", err, out) + } + + // A rejecting verifier fails verify. + registry.DefaultVerifier = &rejectVerifier{} + if _, err := runCmd(t, "template", "verify", "acmepg", "--identity", "^https://github.com/acme/.+$", "--issuer", "https://token.actions.githubusercontent.com"); err == nil { + t.Fatal("verify with a bad signature must fail") + } + + // Without a policy, verify only re-checks the digest pin (no cosign needed). + if out, err := runCmd(t, "template", "verify", "acmepg"); err != nil { + t.Fatalf("digest-only verify should pass without cosign: %v\n%s", err, out) + } +} + +// acceptVerifier / rejectVerifier are fake registry.Verifiers for CLI tests. +type acceptVerifier struct{} + +func (acceptVerifier) Available() bool { return true } +func (acceptVerifier) Verify(context.Context, string, registry.VerifyPolicy) error { + return nil +} + +type rejectVerifier struct{} + +func (rejectVerifier) Available() bool { return true } +func (rejectVerifier) Verify(context.Context, string, registry.VerifyPolicy) error { + return errBadSig +} + +var errBadSig = &cliError{"signature mismatch"} + +type cliError struct{ s string } + +func (e *cliError) Error() string { return e.s } diff --git a/internal/cli/up.go b/internal/cli/up.go index 0e22f80..a308810 100644 --- a/internal/cli/up.go +++ b/internal/cli/up.go @@ -209,7 +209,7 @@ func downProject(ctx context.Context, d orchestrate.UpDeps, project string) erro } } - cp := docker.Compose{Project: "devstack-" + project, File: composeFile, Dir: outDir, Runner: docker.ExecRunner{}} + cp := docker.Compose{Project: "devstack-" + project, File: composeFile, Dir: outDir, Runner: docker.ExecRunner{}, ContextEnv: d.Backend.ComposeEnv()} if err := cp.Down(ctx, false); err != nil { return err } @@ -233,7 +233,11 @@ func buildUpDeps(cmd *cobra.Command) (orchestrate.UpDeps, func(), error) { return zero, nil, err } ctx := cmd.Context() - dc, err := docker.NewClient(ctx) + // Resolve the Docker backend (spec 21): local (default) or a remote context/ + // DOCKER_HOST. The read-only client and the ledger's context key both bind to + // this backend, so remote rows never bleed into the local counts. + backend := backendFor(model) + dc, err := backend.NewClient(ctx) if err != nil { return zero, nil, fmt.Errorf("docker client: %w", err) } @@ -245,7 +249,7 @@ func buildUpDeps(cmd *cobra.Command) (orchestrate.UpDeps, func(), error) { lockPath := filepath.Join(xdg.RuntimeDir(), "devstack.lock") mgr := &workspace.Manager{Model: model, DB: db, Docker: dc, Source: builtinSource(), LockPath: lockPath} d := orchestrate.UpDeps{ - Model: model, DB: db, Docker: dc, Manager: mgr, + Model: model, DB: db, Docker: dc, Manager: mgr, Backend: backend, Source: mgr.Source, LockPath: lockPath, } closeFn := func() { diff --git a/internal/config/backend_test.go b/internal/config/backend_test.go new file mode 100644 index 0000000..6be4361 --- /dev/null +++ b/internal/config/backend_test.go @@ -0,0 +1,91 @@ +package config + +import ( + "strings" + "testing" +) + +// minimalWS is a valid single-file workspace (no projects) with the given +// backend: block spliced in. A shared postgres keeps it non-trivial. +func minimalWS(backendBlock string) string { + return `apiVersion: devstack/v1 +kind: Workspace +name: acme +` + backendBlock + ` +shared: + postgres: + template: postgres + params: + version: "16" +` +} + +func loadWS(t *testing.T, backendBlock string) (*Model, error) { + t.Helper() + root := writeTree(t, map[string]string{"workspace.yaml": minimalWS(backendBlock)}) + return LoadAt(root) +} + +func TestBackendDefaultLocal(t *testing.T) { + m, err := loadWS(t, "") + if err != nil { + t.Fatalf("LoadAt: %v", err) + } + if m.Workspace.Backend != nil { + t.Errorf("Backend = %+v, want nil (default local)", m.Workspace.Backend) + } + if m.Workspace.Backend.IsRemote() { + t.Error("nil Backend.IsRemote() = true, want false") + } +} + +func TestBackendContextValid(t *testing.T) { + m, err := loadWS(t, "backend:\n context: prod-cluster") + if err != nil { + t.Fatalf("LoadAt: %v", err) + } + if m.Workspace.Backend == nil || m.Workspace.Backend.Context != "prod-cluster" { + t.Fatalf("Backend = %+v, want context=prod-cluster", m.Workspace.Backend) + } + if !m.Workspace.Backend.IsRemote() { + t.Error("Backend.IsRemote() = false, want true") + } +} + +func TestBackendHostValid(t *testing.T) { + m, err := loadWS(t, "backend:\n host: ssh://dev@build-host") + if err != nil { + t.Fatalf("LoadAt: %v", err) + } + if m.Workspace.Backend == nil || m.Workspace.Backend.Host != "ssh://dev@build-host" { + t.Fatalf("Backend = %+v, want host=ssh://dev@build-host", m.Workspace.Backend) + } +} + +func TestBackendHostBadScheme(t *testing.T) { + _, err := loadWS(t, "backend:\n host: dev@build-host") + if err == nil { + t.Fatal("expected validation error for a scheme-less DOCKER_HOST") + } + if !strings.Contains(err.Error(), "dockerhost") && !strings.Contains(err.Error(), "Host") { + t.Errorf("error = %q, want it to mention the host field/rule", err) + } +} + +func TestBackendContextAndHostMutuallyExclusive(t *testing.T) { + _, err := loadWS(t, "backend:\n context: prod\n host: ssh://dev@box") + if err == nil { + t.Fatal("expected error when both context and host are set") + } + if !strings.Contains(err.Error(), "not both") { + t.Errorf("error = %q, want it to explain context/host are mutually exclusive", err) + } +} + +func TestBackendHostSchemes(t *testing.T) { + for _, host := range []string{"ssh://u@h", "tcp://1.2.3.4:2376", "unix:///var/run/docker.sock"} { + if _, err := loadWS(t, "backend:\n host: "+host); err != nil { + t.Errorf("host %q rejected: %v", host, err) + } + } +} diff --git a/internal/config/model.go b/internal/config/model.go index 15787a5..8a86b0a 100644 --- a/internal/config/model.go +++ b/internal/config/model.go @@ -31,11 +31,34 @@ type Workspace struct { MemoryBudgetMB int `yaml:"memoryBudgetMB"` // spec 12/18 — warn when active services' memoryMB sum exceeds this Secrets Secrets `yaml:"secrets"` Network Network `yaml:"network"` - Hooks Hooks `yaml:"hooks"` // spec 11 — workspace-scope lifecycle hooks + Backend *BackendConfig `yaml:"backend"` // spec 21 — where the shared stack runs (nil = local) + Hooks Hooks `yaml:"hooks"` // spec 11 — workspace-scope lifecycle hooks Shared map[string]SharedSvc `yaml:"shared" validate:"dive"` Projects []ProjectRef `yaml:"projects" validate:"dive"` } +// BackendConfig selects WHERE the shared stack runs (spec 21). nil / the zero +// value means the LOCAL Docker daemon (the active context / DOCKER_HOST) — the +// default and fully backward-compatible. Exactly one of Context or Host may be +// set to target a remote host: +// - Context names a `docker context` (typically an ssh:// one, which inherits +// the user's SSH config/agent/ProxyJump for free — DECISIONS D9). +// - Host is a raw DOCKER_HOST endpoint (ssh://, tcp://, unix://). +// +// The ledger is already keyed by Docker context (spec 08), so a remote endpoint +// simply keys its own rows — no cross-context count bleed with the local daemon. +// The mutual-exclusion of Context vs Host is enforced by the cross-field +// validator (validateBackend); the host scheme by the `dockerhost` field rule. +type BackendConfig struct { + Context string `yaml:"context"` + Host string `yaml:"host" validate:"omitempty,dockerhost"` +} + +// IsRemote reports whether this backend targets a non-local endpoint. +func (b *BackendConfig) IsRemote() bool { + return b != nil && (b.Context != "" || b.Host != "") +} + // Profiles selects the env OVERLAY (config layering), distinct from the service // slices in spec 12. Default overlay is `dev`. type Profiles struct { diff --git a/internal/config/validate.go b/internal/config/validate.go index e79850a..5d352d1 100644 --- a/internal/config/validate.go +++ b/internal/config/validate.go @@ -44,9 +44,21 @@ func newValidator() *validator.Validate { _ = v.RegisterValidation("platform", func(fl validator.FieldLevel) bool { return platformRE.MatchString(fl.Field().String()) }) + // dockerhost: a DOCKER_HOST endpoint for the remote-backend selector (spec 21). + // Paired with omitempty so an unset field is allowed; only a non-empty value is + // checked against the schemes docker understands. + _ = v.RegisterValidation("dockerhost", func(fl validator.FieldLevel) bool { + return dockerHostRE.MatchString(fl.Field().String()) + }) return v } +// dockerHostRE matches the DOCKER_HOST endpoint schemes the docker CLI accepts +// (spec 21). ssh:// is the primary remote path (inherits the user's SSH setup); +// tcp:// (optionally TLS-guarded) and the local unix://npipe:// sockets round it +// out. Deliberately strict so a typo'd endpoint fails at config-load, not mid-up. +var dockerHostRE = regexp.MustCompile(`^(ssh|tcp|unix|npipe|fd)://.+`) + // platformRE matches a compose `platform:` selector: os/arch with an optional // variant (e.g. linux/amd64, linux/arm64/v8). var platformRE = regexp.MustCompile(`^[a-z0-9]+/[a-z0-9]+(/[a-z0-9]+)?$`) @@ -84,9 +96,30 @@ func validateModel(m *Model, ws *source, projSrc map[string]*source) error { if err := validateResources(m, projSrc); err != nil { return err } + if err := validateBackend(m, ws); err != nil { + return err + } return detectCycles(m) } +// validateBackend enforces the spec-21 remote-backend selector rules that a +// single field tag cannot express: Context and Host are mutually exclusive (both +// set is ambiguous — DOCKER_CONTEXT and DOCKER_HOST would contend). An unset +// block (nil) is the default local backend and always valid. Positioned to +// workspace.yaml so the error renders as file:line:col like every other config +// error. +func validateBackend(m *Model, ws *source) error { + b := m.Workspace.Backend + if b == nil { + return nil + } + if b.Context != "" && b.Host != "" { + return ws.errAt("$.backend", + "backend: set either `context` or `host`, not both (they select the remote Docker endpoint two different ways)") + } + return nil +} + // validateResources checks the spec-27 declarative `resources:` block on each // project: every `uses` targets a declared shared instance, `kind` is one the // target engine's provisioner supports (for known engines — unknown/custom diff --git a/internal/docker/backend.go b/internal/docker/backend.go new file mode 100644 index 0000000..810f5cc --- /dev/null +++ b/internal/docker/backend.go @@ -0,0 +1,194 @@ +package docker + +import ( + "context" + "fmt" + "os/exec" + "strings" + + moby "github.com/moby/moby/client" +) + +// Backend selects WHERE the shared stack (and, in the all-remote topology, the +// project stacks) run — the local Docker daemon or a remote host reached over an +// SSH/TCP `docker context` or a DOCKER_HOST endpoint (spec 21). It is the seam +// that lets internal/workspace stay written against capabilities rather than a +// socket: the ref-counting, provisioning, and DNS/alias model are backend- +// agnostic; a Backend only swaps the Docker endpoint the CLI + read-only SDK +// target, and the reachability strategy for host-side consumers. +// +// The zero value is the LOCAL backend and reproduces today's behavior verbatim +// (default socket / the active docker context, host-routable published ports). +// This is the default path and MUST stay byte-for-byte unchanged. +// +// Exactly one of Context or Host is set for a remote backend: +// - Context names a `docker context` (typically an `ssh://` one, which inherits +// the user's ~/.ssh/config, agent, and ProxyJump for free — DECISIONS D9). +// - Host is a raw DOCKER_HOST endpoint (ssh://, tcp://, unix://). +// +// Scope note (spec 21 is a frontier spec): this pass delivers the single-user, +// SSH-context, all-remote topology with the local flock still serializing a +// single developer's invocations. The distributed lock (two developers, two +// machines) and per-user tenant isolation on a shared cluster are the ~8w +// follow-up and are deliberately NOT implemented here (see the PR body). +type Backend struct { + // Context is the `docker context` name to target (empty = not selected). + Context string + // Host is the DOCKER_HOST endpoint to target (empty = not selected). + Host string +} + +// Reachability classifies how a host-side consumer (host pgx, a psql/GUI) can +// reach the shared services, which drives the port/URL strategy (spec 21 +// §Network reachability). +type Reachability string + +const ( + // HostRoutable: services can publish host ports the developer's machine can + // dial on loopback (the local backend, and Docker Desktop's port proxy). + HostRoutable Reachability = "host-routable" + // ViaProxy: a remote bridge network is on another host entirely and is NOT + // host-routable — host access must go through the proxy/tunnel (an SSH local + // -forward or cloudflared), never a local loopback published port. + ViaProxy Reachability = "via-proxy" +) + +// LocalBackend is the explicit local (default socket / active context) backend. +func LocalBackend() Backend { return Backend{} } + +// IsRemote reports whether this backend targets a non-local endpoint. +func (b Backend) IsRemote() bool { return b.Context != "" || b.Host != "" } + +// Reachability returns the host-reachability class for this backend. Remote +// backends are ViaProxy (the remote bridge is not host-routable); the local +// backend is HostRoutable. +func (b Backend) Reachability() Reachability { + if b.IsRemote() { + return ViaProxy + } + return HostRoutable +} + +// String is a short, stable description for log/doctor output. +func (b Backend) String() string { + switch { + case b.Host != "": + return "remote host " + b.Host + case b.Context != "": + return "remote context " + b.Context + default: + return "local" + } +} + +// ComposeEnv returns the extra environment entries that pin the `docker compose` +// CLI (and any `docker` invocation) to this backend's endpoint. They are appended +// to the child process env by the Runner (exec.Cmd.Env), never written to disk — +// exactly how resolved secrets are threaded (§7.5). Empty for the local backend +// so the default path stays byte-for-byte unchanged. +// +// A remote Host sets DOCKER_HOST; a remote Context sets DOCKER_CONTEXT. They are +// mutually exclusive (validated in config) so the two env vars never contend. +func (b Backend) ComposeEnv() []string { + switch { + case b.Host != "": + return []string{"DOCKER_HOST=" + b.Host} + case b.Context != "": + return []string{"DOCKER_CONTEXT=" + b.Context} + default: + return nil + } +} + +// NewClient builds the READ-ONLY Engine SDK client bound to this backend's +// endpoint. The local backend defers to the package NewClient (FromEnv), so its +// context-name resolution — and therefore the ledger key — is unchanged. A remote +// backend binds moby to the resolved endpoint and keys the ledger by the context +// name (or the host endpoint) so remote rows never bleed into the local counts, +// exactly like the WSL2 Desktop-vs-dockerd split (DECISIONS D6, spec 08). +// +// Construction does NOT contact the daemon — call Ping (or the doctor reachability +// probe) to verify the endpoint is reachable. This matters more for a remote +// backend: an unreachable SSH host must fail with a clear, actionable error, not a +// crash mid-up. +func (b Backend) NewClient(ctx context.Context) (Client, error) { + switch { + case b.Host != "": + cli, err := moby.New(moby.WithHost(b.Host), moby.WithAPIVersionNegotiation()) + if err != nil { + return nil, fmt.Errorf("connect to remote docker host %q: %w", b.Host, err) + } + return &mobyClient{cli: cli, ctxName: b.Host}, nil + case b.Context != "": + endpoint, err := contextEndpoint(ctx, b.Context) + if err != nil { + return nil, err + } + cli, err := moby.New(moby.WithHost(endpoint), moby.WithAPIVersionNegotiation()) + if err != nil { + return nil, fmt.Errorf("connect to docker context %q (%s): %w", b.Context, endpoint, err) + } + // Key the ledger by the context NAME (stable across endpoint edits), not the + // resolved endpoint, so `docker context` rebinds don't fork the counts. + return &mobyClient{cli: cli, ctxName: b.Context}, nil + default: + return NewClient(ctx) + } +} + +// contextEndpoint resolves a `docker context`'s Docker endpoint (e.g. +// ssh://user@host) by shelling the docker CLI, which already knows how to read +// ~/.docker/contexts. Kept behind the CLI (rather than re-parsing the context +// store) so the user's existing context definitions are the single source of +// truth. A missing/mistyped context surfaces the CLI's own error verbatim. +func contextEndpoint(ctx context.Context, name string) (string, error) { + out, err := exec.CommandContext(ctx, "docker", "context", "inspect", name, + "--format", "{{.Endpoints.docker.Host}}").Output() + if err != nil { + return "", &CmdError{ + Cmd: "docker context inspect " + name, + Err: err, + Stderr: fmt.Sprintf("docker context %q not found or has no docker endpoint; "+ + "create it with `docker context create %s --docker host=ssh://user@host`", name, name), + } + } + endpoint := strings.TrimSpace(string(out)) + if endpoint == "" { + return "", fmt.Errorf("docker context %q resolves to an empty endpoint", name) + } + return endpoint, nil +} + +// RemoteReachable probes a remote backend's endpoint and returns a doctor Check. +// For the local backend it returns a zero Check with an empty Name (callers skip +// empty-named checks) — there is nothing remote to probe. The client should be +// one built via Backend.NewClient; a nil client means construction already failed. +func (b Backend) RemoteReachable(ctx context.Context, client Client) Check { + if !b.IsRemote() { + return Check{} + } + name := "remote backend (" + b.String() + ")" + remediation := "verify the endpoint: `docker --context " + b.Context + " info` (or DOCKER_HOST) works, " + + "the remote daemon is running, and your SSH access (~/.ssh/config, agent) is set up" + if b.Host != "" { + remediation = "verify `docker -H " + b.Host + " info` works and the remote daemon is reachable" + } + if client == nil { + return Check{ + Name: name, ID: "backend.remote", Category: "critical", + Status: StatusFail, Detail: "could not construct a docker client for the remote endpoint", + Remediation: remediation, + } + } + if err := client.Ping(ctx); err != nil { + return Check{ + Name: name, ID: "backend.remote", Category: "critical", + Status: StatusFail, Detail: err.Error(), Remediation: remediation, + } + } + detail := "reachable" + if sv, err := client.ServerVersion(ctx); err == nil { + detail = "Engine v" + sv + } + return Check{Name: name, ID: "backend.remote", Category: "critical", Status: StatusOK, Detail: detail} +} diff --git a/internal/docker/backend_test.go b/internal/docker/backend_test.go new file mode 100644 index 0000000..dcdf0f6 --- /dev/null +++ b/internal/docker/backend_test.go @@ -0,0 +1,163 @@ +package docker + +import ( + "context" + "errors" + "testing" +) + +func TestBackendIsRemoteAndReachability(t *testing.T) { + tests := []struct { + name string + b Backend + remote bool + reach Reachability + compose []string + }{ + {"local zero value", Backend{}, false, HostRoutable, nil}, + {"remote context", Backend{Context: "prod"}, true, ViaProxy, []string{"DOCKER_CONTEXT=prod"}}, + {"remote host", Backend{Host: "ssh://dev@box"}, true, ViaProxy, []string{"DOCKER_HOST=ssh://dev@box"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.b.IsRemote(); got != tt.remote { + t.Errorf("IsRemote() = %v, want %v", got, tt.remote) + } + if got := tt.b.Reachability(); got != tt.reach { + t.Errorf("Reachability() = %v, want %v", got, tt.reach) + } + got := tt.b.ComposeEnv() + if len(got) != len(tt.compose) { + t.Fatalf("ComposeEnv() = %v, want %v", got, tt.compose) + } + for i := range got { + if got[i] != tt.compose[i] { + t.Errorf("ComposeEnv()[%d] = %q, want %q", i, got[i], tt.compose[i]) + } + } + }) + } +} + +// TestLocalBackendComposeEnvEmpty pins the default-path invariant: the local +// backend adds NOTHING to the compose environment, so local behavior is +// byte-for-byte unchanged. +func TestLocalBackendComposeEnvEmpty(t *testing.T) { + if env := LocalBackend().ComposeEnv(); env != nil { + t.Fatalf("local ComposeEnv() = %v, want nil", env) + } + if LocalBackend().IsRemote() { + t.Fatal("LocalBackend().IsRemote() = true, want false") + } +} + +func TestBackendString(t *testing.T) { + tests := []struct { + b Backend + want string + }{ + {Backend{}, "local"}, + {Backend{Context: "prod"}, "remote context prod"}, + {Backend{Host: "ssh://dev@box"}, "remote host ssh://dev@box"}, + } + for _, tt := range tests { + if got := tt.b.String(); got != tt.want { + t.Errorf("Backend%+v.String() = %q, want %q", tt.b, got, tt.want) + } + } +} + +// TestComposeContextEnvMerge verifies the ContextEnv threads through to the +// runner AFTER the secret Env, so a remote endpoint pins compose without +// clobbering resolved-secret values — using a fake runner (no real docker). +func TestComposeContextEnvMerge(t *testing.T) { + fr := &fakeEnvRunner{} + cp := Compose{ + Project: "devstack-app", + File: "compose.yaml", + Dir: "/tmp/x", + Env: []string{"SECRET=shh"}, + ContextEnv: Backend{Context: "prod"}.ComposeEnv(), + Runner: fr, + } + if err := cp.Up(context.Background()); err != nil { + t.Fatalf("Up: %v", err) + } + want := []string{"SECRET=shh", "DOCKER_CONTEXT=prod"} + if len(fr.env) != len(want) { + t.Fatalf("runner env = %v, want %v", fr.env, want) + } + for i := range want { + if fr.env[i] != want[i] { + t.Errorf("env[%d] = %q, want %q", i, fr.env[i], want[i]) + } + } + // The docker CLI args are unchanged by the backend selection (endpoint is env, + // not an arg) — the up verb is still `compose -p ... -f ... up -d`. + if len(fr.args) == 0 || fr.args[0] != "compose" { + t.Errorf("args = %v, want to start with compose", fr.args) + } +} + +// TestComposeLocalEnvUnchanged pins that a local Compose (no ContextEnv) passes +// exactly its secret Env — the default path is untouched. +func TestComposeLocalEnvUnchanged(t *testing.T) { + fr := &fakeEnvRunner{} + cp := Compose{Project: "p", File: "f", Env: []string{"SECRET=shh"}, Runner: fr} + if err := cp.Down(context.Background(), false); err != nil { + t.Fatalf("Down: %v", err) + } + if len(fr.env) != 1 || fr.env[0] != "SECRET=shh" { + t.Fatalf("runner env = %v, want [SECRET=shh]", fr.env) + } +} + +func TestRemoteReachable(t *testing.T) { + ctx := context.Background() + + // Local backend → empty check (nothing remote to probe). + if c := LocalBackend().RemoteReachable(ctx, &MockClient{}); c.Name != "" { + t.Errorf("local RemoteReachable Name = %q, want empty", c.Name) + } + + b := Backend{Context: "prod"} + + // Nil client → fail (construction already failed upstream). + if c := b.RemoteReachable(ctx, nil); c.Status != StatusFail || c.ID != "backend.remote" { + t.Errorf("nil-client check = %+v, want fail/backend.remote", c) + } + + // Ping error → fail with the error detail and a remediation. + pingErr := &MockClient{PingErr: errors.New("dial tcp: connection refused")} + if c := b.RemoteReachable(ctx, pingErr); c.Status != StatusFail || c.Remediation == "" { + t.Errorf("ping-error check = %+v, want fail with remediation", c) + } + + // Reachable → ok, reporting the engine version. + ok := &MockClient{Server: "27.1.1"} + c := b.RemoteReachable(ctx, ok) + if c.Status != StatusOK { + t.Fatalf("reachable check = %+v, want ok", c) + } + if c.Detail != "Engine v27.1.1" { + t.Errorf("detail = %q, want Engine v27.1.1", c.Detail) + } +} + +// fakeEnvRunner records the env + args it was called with (no real exec). +type fakeEnvRunner struct { + env []string + args []string +} + +func (f *fakeEnvRunner) Run(_ context.Context, env []string, _ /*dir*/, _ /*name*/ string, args ...string) error { + f.env = env + f.args = args + return nil +} + +func (f *fakeEnvRunner) Output(_ context.Context, env []string, _, _ string, args ...string) ([]byte, error) { + f.env = env + f.args = args + return nil, nil +} diff --git a/internal/docker/compose.go b/internal/docker/compose.go index c9fd78e..0c3dacb 100644 --- a/internal/docker/compose.go +++ b/internal/docker/compose.go @@ -101,7 +101,25 @@ type Compose struct { Overrides []string // additional -f overlays, applied in order after File (up-time only) Dir string // working dir (build contexts resolve relative to it) Env []string // extra env (resolved secrets), appended to os.Environ - Runner Runner + // ContextEnv pins the compose CLI to a specific Docker backend/endpoint + // (DOCKER_HOST / DOCKER_CONTEXT), from Backend.ComposeEnv() (spec 21). Empty for + // the local backend. Kept separate from Env so a remote endpoint and resolved + // secrets compose cleanly; both are appended to the child env, never written to + // a file. ContextEnv is applied AFTER Env so the endpoint selection always wins. + ContextEnv []string + Runner Runner +} + +// env returns the extra environment the Runner appends to os.Environ for every +// verb: the resolved-secret Env followed by the backend-selecting ContextEnv. +func (c *Compose) env() []string { + if len(c.ContextEnv) == 0 { + return c.Env + } + out := make([]string, 0, len(c.Env)+len(c.ContextEnv)) + out = append(out, c.Env...) + out = append(out, c.ContextEnv...) + return out } // NewCompose builds a Compose driver using the real exec runner. @@ -131,7 +149,7 @@ func (c *Compose) base() []string { func (c *Compose) Up(ctx context.Context, services ...string) error { args := append(c.base(), "up", "-d") args = append(args, services...) - return c.Runner.Run(ctx, c.Env, c.Dir, "docker", args...) + return c.Runner.Run(ctx, c.env(), c.Dir, "docker", args...) } // Down stops and removes the stack's containers (and its default network). @@ -142,7 +160,7 @@ func (c *Compose) Down(ctx context.Context, volumes bool) error { if volumes { args = append(args, "--volumes") } - return c.Runner.Run(ctx, c.Env, c.Dir, "docker", args...) + return c.Runner.Run(ctx, c.env(), c.Dir, "docker", args...) } // Stop pauses the stack (or named services) without removing containers — used @@ -150,7 +168,7 @@ func (c *Compose) Down(ctx context.Context, volumes bool) error { func (c *Compose) Stop(ctx context.Context, services ...string) error { args := append(c.base(), "stop") args = append(args, services...) - return c.Runner.Run(ctx, c.Env, c.Dir, "docker", args...) + return c.Runner.Run(ctx, c.env(), c.Dir, "docker", args...) } // Exec runs `docker compose exec` for a service, letting compose resolve the @@ -170,7 +188,7 @@ func (c *Compose) Exec(ctx context.Context, service string, interactive bool, cm } args = append(args, service) args = append(args, cmd...) - return c.Runner.Run(ctx, c.Env, c.Dir, "docker", args...) + return c.Runner.Run(ctx, c.env(), c.Dir, "docker", args...) } // Build rebuilds the named services (with --no-cache for the selective-rebuild @@ -181,5 +199,5 @@ func (c *Compose) Build(ctx context.Context, noCache bool, services ...string) e args = append(args, "--no-cache") } args = append(args, services...) - return c.Runner.Run(ctx, c.Env, c.Dir, "docker", args...) + return c.Runner.Run(ctx, c.env(), c.Dir, "docker", args...) } diff --git a/internal/orchestrate/backend_up_test.go b/internal/orchestrate/backend_up_test.go new file mode 100644 index 0000000..9381ee7 --- /dev/null +++ b/internal/orchestrate/backend_up_test.go @@ -0,0 +1,125 @@ +package orchestrate + +import ( + "context" + "strings" + "testing" + + "github.com/open-source-cloud/devstack/internal/docker" + "github.com/open-source-cloud/devstack/internal/generate" +) + +// TestBuildUpRemoteThreadsContextEnv: with a remote backend and --no-provision +// (so the host-provisioning guard doesn't fire), every compose verb — the shared +// stack up AND the project up — is pinned to the remote endpoint via DOCKER_CONTEXT, +// while the local default path adds nothing. Uses the mock client + fake runner +// (no real remote). +func TestBuildUpRemoteThreadsContextEnv(t *testing.T) { + d, fr, db := upFixture(t) + d.Backend = docker.Backend{Context: "prod-cluster"} + d.NoProvision = true // host-side provisioning over remote is the flagged follow-up + + phases, err := BuildUp(d) + if err != nil { + t.Fatalf("BuildUp: %v", err) + } + saga := &Saga{Workspace: d.Model.Workspace.Name, DB: db, LockPath: d.LockPath} + recs, err := saga.Run(context.Background(), phases) + if err != nil { + t.Fatalf("saga: %v\n%+v", err, recs) + } + if AnyFailed(recs) { + t.Fatalf("a phase failed: %+v", recs) + } + + // Both the shared stack up and the project (devstack-app) up ran, and each was + // pinned to the remote endpoint via DOCKER_CONTEXT. + if env := fr.envForUp("devstack-app"); env == nil { + t.Fatal("expected the project stack to be brought up") + } else if !contains(env, "DOCKER_CONTEXT=prod-cluster") { + t.Errorf("project up env = %v, want DOCKER_CONTEXT=prod-cluster", env) + } + if env := fr.envForUp(generate.SharedStackName); env == nil { + t.Fatal("expected the shared stack to be brought up") + } else if !contains(env, "DOCKER_CONTEXT=prod-cluster") { + t.Errorf("shared up env = %v, want DOCKER_CONTEXT=prod-cluster", env) + } + // No DOCKER_HOST leaked (context path only). + for _, env := range fr.envs { + for _, e := range env { + if strings.HasPrefix(e, "DOCKER_HOST=") { + t.Errorf("unexpected DOCKER_HOST entry %q for a context-selected backend", e) + } + } + } +} + +// TestBuildUpLocalAddsNoBackendEnv pins the default-path invariant: a local +// backend threads NO DOCKER_CONTEXT/DOCKER_HOST into compose — byte-for-byte +// unchanged from before spec 21. +func TestBuildUpLocalAddsNoBackendEnv(t *testing.T) { + d, fr, db := upFixture(t) + // d.Backend is the zero value (local) — set explicitly for clarity. + d.Backend = docker.LocalBackend() + + phases, err := BuildUp(d) + if err != nil { + t.Fatalf("BuildUp: %v", err) + } + saga := &Saga{Workspace: d.Model.Workspace.Name, DB: db, LockPath: d.LockPath} + if _, err := saga.Run(context.Background(), phases); err != nil { + t.Fatalf("saga: %v", err) + } + for _, env := range fr.envs { + for _, e := range env { + if strings.HasPrefix(e, "DOCKER_CONTEXT=") || strings.HasPrefix(e, "DOCKER_HOST=") { + t.Errorf("local backend leaked backend env %q", e) + } + } + } +} + +// TestBuildUpRemoteProvisionGuard: a remote backend with host-side provisioning +// required (a project uses shared Postgres, provisioning NOT skipped) fails the +// shared phase with a clear, actionable error rather than silently publishing a +// loopback port the laptop cannot reach (spec 21 §Network reachability). +func TestBuildUpRemoteProvisionGuard(t *testing.T) { + d, _, db := upFixture(t) + d.Backend = docker.Backend{Host: "ssh://dev@build-host"} + // NoProvision stays false → the app's shared Postgres is a provision target. + + phases, err := BuildUp(d) + if err != nil { + t.Fatalf("BuildUp: %v", err) + } + saga := &Saga{Workspace: d.Model.Workspace.Name, DB: db, LockPath: d.LockPath} + recs, err := saga.Run(context.Background(), phases) + if err == nil { + t.Fatal("expected the saga to fail on the remote-provision guard") + } + var sharedRec *Record + for i := range recs { + if recs[i].Phase == "shared" { + sharedRec = &recs[i] + } + } + if sharedRec == nil || sharedRec.Status != StatusFailed { + t.Fatalf("shared phase = %+v, want failed", sharedRec) + } + if sharedRec.Error == nil { + t.Fatal("failed shared phase carried no error message") + } + msg := *sharedRec.Error + if !strings.Contains(msg, "not supported yet") || !strings.Contains(msg, "--no-provision") { + t.Errorf("guard error = %q, want it to flag the follow-up and suggest --no-provision", msg) + } +} + +func contains(s []string, want string) bool { + for _, x := range s { + if x == want { + return true + } + } + return false +} diff --git a/internal/orchestrate/up.go b/internal/orchestrate/up.go index 68b34f1..4b66b0f 100644 --- a/internal/orchestrate/up.go +++ b/internal/orchestrate/up.go @@ -46,7 +46,14 @@ type UpDeps struct { Source template.TemplateSource LockPath string - Runner docker.Runner // compose CLI runner (nil → docker.ExecRunner) + Runner docker.Runner // compose CLI runner (nil → docker.ExecRunner) + // Backend selects WHERE the stack runs (spec 21): the local daemon (zero value, + // default) or a remote `docker context`/DOCKER_HOST endpoint. Its ComposeEnv() + // pins every compose invocation to that endpoint; its Reachability() gates the + // host-port publish (a remote bridge is not host-routable). The read-only + // d.Docker client and the state ledger's context key are already bound to this + // backend by the caller (buildUpDeps). + Backend docker.Backend Env map[string]string // generate env (nil → process env, via generate default) Profile string // env-overlay profile for ${profile} (generate); "" → workspace default Profiles []string // spec-12 SERVICE SLICES (--profile, repeatable); empty → defaultProfile/all @@ -401,6 +408,21 @@ func sharedPhase(d UpDeps, projects, names, provInstances []string) Phase { Project: generate.SharedStackName, File: filepath.Join(outDir, generate.ComposeFile), Dir: outDir, Runner: d.Runner, + ContextEnv: d.Backend.ComposeEnv(), + } + // Remote backend: a remote bridge network is NOT host-routable, so we do + // NOT publish a loopback provision port — host-side pgx cannot reach it + // (spec 21 §Network reachability). Provisioning the shared Postgres over a + // remote backend needs an SSH local-forward/tunnel, which is the flagged + // follow-up; fail clearly rather than silently publish a port the laptop + // can't dial. `up` still works for the non-provisioned remote shared stack + // (redis/minio, or postgres with no host-provisioned consumer) and with + // --no-provision. + if len(prov) > 0 && d.Backend.Reachability() == docker.ViaProxy { + return nil, fmt.Errorf("provisioning the shared Postgres over a %s is not supported yet: "+ + "a remote shared network is not host-routable, so host-side provisioning needs an SSH tunnel "+ + "(spec 21 follow-up). Re-run with --no-provision, or provision from the remote host", + d.Backend.String()) } // Publish each provisioned Postgres on 127.0.0.1: via an // up-time overlay so host-side pgx (the provision phase) can reach it, @@ -491,6 +513,8 @@ func composeUpPhase(d UpDeps, project string, secretEnv map[string][]string, ser // compose-up process env (Compose substitutes the valueless keys); they // are never written to a file (§7.5). Env: secretEnv[project], + // Pin compose to the selected backend endpoint (spec 21); empty for local. + ContextEnv: d.Backend.ComposeEnv(), } } return Phase{ diff --git a/internal/registry/bundle.go b/internal/registry/bundle.go new file mode 100644 index 0000000..1db8bb6 --- /dev/null +++ b/internal/registry/bundle.go @@ -0,0 +1,206 @@ +package registry + +import ( + "archive/tar" + "bytes" + "fmt" + "io" + "io/fs" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/goccy/go-yaml" +) + +// bundleEpoch is the fixed modification time stamped on every packed tar entry so +// the artifact digest never depends on filesystem timestamps. A constant non-zero +// epoch (rather than the tar zero value) keeps the PAX header stable across +// archive/tar versions. +func bundleEpoch() time.Time { return time.Unix(0, 0).UTC() } + +// TemplateFile is the manifest at the root of every template bundle. Duplicated +// from internal/template to avoid an import cycle (template imports nothing from +// registry, but the CLI wires them together). +const TemplateFile = "template.yaml" + +// bundleMode is the fixed file mode stamped on every packed entry so the tar (and +// thus the artifact digest) is independent of the author's umask. Directories get +// the exec bit; regular files are 0644. +const ( + bundleFileMode = 0o644 + bundleDirMode = 0o755 +) + +// maxBundleBytes caps the unpacked bundle size to defuse a decompression/pull +// bomb from an untrusted registry (templates are small — a few KB). +const maxBundleBytes = 32 << 20 // 32 MiB + +// bundleMeta is the subset of template.yaml the registry cares about: the +// bundle-schema version gate (spec 19 §"What travels"). It is parsed from the +// UNrendered manifest (the meta fields never carry template actions). +type bundleMeta struct { + SchemaVersion int `yaml:"schemaVersion"` +} + +// PackBundle reads a template directory and returns a DETERMINISTIC tar of it, +// rooted at "/…" where name is dir's base (so the unpacked cache dir has a +// single top-level template-name directory, exactly like the embedded FSSource +// layout). Entries are emitted in sorted path order with zeroed timestamps and +// fixed ownership/mode so re-packing identical content yields byte-identical bytes +// — the artifact digest is a pure function of content (spec 19 determinism AC). +// +// The directory MUST contain a template.yaml; its schemaVersion is recorded so a +// consumer can reject a bundle newer than it understands. +func PackBundle(dir string) (data []byte, name string, schemaVersion int, err error) { + abs, err := filepath.Abs(dir) + if err != nil { + return nil, "", 0, err + } + name = filepath.Base(abs) + if !isValidName(name) { + return nil, "", 0, fmt.Errorf("template bundle dir %q is not a valid template name (single path segment, no slash/..)", name) + } + + manifest, err := os.ReadFile(filepath.Join(abs, TemplateFile)) + if err != nil { + return nil, "", 0, fmt.Errorf("template bundle %q: missing %s: %w", name, TemplateFile, err) + } + var meta bundleMeta + if err := yaml.Unmarshal(manifest, &meta); err != nil { + return nil, "", 0, fmt.Errorf("template bundle %q: %s is not valid YAML: %w", name, TemplateFile, err) + } + schemaVersion = meta.SchemaVersion + + // Collect every regular file under dir, keyed by its slash path relative to + // dir, then sort for deterministic emission. + files := map[string][]byte{} + err = filepath.WalkDir(abs, func(p string, d fs.DirEntry, werr error) error { + if werr != nil { + return werr + } + if d.IsDir() { + return nil + } + rel, rerr := filepath.Rel(abs, p) + if rerr != nil { + return rerr + } + b, rerr := os.ReadFile(p) + if rerr != nil { + return rerr + } + files[filepath.ToSlash(rel)] = b + return nil + }) + if err != nil { + return nil, "", 0, err + } + + rels := make([]string, 0, len(files)) + for k := range files { + rels = append(rels, k) + } + sort.Strings(rels) + + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + for _, rel := range rels { + body := files[rel] + hdr := &tar.Header{ + Name: path.Join(name, rel), + Mode: bundleFileMode, + Size: int64(len(body)), + Typeflag: tar.TypeReg, + // Zeroed metadata for reproducibility. + ModTime: bundleEpoch(), + Format: tar.FormatPAX, + } + if err := tw.WriteHeader(hdr); err != nil { + return nil, "", 0, err + } + if _, err := tw.Write(body); err != nil { + return nil, "", 0, err + } + } + if err := tw.Close(); err != nil { + return nil, "", 0, err + } + return buf.Bytes(), name, schemaVersion, nil +} + +// UnpackBundle extracts a bundle tar into destDir (which must not yet exist), +// creating a "/…" tree. It refuses path traversal and oversize payloads. It +// returns the template name (the single top-level directory in the tar). +// +// The caller is responsible for atomicity (unpack into a temp dir, then rename) so +// a kill mid-pull never leaves a half-populated, wrongly-trusted cache dir +// (spec 19 §"OCI digests are over the canonical manifest"). +func UnpackBundle(data []byte, destDir string) (name string, err error) { + if err := os.MkdirAll(destDir, bundleDirMode); err != nil { + return "", err + } + tr := tar.NewReader(bytes.NewReader(data)) + var total int64 + top := map[string]struct{}{} + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return "", fmt.Errorf("read bundle tar: %w", err) + } + if hdr.Typeflag != tar.TypeReg { + continue // bundles carry only regular files + } + clean := path.Clean(hdr.Name) + if clean == "." || strings.HasPrefix(clean, "..") || strings.HasPrefix(clean, "/") || strings.Contains(clean, "/../") { + return "", fmt.Errorf("bundle entry %q escapes the destination", hdr.Name) + } + seg := strings.SplitN(clean, "/", 2) + top[seg[0]] = struct{}{} + + total += hdr.Size + if total > maxBundleBytes { + return "", fmt.Errorf("bundle exceeds %d bytes — refusing to unpack", maxBundleBytes) + } + target := filepath.Join(destDir, filepath.FromSlash(clean)) + if err := os.MkdirAll(filepath.Dir(target), bundleDirMode); err != nil { + return "", err + } + f, err := os.OpenFile(target, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, bundleFileMode) + if err != nil { + return "", err + } + if _, err := io.CopyN(f, tr, hdr.Size); err != nil { + f.Close() + return "", err + } + if err := f.Close(); err != nil { + return "", err + } + } + if len(top) != 1 { + return "", fmt.Errorf("bundle must contain exactly one top-level template directory, found %d", len(top)) + } + for n := range top { + name = n + } + if _, err := os.Stat(filepath.Join(destDir, name, TemplateFile)); err != nil { + return "", fmt.Errorf("bundle %q has no %s", name, TemplateFile) + } + return name, nil +} + +// isValidName mirrors template.ValidRef without importing it (a template name is a +// single path segment; dots allowed, no slash/".."). +func isValidName(name string) bool { + if name == "" || name == "." || name == ".." { + return false + } + return name == path.Base(name) && !filepath.IsAbs(name) && !strings.ContainsAny(name, "/\\") +} diff --git a/internal/registry/pull.go b/internal/registry/pull.go new file mode 100644 index 0000000..da6b674 --- /dev/null +++ b/internal/registry/pull.go @@ -0,0 +1,92 @@ +package registry + +import ( + "context" + "encoding/json" + "fmt" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" + "oras.land/oras-go/v2/content" +) + +// Pulled is the outcome of a Pull: the resolved Descriptor plus the raw bundle tar +// bytes (the single layer). The caller unpacks the tar atomically into the digest +// cache. +type Pulled struct { + Descriptor Descriptor + // Tar is the bundle tar (the artifact's single layer), verified against the + // layer digest by oras during fetch. + Tar []byte +} + +// Pull resolves ref, fetches its manifest + single bundle layer, and returns the +// tar bytes. Digest verification is MANDATORY and always-on: when ref carries a +// pinned digest, the resolved manifest digest MUST equal it or the pull is refused +// (nothing is unpacked). oras additionally verifies every fetched blob against its +// descriptor digest, so a corrupt/tampered layer is rejected before it reaches the +// caller (spec 19 §"Digest verification is mandatory"). +func (c *Client) Pull(ctx context.Context, ref Reference) (Pulled, error) { + target, err := c.newTarget(ctx, ref) + if err != nil { + return Pulled{}, err + } + + manDesc, manBytes, err := oras.FetchBytes(ctx, target, ref.shortRef(), oras.DefaultFetchBytesOptions) + if err != nil { + return Pulled{}, wrapAuth(err, ref, "fetch manifest") + } + // Pin enforcement: a pinned ref must resolve to exactly that manifest digest. + if ref.Digest != "" && manDesc.Digest.String() != ref.Digest { + return Pulled{}, fmt.Errorf("digest mismatch for %s: expected %s, registry served %s — refusing (tampered or re-pushed tag)", ref.Name(), ref.Digest, manDesc.Digest.String()) + } + + var man ocispec.Manifest + if err := json.Unmarshal(manBytes, &man); err != nil { + return Pulled{}, fmt.Errorf("parse manifest for %s: %w", ref.Name(), err) + } + if man.ArtifactType != "" && man.ArtifactType != ArtifactType && man.Config.MediaType != ArtifactType { + return Pulled{}, fmt.Errorf("%s is not a devstack template artifact (artifactType %q)", ref.Name(), man.ArtifactType) + } + layer, err := bundleLayer(man) + if err != nil { + return Pulled{}, fmt.Errorf("%s: %w", ref.Name(), err) + } + + tarData, err := content.FetchAll(ctx, target, layer) + if err != nil { + return Pulled{}, wrapAuth(err, ref, "fetch bundle layer") + } + + desc := Descriptor{ + Ref: refWithDigest(ref, manDesc.Digest.String()).String(), + Repository: ref.Name(), + Tag: ref.Tag, + Digest: manDesc.Digest.String(), + SchemaVersion: schemaVersionFromManifest(man), + Name: man.Annotations[annotationTemplateName], + Size: manDesc.Size, + } + return Pulled{Descriptor: desc, Tar: tarData}, nil +} + +// bundleLayer returns the single template-tar layer of a bundle manifest. +func bundleLayer(man ocispec.Manifest) (ocispec.Descriptor, error) { + for _, l := range man.Layers { + if l.MediaType == LayerMediaType { + return l, nil + } + } + if len(man.Layers) == 1 { + return man.Layers[0], nil + } + return ocispec.Descriptor{}, fmt.Errorf("manifest has no %s layer", LayerMediaType) +} + +// schemaVersionFromManifest reads the bundle schemaVersion annotation (0 if absent +// or malformed — the caller re-reads it from the unpacked template.yaml too). +func schemaVersionFromManifest(man ocispec.Manifest) int { + var v int + _, _ = fmt.Sscanf(man.Annotations[annotationSchemaVersion], "%d", &v) + return v +} diff --git a/internal/registry/reference.go b/internal/registry/reference.go new file mode 100644 index 0000000..f24b8c5 --- /dev/null +++ b/internal/registry/reference.go @@ -0,0 +1,174 @@ +// Package registry is devstack's OCI template-registry seam (spec 19). It packages +// a template bundle (template.yaml + optional build/ tree + golden.yaml) as a +// deterministic OCI artifact, pushes/pulls it addressed by name:tag and pinned by +// the manifest digest, and verifies its signature — all behind a small interface +// so the CLI never touches oras directly and tests round-trip through an in-memory +// store with no network. +// +// The pure-Go client is oras.land/oras-go/v2 (Apache-2.0, CGO-free), preserving +// the single-static-binary constraint. Registry auth rides the existing docker / +// ORAS credential store (~/.docker/config.json, OS helpers, GITHUB_TOKEN) — we do +// NOT invent a devstack token store (spec 19 §"Verified constraints"). +// +// Determinism is the whole point: a render is reproducible across a team only if +// template content is addressed by DIGEST, not a moving tag. Pull verifies the +// pulled manifest's computed digest against the pinned digest before anything is +// unpacked, and the bundle tar is byte-deterministic (see bundle.go). +package registry + +import ( + "fmt" + "strings" + + "github.com/opencontainers/go-digest" +) + +// Scheme is the optional URI scheme accepted on a template ref. +const Scheme = "oci://" + +// Reference is a parsed OCI reference: a registry host, a repository path, and +// EITHER a tag OR a digest (a ref may carry both — `repo:tag@sha256:…` — in which +// case both are populated and the digest wins for fetching). +type Reference struct { + // Registry is the host[:port] (e.g. "ghcr.io", "localhost:5000"). + Registry string + // Repository is the path under the registry (e.g. "acme/templates"). + Repository string + // Tag is the human-facing tag (e.g. "1.4.0"), possibly empty when a bare + // digest ref was given. + Tag string + // Digest is the pinned manifest digest (e.g. "sha256:9c8b…"), possibly empty + // when only a tag was given (it is resolved+pinned at `template add`). + Digest string +} + +// ParseReference parses an `oci://host/repo:tag`, `host/repo:tag`, +// `host/repo@sha256:…`, or `host/repo:tag@sha256:…` reference. The `oci://` +// scheme is optional and stripped. Registry and repository are required. +func ParseReference(ref string) (Reference, error) { + orig := ref + ref = strings.TrimPrefix(ref, Scheme) + if ref == "" { + return Reference{}, fmt.Errorf("empty registry reference") + } + + var r Reference + // Split off an optional @digest first (it always trails). + if at := strings.LastIndex(ref, "@"); at >= 0 { + dig := ref[at+1:] + if err := digest.Digest(dig).Validate(); err != nil { + return Reference{}, fmt.Errorf("invalid digest %q in %q: %w", dig, orig, err) + } + r.Digest = dig + ref = ref[:at] + } + + // The registry host is the first path segment and MUST look like a host + // (contain a "." or ":" or be "localhost") so `repo:tag` is not mistaken for + // `host/repo`. + slash := strings.IndexByte(ref, '/') + if slash < 0 { + return Reference{}, fmt.Errorf("reference %q is missing a registry host (want host/repo:tag)", orig) + } + r.Registry = ref[:slash] + rest := ref[slash+1:] + if !looksLikeHost(r.Registry) { + return Reference{}, fmt.Errorf("reference %q has no registry host — %q is not a hostname (want e.g. ghcr.io/owner/name:tag)", orig, r.Registry) + } + + // A tag, if present, trails the LAST colon in the remaining path (repo paths + // never contain colons; ports were already consumed by the registry host). + if colon := strings.LastIndex(rest, ":"); colon >= 0 { + r.Tag = rest[colon+1:] + rest = rest[:colon] + } + r.Repository = rest + if r.Repository == "" { + return Reference{}, fmt.Errorf("reference %q is missing a repository path", orig) + } + if r.Tag == "" && r.Digest == "" { + return Reference{}, fmt.Errorf("reference %q must carry a tag or a digest", orig) + } + return r, nil +} + +// ParseRepository parses a bare `oci://host/repo` (or `host/repo`) with NO tag or +// digest — the form recorded in the lockfile's `source:` field. `update`/`diff`/ +// `verify` parse it and then re-attach the recorded tag/digest before fetching. +func ParseRepository(s string) (Reference, error) { + s = strings.TrimPrefix(s, Scheme) + slash := strings.IndexByte(s, '/') + if slash < 0 { + return Reference{}, fmt.Errorf("repository %q is missing a registry host (want host/repo)", s) + } + r := Reference{Registry: s[:slash], Repository: s[slash+1:]} + if !looksLikeHost(r.Registry) || r.Repository == "" { + return Reference{}, fmt.Errorf("invalid repository %q (want e.g. ghcr.io/owner/name)", s) + } + return r, nil +} + +// looksLikeHost reports whether s is plausibly a registry host: it contains a dot +// or a port colon, or is exactly "localhost". +func looksLikeHost(s string) bool { + return s == "localhost" || strings.ContainsAny(s, ".:") +} + +// IsFloatingTag reports whether the tag is a mutable "floating" tag (":latest") +// that breaks determinism when fetched at generation. `template add` rejects it +// without --allow-floating (spec 19 §"A moving tag breaks determinism"). +func (r Reference) IsFloatingTag() bool { + return r.Tag == "latest" +} + +// Name is the "registry/repository" identity without any tag or digest — the key +// used to build an oras Target for the repository. +func (r Reference) Name() string { + return r.Registry + "/" + r.Repository +} + +// TagRef renders "registry/repository:tag" (used as the oras copy destination +// reference on push and the resolve reference on pull-by-tag). +func (r Reference) TagRef() string { + return r.Name() + ":" + r.Tag +} + +// DigestRef renders "registry/repository@digest" (the pinned pull reference). +func (r Reference) DigestRef() string { + return r.Name() + "@" + r.Digest +} + +// FetchRef returns the reference oras should resolve: the digest when pinned +// (reproducible), else the tag. Generation always pins a digest; `add`/`diff` +// may resolve a bare tag. +func (r Reference) FetchRef() string { + if r.Digest != "" { + return r.DigestRef() + } + return r.TagRef() +} + +// shortRef is the tag-or-digest oras resolves against a repository Target (the +// registry/repo is implied by the Target). A bare tag resolves uniformly across +// remote registries and the local memory/OCI stores tests use; a bare digest is +// prefixed with "@" so a live registry treats it as a digest. Pull always has a +// tag (a pin is minted FROM a tag), so it resolves by tag then asserts the digest +// matches the pin — never depending on a store resolving a raw digest. +func (r Reference) shortRef() string { + if r.Tag != "" { + return r.Tag + } + return "@" + r.Digest +} + +// String renders the canonical `oci://` form for display and lockfile provenance. +func (r Reference) String() string { + out := Scheme + r.Name() + if r.Tag != "" { + out += ":" + r.Tag + } + if r.Digest != "" { + out += "@" + r.Digest + } + return out +} diff --git a/internal/registry/reference_test.go b/internal/registry/reference_test.go new file mode 100644 index 0000000..1c6d693 --- /dev/null +++ b/internal/registry/reference_test.go @@ -0,0 +1,70 @@ +package registry + +import "testing" + +func TestParseReference(t *testing.T) { + tests := []struct { + in string + wantReg string + wantRepo string + wantTag string + wantDig string + wantErr bool + }{ + {in: "oci://ghcr.io/acme/templates:1.4.0", wantReg: "ghcr.io", wantRepo: "acme/templates", wantTag: "1.4.0"}, + {in: "ghcr.io/acme/templates:1.4.0", wantReg: "ghcr.io", wantRepo: "acme/templates", wantTag: "1.4.0"}, + {in: "localhost:5000/t:1.0", wantReg: "localhost:5000", wantRepo: "t", wantTag: "1.0"}, + { + in: "ghcr.io/acme/postgres@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + wantReg: "ghcr.io", wantRepo: "acme/postgres", + wantDig: "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + { + in: "ghcr.io/acme/postgres:1.0@sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + wantReg: "ghcr.io", wantRepo: "acme/postgres", wantTag: "1.0", + wantDig: "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + }, + {in: "", wantErr: true}, + {in: "templates:1.4.0", wantErr: true}, // no registry host + {in: "ghcr.io/acme/templates", wantErr: true}, // no tag or digest + {in: "ghcr.io/acme/x@sha256:short", wantErr: true}, // bad digest + } + for _, tt := range tests { + t.Run(tt.in, func(t *testing.T) { + r, err := ParseReference(tt.in) + if tt.wantErr { + if err == nil { + t.Fatalf("want error for %q, got %+v", tt.in, r) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if r.Registry != tt.wantReg || r.Repository != tt.wantRepo || r.Tag != tt.wantTag || r.Digest != tt.wantDig { + t.Errorf("ParseReference(%q) = %+v, want reg=%s repo=%s tag=%s dig=%s", + tt.in, r, tt.wantReg, tt.wantRepo, tt.wantTag, tt.wantDig) + } + }) + } +} + +func TestFetchRefPrefersDigest(t *testing.T) { + r := Reference{Registry: "ghcr.io", Repository: "a/b", Tag: "1.0", Digest: "sha256:deadbeef"} + if got := r.FetchRef(); got != "ghcr.io/a/b@sha256:deadbeef" { + t.Errorf("FetchRef = %q, want the digest ref", got) + } + r.Digest = "" + if got := r.FetchRef(); got != "ghcr.io/a/b:1.0" { + t.Errorf("FetchRef = %q, want the tag ref when unpinned", got) + } +} + +func TestIsFloatingTag(t *testing.T) { + if !(Reference{Tag: "latest"}).IsFloatingTag() { + t.Error(":latest must be flagged as floating") + } + if (Reference{Tag: "1.4.0"}).IsFloatingTag() { + t.Error("a pinned version tag is not floating") + } +} diff --git a/internal/registry/registry.go b/internal/registry/registry.go new file mode 100644 index 0000000..91e040c --- /dev/null +++ b/internal/registry/registry.go @@ -0,0 +1,142 @@ +package registry + +import ( + "context" + "fmt" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/v2" +) + +// Media types for a devstack template artifact. The custom artifactType marks the +// artifact as opaque to non-devstack tooling (so a registry UI won't mis-render it +// as a runnable image), and each layer's mediaType is our own tar type. +const ( + ArtifactType = "application/vnd.devstack.template.v1" + ConfigMediaType = "application/vnd.devstack.template.config.v1+json" + LayerMediaType = "application/vnd.devstack.template.layer.v1.tar" +) + +// createdAnnotation is pinned to a fixed value so PackManifest is reproducible +// (spec 19 determinism AC): the manifest digest is a pure function of content. +const createdAnnotation = "1970-01-01T00:00:00Z" + +// Manifest annotations carrying the bundle's identity + schema gate. +const ( + annotationTemplateName = "land.devstack.template.name" + annotationSchemaVersion = "land.devstack.template.schemaVersion" +) + +// Descriptor is the resolved identity of a pushed/pulled artifact: the human tag +// (provenance only) plus the manifest digest that is actually fetched and verified. +type Descriptor struct { + Ref string `json:"ref"` // canonical oci:// form (with digest) + Repository string `json:"repository"` // registry/repository + Tag string `json:"tag,omitempty"` // human tag (NOT used to fetch) + Digest string `json:"digest"` // sha256:… manifest digest (the pin) + SchemaVersion int `json:"schemaVersion"` // template-bundle schemaVersion + Name string `json:"name"` // template name inside the bundle + Size int64 `json:"size"` // manifest size in bytes +} + +// Target is the subset of oras a registry client needs: a content store that can +// be tagged and resolved. A live *remote.Repository and an in-memory store both +// satisfy it, so tests round-trip with no network. +type Target = oras.Target + +// TargetResolver returns an oras Target for a repository (host/path, no tag). The +// production resolver builds an authenticated *remote.Repository; tests return a +// shared store so push/pull round-trips offline. Behind this seam the CLI never +// constructs a remote.Repository itself. +type TargetResolver func(ctx context.Context, ref Reference) (Target, error) + +// Client packages, pushes, resolves and pulls template bundles. It is stateless +// apart from the TargetResolver seam. +type Client struct { + newTarget TargetResolver +} + +// New returns a Client whose TargetResolver talks to real registries, reading +// credentials from the docker/ORAS credential store (~/.docker/config.json + OS +// helpers) with a GITHUB_TOKEN fallback for GHCR. No devstack-specific token store. +func New() (*Client, error) { + res, err := defaultTargetResolver() + if err != nil { + return nil, err + } + return &Client{newTarget: res}, nil +} + +// NewWithResolver returns a Client backed by a custom TargetResolver (tests inject +// a memory/OCI store). +func NewWithResolver(r TargetResolver) *Client { + return &Client{newTarget: r} +} + +// Push packages the template directory at dir into a deterministic OCI artifact, +// pushes it to ref's repository, tags it with ref's tag, and returns the resolved +// manifest Descriptor (digest pin). +func (c *Client) Push(ctx context.Context, ref Reference, dir string) (Descriptor, error) { + if ref.Tag == "" { + return Descriptor{}, fmt.Errorf("push requires a tag (got %s)", ref) + } + tarData, name, schemaVersion, err := PackBundle(dir) + if err != nil { + return Descriptor{}, err + } + target, err := c.newTarget(ctx, ref) + if err != nil { + return Descriptor{}, err + } + + layerDesc, err := oras.PushBytes(ctx, target, LayerMediaType, tarData) + if err != nil { + return Descriptor{}, wrapAuth(err, ref, "push layer") + } + layerDesc.Annotations = map[string]string{ocispec.AnnotationTitle: name} + + manifestDesc, err := oras.PackManifest(ctx, target, oras.PackManifestVersion1_1, ArtifactType, oras.PackManifestOptions{ + Layers: []ocispec.Descriptor{layerDesc}, + ManifestAnnotations: map[string]string{ + ocispec.AnnotationCreated: createdAnnotation, + annotationTemplateName: name, + annotationSchemaVersion: fmt.Sprintf("%d", schemaVersion), + }, + }) + if err != nil { + return Descriptor{}, wrapAuth(err, ref, "pack manifest") + } + if err := target.Tag(ctx, manifestDesc, ref.Tag); err != nil { + return Descriptor{}, wrapAuth(err, ref, "tag manifest") + } + + return Descriptor{ + Ref: refWithDigest(ref, manifestDesc.Digest.String()).String(), + Repository: ref.Name(), + Tag: ref.Tag, + Digest: manifestDesc.Digest.String(), + SchemaVersion: schemaVersion, + Name: name, + Size: manifestDesc.Size, + }, nil +} + +// ResolveDigest resolves ref (by tag or digest) to its manifest Descriptor WITHOUT +// fetching the layers — the cheap tag→digest step used by `add`/`update`. +func (c *Client) ResolveDigest(ctx context.Context, ref Reference) (Descriptor, error) { + target, err := c.newTarget(ctx, ref) + if err != nil { + return Descriptor{}, err + } + desc, err := target.Resolve(ctx, ref.shortRef()) + if err != nil { + return Descriptor{}, wrapAuth(err, ref, "resolve") + } + return Descriptor{ + Ref: refWithDigest(ref, desc.Digest.String()).String(), + Repository: ref.Name(), + Tag: ref.Tag, + Digest: desc.Digest.String(), + Size: desc.Size, + }, nil +} diff --git a/internal/registry/registry_test.go b/internal/registry/registry_test.go new file mode 100644 index 0000000..346ac4b --- /dev/null +++ b/internal/registry/registry_test.go @@ -0,0 +1,150 @@ +package registry + +import ( + "bytes" + "context" + "os" + "path/filepath" + "strings" + "testing" + + "oras.land/oras-go/v2/content/memory" +) + +// memResolver returns a TargetResolver backed by a single shared in-memory oras +// store, so push/pull round-trips with no network (spec 19 test guidance). +func memResolver() TargetResolver { + store := memory.New() + return func(_ context.Context, _ Reference) (Target, error) { return store, nil } +} + +// writeBundle scaffolds a minimal-but-complete template bundle on disk and returns +// its directory. name becomes the bundle's top-level template directory. +func writeBundle(t *testing.T, name, version string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), name) + if err := os.MkdirAll(filepath.Join(dir, "build"), 0o755); err != nil { + t.Fatal(err) + } + files := map[string]string{ + "template.yaml": "schemaVersion: 1\ndescription: \"test\"\nparams:\n version:\n type: string\n default: \"" + version + "\"\nservice:\n image: alpine:[[ .params.version ]]\n", + "build/Dockerfile": "FROM alpine:[[ .params.version ]]\n", + "golden.yaml": "services:\n test: {}\n", + } + for rel, body := range files { + if err := os.WriteFile(filepath.Join(dir, rel), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + return dir +} + +func TestPushPullRoundTrip(t *testing.T) { + ctx := context.Background() + dir := writeBundle(t, "postgres", "16") + c := NewWithResolver(memResolver()) + + ref, err := ParseReference("oci://ghcr.io/acme/postgres:1.0.0") + if err != nil { + t.Fatal(err) + } + desc, err := c.Push(ctx, ref, dir) + if err != nil { + t.Fatalf("push: %v", err) + } + if desc.Digest == "" || desc.Name != "postgres" || desc.SchemaVersion != 1 { + t.Fatalf("unexpected descriptor: %+v", desc) + } + + // Pull by the resolved digest (the reproducible path) and unpack. + pinned := refWithDigest(ref, desc.Digest) + pulled, err := c.Pull(ctx, pinned) + if err != nil { + t.Fatalf("pull: %v", err) + } + if pulled.Descriptor.Digest != desc.Digest { + t.Errorf("pulled digest %s != pushed %s", pulled.Descriptor.Digest, desc.Digest) + } + + dest := t.TempDir() + gotName, err := UnpackBundle(pulled.Tar, dest) + if err != nil { + t.Fatalf("unpack: %v", err) + } + if gotName != "postgres" { + t.Errorf("unpacked name %q, want postgres", gotName) + } + // Round-trip fidelity: the unpacked template.yaml matches the source. + want, _ := os.ReadFile(filepath.Join(dir, "template.yaml")) + got, err := os.ReadFile(filepath.Join(dest, "postgres", "template.yaml")) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(want, got) { + t.Errorf("template.yaml round-trip mismatch:\n want %q\n got %q", want, got) + } + for _, f := range []string{"build/Dockerfile", "golden.yaml"} { + if _, err := os.Stat(filepath.Join(dest, "postgres", f)); err != nil { + t.Errorf("missing %s after round-trip: %v", f, err) + } + } +} + +func TestPackBundleDeterministic(t *testing.T) { + dir := writeBundle(t, "redis", "7") + a, _, _, err := PackBundle(dir) + if err != nil { + t.Fatal(err) + } + b, _, _, err := PackBundle(dir) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(a, b) { + t.Fatal("PackBundle is not deterministic: two packs of identical content differ") + } + + // Same content in a different directory (different mtimes/paths) packs identically. + dir2 := writeBundle(t, "redis", "7") + c, _, _, err := PackBundle(dir2) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(a, c) { + t.Fatal("PackBundle output depends on filesystem metadata — digest would not be content-addressed") + } +} + +func TestPullDigestMismatchRefused(t *testing.T) { + ctx := context.Background() + dir := writeBundle(t, "postgres", "16") + c := NewWithResolver(memResolver()) + ref, _ := ParseReference("oci://ghcr.io/acme/postgres:1.0.0") + if _, err := c.Push(ctx, ref, dir); err != nil { + t.Fatal(err) + } + + // Pin a bogus digest: the resolved manifest digest won't match → refused. + bad := refWithDigest(ref, "sha256:"+strings.Repeat("0", 64)) + if _, err := c.Pull(ctx, bad); err == nil { + t.Fatal("want a digest-mismatch error, got nil") + } +} + +func TestResolveDigest(t *testing.T) { + ctx := context.Background() + dir := writeBundle(t, "minio", "latest") + c := NewWithResolver(memResolver()) + ref, _ := ParseReference("oci://ghcr.io/acme/minio:2.0.0") + pushed, err := c.Push(ctx, ref, dir) + if err != nil { + t.Fatal(err) + } + got, err := c.ResolveDigest(ctx, ref) + if err != nil { + t.Fatalf("resolve: %v", err) + } + if got.Digest != pushed.Digest { + t.Errorf("resolve digest %s != pushed %s", got.Digest, pushed.Digest) + } +} diff --git a/internal/registry/target.go b/internal/registry/target.go new file mode 100644 index 0000000..c8fdc63 --- /dev/null +++ b/internal/registry/target.go @@ -0,0 +1,128 @@ +package registry + +import ( + "context" + "fmt" + "os" + + "oras.land/oras-go/v2/registry/remote" + "oras.land/oras-go/v2/registry/remote/auth" + "oras.land/oras-go/v2/registry/remote/credentials" + "oras.land/oras-go/v2/registry/remote/retry" +) + +// refWithDigest returns a copy of ref with its Digest set (Tag preserved). +func refWithDigest(ref Reference, dig string) Reference { + ref.Digest = dig + return ref +} + +// wrapAuth maps an oras authorization failure to a one-line remediation (the +// ARCHITECTURE §7.6 error contract) and otherwise annotates the failing step. +func wrapAuth(err error, ref Reference, step string) error { + if err == nil { + return nil + } + if isAuthMessage(err) { + return fmt.Errorf("not authorized for %s (%s): log in with `docker login %s` (or set GITHUB_TOKEN for ghcr.io) — devstack reads ~/.docker/config.json: %w", + ref.Name(), step, ref.Registry, err) + } + return fmt.Errorf("%s %s: %w", step, ref.Name(), err) +} + +// isAuthMessage catches auth failures that some registries surface without the +// typed errdef.ErrUnauthorized (e.g. a 403 on push to a repo you can read). +func isAuthMessage(err error) bool { + msg := err.Error() + for _, s := range []string{"401", "403", "unauthorized", "denied", "forbidden"} { + if containsFold(msg, s) { + return true + } + } + return false +} + +func containsFold(s, sub string) bool { + // tiny ASCII-fold contains to avoid pulling strings for two call sites + ls, lsub := toLowerASCII(s), toLowerASCII(sub) + return len(lsub) == 0 || indexOf(ls, lsub) >= 0 +} + +func toLowerASCII(s string) string { + b := []byte(s) + for i := range b { + if b[i] >= 'A' && b[i] <= 'Z' { + b[i] += 'a' - 'A' + } + } + return string(b) +} + +func indexOf(s, sub string) int { + n, m := len(s), len(sub) + for i := 0; i+m <= n; i++ { + if s[i:i+m] == sub { + return i + } + } + return -1 +} + +// defaultTargetResolver builds authenticated *remote.Repository targets, reading +// credentials from the docker/ORAS credential store (~/.docker/config.json + OS +// helpers). A GITHUB_TOKEN in the environment is used for ghcr.io when the docker +// store has no entry, so CI publishes without a `docker login` step. +func defaultTargetResolver() (TargetResolver, error) { + credStore, err := credentials.NewStoreFromDocker(credentials.StoreOptions{ + AllowPlaintextPut: false, + DetectDefaultNativeStore: true, + }) + if err != nil { + return nil, fmt.Errorf("open docker credential store: %w", err) + } + credFn := credentials.Credential(credStore) + + return func(_ context.Context, ref Reference) (Target, error) { + repo, err := remote.NewRepository(ref.Name()) + if err != nil { + return nil, fmt.Errorf("invalid repository %q: %w", ref.Name(), err) + } + // Allow plain HTTP for localhost registries (the `template push + // oci://localhost:5000/…` dev/testing path); everything else is HTTPS. + if isLocalhost(ref.Registry) { + repo.PlainHTTP = true + } + repo.Client = &auth.Client{ + Client: retry.DefaultClient, + Cache: auth.NewCache(), + Credential: func(ctx context.Context, hostport string) (auth.Credential, error) { + if c, err := credFn(ctx, hostport); err == nil && c != (auth.Credential{}) { + return c, nil + } + if tok := ghcrToken(hostport); tok != "" { + return auth.Credential{Username: "oauth2", Password: tok}, nil + } + return auth.Credential{}, nil + }, + } + return repo, nil + }, nil +} + +// ghcrToken returns a GITHUB_TOKEN (or GHCR_TOKEN) for ghcr.io hosts, else "". +func ghcrToken(hostport string) string { + if hostport != "ghcr.io" { + return "" + } + for _, env := range []string{"GHCR_TOKEN", "GITHUB_TOKEN"} { + if v := os.Getenv(env); v != "" { + return v + } + } + return "" +} + +func isLocalhost(host string) bool { + return host == "localhost" || host == "127.0.0.1" || host == "::1" || + len(host) >= 10 && host[:10] == "localhost:" +} diff --git a/internal/registry/verify.go b/internal/registry/verify.go new file mode 100644 index 0000000..4b7af2b --- /dev/null +++ b/internal/registry/verify.go @@ -0,0 +1,160 @@ +package registry + +import ( + "context" + "fmt" + "os/exec" + "strings" +) + +// Signature verification (spec 19 §"Supply-chain posture"). Digest verification is +// mandatory and always-on (see Pull); SIGNATURE verification is optional but +// first-class: with a VerifyPolicy configured, `add`/`update`/`verify` require a +// valid cosign signature over the manifest digest before the artifact is trusted. +// +// Consistent with internal/selfupdate (which deliberately shells the external +// `cosign` binary behind an interface to keep the release binary CGO-free rather +// than vendor the heavy sigstore-go tree), the real Verifier shells `cosign +// verify`. The seam is mock-able so tests never touch the network or need cosign. +// The pure-Go sigstore-go path (spec 19 §"no cosign binary required") is scoped +// out for this milestone and tracked in the PR — the seam makes it a drop-in swap. + +// VerifyPolicy pins the accepted signer identity for keyless cosign, or a public +// key for keyed cosign. A zero policy means "no signature policy" — the artifact +// is digest-pinned but unsigned (a loud one-line notice is printed on add). +type VerifyPolicy struct { + // Enabled turns signature verification on. When false, only the digest is + // verified (mandatory) and a "digest-pinned but unsigned" notice is shown. + Enabled bool + // IdentityRegexp pins the keyless signer identity (the Fulcio cert SAN, e.g. a + // GitHub Actions workflow ref). Required in keyless mode. + IdentityRegexp string + // OIDCIssuer pins the keyless OIDC issuer (e.g. GitHub Actions' token issuer). + // Required in keyless mode. + OIDCIssuer string + // KeyPath points at a cosign public key for KEYED (offline) verification. When + // set, keyless identity/issuer are ignored. + KeyPath string +} + +// IsZero reports whether no signature policy is configured. +func (p VerifyPolicy) IsZero() bool { return !p.Enabled && p.KeyPath == "" } + +// Keyless reports whether the policy verifies via keyless (Fulcio/Rekor) rather +// than a distributed public key. +func (p VerifyPolicy) Keyless() bool { return p.KeyPath == "" } + +// Verifier verifies a cosign signature over an OCI artifact addressed by its +// digest reference. Behind an interface so tests inject a fake and the real +// implementation shells cosign. +type Verifier interface { + // Available reports whether the verification backend (the cosign binary) is + // usable on this host. + Available() bool + // Verify returns nil when digestRef carries a valid cosign signature satisfying + // policy; any other outcome (unsigned, wrong identity, tampered) is an error. + Verify(ctx context.Context, digestRef string, policy VerifyPolicy) error +} + +// DefaultVerifier is the cosign-backed Verifier used when none is injected. +var DefaultVerifier Verifier = cosignVerifier{} + +// cosignVerifier shells `cosign verify`. +type cosignVerifier struct{} + +const cosignBin = "cosign" + +func (cosignVerifier) Available() bool { + _, err := exec.LookPath(cosignBin) + return err == nil +} + +func (cosignVerifier) Verify(ctx context.Context, digestRef string, policy VerifyPolicy) error { + args := []string{"verify"} + if policy.Keyless() { + if policy.IdentityRegexp == "" || policy.OIDCIssuer == "" { + return fmt.Errorf("keyless cosign policy requires both an identity regexp and an OIDC issuer") + } + args = append(args, + "--certificate-identity-regexp", policy.IdentityRegexp, + "--certificate-oidc-issuer", policy.OIDCIssuer, + ) + } else { + args = append(args, "--key", policy.KeyPath) + } + args = append(args, digestRef) + + // #nosec G204 -- args are fixed flags + a validated digest reference. + cmd := exec.CommandContext(ctx, cosignBin, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("cosign signature verification failed for %s — refusing to trust the artifact (unsigned, tampered, or signed by an untrusted identity): %w: %s", + digestRef, err, strings.TrimSpace(string(out))) + } + return nil +} + +// Signer signs an OCI artifact addressed by its digest reference. Behind an +// interface for symmetry with Verifier; the real implementation shells `cosign +// sign` (keyless by default). Scoped for manual/CI use — see the package doc. +type Signer interface { + Available() bool + Sign(ctx context.Context, digestRef string, keyPath string) error +} + +// DefaultSigner is the cosign-backed Signer. +var DefaultSigner Signer = cosignSigner{} + +type cosignSigner struct{} + +func (cosignSigner) Available() bool { + _, err := exec.LookPath(cosignBin) + return err == nil +} + +func (cosignSigner) Sign(ctx context.Context, digestRef, keyPath string) error { + args := []string{"sign", "--yes"} + if keyPath != "" { + args = append(args, "--key", keyPath) + } + args = append(args, digestRef) + // #nosec G204 -- fixed flags + a validated digest reference + operator key path. + cmd := exec.CommandContext(ctx, cosignBin, args...) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("cosign sign failed for %s: %w: %s", digestRef, err, strings.TrimSpace(string(out))) + } + return nil +} + +// SignArtifact signs desc with s (or DefaultSigner). A missing cosign binary is a +// clear error with a remediation. +func SignArtifact(ctx context.Context, s Signer, desc Descriptor, keyPath string) error { + if s == nil { + s = DefaultSigner + } + if !s.Available() { + return fmt.Errorf("cosign not found: --sign requires the `cosign` binary (https://docs.sigstore.dev/system_config/installation/)") + } + return s.Sign(ctx, desc.Repository+"@"+desc.Digest, keyPath) +} + +// VerifySignature verifies the artifact at desc.Ref against policy using v (or the +// DefaultVerifier when nil). A no-op (nil) when the policy is zero — the caller is +// responsible for printing the unsigned notice. A missing cosign binary with an +// enabled policy is a hard error with a remediation (never a silent downgrade). +func VerifySignature(ctx context.Context, v Verifier, desc Descriptor, policy VerifyPolicy) error { + if policy.IsZero() { + return nil + } + if v == nil { + v = DefaultVerifier + } + if !v.Available() { + return fmt.Errorf("cosign not found: template signature verification is required by policy but the `cosign` binary is not installed.\n" + + " Install it (https://docs.sigstore.dev/system_config/installation/) and retry,\n" + + " or remove the verify policy to trust the digest pin alone (digest is still enforced).") + } + digestRef := desc.Repository + "@" + desc.Digest + return v.Verify(ctx, digestRef, policy) +} diff --git a/internal/registry/verify_test.go b/internal/registry/verify_test.go new file mode 100644 index 0000000..d595f0a --- /dev/null +++ b/internal/registry/verify_test.go @@ -0,0 +1,80 @@ +package registry + +import ( + "context" + "errors" + "testing" +) + +// fakeVerifier is a mock Verifier: it records what it was asked to verify and +// returns a configurable availability + result, so tests exercise the trust gate +// without shelling cosign or touching the network. +type fakeVerifier struct { + available bool + err error + called bool + gotRef string + gotPolicy VerifyPolicy +} + +func (f *fakeVerifier) Available() bool { return f.available } +func (f *fakeVerifier) Verify(_ context.Context, digestRef string, policy VerifyPolicy) error { + f.called = true + f.gotRef = digestRef + f.gotPolicy = policy + return f.err +} + +var keylessPolicy = VerifyPolicy{ + Enabled: true, + IdentityRegexp: `^https://github\.com/acme/.+$`, + OIDCIssuer: "https://token.actions.githubusercontent.com", +} + +func desc() Descriptor { + return Descriptor{Repository: "ghcr.io/acme/postgres", Digest: "sha256:abc"} +} + +func TestVerifyGoodSignatureAccepted(t *testing.T) { + v := &fakeVerifier{available: true} + if err := VerifySignature(context.Background(), v, desc(), keylessPolicy); err != nil { + t.Fatalf("good signature should be accepted: %v", err) + } + if !v.called { + t.Fatal("verifier was never invoked") + } + if v.gotRef != "ghcr.io/acme/postgres@sha256:abc" { + t.Errorf("verified %q, want the digest ref", v.gotRef) + } + if v.gotPolicy.IdentityRegexp != keylessPolicy.IdentityRegexp { + t.Errorf("policy identity not propagated: %+v", v.gotPolicy) + } +} + +func TestVerifyTamperedSignatureRejected(t *testing.T) { + v := &fakeVerifier{available: true, err: errors.New("signature mismatch")} + if err := VerifySignature(context.Background(), v, desc(), keylessPolicy); err == nil { + t.Fatal("a tampered/invalid signature must be rejected") + } +} + +func TestVerifyZeroPolicyIsNoop(t *testing.T) { + v := &fakeVerifier{available: false} // would fail if consulted + if err := VerifySignature(context.Background(), v, desc(), VerifyPolicy{}); err != nil { + t.Fatalf("a zero policy verifies nothing (digest-pinned only): %v", err) + } + if v.called { + t.Error("no verification should happen without a policy") + } +} + +func TestVerifyUnavailableCosignAborts(t *testing.T) { + v := &fakeVerifier{available: false} + err := VerifySignature(context.Background(), v, desc(), keylessPolicy) + if err == nil { + t.Fatal("an enabled policy with no cosign backend must abort") + } + if v.called { + t.Error("Verify must not be called when the backend is unavailable") + } +} diff --git a/internal/store/store.go b/internal/store/store.go index fd2d2f7..84a9e21 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -88,6 +88,13 @@ type Config struct { APIVersion string `yaml:"apiVersion"` Kind string `yaml:"kind"` Shared map[string]config.SharedSvc `yaml:"shared"` + // Backend is the machine-global default for WHERE the shared stack runs (spec + // 21): a `docker context` or DOCKER_HOST endpoint. nil = the local daemon. A + // workspace.yaml `backend:` block, when present, overrides this per workspace. + Backend *config.BackendConfig `yaml:"backend,omitempty"` + // Templates is the digest-pinned registry lockfile (spec 19): remote template + // sources registered by `template add`, resolved by name in the source chain. + Templates []RemoteTemplate `yaml:"templates,omitempty"` // Telemetry is the per-user/per-machine opt-in usage-telemetry consent // (spec 20). It lives here — never in workspace.yaml (must not be committed) // and never in state.db (it's user policy, not ledger state). Default OFF: a diff --git a/internal/store/templates.go b/internal/store/templates.go new file mode 100644 index 0000000..dd43fd6 --- /dev/null +++ b/internal/store/templates.go @@ -0,0 +1,89 @@ +package store + +import ( + "fmt" + "os" + "sort" + + "github.com/opencontainers/go-digest" + + "github.com/open-source-cloud/devstack/internal/xdg" +) + +// RemoteTemplate is one registered remote template source in the store config +// (spec 19). It is the lockfile row: a local `name` resolvable in the template +// source chain, the `source` registry repository, the human `version` tag (kept +// for provenance / diffs only), and the `digest` that is ACTUALLY fetched and +// verified. `schemaVersion` is the template-bundle schema the pinned content +// declares — a bundle newer than this binary understands is refused on add. +// +// The task-scoped v2 slice keeps this lockfile in the store config +// (~/.devstack/config.yaml). The full spec-19 target is the committed +// `templates:` block in workspace.yaml; the shape here is deliberately identical +// so that migration is a move, not a redesign. +type RemoteTemplate struct { + Name string `yaml:"name"` + Source string `yaml:"source"` + Version string `yaml:"version"` + Digest string `yaml:"digest"` + SchemaVersion int `yaml:"schemaVersion,omitempty"` +} + +// TemplateCacheRoot is the digest-keyed template cache root: +// $XDG_CACHE_HOME/devstack/templates. Content is addressed by digest (never by +// tag) so a re-pushed tag lands under a NEW key and a pinned workspace never +// silently reaches poisoned content (spec 19 §"Cache poisoning if the key is the +// tag"). The cache lives on the Linux side under WSL2 (xdg refuses /mnt/*). +func TemplateCacheRoot() string { + return xdg.CacheHome() + string(os.PathSeparator) + "templates" +} + +// TemplateCacheDir returns the cache directory for a manifest digest: +// //. The unpacked bundle's single "/…" tree lives +// under it, so an FSSource rooted here lists the template by name. +func TemplateCacheDir(dig string) (string, error) { + d, err := digest.Parse(dig) + if err != nil { + return "", fmt.Errorf("invalid digest %q: %w", dig, err) + } + return TemplateCacheRoot() + string(os.PathSeparator) + d.Algorithm().String() + string(os.PathSeparator) + d.Encoded(), nil +} + +// Template returns the registered remote template with the given name. +func (c *Config) Template(name string) (RemoteTemplate, bool) { + for _, t := range c.Templates { + if t.Name == name { + return t, true + } + } + return RemoteTemplate{}, false +} + +// UpsertTemplate inserts or replaces a remote-template entry by name and returns +// whether an existing entry was replaced. Callers hold the flock (this mutates +// machine-global state) and Save afterwards. +func (c *Config) UpsertTemplate(rt RemoteTemplate) (replaced bool) { + for i, t := range c.Templates { + if t.Name == rt.Name { + c.Templates[i] = rt + return true + } + } + c.Templates = append(c.Templates, rt) + sort.Slice(c.Templates, func(i, j int) bool { return c.Templates[i].Name < c.Templates[j].Name }) + return false +} + +// RemoveTemplate drops the entry with the given name, reporting whether it existed. +func (c *Config) RemoveTemplate(name string) (removed bool) { + out := c.Templates[:0] + for _, t := range c.Templates { + if t.Name == name { + removed = true + continue + } + out = append(out, t) + } + c.Templates = out + return removed +}