diff --git a/internal/orchestrate/orchestrate.go b/internal/orchestrate/orchestrate.go new file mode 100644 index 0000000..0b8ab3f --- /dev/null +++ b/internal/orchestrate/orchestrate.go @@ -0,0 +1,282 @@ +// Package orchestrate sequences the devstack subsystems into the resumable, +// crash-safe `up` saga (spec 09). It owns no domain logic — it drives ordered, +// named Phases under the concurrency spine (internal/lock + internal/state), and +// makes the multi-phase operation resumable (skip phases already satisfied for an +// unchanged input fingerprint), crash-safe (a phase interrupted mid-run re-runs +// because only `satisfied` skips), and observable (a Record per phase for the +// plain/--json contract + an event_log trail). +// +// This file is the engine (C5a): the Phase model and the Saga driver. The +// concrete daemon phases (clone/network/shared/provision/generate/compose-up/ +// hooks) that wire the real modules are assembled on top (C5b). +package orchestrate + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "time" + + "github.com/open-source-cloud/devstack/internal/lock" + "github.com/open-source-cloud/devstack/internal/state" +) + +// Phase statuses in the output contract (spec 09 §output-contract). +const ( + StatusOK = "ok" + StatusSkipped = "skipped" + StatusFailed = "failed" +) + +// Phase is one named, idempotent, resumable step of the saga (spec 09 §phases). +type Phase struct { + // Name is the saga_phase key + the Record label (preflight|clone|network|…). + Name string + // Scope is "" for a workspace-wide phase or a project name for a per-project one. + Scope string + // Mutating marks a phase that changed global state; only mutating phases with a + // Compensate are unwound (in reverse) when a LATER phase fails. + Mutating bool + // AlwaysRun forces the phase to run every time, never skipped or fingerprinted + // (the `secrets` phase: values are resolved in memory and never cached, spec 09). + AlwaysRun bool + // Fingerprint returns the SHA-256-able digest of the phase's resolved inputs; a + // changed digest re-arms the phase. nil ⇒ the empty fingerprint (a config-free + // phase like network-ensure, which is idempotent regardless). + Fingerprint func(context.Context) (string, error) + // Run executes the phase. It manages its own short, lock-held mutating critical + // sections (network/port/ref/provision); long work (pulls, clones, health + // polling) runs lock-free — the saga does not hold the flock across Run. + Run func(context.Context) (detail any, err error) + // Compensate undoes this phase's global mutation when a downstream phase fails. + // nil ⇒ no compensation (non-mutating phases, or mutations intentionally kept, + // e.g. the shared network and provisioned data — spec 09 §compensation). + Compensate func(context.Context) error +} + +// Record is one phase's machine-readable outcome (spec 09 §output-contract). It +// serializes to the documented `--json` element; Error is null on success. +type Record struct { + Phase string `json:"phase"` + Scope string `json:"scope,omitempty"` + Status string `json:"status"` // ok | skipped | failed + DurationMs int64 `json:"durationMs"` + Error *string `json:"error"` + Detail any `json:"detail,omitempty"` + + fingerprint string // internal: carried to the satisfied row +} + +// Saga drives an ordered phase list with resumability + compensation. +type Saga struct { + Workspace string + DB *state.DB + LockPath string + // Emit, if set, receives each Record as it completes (live streaming for the + // CLI checklist); rendering must never block — keep it cheap. + Emit func(Record) + // clock is injectable for tests; defaults to time.Now. + clock func() time.Time +} + +func (s *Saga) now() time.Time { + if s.clock != nil { + return s.clock() + } + return time.Now() +} + +// Run executes the phases in order. It returns the Record for every attempted +// phase and a non-nil error iff a phase failed (after compensation has unwound +// the mutating phases that had succeeded, in reverse order). Phases after a +// failure are not attempted. +func (s *Saga) Run(ctx context.Context, phases []Phase) ([]Record, error) { + records := make([]Record, 0, len(phases)) + var done []Phase // succeeded mutating phases, in execution order (for unwind) + + for _, p := range phases { + rec, err := s.runPhase(ctx, p) + records = append(records, rec) + s.emit(rec) + + if err != nil { + s.compensate(ctx, done) + return records, fmt.Errorf("phase %q failed: %w", p.Name, err) + } + if rec.Status == StatusOK && p.Mutating && p.Compensate != nil { + done = append(done, p) + } + } + return records, nil +} + +func (s *Saga) runPhase(ctx context.Context, p Phase) (Record, error) { + start := s.now() + rec := Record{Phase: p.Name, Scope: p.Scope} + + fp, err := s.fingerprint(ctx, p) + if err != nil { + return s.finishFail(rec, start, fmt.Errorf("fingerprint: %w", err)), err + } + rec.fingerprint = fp + + // Skip iff already satisfied for this exact fingerprint (spec 09 resumability). + if !p.AlwaysRun { + satisfied, err := s.DB.PhaseSatisfied(s.Workspace, p.Scope, p.Name, fp) + if err != nil { + return s.finishFail(rec, start, err), err + } + if satisfied { + rec.Status = StatusSkipped + rec.DurationMs = s.since(start) + return rec, nil + } + } + + // Mark started (under the flock) — a `started`-but-not-`satisfied` row is what a + // crashed run re-runs on the next invocation. + if err := s.withLock(ctx, func() error { + return s.DB.StartPhase(s.Workspace, p.Scope, p.Name, fp) + }); err != nil { + return s.finishFail(rec, start, err), err + } + s.DB.LogEvent("saga", s.qualified(p), "started") + + detail, runErr := s.runBody(ctx, p) + rec.Detail = detail + if runErr != nil { + _ = s.withLock(ctx, func() error { + return s.DB.FailPhase(s.Workspace, p.Scope, p.Name, runErr.Error()) + }) + s.DB.LogEvent("saga", s.qualified(p), "failed: "+runErr.Error()) + return s.finishFail(rec, start, runErr), runErr + } + + if err := s.withLock(ctx, func() error { + return s.DB.SatisfyPhase(s.Workspace, p.Scope, p.Name) + }); err != nil { + return s.finishFail(rec, start, err), err + } + rec.Status = StatusOK + rec.DurationMs = s.since(start) + s.DB.LogEvent("saga", s.qualified(p), fmt.Sprintf("satisfied in %dms", rec.DurationMs)) + return rec, nil +} + +// runBody invokes a phase's Run, converting a panic into an error so one phase +// can never crash the whole CLI (the saga must always reach compensation). +func (s *Saga) runBody(ctx context.Context, p Phase) (detail any, err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic in phase %q: %v", p.Name, r) + } + }() + if p.Run == nil { + return nil, nil + } + return p.Run(ctx) +} + +// compensate unwinds the succeeded mutating phases in reverse, clearing each +// phase row so a re-run redoes it. A compensation error is logged, not fatal — +// best-effort cleanup must not mask the original failure. +func (s *Saga) compensate(ctx context.Context, done []Phase) { + for i := len(done) - 1; i >= 0; i-- { + p := done[i] + if p.Compensate != nil { + if err := p.Compensate(ctx); err != nil { + s.DB.LogEvent("saga", s.qualified(p), "compensation failed: "+err.Error()) + } else { + s.DB.LogEvent("saga", s.qualified(p), "compensated") + } + } + _ = s.withLock(ctx, func() error { + return s.DB.ClearPhase(s.Workspace, p.Scope, p.Name) + }) + } +} + +func (s *Saga) fingerprint(ctx context.Context, p Phase) (string, error) { + if p.AlwaysRun || p.Fingerprint == nil { + return "", nil + } + return p.Fingerprint(ctx) +} + +func (s *Saga) withLock(ctx context.Context, fn func() error) error { + if s.LockPath == "" { + return fn() // tests without a lock path run unlocked + } + return lock.WithLock(ctx, s.LockPath, fn) +} + +func (s *Saga) finishFail(rec Record, start time.Time, err error) Record { + rec.Status = StatusFailed + rec.DurationMs = s.since(start) + msg := err.Error() + rec.Error = &msg + return rec +} + +func (s *Saga) since(start time.Time) int64 { return s.now().Sub(start).Milliseconds() } + +func (s *Saga) emit(rec Record) { + if s.Emit != nil { + s.Emit(rec) + } +} + +func (s *Saga) qualified(p Phase) string { + if p.Scope == "" { + return p.Name + } + return p.Scope + "/" + p.Name +} + +// Fingerprint hashes its parts into a stable hex digest for a phase's +// Fingerprint func (config bytes, params, resolved versions). Order-sensitive. +func Fingerprint(parts ...string) string { + h := sha256.New() + for _, p := range parts { + _, _ = io.WriteString(h, p) + _, _ = h.Write([]byte{0}) // length-independent separator + } + return hex.EncodeToString(h.Sum(nil)) +} + +// FormatPlain renders one Record as a single non-TTY line (spec 09 plain mode): +// +// [ok] network (12ms) +// [skipped] generate +// [failed] compose-up: service api exited (1) +func FormatPlain(r Record) string { + label := r.Phase + if r.Scope != "" { + label = r.Scope + "/" + r.Phase + } + switch r.Status { + case StatusSkipped: + return fmt.Sprintf("[skipped] %s", label) + case StatusFailed: + msg := "" + if r.Error != nil { + msg = ": " + *r.Error + } + return fmt.Sprintf("[failed] %s (%dms)%s", label, r.DurationMs, msg) + default: + return fmt.Sprintf("[ok] %s (%dms)", label, r.DurationMs) + } +} + +// AnyFailed reports whether any record failed (the saga's process exit code is +// non-zero iff this is true, spec 09). +func AnyFailed(records []Record) bool { + for _, r := range records { + if r.Status == StatusFailed { + return true + } + } + return false +} diff --git a/internal/orchestrate/orchestrate_test.go b/internal/orchestrate/orchestrate_test.go new file mode 100644 index 0000000..7ac8e62 --- /dev/null +++ b/internal/orchestrate/orchestrate_test.go @@ -0,0 +1,287 @@ +package orchestrate + +import ( + "context" + "encoding/json" + "errors" + "path/filepath" + "strings" + "testing" + + "github.com/open-source-cloud/devstack/internal/state" +) + +func newSaga(t *testing.T) *Saga { + t.Helper() + dir := t.TempDir() + db, err := state.Open(context.Background(), dir, "ctx") + if err != nil { + t.Fatalf("open state: %v", err) + } + t.Cleanup(func() { db.Close() }) + return &Saga{Workspace: "demo", DB: db, LockPath: filepath.Join(dir, "lock")} +} + +// recordingPhase builds a Phase that counts its runs/compensations via the given +// pointers, with a fixed fingerprint. +func recordingPhase(name string, runs, comps *int, fp string, fail bool) Phase { + return Phase{ + Name: name, + Mutating: true, + Fingerprint: func(context.Context) (string, error) { + if fp == "" { + return "", nil + } + return Fingerprint(fp), nil + }, + Run: func(context.Context) (any, error) { + *runs++ + if fail { + return nil, errors.New("boom in " + name) + } + return map[string]any{"name": name}, nil + }, + Compensate: func(context.Context) error { + *comps++ + return nil + }, + } +} + +func TestSagaHappyPath(t *testing.T) { + s := newSaga(t) + var aRuns, aComp, bRuns, bComp int + phases := []Phase{ + recordingPhase("network", &aRuns, &aComp, "fp-a", false), + recordingPhase("shared", &bRuns, &bComp, "fp-b", false), + } + recs, err := s.Run(context.Background(), phases) + if err != nil { + t.Fatalf("Run: %v", err) + } + if len(recs) != 2 || recs[0].Status != StatusOK || recs[1].Status != StatusOK { + t.Fatalf("records = %+v, want two ok", recs) + } + if aRuns != 1 || bRuns != 1 { + t.Errorf("runs a=%d b=%d, want 1 each", aRuns, bRuns) + } + if aComp != 0 || bComp != 0 { + t.Errorf("no compensation on success; got a=%d b=%d", aComp, bComp) + } + if recs[0].Error != nil { + t.Error("ok record should have null error") + } +} + +func TestSagaSkipsSatisfiedOnRerun(t *testing.T) { + s := newSaga(t) + var runs, comps int + mk := func() []Phase { + return []Phase{recordingPhase("generate", &runs, &comps, "fp", false)} + } + if _, err := s.Run(context.Background(), mk()); err != nil { + t.Fatal(err) + } + recs, err := s.Run(context.Background(), mk()) + if err != nil { + t.Fatal(err) + } + if recs[0].Status != StatusSkipped { + t.Errorf("re-run status = %q, want skipped", recs[0].Status) + } + if runs != 1 { + t.Errorf("Run executed %d times, want 1 (second was skipped)", runs) + } +} + +func TestSagaFingerprintRearm(t *testing.T) { + s := newSaga(t) + var runs, comps int + first := []Phase{recordingPhase("generate", &runs, &comps, "v1", false)} + if _, err := s.Run(context.Background(), first); err != nil { + t.Fatal(err) + } + // Same name, different fingerprint → must re-run. + second := []Phase{recordingPhase("generate", &runs, &comps, "v2", false)} + recs, err := s.Run(context.Background(), second) + if err != nil { + t.Fatal(err) + } + if recs[0].Status != StatusOK || runs != 2 { + t.Errorf("changed fingerprint should re-run: status=%q runs=%d", recs[0].Status, runs) + } +} + +func TestSagaAlwaysRun(t *testing.T) { + s := newSaga(t) + runs := 0 + p := Phase{Name: "secrets", AlwaysRun: true, Run: func(context.Context) (any, error) { + runs++ + return nil, nil + }} + for range 3 { + if _, err := s.Run(context.Background(), []Phase{p}); err != nil { + t.Fatal(err) + } + } + if runs != 3 { + t.Errorf("AlwaysRun executed %d times, want 3", runs) + } +} + +func TestSagaFailureCompensatesInReverse(t *testing.T) { + s := newSaga(t) + var aRuns, aComp, bRuns, bComp, cRuns, cComp int + var order []string + mkComp := func(name string, comp *int) func(context.Context) error { + return func(context.Context) error { + *comp++ + order = append(order, name) + return nil + } + } + a := recordingPhase("network", &aRuns, &aComp, "a", false) + a.Compensate = mkComp("network", &aComp) + b := recordingPhase("shared", &bRuns, &bComp, "b", false) + b.Compensate = mkComp("shared", &bComp) + c := recordingPhase("compose-up", &cRuns, &cComp, "c", true) // fails + + recs, err := s.Run(context.Background(), []Phase{a, b, c}) + if err == nil { + t.Fatal("expected the saga to fail") + } + if len(recs) != 3 || recs[2].Status != StatusFailed { + t.Fatalf("records = %+v, want a,b ok + c failed", recs) + } + // Compensation runs the SUCCEEDED mutating phases in reverse: shared then network. + if len(order) != 2 || order[0] != "shared" || order[1] != "network" { + t.Fatalf("compensation order = %v, want [shared network]", order) + } + // The compensated phases' rows are cleared → a re-run redoes them. + for _, name := range []string{"network", "shared"} { + if ok, _ := s.DB.PhaseSatisfied("demo", "", name, Fingerprint(name[:1])); ok { + t.Errorf("phase %q should not be satisfied after compensation", name) + } + } +} + +func TestSagaResumesAfterFailure(t *testing.T) { + s := newSaga(t) + var aRuns, aComp, bRuns, bComp int + // First run: A ok, B fails. + a1 := recordingPhase("network", &aRuns, &aComp, "a", false) + a1.Compensate = nil // keep A's row (network is never auto-removed, spec 09) + a1.Mutating = false + b1 := recordingPhase("shared", &bRuns, &bComp, "b", true) + if _, err := s.Run(context.Background(), []Phase{a1, b1}); err == nil { + t.Fatal("first run should fail at shared") + } + // Second run: A skips (satisfied, unchanged), B now succeeds. + a2 := recordingPhase("network", &aRuns, &aComp, "a", false) + a2.Compensate = nil + a2.Mutating = false + b2 := recordingPhase("shared", &bRuns, &bComp, "b", false) + recs, err := s.Run(context.Background(), []Phase{a2, b2}) + if err != nil { + t.Fatalf("resume run: %v", err) + } + if recs[0].Status != StatusSkipped { + t.Errorf("network should skip on resume, got %q", recs[0].Status) + } + if recs[1].Status != StatusOK { + t.Errorf("shared should run on resume, got %q", recs[1].Status) + } + if aRuns != 1 { + t.Errorf("network ran %d times, want 1 (resumed = skipped)", aRuns) + } + if bRuns != 2 { + t.Errorf("shared ran %d times, want 2 (failed then succeeded)", bRuns) + } +} + +func TestSagaPanicBecomesFailure(t *testing.T) { + s := newSaga(t) + comp := 0 + a := Phase{Name: "network", Mutating: true, + Run: func(context.Context) (any, error) { return nil, nil }, + Compensate: func(context.Context) error { comp++; return nil }} + boom := Phase{Name: "shared", Run: func(context.Context) (any, error) { panic("kaboom") }} + recs, err := s.Run(context.Background(), []Phase{a, boom}) + if err == nil { + t.Fatal("a panicking phase must fail the saga") + } + if recs[1].Status != StatusFailed || recs[1].Error == nil || !strings.Contains(*recs[1].Error, "panic") { + t.Errorf("panic record = %+v, want failed with panic error", recs[1]) + } + if comp != 1 { + t.Errorf("compensation should run after a panic; comp=%d", comp) + } +} + +func TestRecordJSONContract(t *testing.T) { + ok := Record{Phase: "clone", Status: StatusOK, DurationMs: 1200} + b, _ := json.Marshal(ok) + if !strings.Contains(string(b), `"error":null`) { + t.Errorf("ok record JSON = %s, want error:null", b) + } + msg := "service api exited (1)" + fail := Record{Phase: "compose-up", Status: StatusFailed, DurationMs: 900, Error: &msg} + b2, _ := json.Marshal(fail) + if !strings.Contains(string(b2), `"status":"failed"`) || !strings.Contains(string(b2), msg) { + t.Errorf("failed record JSON = %s", b2) + } +} + +func TestFormatPlain(t *testing.T) { + msg := "boom" + cases := []struct { + rec Record + want string + }{ + {Record{Phase: "network", Status: StatusOK, DurationMs: 12}, "[ok] network (12ms)"}, + {Record{Phase: "generate", Status: StatusSkipped}, "[skipped] generate"}, + {Record{Phase: "compose-up", Scope: "api", Status: StatusFailed, DurationMs: 5, Error: &msg}, "[failed] api/compose-up (5ms): boom"}, + } + for _, c := range cases { + if got := FormatPlain(c.rec); got != c.want { + t.Errorf("FormatPlain = %q, want %q", got, c.want) + } + } +} + +func TestFingerprintStable(t *testing.T) { + ab := Fingerprint("a", "b") + if ab != Fingerprint("a", "b") { + t.Error("Fingerprint not stable for equal inputs") + } + // Order- and boundary-sensitive: ["a","b"] != ["ab"] != ["b","a"]. + if ab == Fingerprint("ab") { + t.Error("Fingerprint collides across part boundaries") + } + if ab == Fingerprint("b", "a") { + t.Error("Fingerprint should be order-sensitive") + } +} + +func TestAnyFailed(t *testing.T) { + if AnyFailed([]Record{{Status: StatusOK}, {Status: StatusSkipped}}) { + t.Error("no failed records → AnyFailed false") + } + if !AnyFailed([]Record{{Status: StatusOK}, {Status: StatusFailed}}) { + t.Error("a failed record → AnyFailed true") + } +} + +func TestEmitStreamsRecords(t *testing.T) { + s := newSaga(t) + var emitted []string + s.Emit = func(r Record) { emitted = append(emitted, r.Phase+":"+r.Status) } + var runs, comps int + _, err := s.Run(context.Background(), []Phase{recordingPhase("network", &runs, &comps, "a", false)}) + if err != nil { + t.Fatal(err) + } + if len(emitted) != 1 || emitted[0] != "network:ok" { + t.Errorf("emitted = %v, want [network:ok]", emitted) + } +}