simulate: retry still-failing scenarios in CI with --retries (default 3)

cmd/lk/simulate.go

@@ -105,6 +105,11 @@ var simulateCommand = &cli.Command{
 			Name:  "baseline",
 			Usage: "Compare failures against the finished run with run `ID`: only scenarios that pass there and fail here fail the exit code. Non-interactive (CI) runs only",
 		},
+		&cli.IntFlag{
+			Name:  "retries",
+			Value: 3,
+			Usage: "Times to re-run scenarios that still fail before giving up (0 disables). A scenario passes when any attempt passes. Non-interactive (CI) runs only",
+		},
 		&cli.StringFlag{
 			Name:  "agent-name",
 			Usage: "Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or \"\" to target the project's default agent (the one that auto-joins every room). Requires --scenarios.",
@@ -219,6 +224,7 @@ type simulateConfig struct {
 	scenariosPath  string   // path to the --scenarios file (empty when generating from source)
 	viewModeRunID  string   // non-empty when --view opens a pre-existing run
 	baselineRunID  string   // --baseline: failures this run also has don't fail CI
+	retries        int      // --retries: times to re-run still-failing scenarios (CI only)
 	liveAgent      bool     // --agent-name: run against an already-running agent, don't spawn one
 	warnings       []string // config-level warnings surfaced at setup (e.g. ignored flags)
 
@@ -422,6 +428,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S
 		scenariosPath:  scenariosPath,
 		viewModeRunID:  runID,
 		baselineRunID:  cmd.String("baseline"),
+		retries:        int(cmd.Int("retries")),
 		liveAgent:      liveAgent,
 		warnings:       simulateConfigWarnings(mode, numSimulations),
 	}

cmd/lk/simulate_ci.go

@@ -139,86 +139,152 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error {
 
 	// --- Poll until terminal ---
 
-	brokenAgent := false
-	quotaWarned := false
-	peakRunning := 0
+	poller := &ciRunPoller{config: config, report: report, agent: agent}
+	run, err = poller.poll(ctx, runID)
+	if err != nil {
+		return err
+	}
+	runFinished = true
+	firstRunID := runID
+
+	// --- Retry still-failing scenarios ---
+	//
+	// Each retry re-runs only the scenarios still failing, against the same
+	// already-registered agent. Systemic conditions (broken agent, quota
+	// exhaustion) fail the same way again, so they are not retried.
+	attempts := []*livekit.SimulationRun{run}
+	for len(attempts)-1 < config.retries && !poller.brokenAgent && !poller.quotaWarned {
+		failed := failedScenarioKeys(mergedFinalRun(attempts))
+		if len(failed) == 0 {
+			break
+		}
+		group := retryScenarioGroup(run, config.scenarioGroup, failed)
+		if group == nil {
+			break
+		}
+
+		report.RetryingFailed(len(attempts), config.retries, failed)
+
+		retryCfg := *config
+		retryCfg.mode = modeScenarios
+		retryCfg.scenarioGroup = group
+		retryID, _, err := createSimulationRun(ctx, &retryCfg)
+		if err != nil {
+			out.Warnf("Warning: could not create the retry run: %v", err)
+			break
+		}
+		runID, runFinished = retryID, false
+		report.RunCreated(runID, simulationDashboardURL(config.pc.ProjectId, runID))
+
+		retryRun, err := poller.poll(ctx, runID)
+		if err != nil {
+			return err
+		}
+		runFinished = true
+		attempts = append(attempts, retryRun)
+	}
+	brokenAgent := poller.brokenAgent
+	finalRun := mergedFinalRun(attempts)
+
+	// --- Results ---
+
+	if !out.Interactive() {
+		report.ResultsAll(attempts, agent)
+	} else {
+		// A terminal is watching; we just couldn't open the TUI (e.g. stdin
+		// isn't a TTY). Keep it to counts and pointers, the per-scenario
+		// transcripts go to a report file like the TUI's.
+		dashboardURL := simulationDashboardURL(config.pc.ProjectId, firstRunID)
+		if path := newRunReporter().FinishAll(attempts, agent, brokenAgent, dashboardURL); path != "" {
+			out.Statusf("Run report: %s", path)
+		}
+		total, _, passed, failedN := simulationJobCounts(finalRun)
+		line := fmt.Sprintf("%d total, %d passed, %d failed", total, passed, failedN)
+		if flaky := passedOnRetry(attempts); len(flaky) > 0 {
+			line += fmt.Sprintf(" (%d passed on retry)", len(flaky))
+		}
+		fmt.Fprintln(out.ResultWriter(), line)
+	}
+
+	if brokenAgent && agent != nil {
+		writeBrokenAgentNote(out.WarnWriter(), agent)
+	}
+
+	if url := simulationDashboardURL(config.pc.ProjectId, firstRunID); url != "" {
+		out.Statusf("Dashboard:  %s", url)
+	}
+
+	return baselineFailureError(ctx, config, finalRun)
+}
+
+// ciRunPoller polls a run until it reaches a terminal state. Detection state
+// lives on the struct because it must outlive a single run: the quota warning
+// fires at most once per invocation, and peak concurrency is observed across
+// everything the invocation runs.
+type ciRunPoller struct {
+	config      *simulateConfig
+	report      *simLog
+	agent       *AgentProcess
+	brokenAgent bool
+	quotaWarned bool
+	peakRunning int
+}
+
+// poll returns the run in its terminal state, or the last state seen when ctx
+// is cancelled (so the caller's cleanup can still act on it). A broken agent
+// cancels the run and returns it without error; the caller reads brokenAgent.
+func (p *ciRunPoller) poll(ctx context.Context, runID string) (*livekit.SimulationRun, error) {
 	ticker := time.NewTicker(simulationPollInterval)
 	defer ticker.Stop()
 
+	var run *livekit.SimulationRun
+	var err error
 	for {
 		pollCtx, pollCancel := context.WithTimeout(ctx, simulationAPITimeout)
-		run, err = getSimulationRun(pollCtx, config.client, runID, config.pc.ProjectId)
+		run, err = getSimulationRun(pollCtx, p.config.client, runID, p.config.pc.ProjectId)
 		pollCancel()
 
 		if err != nil {
 			if ctx.Err() != nil {
-				return ctx.Err()
+				return run, ctx.Err()
 			}
 			out.Warnf("Warning: poll failed: %v", err)
 		} else {
-			if running := runningJobCount(run); running > peakRunning {
-				peakRunning = running
+			if running := runningJobCount(run); running > p.peakRunning {
+				p.peakRunning = running
 			}
-			if !quotaWarned && agent != nil {
-				if info := detectQuotaExceeded(agent.RecentLogs(0)); info != nil {
-					quotaWarned = true
-					suggested := suggestConcurrency(config.concurrency, peakRunning)
+			if !p.quotaWarned && p.agent != nil {
+				if info := detectQuotaExceeded(p.agent.RecentLogs(0)); info != nil {
+					p.quotaWarned = true
+					suggested := suggestConcurrency(p.config.concurrency, p.peakRunning)
 					out.Warnf("Warning: inference quota exceeded — this project is hitting its %s; LLM completions are failing with 429s. Suggested fix: re-run with --concurrency %d",
 						info.describe(), suggested)
-					report.QuotaExceeded(info.describe(), suggested)
+					p.report.QuotaExceeded(info.describe(), suggested)
 				}
 			}
 
 			// the worker is failing systemically (or, in live-agent mode, the
 			// agent never joined): stop early and surface its log
-			if !brokenAgent && agentBroken(run, agent) {
-				brokenAgent = true
-				report.BrokenAgent()
-				cancelSimulationRun(config.client, runID)
-				runFinished = true
-				break
+			if !p.brokenAgent && agentBroken(run, p.agent) {
+				p.brokenAgent = true
+				p.report.BrokenAgent()
+				cancelSimulationRun(p.config.client, runID)
+				return run, nil
 			}
 
-			report.RunUpdate(run, config.numSimulations)
+			p.report.RunUpdate(run, p.config.numSimulations)
 
 			if isTerminalRunStatus(run.Status) {
-				runFinished = true
-				break
+				return run, nil
 			}
 		}
 
 		select {
 		case <-ticker.C:
 		case <-ctx.Done():
-			return ctx.Err()
+			return run, ctx.Err()
 		}
 	}
-
-	// --- Results ---
-
-	if !out.Interactive() {
-		report.Results(run, agent)
-	} else {
-		// A terminal is watching; we just couldn't open the TUI (e.g. stdin
-		// isn't a TTY). Keep it to counts and pointers, the per-scenario
-		// transcripts go to a report file like the TUI's.
-		dashboardURL := simulationDashboardURL(config.pc.ProjectId, runID)
-		if path := newRunReporter().Finish(run, agent, brokenAgent, dashboardURL); path != "" {
-			out.Statusf("Run report: %s", path)
-		}
-		total, _, passed, failedN := simulationJobCounts(run)
-		fmt.Fprintf(out.ResultWriter(), "%d total, %d passed, %d failed\n", total, passed, failedN)
-	}
-
-	if brokenAgent && agent != nil {
-		writeBrokenAgentNote(out.WarnWriter(), agent)
-	}
-
-	if url := simulationDashboardURL(config.pc.ProjectId, runID); url != "" {
-		out.Statusf("Dashboard:  %s", url)
-	}
-
-	return baselineFailureError(ctx, config, run)
 }
 
 // baselineFailureError fetches the --baseline run when one was given and

cmd/lk/simulate_report.go

@@ -127,8 +127,48 @@ func (l *simLog) BrokenAgent() {
 	fmt.Fprintln(l.info, "The agent is failing to run jobs; cancelling the run.")
 }
 
+// RetryingFailed announces the next retry run and resets per-run progress
+// tracking so the retry's counts print from zero.
+func (l *simLog) RetryingFailed(retryNum, maxRetries int, keys []string) {
+	l.prevStatus = livekit.SimulationRun_Status(-1)
+	l.prevDone = 0
+	fmt.Fprintln(l.out)
+	fmt.Fprintf(l.out, "%d scenario(s) still failing, retrying (%d of %d): %s\n",
+		len(keys), retryNum, maxRetries, strings.Join(keys, ", "))
+}
+
 func (l *simLog) Results(run *livekit.SimulationRun, ap *AgentProcess) {
-	writeRunResults(l.out, run, ap)
+	l.ResultsAll([]*livekit.SimulationRun{run}, ap)
+}
+
+// ResultsAll writes each attempt's results in order. A job that failed but
+// passed on a later attempt keeps its transcript and loses only the ::error::
+// annotation; the merged counts at the end are the run's verdict.
+func (l *simLog) ResultsAll(attempts []*livekit.SimulationRun, ap *AgentProcess) {
+	if len(attempts) == 0 {
+		return
+	}
+	flaky := passedOnRetry(attempts)
+	resolved := make(map[string]bool, len(flaky))
+	for _, key := range flaky {
+		resolved[key] = true
+	}
+	for i, run := range attempts {
+		if i > 0 {
+			fmt.Fprintln(l.out)
+			fmt.Fprintf(l.out, "--- Retry %d ---\n", i)
+		}
+		writeRunResults(l.out, run, ap, resolved)
+	}
+	if len(attempts) > 1 {
+		total, _, passed, failed := simulationJobCounts(mergedFinalRun(attempts))
+		fmt.Fprintln(l.out)
+		fmt.Fprintf(l.out, "After retries: %d total, %d passed, %d failed", total, passed, failed)
+		if len(flaky) > 0 {
+			fmt.Fprintf(l.out, " (passed on retry: %s)", strings.Join(flaky, ", "))
+		}
+		fmt.Fprintln(l.out)
+	}
 	if l.quotaNote != "" {
 		fmt.Fprintf(l.out, "\n⚠ %s\n", l.quotaNote)
 	}
@@ -161,8 +201,10 @@ func (a asciiWriter) Write(p []byte) (int, error) {
 }
 
 // writeRunResults writes the per-job results and the run summary, with GitHub
-// group markers (a useful delimiter outside GitHub too).
-func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) {
+// group markers (a useful delimiter outside GitHub too). Failed jobs whose
+// scenario is in passedOnRetry get no ::error:: annotation — a later attempt
+// passed them.
+func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess, passedOnRetry map[string]bool) {
 	if run == nil {
 		return
 	}
@@ -227,7 +269,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess)
 
 		fmt.Fprintln(w, "::endgroup::")
 
-		if job.Status == livekit.SimulationRun_Job_STATUS_FAILED {
+		if job.Status == livekit.SimulationRun_Job_STATUS_FAILED && !passedOnRetry[scenarioKey(job)] {
 			firstLine, _, _ := strings.Cut(job.Error, "\n")
 			fmt.Fprintf(w, "::error::Job %d failed: %s\n", i+1, firstLine)
 		}
@@ -340,12 +382,18 @@ func newRunReporter() *runReporter {
 }
 
 func (r *runReporter) Finish(run *livekit.SimulationRun, ap *AgentProcess, brokenAgent bool, dashboardURL string) string {
+	var attempts []*livekit.SimulationRun
+	if run != nil {
+		attempts = []*livekit.SimulationRun{run}
+	}
+	return r.FinishAll(attempts, ap, brokenAgent, dashboardURL)
+}
+
+func (r *runReporter) FinishAll(attempts []*livekit.SimulationRun, ap *AgentProcess, brokenAgent bool, dashboardURL string) string {
 	if r.f == nil {
 		return ""
 	}
-	if run != nil {
-		r.Results(run, ap)
-	}
+	r.ResultsAll(attempts, ap)
 	if brokenAgent && ap != nil {
 		writeBrokenAgentNote(r.info, ap)
 	}