From 5998abf6e336269d2a908d72774aecf318df4b73 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 31 Aug 2026 13:31:41 -0400 Subject: [PATCH 1/2] refactor(simulate): extract the CI poll loop into ciRunPoller Pure move: the loop body is unchanged, its cross-run detection state (broken agent, quota warning, peak concurrency) now lives on a struct so a follow-up can poll more than one run per invocation. --- cmd/lk/simulate_ci.go | 119 +++++++++++++++++++++++++----------------- 1 file changed, 71 insertions(+), 48 deletions(-) diff --git a/cmd/lk/simulate_ci.go b/cmd/lk/simulate_ci.go index ae9cc525..a2028534 100644 --- a/cmd/lk/simulate_ci.go +++ b/cmd/lk/simulate_ci.go @@ -139,86 +139,109 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { // --- Poll until terminal --- - brokenAgent := false - quotaWarned := false - peakRunning := 0 + poller := &ciRunPoller{config: config, report: report, agent: agent} + run, err = poller.poll(ctx, runID) + if err != nil { + return err + } + runFinished = true + brokenAgent := poller.brokenAgent + + // --- Results --- + + if !out.Interactive() { + report.Results(run, agent) + } else { + // A terminal is watching; we just couldn't open the TUI (e.g. stdin + // isn't a TTY). Keep it to counts and pointers, the per-scenario + // transcripts go to a report file like the TUI's. + dashboardURL := simulationDashboardURL(config.pc.ProjectId, runID) + if path := newRunReporter().Finish(run, agent, brokenAgent, dashboardURL); path != "" { + out.Statusf("Run report: %s", path) + } + total, _, passed, failedN := simulationJobCounts(run) + fmt.Fprintf(out.ResultWriter(), "%d total, %d passed, %d failed\n", total, passed, failedN) + } + + if brokenAgent && agent != nil { + writeBrokenAgentNote(out.WarnWriter(), agent) + } + + if url := simulationDashboardURL(config.pc.ProjectId, runID); url != "" { + out.Statusf("Dashboard: %s", url) + } + + return baselineFailureError(ctx, config, run) +} + +// ciRunPoller polls a run until it reaches a terminal state. Detection state +// lives on the struct because it must outlive a single run: the quota warning +// fires at most once per invocation, and peak concurrency is observed across +// everything the invocation runs. +type ciRunPoller struct { + config *simulateConfig + report *simLog + agent *AgentProcess + brokenAgent bool + quotaWarned bool + peakRunning int +} + +// poll returns the run in its terminal state, or the last state seen when ctx +// is cancelled (so the caller's cleanup can still act on it). A broken agent +// cancels the run and returns it without error; the caller reads brokenAgent. +func (p *ciRunPoller) poll(ctx context.Context, runID string) (*livekit.SimulationRun, error) { ticker := time.NewTicker(simulationPollInterval) defer ticker.Stop() + var run *livekit.SimulationRun + var err error for { pollCtx, pollCancel := context.WithTimeout(ctx, simulationAPITimeout) - run, err = getSimulationRun(pollCtx, config.client, runID, config.pc.ProjectId) + run, err = getSimulationRun(pollCtx, p.config.client, runID, p.config.pc.ProjectId) pollCancel() if err != nil { if ctx.Err() != nil { - return ctx.Err() + return run, ctx.Err() } out.Warnf("Warning: poll failed: %v", err) } else { - if running := runningJobCount(run); running > peakRunning { - peakRunning = running + if running := runningJobCount(run); running > p.peakRunning { + p.peakRunning = running } - if !quotaWarned && agent != nil { - if info := detectQuotaExceeded(agent.RecentLogs(0)); info != nil { - quotaWarned = true - suggested := suggestConcurrency(config.concurrency, peakRunning) + if !p.quotaWarned && p.agent != nil { + if info := detectQuotaExceeded(p.agent.RecentLogs(0)); info != nil { + p.quotaWarned = true + suggested := suggestConcurrency(p.config.concurrency, p.peakRunning) out.Warnf("Warning: inference quota exceeded ā€” this project is hitting its %s; LLM completions are failing with 429s. Suggested fix: re-run with --concurrency %d", info.describe(), suggested) - report.QuotaExceeded(info.describe(), suggested) + p.report.QuotaExceeded(info.describe(), suggested) } } // the worker is failing systemically (or, in live-agent mode, the // agent never joined): stop early and surface its log - if !brokenAgent && agentBroken(run, agent) { - brokenAgent = true - report.BrokenAgent() - cancelSimulationRun(config.client, runID) - runFinished = true - break + if !p.brokenAgent && agentBroken(run, p.agent) { + p.brokenAgent = true + p.report.BrokenAgent() + cancelSimulationRun(p.config.client, runID) + return run, nil } - report.RunUpdate(run, config.numSimulations) + p.report.RunUpdate(run, p.config.numSimulations) if isTerminalRunStatus(run.Status) { - runFinished = true - break + return run, nil } } select { case <-ticker.C: case <-ctx.Done(): - return ctx.Err() - } - } - - // --- Results --- - - if !out.Interactive() { - report.Results(run, agent) - } else { - // A terminal is watching; we just couldn't open the TUI (e.g. stdin - // isn't a TTY). Keep it to counts and pointers, the per-scenario - // transcripts go to a report file like the TUI's. - dashboardURL := simulationDashboardURL(config.pc.ProjectId, runID) - if path := newRunReporter().Finish(run, agent, brokenAgent, dashboardURL); path != "" { - out.Statusf("Run report: %s", path) + return run, ctx.Err() } - total, _, passed, failedN := simulationJobCounts(run) - fmt.Fprintf(out.ResultWriter(), "%d total, %d passed, %d failed\n", total, passed, failedN) - } - - if brokenAgent && agent != nil { - writeBrokenAgentNote(out.WarnWriter(), agent) - } - - if url := simulationDashboardURL(config.pc.ProjectId, runID); url != "" { - out.Statusf("Dashboard: %s", url) } - - return baselineFailureError(ctx, config, run) } // baselineFailureError fetches the --baseline run when one was given and From 30b901b367ac3f3c4e82de157f8071c858d034b5 Mon Sep 17 00:00:00 2001 From: Jason Lernerman Date: Mon, 31 Aug 2026 13:37:38 -0400 Subject: [PATCH 2/2] feat(simulate): retry still-failing scenarios in CI with --retries (default 3) After a run finishes with failures, CI mode re-runs only the failing scenarios against the same already-registered agent, up to --retries times; a scenario passes when any attempt does. The scenarios come from the finished run itself, so generated-from-source runs retry without re-uploading or regenerating. Systemic conditions (broken agent, quota exhaustion) are never retried. The verdict (counts, --baseline comparison, exit error) reads each scenario's outcome from the last attempt that ran it to a terminal state, so a cancelled retry cannot launder an earlier failure. Every attempt's transcript still prints; a failure that later passed keeps its transcript but loses its ::error:: annotation, and the final counts name the scenarios that passed on retry so flakes stay visible. --- cmd/lk/simulate.go | 7 ++ cmd/lk/simulate_ci.go | 57 ++++++++++++++-- cmd/lk/simulate_report.go | 62 ++++++++++++++++-- cmd/lk/simulate_retry.go | 105 ++++++++++++++++++++++++++++++ cmd/lk/simulate_retry_test.go | 118 ++++++++++++++++++++++++++++++++++ 5 files changed, 335 insertions(+), 14 deletions(-) create mode 100644 cmd/lk/simulate_retry.go create mode 100644 cmd/lk/simulate_retry_test.go diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index 526dda1c..7bb43cb4 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -105,6 +105,11 @@ var simulateCommand = &cli.Command{ Name: "baseline", Usage: "Compare failures against the finished run with run `ID`: only scenarios that pass there and fail here fail the exit code. Non-interactive (CI) runs only", }, + &cli.IntFlag{ + Name: "retries", + Value: 3, + Usage: "Times to re-run scenarios that still fail before giving up (0 disables). A scenario passes when any attempt passes. Non-interactive (CI) runs only", + }, &cli.StringFlag{ Name: "agent-name", Usage: "Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or \"\" to target the project's default agent (the one that auto-joins every room). Requires --scenarios.", @@ -219,6 +224,7 @@ type simulateConfig struct { scenariosPath string // path to the --scenarios file (empty when generating from source) viewModeRunID string // non-empty when --view opens a pre-existing run baselineRunID string // --baseline: failures this run also has don't fail CI + retries int // --retries: times to re-run still-failing scenarios (CI only) liveAgent bool // --agent-name: run against an already-running agent, don't spawn one warnings []string // config-level warnings surfaced at setup (e.g. ignored flags) @@ -422,6 +428,7 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S scenariosPath: scenariosPath, viewModeRunID: runID, baselineRunID: cmd.String("baseline"), + retries: int(cmd.Int("retries")), liveAgent: liveAgent, warnings: simulateConfigWarnings(mode, numSimulations), } diff --git a/cmd/lk/simulate_ci.go b/cmd/lk/simulate_ci.go index a2028534..6a89ebae 100644 --- a/cmd/lk/simulate_ci.go +++ b/cmd/lk/simulate_ci.go @@ -145,33 +145,76 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { return err } runFinished = true + firstRunID := runID + + // --- Retry still-failing scenarios --- + // + // Each retry re-runs only the scenarios still failing, against the same + // already-registered agent. Systemic conditions (broken agent, quota + // exhaustion) fail the same way again, so they are not retried. + attempts := []*livekit.SimulationRun{run} + for len(attempts)-1 < config.retries && !poller.brokenAgent && !poller.quotaWarned { + failed := failedScenarioKeys(mergedFinalRun(attempts)) + if len(failed) == 0 { + break + } + group := retryScenarioGroup(run, config.scenarioGroup, failed) + if group == nil { + break + } + + report.RetryingFailed(len(attempts), config.retries, failed) + + retryCfg := *config + retryCfg.mode = modeScenarios + retryCfg.scenarioGroup = group + retryID, _, err := createSimulationRun(ctx, &retryCfg) + if err != nil { + out.Warnf("Warning: could not create the retry run: %v", err) + break + } + runID, runFinished = retryID, false + report.RunCreated(runID, simulationDashboardURL(config.pc.ProjectId, runID)) + + retryRun, err := poller.poll(ctx, runID) + if err != nil { + return err + } + runFinished = true + attempts = append(attempts, retryRun) + } brokenAgent := poller.brokenAgent + finalRun := mergedFinalRun(attempts) // --- Results --- if !out.Interactive() { - report.Results(run, agent) + report.ResultsAll(attempts, agent) } else { // A terminal is watching; we just couldn't open the TUI (e.g. stdin // isn't a TTY). Keep it to counts and pointers, the per-scenario // transcripts go to a report file like the TUI's. - dashboardURL := simulationDashboardURL(config.pc.ProjectId, runID) - if path := newRunReporter().Finish(run, agent, brokenAgent, dashboardURL); path != "" { + dashboardURL := simulationDashboardURL(config.pc.ProjectId, firstRunID) + if path := newRunReporter().FinishAll(attempts, agent, brokenAgent, dashboardURL); path != "" { out.Statusf("Run report: %s", path) } - total, _, passed, failedN := simulationJobCounts(run) - fmt.Fprintf(out.ResultWriter(), "%d total, %d passed, %d failed\n", total, passed, failedN) + total, _, passed, failedN := simulationJobCounts(finalRun) + line := fmt.Sprintf("%d total, %d passed, %d failed", total, passed, failedN) + if flaky := passedOnRetry(attempts); len(flaky) > 0 { + line += fmt.Sprintf(" (%d passed on retry)", len(flaky)) + } + fmt.Fprintln(out.ResultWriter(), line) } if brokenAgent && agent != nil { writeBrokenAgentNote(out.WarnWriter(), agent) } - if url := simulationDashboardURL(config.pc.ProjectId, runID); url != "" { + if url := simulationDashboardURL(config.pc.ProjectId, firstRunID); url != "" { out.Statusf("Dashboard: %s", url) } - return baselineFailureError(ctx, config, run) + return baselineFailureError(ctx, config, finalRun) } // ciRunPoller polls a run until it reaches a terminal state. Detection state diff --git a/cmd/lk/simulate_report.go b/cmd/lk/simulate_report.go index 5344ab58..b0a4fb04 100644 --- a/cmd/lk/simulate_report.go +++ b/cmd/lk/simulate_report.go @@ -127,8 +127,48 @@ func (l *simLog) BrokenAgent() { fmt.Fprintln(l.info, "The agent is failing to run jobs; cancelling the run.") } +// RetryingFailed announces the next retry run and resets per-run progress +// tracking so the retry's counts print from zero. +func (l *simLog) RetryingFailed(retryNum, maxRetries int, keys []string) { + l.prevStatus = livekit.SimulationRun_Status(-1) + l.prevDone = 0 + fmt.Fprintln(l.out) + fmt.Fprintf(l.out, "%d scenario(s) still failing, retrying (%d of %d): %s\n", + len(keys), retryNum, maxRetries, strings.Join(keys, ", ")) +} + func (l *simLog) Results(run *livekit.SimulationRun, ap *AgentProcess) { - writeRunResults(l.out, run, ap) + l.ResultsAll([]*livekit.SimulationRun{run}, ap) +} + +// ResultsAll writes each attempt's results in order. A job that failed but +// passed on a later attempt keeps its transcript and loses only the ::error:: +// annotation; the merged counts at the end are the run's verdict. +func (l *simLog) ResultsAll(attempts []*livekit.SimulationRun, ap *AgentProcess) { + if len(attempts) == 0 { + return + } + flaky := passedOnRetry(attempts) + resolved := make(map[string]bool, len(flaky)) + for _, key := range flaky { + resolved[key] = true + } + for i, run := range attempts { + if i > 0 { + fmt.Fprintln(l.out) + fmt.Fprintf(l.out, "--- Retry %d ---\n", i) + } + writeRunResults(l.out, run, ap, resolved) + } + if len(attempts) > 1 { + total, _, passed, failed := simulationJobCounts(mergedFinalRun(attempts)) + fmt.Fprintln(l.out) + fmt.Fprintf(l.out, "After retries: %d total, %d passed, %d failed", total, passed, failed) + if len(flaky) > 0 { + fmt.Fprintf(l.out, " (passed on retry: %s)", strings.Join(flaky, ", ")) + } + fmt.Fprintln(l.out) + } if l.quotaNote != "" { fmt.Fprintf(l.out, "\nāš  %s\n", l.quotaNote) } @@ -161,8 +201,10 @@ func (a asciiWriter) Write(p []byte) (int, error) { } // writeRunResults writes the per-job results and the run summary, with GitHub -// group markers (a useful delimiter outside GitHub too). -func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) { +// group markers (a useful delimiter outside GitHub too). Failed jobs whose +// scenario is in passedOnRetry get no ::error:: annotation ā€” a later attempt +// passed them. +func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess, passedOnRetry map[string]bool) { if run == nil { return } @@ -227,7 +269,7 @@ func writeRunResults(w io.Writer, run *livekit.SimulationRun, ap *AgentProcess) fmt.Fprintln(w, "::endgroup::") - if job.Status == livekit.SimulationRun_Job_STATUS_FAILED { + if job.Status == livekit.SimulationRun_Job_STATUS_FAILED && !passedOnRetry[scenarioKey(job)] { firstLine, _, _ := strings.Cut(job.Error, "\n") fmt.Fprintf(w, "::error::Job %d failed: %s\n", i+1, firstLine) } @@ -340,12 +382,18 @@ func newRunReporter() *runReporter { } func (r *runReporter) Finish(run *livekit.SimulationRun, ap *AgentProcess, brokenAgent bool, dashboardURL string) string { + var attempts []*livekit.SimulationRun + if run != nil { + attempts = []*livekit.SimulationRun{run} + } + return r.FinishAll(attempts, ap, brokenAgent, dashboardURL) +} + +func (r *runReporter) FinishAll(attempts []*livekit.SimulationRun, ap *AgentProcess, brokenAgent bool, dashboardURL string) string { if r.f == nil { return "" } - if run != nil { - r.Results(run, ap) - } + r.ResultsAll(attempts, ap) if brokenAgent && ap != nil { writeBrokenAgentNote(r.info, ap) } diff --git a/cmd/lk/simulate_retry.go b/cmd/lk/simulate_retry.go new file mode 100644 index 00000000..a71ba394 --- /dev/null +++ b/cmd/lk/simulate_retry.go @@ -0,0 +1,105 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/livekit/protocol/livekit" + "google.golang.org/protobuf/proto" +) + +// scenarioKey identifies a scenario across runs: the label, or the +// instructions for generated jobs that carry no label. +func scenarioKey(job *livekit.SimulationRun_Job) string { + if l := job.GetLabel(); l != "" { + return l + } + return job.GetInstructions() +} + +// retryScenarioGroup returns the scenarios matching keys, taken from the +// finished run (which carries its scenarios in both --scenarios and generated +// modes) or from fallback when the run carries none. Nil when nothing matches: +// there is nothing to re-run. +func retryScenarioGroup(run *livekit.SimulationRun, fallback *livekit.ScenarioGroup, keys []string) *livekit.ScenarioGroup { + group := run.GetScenarioGroup() + if len(group.GetScenarios()) == 0 { + group = fallback + } + want := make(map[string]bool, len(keys)) + for _, k := range keys { + want[k] = true + } + out := &livekit.ScenarioGroup{Name: group.GetName()} + for _, s := range group.GetScenarios() { + key := s.GetLabel() + if key == "" { + key = s.GetInstructions() + } + if want[key] { + out.Scenarios = append(out.Scenarios, s) + } + } + if len(out.Scenarios) == 0 { + return nil + } + return out +} + +// mergedFinalRun folds retry attempts into the first run: each job's outcome +// comes from the last attempt that ran its scenario to a terminal state, so a +// cancelled retry cannot launder an earlier failure. The CI verdict (counts, +// baseline comparison, exit error) reads final outcomes; the printed +// per-attempt results stay verbatim. +func mergedFinalRun(attempts []*livekit.SimulationRun) *livekit.SimulationRun { + if len(attempts) == 1 { + return attempts[0] + } + final := make(map[string]*livekit.SimulationRun_Job) + for _, run := range attempts[1:] { + for _, job := range run.GetJobs() { + if isTerminalJobStatus(job.GetStatus()) { + final[scenarioKey(job)] = job + } + } + } + merged := proto.Clone(attempts[0]).(*livekit.SimulationRun) + for i, job := range merged.Jobs { + if f, ok := final[scenarioKey(job)]; ok { + merged.Jobs[i] = f + } + } + return merged +} + +// passedOnRetry returns the scenarios that failed on some attempt but passed +// on a later one, in first-run job order. +func passedOnRetry(attempts []*livekit.SimulationRun) []string { + failedEver := make(map[string]bool) + for _, run := range attempts { + for _, key := range failedScenarioKeys(run) { + failedEver[key] = true + } + } + seen := make(map[string]bool) + var flaky []string + for _, job := range mergedFinalRun(attempts).GetJobs() { + key := scenarioKey(job) + if failedEver[key] && !seen[key] && job.GetStatus() == livekit.SimulationRun_Job_STATUS_COMPLETED { + seen[key] = true + flaky = append(flaky, key) + } + } + return flaky +} diff --git a/cmd/lk/simulate_retry_test.go b/cmd/lk/simulate_retry_test.go new file mode 100644 index 00000000..2169a0c5 --- /dev/null +++ b/cmd/lk/simulate_retry_test.go @@ -0,0 +1,118 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "testing" + + "github.com/livekit/protocol/livekit" + "github.com/stretchr/testify/require" +) + +func retryJob(label string, status livekit.SimulationRun_Job_Status) *livekit.SimulationRun_Job { + return &livekit.SimulationRun_Job{Label: label, Status: status} +} + +func retryRunOf(jobs ...*livekit.SimulationRun_Job) *livekit.SimulationRun { + return &livekit.SimulationRun{ + Status: livekit.SimulationRun_STATUS_COMPLETED, + Jobs: jobs, + } +} + +const ( + jobPassed = livekit.SimulationRun_Job_STATUS_COMPLETED + jobFailed = livekit.SimulationRun_Job_STATUS_FAILED + jobRunning = livekit.SimulationRun_Job_STATUS_RUNNING +) + +func TestMergedFinalRun(t *testing.T) { + first := retryRunOf( + retryJob("a", jobPassed), + retryJob("b", jobFailed), + retryJob("c", jobFailed), + ) + + t.Run("single attempt is returned as-is", func(t *testing.T) { + require.Same(t, first, mergedFinalRun([]*livekit.SimulationRun{first})) + }) + + t.Run("last terminal outcome wins", func(t *testing.T) { + retry1 := retryRunOf(retryJob("b", jobPassed), retryJob("c", jobFailed)) + retry2 := retryRunOf(retryJob("c", jobPassed)) + merged := mergedFinalRun([]*livekit.SimulationRun{first, retry1, retry2}) + require.Equal(t, []string(nil), failedScenarioKeys(merged)) + total, _, passed, failed := simulationJobCounts(merged) + require.Equal(t, 3, total) + require.Equal(t, 3, passed) + require.Equal(t, 0, failed) + }) + + t.Run("a cancelled retry cannot launder a failure", func(t *testing.T) { + // the retry was cancelled with the job still running: non-terminal + // outcomes are ignored, the first run's failure stands + retry := retryRunOf(retryJob("b", jobRunning), retryJob("c", jobPassed)) + merged := mergedFinalRun([]*livekit.SimulationRun{first, retry}) + require.Equal(t, []string{"b"}, failedScenarioKeys(merged)) + }) + + t.Run("generated jobs without labels merge by instructions", func(t *testing.T) { + f := retryRunOf(&livekit.SimulationRun_Job{Instructions: "ask for hours", Status: jobFailed}) + r := retryRunOf(&livekit.SimulationRun_Job{Instructions: "ask for hours", Status: jobPassed}) + merged := mergedFinalRun([]*livekit.SimulationRun{f, r}) + require.Empty(t, failedScenarioKeys(merged)) + }) +} + +func TestPassedOnRetry(t *testing.T) { + first := retryRunOf( + retryJob("a", jobPassed), + retryJob("b", jobFailed), + retryJob("c", jobFailed), + ) + retry := retryRunOf(retryJob("b", jobPassed), retryJob("c", jobFailed)) + + require.Empty(t, passedOnRetry([]*livekit.SimulationRun{first})) + require.Equal(t, []string{"b"}, passedOnRetry([]*livekit.SimulationRun{first, retry})) +} + +func TestRetryScenarioGroup(t *testing.T) { + group := &livekit.ScenarioGroup{ + Name: "g", + Scenarios: []*livekit.Scenario{ + {Label: "a"}, + {Label: "b"}, + {Instructions: "unlabeled"}, + }, + } + + t.Run("prefers the run's own group", func(t *testing.T) { + run := &livekit.SimulationRun{ScenarioGroup: group} + got := retryScenarioGroup(run, nil, []string{"b", "unlabeled"}) + require.Len(t, got.Scenarios, 2) + require.Equal(t, "g", got.Name) + }) + + t.Run("falls back when the run carries no scenarios", func(t *testing.T) { + got := retryScenarioGroup(&livekit.SimulationRun{}, group, []string{"a"}) + require.Len(t, got.Scenarios, 1) + require.Equal(t, "a", got.Scenarios[0].Label) + }) + + t.Run("nil when nothing matches", func(t *testing.T) { + run := &livekit.SimulationRun{ScenarioGroup: group} + require.Nil(t, retryScenarioGroup(run, nil, []string{"missing"})) + }) +}