diff --git a/README.md b/README.md index 05820013..45ea3fc0 100644 --- a/README.md +++ b/README.md @@ -455,6 +455,20 @@ The above simulates 5 concurrent rooms, where each room has: Once the specified duration is over (or if the load test is manually stopped), the load test statistics will be displayed in the form of a table. +## Agent simulations in CI + +Use a finished simulation run as a baseline so known failures are reported without failing CI, while regressions still return a nonzero exit code: + +```shell +lk agent simulate \ + --scenarios scenarios.yaml \ + --baseline "$SIMULATION_BASELINE_RUN_ID" \ + --run-id-file "$RUNNER_TEMP/simulation-run-id" +``` + +The CLI writes the new run ID to `--run-id-file` as soon as the run is created, even if the simulation later fails. Store the ID in your CI provider's variable or artifact store. Only a successful run on the main branch should replace the stored baseline; pull requests and failed main runs should leave it unchanged. + +For the first run, omit `--baseline`, inspect and accept its results, then store the ID written to the file. A missing, unfinished, or inaccessible baseline fails CI rather than silently using strict comparison. ## Browsing documentation diff --git a/autocomplete/fish_autocomplete b/autocomplete/fish_autocomplete index d7df2680..1cd402de 100644 --- a/autocomplete/fish_autocomplete +++ b/autocomplete/fish_autocomplete @@ -226,6 +226,8 @@ complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcomma complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l yes -s y -d 'Skip the source-upload confirmation prompt (required for non-interactive runs that generate from source)' complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l view -r -d 'Open a pre-existing simulation' complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l export -r -d 'Print the run with run `ID` and its exact per-job chat contexts as JSON. Nothing is run or polled: the run must already be finished' +complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l baseline -r -d 'Compare failures against the finished run with run `ID`: only scenarios that pass there and fail here fail the exit code. Non-interactive (CI) runs only' +complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l run-id-file -r -d 'Write the simulation run ID to `FILE` as soon as it is available. Non-interactive (CI) runs only' complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l agent-name -r -d 'Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or "" to target the project\'s default agent (the one that auto-joins every room). Requires --scenarios.' complete -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate' -f -l help -s h -d 'show help' complete -x -c lk -n '__fish_seen_subcommand_from agent a; and __fish_seen_subcommand_from simulate; and not __fish_seen_subcommand_from audio' -a 'audio' -d 'Simulate speech-to-speech interactions using the agent\'s full audio pipeline' diff --git a/cmd/lk/simulate.go b/cmd/lk/simulate.go index ef1cd6ba..6d355a30 100644 --- a/cmd/lk/simulate.go +++ b/cmd/lk/simulate.go @@ -101,6 +101,14 @@ var simulateCommand = &cli.Command{ Name: "export", Usage: "Print the run with run `ID` and its exact per-job chat contexts as JSON. Nothing is run or polled: the run must already be finished", }, + &cli.StringFlag{ + Name: "baseline", + Usage: "Compare failures against the finished run with run `ID`: only scenarios that pass there and fail here fail the exit code. Non-interactive (CI) runs only", + }, + &cli.StringFlag{ + Name: "run-id-file", + Usage: "Write the simulation run ID to `FILE` as soon as it is available. Non-interactive (CI) runs only", + }, &cli.StringFlag{ Name: "agent-name", Usage: "Run against an already-running agent instead of spawning one locally. Pass the registered `NAME`, or \"\" to target the project's default agent (the one that auto-joins every room). Requires --scenarios.", @@ -214,6 +222,8 @@ type simulateConfig struct { scenarioGroup *livekit.ScenarioGroup scenariosPath string // path to the --scenarios file (empty when generating from source) viewModeRunID string // non-empty when --view opens a pre-existing run + baselineRunID string // --baseline: failures this run also has don't fail CI + runIDFile string // --run-id-file: machine-readable handoff to CI liveAgent bool // --agent-name: run against an already-running agent, don't spawn one warnings []string // config-level warnings surfaced at setup (e.g. ignored flags) @@ -416,6 +426,8 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S scenarioGroup: scenarioGroup, scenariosPath: scenariosPath, viewModeRunID: runID, + baselineRunID: cmd.String("baseline"), + runIDFile: cmd.String("run-id-file"), liveAgent: liveAgent, warnings: simulateConfigWarnings(mode, numSimulations), } @@ -429,6 +441,12 @@ func runSimulate(ctx context.Context, cmd *cli.Command, simulationMode livekit.S if !isInteractive() { return runSimulateCI(ctx, simCfg) } + if simCfg.baselineRunID != "" { + return fmt.Errorf("--baseline only applies to non-interactive (CI) runs; the TUI already shows every failure") + } + if simCfg.runIDFile != "" { + return fmt.Errorf("--run-id-file only applies to non-interactive (CI) runs; the TUI already shows the run ID") + } return runSimulateTUI(simCfg) } diff --git a/cmd/lk/simulate_ci.go b/cmd/lk/simulate_ci.go index 1514bde8..9dd3af79 100644 --- a/cmd/lk/simulate_ci.go +++ b/cmd/lk/simulate_ci.go @@ -20,6 +20,7 @@ import ( "io" "os" "os/signal" + "strings" "sync/atomic" "time" @@ -119,6 +120,11 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { report.EndSetup() return err } + if err := writeSimulationRunID(config.runIDFile, runID); err != nil { + report.SetupFailed(err) + report.EndSetup() + return err + } report.SimulationCreated(time.Since(start)) if config.mode == modeGenerateFromSource { @@ -217,23 +223,118 @@ func runSimulateCI(ctx context.Context, config *simulateConfig) error { out.Statusf("Dashboard: %s", url) } - return runFailureError(run) + return baselineFailureError(ctx, config, run) +} + +func writeSimulationRunID(path, runID string) error { + if path == "" { + return nil + } + if err := os.WriteFile(path, []byte(runID+"\n"), 0o644); err != nil { + return fmt.Errorf("write simulation run ID to %s: %w", path, err) + } + return nil +} + +// baselineFailureError fetches the --baseline run when one was given and +// reports which failures it already had before deciding the exit error. A +// baseline that can't be fetched fails CI loudly rather than silently +// falling back to strict comparison. +func baselineFailureError(ctx context.Context, config *simulateConfig, run *livekit.SimulationRun) error { + var baseline *livekit.SimulationRun + if config.baselineRunID != "" { + fetchCtx, cancel := context.WithTimeout(ctx, simulationAPITimeout) + defer cancel() + var err error + baseline, err = getSimulationRun(fetchCtx, config.client, config.baselineRunID, config.pc.ProjectId) + if err != nil { + return fmt.Errorf("fetch baseline run %s: %w", config.baselineRunID, err) + } + if !isTerminalRunStatus(baseline.GetStatus()) { + return fmt.Errorf("baseline run %s is still in progress", config.baselineRunID) + } + if cmp := compareToBaseline(run, baseline); len(cmp.knownFailures) > 0 { + out.Statusf("%d failure(s) already failing in baseline %s (not failing CI): %s", + len(cmp.knownFailures), config.baselineRunID, strings.Join(cmp.knownFailures, ", ")) + } + } + return runFailureError(run, baseline) } // runFailureError converts a terminal run's failures into the CI exit error; // the error is printed by main and reports the failure — the counts line / // full dump above already carries the detail. Returns nil when everything -// passed. -func runFailureError(run *livekit.SimulationRun) error { +// passed. With a baseline run, scenario failures the baseline already had +// don't fail CI; a run-level STATUS_FAILED is systemic (the server only sets +// it when generation/submission breaks, never for scenario failures) and +// fails regardless of baseline. +func runFailureError(run, baseline *livekit.SimulationRun) error { _, _, _, failed := simulationJobCounts(run) - if failed > 0 || run.Status == livekit.SimulationRun_STATUS_FAILED { - if run.Status == livekit.SimulationRun_STATUS_FAILED && len(run.Jobs) == 0 { + if failed == 0 && run.Status != livekit.SimulationRun_STATUS_FAILED { + return nil + } + if run.Status == livekit.SimulationRun_STATUS_FAILED { + if len(run.Jobs) == 0 { return fmt.Errorf("simulation failed: %s", run.Error) } return fmt.Errorf("%d of %d simulations failed", failed, len(run.Jobs)) } + if baseline == nil { + return fmt.Errorf("%d of %d simulations failed", failed, len(run.Jobs)) + } - return nil + cmp := compareToBaseline(run, baseline) + if len(cmp.newFailures) == 0 { + return nil + } + return fmt.Errorf("%d new simulation failure(s) not failing in the baseline: %s", + len(cmp.newFailures), strings.Join(cmp.newFailures, ", ")) +} + +// baselineComparison splits the run's failed scenarios by whether the +// baseline run already failed them. +type baselineComparison struct { + newFailures []string + knownFailures []string +} + +func compareToBaseline(run, baseline *livekit.SimulationRun) baselineComparison { + known := make(map[string]bool) + for _, key := range failedScenarioKeys(baseline) { + known[key] = true + } + var cmp baselineComparison + for _, key := range failedScenarioKeys(run) { + if known[key] { + cmp.knownFailures = append(cmp.knownFailures, key) + } else { + cmp.newFailures = append(cmp.newFailures, key) + } + } + return cmp +} + +// failedScenarioKeys returns each failed scenario once, in job order. The +// label (scenario name) identifies a scenario across runs; generated jobs may +// carry only instructions. Repeats of one scenario (--num-simulations) share +// a key, so any failed repeat marks the scenario failed. +func failedScenarioKeys(run *livekit.SimulationRun) []string { + seen := make(map[string]bool) + var keys []string + for _, job := range run.GetJobs() { + if job.GetStatus() != livekit.SimulationRun_Job_STATUS_FAILED { + continue + } + key := job.GetLabel() + if key == "" { + key = job.GetInstructions() + } + if !seen[key] { + seen[key] = true + keys = append(keys, key) + } + } + return keys } // runSimulateCIView handles --view in non-interactive mode: it fetches the @@ -246,6 +347,9 @@ func runSimulateCIView(ctx context.Context, config *simulateConfig) error { report := newSimLog(out.ResultWriter(), out.StatusWriter()) runID := config.viewModeRunID + if err := writeSimulationRunID(config.runIDFile, runID); err != nil { + return err + } ticker := time.NewTicker(simulationPollInterval) defer ticker.Stop() @@ -281,5 +385,5 @@ func runSimulateCIView(ctx context.Context, config *simulateConfig) error { out.Statusf("Dashboard: %s", url) } - return runFailureError(run) + return baselineFailureError(ctx, config, run) } diff --git a/cmd/lk/simulate_ci_test.go b/cmd/lk/simulate_ci_test.go new file mode 100644 index 00000000..05943bf7 --- /dev/null +++ b/cmd/lk/simulate_ci_test.go @@ -0,0 +1,187 @@ +// Copyright 2026 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/livekit/protocol/livekit" +) + +func TestWriteSimulationRunID(t *testing.T) { + path := filepath.Join(t.TempDir(), "run-id") + + if err := writeSimulationRunID(path, "SR_test123"); err != nil { + t.Fatalf("writeSimulationRunID: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read run ID file: %v", err) + } + if string(got) != "SR_test123\n" { + t.Errorf("run ID file = %q, want %q", got, "SR_test123\\n") + } +} + +func simJob(label string, failed bool) *livekit.SimulationRun_Job { + status := livekit.SimulationRun_Job_STATUS_COMPLETED + if failed { + status = livekit.SimulationRun_Job_STATUS_FAILED + } + return &livekit.SimulationRun_Job{Label: label, Status: status} +} + +func simRun(jobs ...*livekit.SimulationRun_Job) *livekit.SimulationRun { + return &livekit.SimulationRun{ + Status: livekit.SimulationRun_STATUS_COMPLETED, + Jobs: jobs, + } +} + +func TestCompareToBaseline(t *testing.T) { + tests := []struct { + name string + run *livekit.SimulationRun + baseline *livekit.SimulationRun + wantNew []string + wantKnown []string + }{ + { + name: "failure also failing in baseline is known", + run: simRun(simJob("greeting", true)), + baseline: simRun(simJob("greeting", true)), + wantKnown: []string{"greeting"}, + }, + { + name: "failure that passed in baseline is new", + run: simRun(simJob("greeting", true)), + baseline: simRun(simJob("greeting", false)), + wantNew: []string{"greeting"}, + }, + { + name: "failure absent from baseline is new", + run: simRun(simJob("greeting", true)), + baseline: simRun(simJob("other", true)), + wantNew: []string{"greeting"}, + }, + { + name: "mixed failures split into new and known", + run: simRun(simJob("a", true), simJob("b", true), simJob("c", false)), + baseline: simRun(simJob("a", true), simJob("b", false)), + wantNew: []string{"b"}, + wantKnown: []string{"a"}, + }, + { + name: "any failed job among repeats marks the scenario failed", + run: simRun(simJob("a", false), simJob("a", true)), + baseline: simRun(simJob("a", true), simJob("a", false)), + wantKnown: []string{"a"}, + }, + { + name: "repeated failing label reported once", + run: simRun(simJob("a", true), simJob("a", true)), + baseline: simRun(simJob("b", true)), + wantNew: []string{"a"}, + }, + { + name: "unlabeled jobs match on instructions", + run: simRun(&livekit.SimulationRun_Job{ + Instructions: "ask for a refund", + Status: livekit.SimulationRun_Job_STATUS_FAILED, + }), + baseline: simRun(&livekit.SimulationRun_Job{ + Instructions: "ask for a refund", + Status: livekit.SimulationRun_Job_STATUS_FAILED, + }), + wantKnown: []string{"ask for a refund"}, + }, + { + name: "baseline-only failure that passes now is ignored", + run: simRun(simJob("a", false)), + baseline: simRun(simJob("a", true), simJob("gone", true)), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cmp := compareToBaseline(tt.run, tt.baseline) + if !slicesEqual(cmp.newFailures, tt.wantNew) { + t.Errorf("newFailures = %v, want %v", cmp.newFailures, tt.wantNew) + } + if !slicesEqual(cmp.knownFailures, tt.wantKnown) { + t.Errorf("knownFailures = %v, want %v", cmp.knownFailures, tt.wantKnown) + } + }) + } +} + +func slicesEqual(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} + +func TestRunFailureErrorWithoutBaseline(t *testing.T) { + if err := runFailureError(simRun(simJob("a", false)), nil); err != nil { + t.Errorf("all passed: got %v, want nil", err) + } + if err := runFailureError(simRun(simJob("a", true)), nil); err == nil { + t.Error("failure without baseline: got nil, want error") + } +} + +func TestRunFailureErrorWithBaseline(t *testing.T) { + t.Run("known failures alone do not fail", func(t *testing.T) { + err := runFailureError(simRun(simJob("a", true)), simRun(simJob("a", true))) + if err != nil { + t.Errorf("got %v, want nil", err) + } + }) + + t.Run("new failure fails and names the scenario", func(t *testing.T) { + err := runFailureError( + simRun(simJob("a", true), simJob("b", true)), + simRun(simJob("a", true)), + ) + if err == nil { + t.Fatal("got nil, want error") + } + if !strings.Contains(err.Error(), "b") { + t.Errorf("error %q does not name new failure \"b\"", err) + } + if strings.Contains(err.Error(), "\"a\"") { + t.Errorf("error %q names known failure \"a\"", err) + } + }) + + t.Run("systemic run failure fails regardless of baseline", func(t *testing.T) { + run := &livekit.SimulationRun{ + Status: livekit.SimulationRun_STATUS_FAILED, + Error: "worker crashed", + } + if err := runFailureError(run, simRun(simJob("a", true))); err == nil { + t.Error("got nil, want error") + } + }) +}