diff --git a/Makefile b/Makefile index cffcd53..57e30c2 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Developer commands for fly. Run "make help" for the list. # Keep this file compatible with GNU Make 3.81, the version on macOS. -.PHONY: build test vet lint vuln fmt fmt-check check release dev-version dev-release clean help +.PHONY: build test vet lint vuln fmt fmt-check check release dev-version dev-release release-key sign-release clean help BINARY := fly PKG := github.com/flywp/server-cli @@ -10,7 +10,10 @@ PKG := github.com/flywp/server-cli # make release VERSION=v0.2.0 VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) COMMIT := $(shell git rev-parse HEAD 2>/dev/null || echo unknown) -BUILD_DATE := $(shell date -u +%Y-%m-%d) +# The date of the commit, not of the build: the same commit gives the same +# binary on any computer, so make sign-release can build a release again and +# compare it with the one that CI published. +BUILD_DATE := $(shell TZ=UTC git log -1 --date=format-local:%Y-%m-%d --format=%cd 2>/dev/null || date -u +%Y-%m-%d) LDFLAGS := -X $(PKG)/internal/version.Version=$(VERSION) \ -X $(PKG)/internal/version.CommitHash=$(COMMIT) \ @@ -90,6 +93,27 @@ dev-release: ## Tag HEAD as a dev pre-release and push the tag (CI publishes it) echo "Pushed $(DEV_VERSION). The Release workflow publishes it as a pre-release:"; \ echo " https://github.com/flywp/server-cli/releases/tag/$(DEV_VERSION)" +# The agent installs a release by itself only when checksums.txt has a +# signature from a key that is kept outside GitHub. Make the key one time, put +# the printed public key line in internal/release/keys.go, and keep the +# private key in a password manager, with a backup. Never commit it. +# COMMENT names the key, in the key file and next to its line in keys.go. +COMMENT ?= server-cli release key for flywp +release-key: ## Make the release signing key: make release-key KEY= [COMMENT=...] + @test -n "$(KEY)" || { echo "Usage: make release-key KEY= [COMMENT=\"...\"]"; exit 1; } + go run ./tools/releasesign keygen -out "$(KEY)" -comment "$(COMMENT)" + +# Run this after the Release workflow publishes VERSION, on your own +# computer. tools/sign-release.sh builds the release again from your local +# tag, checks that GitHub serves the same binaries, signs checksums.txt, +# checks the signature with the keys of the tag, and uploads +# checksums.txt.sig. Agents install the release 24 hours after the +# signature. KEY=- reads the key from stdin. +sign-release: ## Sign a published release: make sign-release VERSION=v0.2.1 KEY= + @if [ "$(origin VERSION)" != "command line" ] || [ -z "$(KEY)" ]; then \ + echo "Usage: make sign-release VERSION=v0.2.1 KEY="; exit 1; fi + @tools/sign-release.sh "$(VERSION)" "$(KEY)" + clean: ## Remove bin/ and build/ rm -rf bin/ build/ diff --git a/README.md b/README.md index 464becc..d055ac7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Easy CLI tool for servers managed by FlyWP. -Conforms to the FlyWP monitoring agent contract v0.2.1. +Conforms to the FlyWP monitoring agent contract v0.3.1. ## Installation @@ -107,6 +107,14 @@ All arguments after the WP-CLI command (or after the command for `fly exec`) go It reads `FLY_AGENT_URL` (https), `FLY_AGENT_TOKEN` and `FLY_AGENT_SERVER_ID` from `/etc/fly/agent.env`, and keeps its state in `STATE_DIRECTORY` (`/var/lib/fly-agent`). +The agent also updates itself. Once a day, at a time set by the server id, it checks the latest release on GitHub. It installs the release only when: + +- the release is newer than the running version (the agent never downgrades) +- the release has a valid signature from the FlyWP release key (see [Releasing](#releasing)) +- the signature is more than 24 hours old + +To turn this off on one server, add `FLY_AGENT_AUTO_UPDATE=off` to `/etc/fly/agent.env` and restart the agent. Updates that FlyWP sends still work. + ```bash systemctl status fly-agent # is the agent running? journalctl -u fly-agent -f # the agent log @@ -137,7 +145,7 @@ make release # static linux/amd64 and linux/arm64 archives + checksums.txt in make help # lists all targets ``` -`make release VERSION=v0.2.0` stamps a specific version. The release archives must keep the names `fly-linux-.tar.gz` with the binary `fly-linux-` inside: installed CLIs look for these names when they run `fly update`. +`make release VERSION=v0.2.0` stamps a specific version. The build date is the date of the commit, so the same commit and Go version give the same binary on any computer. The release archives must keep the names `fly-linux-.tar.gz` with the binary `fly-linux-` inside: installed CLIs look for these names when they run `fly update`. CI runs `make check` and `make release` on every pull request and on every push to `develop` and `main`. @@ -154,6 +162,23 @@ The Release workflow checks that the tag is on `main`, runs `make check`, builds `install.sh` and `fly update` install only a release that has `checksums.txt`. Releases before v0.2.0 have none, so push the tag right after the merge into `main`: until the release is published, `install.sh` from `main` stops. +After the workflow publishes the release, sign it on your own computer: + +```bash +make sign-release VERSION=v0.2.0 KEY= # KEY=- reads the key from stdin +``` + +The signature says "this release is the code of my tag", so the command signs only what it can build again: + +1. It shows the commit of your **local** tag and asks you to type the tag. Review that commit first: the signature is the approval. +2. It downloads the archives and `checksums.txt`, and checks the archives against the sums. +3. It builds the release again from your local tag, with the Go version of the CI build, and compares the binaries byte for byte. A binary holds its commit, so this also proves that CI built your tag. +4. It signs `checksums.txt`, checks the signature with the keys of the tag, and uploads `checksums.txt.sig`. + +If GitHub serves a swapped archive, or the tag on GitHub moved, step 2 or 3 stops before the signature. Agents install a release by themselves only when it has a valid signature, and only 24 hours after it was signed. Each server then installs it at its own time of day. To stop a bad release in those 24 hours, mark it as a pre-release on GitHub. + +The signing key is kept outside GitHub, so that a push to GitHub alone cannot reach every server. `make release-key KEY=` makes a key and prints its public key line for `internal/release/keys.go`. `COMMENT=` names the key, in the key file and next to its line in `keys.go` (the default is "server-cli release key for flywp"). Keep the private key in a password manager, with a backup. Never commit it, and never put it in a GitHub secret. If the key is lost or leaked, ship a binary with a new key through a FlyWP update: that path does not use the signature. + ### Dev pre-releases To test a branch on real servers before it merges, publish a dev pre-release of its current commit: diff --git a/agent_test.go b/agent_test.go index 0a21e3e..ab03ddc 100644 --- a/agent_test.go +++ b/agent_test.go @@ -8,6 +8,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "crypto/ed25519" "crypto/sha256" "encoding/hex" "encoding/json" @@ -45,6 +46,8 @@ func agentEnv(t *testing.T) []string { "FLY_AGENT_TOKEN=" + testToken, "FLY_AGENT_SERVER_ID=17", "STATE_DIRECTORY=" + t.TempDir(), + // A test never asks the real GitHub for a release. + "FLY_AGENT_AUTO_UPDATE=off", } } @@ -260,6 +263,98 @@ func TestAgentUpdatesItself(t *testing.T) { } } +func TestAgentInstallsASignedReleaseByItself(t *testing.T) { + // A release has archives for Linux only. + if runtime.GOOS != "linux" { + t.Skip("the releases have binaries for Linux only") + } + environ := agentEnv(t) + + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + + // The new release: fly built as v9.9.9, signed 25 hours ago. + newBin := filepath.Join(t.TempDir(), "fly") + build := exec.Command("go", "build", "-ldflags", "-X github.com/flywp/server-cli/internal/version.Version=v9.9.9", "-o", newBin, ".") + if out, err := build.CombinedOutput(); err != nil { + t.Fatalf("building the new release: %v\n%s", err, out) + } + archiveName := release.BinaryName(runtime.GOOS, runtime.GOARCH) + ".tar.gz" + tarball := releaseArchive(t, newBin, release.BinaryName(runtime.GOOS, runtime.GOARCH)) + sum := sha256.Sum256(tarball) + checksums := []byte(hex.EncodeToString(sum[:]) + " " + archiveName + "\n") + sigFile := release.Sign(priv, "v9.9.9", time.Now().Add(-25*time.Hour), checksums) + + var github *httptest.Server + github = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/releases/latest": + assets := []map[string]string{} + for _, name := range []string{archiveName, release.ChecksumsAsset, release.SignatureAsset} { + assets = append(assets, map[string]string{"name": name, "browser_download_url": github.URL + "/download/" + name}) + } + _ = json.NewEncoder(w).Encode(map[string]any{"tag_name": "v9.9.9", "assets": assets}) + case "/download/" + archiveName: + _, _ = w.Write(tarball) + case "/download/" + release.ChecksumsAsset: + _, _ = w.Write(checksums) + case "/download/" + release.SignatureAsset: + _, _ = w.Write(sigFile) + default: + http.NotFound(w, r) + } + })) + defer github.Close() + + // The running agent: an older release that asks this GitHub and trusts + // the test key. + dir := t.TempDir() + exe := filepath.Join(dir, "fly") + ldflags := strings.Join([]string{ + "-X github.com/flywp/server-cli/internal/version.Version=v0.0.1", + "-X github.com/flywp/server-cli/internal/release.GithubAPI=" + github.URL + "/releases/latest", + "-X github.com/flywp/server-cli/internal/release.trustedKeys=" + release.KeyLine(pub), + }, " ") + if out, err := exec.Command("go", "build", "-ldflags", ldflags, "-o", exe, ".").CombinedOutput(); err != nil { + t.Fatalf("building the old release: %v\n%s", err, out) + } + + srv := httptest.NewServer(&updateServer{closed: true}) + defer srv.Close() + + // Work 3 seconds from now. The agent never checked, so it checks at its + // first tick. + environ = append(environ, "FLY_AGENT_URL="+srv.URL, fmt.Sprintf("FLY_AGENT_SERVER_ID=%d", (time.Now().Second()+3)%60), "FLY_AGENT_AUTO_UPDATE=on") + + agent := exec.Command(exe, "agent", "run") + agent.Env = environ + stderr := &lockedBuffer{} + agent.Stderr = stderr + if err := agent.Start(); err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { done <- agent.Wait() }() + select { + case err := <-done: + if err != nil { + t.Fatalf("the agent exit after the update: %v, want 0. stderr:\n%s", err, stderr) + } + case <-time.After(70 * time.Second): + _ = agent.Process.Kill() + t.Fatalf("the agent did not install the release within 70s. stderr:\n%s", stderr) + } + + if out := runFlyAt(t, exe, "version"); !strings.Contains(out, "v9.9.9") { + t.Fatalf("fly version = %q, want the new release v9.9.9 on disk. stderr:\n%s", out, stderr) + } + if !strings.Contains(stderr.String(), "installed the new release") { + t.Errorf("stderr does not log the install:\n%s", stderr) + } +} + // runFlyAt runs the fly binary at exe and returns its stdout. func runFlyAt(t *testing.T, exe string, args ...string) string { t.Helper() diff --git a/cmd/agent.go b/cmd/agent.go index 0afe1ba..9819443 100644 --- a/cmd/agent.go +++ b/cmd/agent.go @@ -23,7 +23,10 @@ var agentRunCmd = &cobra.Command{ Short: "Run the monitoring agent until it is stopped", Long: `Run the FlyWP monitoring agent until it is stopped. systemd starts this command (fly-agent.service). The agent reads FLY_AGENT_URL, FLY_AGENT_TOKEN, -FLY_AGENT_SERVER_ID and STATE_DIRECTORY from the environment.`, +FLY_AGENT_SERVER_ID and STATE_DIRECTORY from the environment. + +Once a day the agent installs a newer signed release by itself. +FLY_AGENT_AUTO_UPDATE=off stops this.`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { cfg, err := agent.ConfigFromEnv(os.Getenv) diff --git a/internal/agent/agent.go b/internal/agent/agent.go index d86b246..b5a23e0 100644 --- a/internal/agent/agent.go +++ b/internal/agent/agent.go @@ -2,6 +2,7 @@ package agent import ( "context" + "crypto/ed25519" "errors" "io/fs" "log/slog" @@ -41,6 +42,8 @@ type Collector interface { // state is the part of the agent state that is not a queue. type state struct { ReportInterval int `json:"report_interval"` + // LastUpdateCheck is the time of the last check for a new release. + LastUpdateCheck time.Time `json:"last_update_check,omitzero"` } type agent struct { @@ -59,6 +62,14 @@ type agent struct { // last is the time of the last tick. last time.Time + // autoUpdate is true when the agent checks for a new release each day, + // with keys. lastUpdateCheck is the time of the last check, and + // nextUpdateCheck the time of the next one. + autoUpdate bool + keys map[string]ed25519.PublicKey + lastUpdateCheck time.Time + nextUpdateCheck time.Time + // The waits of the events and the metrics requests after a failure, and // whether the last report sent all samples. eventsWait backoff @@ -88,16 +99,22 @@ func run(ctx context.Context, cfg Config, log *slog.Logger, cp ControlPlane, col // A crash during a write can leave a temporary file. statefile.RemoveTemp(cfg.StateDir) + saved := loadState(cfg.StateDir, log) a := &agent{ - cfg: cfg, - log: log, - cp: cp, - collector: collector, - outbox: loadOutbox(cfg.StateDir, log), - ledger: loadLedger(cfg.StateDir, log), - interval: loadInterval(cfg.StateDir, log), + cfg: cfg, + log: log, + cp: cp, + collector: collector, + outbox: loadOutbox(cfg.StateDir, log), + ledger: loadLedger(cfg.StateDir, log), + interval: saved.ReportInterval, + lastUpdateCheck: saved.LastUpdateCheck, } log.Info("agent started", "version", version.Version, "offset", cfg.Offset(), "report_interval", a.interval) + for _, w := range cfg.Warnings { + log.Warn(w) + } + a.startAutoUpdate(time.Now()) // Send the events at once, not at the next tick: after an update or a // restart, they hold the result of the command. @@ -108,7 +125,7 @@ func run(ctx context.Context, cfg Config, log *slog.Logger, cp ControlPlane, col if a.loop(ctx) { // systemd starts the agent again (Restart=always), with the new // binary after an update. - log.Info("agent exits for a command") + log.Info("agent exits; systemd starts it again") return nil } log.Info("agent stopped") @@ -135,13 +152,22 @@ func (a *agent) loop(ctx context.Context) (exit bool) { } } -// nextAfter returns the next tick after now, and never a tick at or before -// last. The timer runs on the monotonic clock, but the tick times come from -// the wall clock: when the wall clock steps back, the timer fires before the -// tick time, and without last the same tick would run two times. +// maxStepBack is the largest step back of the wall clock after which the loop +// still skips the minute that already ran. A larger step means that the clock +// was wrong before (for example a VM that booted with its clock ahead, then +// NTP): the loop follows the new clock at once, or it would wait, silent, for +// the size of the step. A minute that runs two times is harmless: the control +// plane keeps one sample for each minute. +const maxStepBack = 2 * time.Minute + +// nextAfter returns the next tick after now. After a small step back of the +// wall clock, it never returns a tick at or before last. The timer runs on the +// monotonic clock, but the tick times come from the wall clock: when the wall +// clock steps back, the timer fires before the tick time, and without last +// the same tick would run two times. func nextAfter(now, last time.Time, offset time.Duration) time.Time { next := nextTick(now, offset) - if !last.IsZero() && !next.After(last) { + if !last.IsZero() && !next.After(last) && last.Sub(now) < maxStepBack { next = nextTick(last, offset) } @@ -149,8 +175,9 @@ func nextAfter(now, last time.Time, offset time.Duration) time.Time { } // tick does the work of one minute: it takes a sample and, after each -// interval samples, sends a report and runs the new commands. It returns true -// when a command ends the process. +// interval samples, sends a report and runs the new commands. One time each +// day it checks for a new release. It returns true when a command or an +// update ends the process. func (a *agent) tick(ctx context.Context, now time.Time) (exit bool) { a.log.Debug("tick", "at", now) @@ -164,6 +191,21 @@ func (a *agent) tick(ctx context.Context, now time.Time) (exit bool) { } } + if a.report(ctx) { + return true + } + + // On each tick, not only after a report: a long report interval or a + // control plane that is down must not move the check. + if a.autoUpdate && !now.Before(a.nextUpdateCheck) { + return a.checkForUpdate(ctx, now) + } + return false +} + +// report sends a report after each interval samples, and runs the new +// commands. It returns true when a command ends the process. +func (a *agent) report(ctx context.Context) (exit bool) { a.pending++ if a.pending < a.interval { return false @@ -214,8 +256,15 @@ func (a *agent) setInterval(n int) { a.log.Info("report interval changed", "from", a.interval, "to", n) a.interval = n - if err := statefile.Write(filepath.Join(a.cfg.StateDir, "state.json"), state{ReportInterval: n}); err != nil { - a.log.Error("saving the report interval", "error", err) + a.saveState() +} + +// saveState writes all of the state: a write of one field must not remove +// an other. +func (a *agent) saveState() { + s := state{ReportInterval: a.interval, LastUpdateCheck: a.lastUpdateCheck} + if err := statefile.Write(filepath.Join(a.cfg.StateDir, "state.json"), s); err != nil { + a.log.Error("saving the state", "error", err) } } @@ -231,19 +280,21 @@ func nextTick(now time.Time, offset time.Duration) time.Time { return t } -// loadInterval returns the report interval of the last reply, or 1. -func loadInterval(dir string, log *slog.Logger) int { +// loadState returns the saved state. The report interval is the one of the +// last reply, or 1. +func loadState(dir string, log *slog.Logger) state { var s state err := statefile.Read(filepath.Join(dir, "state.json"), &s) switch { case errors.Is(err, fs.ErrNotExist): - return minReportInterval + return state{ReportInterval: minReportInterval} case err != nil: log.Warn("ignoring the saved state", "error", err) - return minReportInterval + return state{ReportInterval: minReportInterval} } - return clampInterval(s.ReportInterval) + s.ReportInterval = clampInterval(s.ReportInterval) + return s } func clampInterval(n int) int { diff --git a/internal/agent/agent_test.go b/internal/agent/agent_test.go index a3faf9e..61a6d4c 100644 --- a/internal/agent/agent_test.go +++ b/internal/agent/agent_test.go @@ -112,6 +112,38 @@ func TestNextAfterAClockStepBack(t *testing.T) { } } +func TestNextAfterALargeClockStepBack(t *testing.T) { + base := time.Date(2026, 9, 22, 10, 0, 0, 0, time.UTC) + offset := 17 * time.Second + + tests := []struct { + name string + step time.Duration + want time.Time + }{ + // A step of 90 s is still small: skip to the minute after the last tick. + {"90 seconds", 90 * time.Second, base.Add(time.Minute + offset)}, + // The clock was one hour ahead. Follow the new clock: the next tick + // comes in less than one minute, not in one hour. + {"one hour", time.Hour, base.Add(-time.Hour + time.Minute + offset)}, + {"one year", 365 * 24 * time.Hour, base.Add(-365*24*time.Hour + time.Minute + offset)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + last := base.Add(offset) + now := last.Add(-tt.step).Add(time.Second) + got := nextAfter(now, last, offset) + if !got.Equal(tt.want) { + t.Errorf("nextAfter() = %s, want %s", got, tt.want) + } + if wait := got.Sub(now); wait <= 0 || wait > tt.step+2*time.Minute { + t.Errorf("the loop waits %v after a step back of %v", wait, tt.step) + } + }) + } +} + func TestLoopTicksAtTheOffsetAndReportsEachInterval(t *testing.T) { synctest.Test(t, func(t *testing.T) { dir := t.TempDir() @@ -167,7 +199,7 @@ func TestRunRefusesASecondAgent(t *testing.T) { } } -func TestLoadInterval(t *testing.T) { +func TestLoadStateInterval(t *testing.T) { tests := []struct { name string saved *state @@ -188,8 +220,8 @@ func TestLoadInterval(t *testing.T) { } } - if got := loadInterval(dir, slog.New(&recorder{})); got != tt.want { - t.Errorf("loadInterval() = %d, want %d", got, tt.want) + if got := loadState(dir, slog.New(&recorder{})).ReportInterval; got != tt.want { + t.Errorf("loadState().ReportInterval = %d, want %d", got, tt.want) } }) } diff --git a/internal/agent/autoupdate.go b/internal/agent/autoupdate.go new file mode 100644 index 0000000..60d2754 --- /dev/null +++ b/internal/agent/autoupdate.go @@ -0,0 +1,165 @@ +package agent + +import ( + "context" + "crypto/ed25519" + "errors" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/flywp/server-cli/internal/agent/wire" + "github.com/flywp/server-cli/internal/release" + "github.com/flywp/server-cli/internal/version" + "golang.org/x/mod/semver" + "golang.org/x/sys/unix" +) + +const ( + // autoUpdateWait is the time between the signature of a release and its + // install by the agents. In this time the maintainer can mark a bad + // release as a pre-release: /releases/latest then stops returning it. + autoUpdateWait = 24 * time.Hour + + // checkGap is the shortest time between two checks. With it, a clock + // step cannot cause two checks in one day. + checkGap = 12 * time.Hour +) + +// The parts of an update by release. Tests replace them. +var ( + latestRelease = release.LatestRelease + signedChecksum = func(ctx context.Context, rel *release.GithubRelease, keys map[string]ed25519.PublicKey, now time.Time) (*release.Signed, error) { + return release.SignedChecksum(ctx, rel, runtime.GOOS, runtime.GOARCH, keys, now) + } + releaseKeys = release.TrustedKeys + binaryDirWritable = func() bool { + exe, err := os.Executable() + if err != nil { + return false + } + if exe, err = filepath.EvalSymlinks(exe); err != nil { + return false + } + return unix.Access(filepath.Dir(exe), unix.W_OK) == nil + } +) + +// updateSlot is the time after 00:00 UTC at which the server checks for a new +// release each day. It spreads the installs of a release over one day, so +// that the first servers show a problem before the others get the release. +func updateSlot(serverID int64) time.Duration { + return time.Duration(serverID%(24*60)) * time.Minute +} + +// nextCheck returns the time of the next check for a new release: the first +// slot that comes more than checkGap after the last check. An agent that +// never checked checks at once. +func nextCheck(last time.Time, slot time.Duration) time.Time { + if last.IsZero() { + return time.Time{} + } + + after := last.Add(checkGap) + // Truncate counts from the zero time, which is a midnight UTC. + t := after.UTC().Truncate(24 * time.Hour).Add(slot) + if !t.After(after) { + t = t.Add(24 * time.Hour) + } + + return t +} + +// startAutoUpdate decides at start whether the agent checks for new releases, +// and when it checks first. +func (a *agent) startAutoUpdate(now time.Time) { + switch keys, err := releaseKeys(); { + case !a.cfg.AutoUpdate: + a.log.Info("auto-update is off", "key", EnvAutoUpdate) + return + case !semver.IsValid(version.Version) || release.IsLocalBuild(version.Version): + // A build without a release tag (go build gives "dev") has no + // place in the order of the releases. A make build of a commit + // after a tag (v0.2.0-3-gabcdef1) sorts before that tag, so the + // release would replace code that is newer. + a.log.Info("auto-update is off: this build has no release version", "version", version.Version) + return + case err != nil: + a.log.Error("auto-update is off: the release keys of this build are not valid", "error", err) + return + case len(keys) == 0: + a.log.Warn("auto-update is off: this build trusts no release key") + return + case !binaryDirWritable(): + a.log.Warn("auto-update is off: the agent cannot write the folder of its binary") + return + default: + a.keys = keys + } + + // A check time after now comes from a clock that was wrong. Without + // this, the agent could wait for a day that is years away. + if a.lastUpdateCheck.After(now) { + a.log.Warn("the time of the last release check is in the future; checking again", "last", a.lastUpdateCheck) + a.lastUpdateCheck = time.Time{} + } + + a.autoUpdate = true + a.nextUpdateCheck = nextCheck(a.lastUpdateCheck, updateSlot(a.cfg.ServerID)) +} + +// checkForUpdate installs the latest release when it is newer, has a valid +// signature and waited autoUpdateWait after the signature. It returns true +// when it installed the release: the process must exit, so that systemd +// starts the new binary. +func (a *agent) checkForUpdate(ctx context.Context, now time.Time) (exit bool) { + // Save the time first: a crash or a failure waits for the next day, and + // does not check at each restart. + a.lastUpdateCheck = now + a.nextUpdateCheck = nextCheck(now, updateSlot(a.cfg.ServerID)) + a.saveState() + + log := a.log.With("version", version.Version, "next_check", a.nextUpdateCheck) + + ctx, cancel := context.WithTimeout(ctx, updateTimeout) + defer cancel() + + rel, err := latestRelease(ctx) + if err != nil { + log.Warn("checking for a new release failed", "error", err) + return false + } + log = log.With("release", rel.TagName) + + // The agent never downgrades. A version without an order (a dev build) + // does not update by itself. + if cmp, ok := compareVersions(version.Version, rel.TagName); !ok || cmp >= 0 { + log.Info("checked for a new release: none to install") + return false + } + + signed, err := signedChecksum(ctx, rel, a.keys, now) + switch { + case errors.Is(err, release.ErrNotSigned): + log.Info("a new release waits for its signature") + return false + case err != nil: + log.Error("not installing a new release", "error", err) + return false + } + + if ready := signed.SignedAt.Add(autoUpdateWait); now.Before(ready) { + log.Info("a new release installs after its wait", "signed_at", signed.SignedAt, "ready_at", ready) + return false + } + + log.Info("installing a new release") + if err := updateBinary(ctx, wire.UpdateArgs{URL: signed.URL, Version: rel.TagName, SHA256: signed.SHA256}); err != nil { + log.Error("the update failed; the old binary continues", "error", err) + return false + } + + log.Info("installed the new release; exiting so that systemd starts it") + return true +} diff --git a/internal/agent/autoupdate_test.go b/internal/agent/autoupdate_test.go new file mode 100644 index 0000000..e4d2d11 --- /dev/null +++ b/internal/agent/autoupdate_test.go @@ -0,0 +1,402 @@ +package agent + +import ( + "context" + "crypto/ed25519" + "errors" + "log/slog" + "path/filepath" + "sync" + "testing" + "testing/synctest" + "time" + + "github.com/flywp/server-cli/internal/agent/wire" + "github.com/flywp/server-cli/internal/release" + "github.com/flywp/server-cli/internal/statefile" +) + +func TestUpdateSlot(t *testing.T) { + tests := map[int64]time.Duration{ + 0: 0, + 17: 17 * time.Minute, + 1439: 1439 * time.Minute, + 1445: 5 * time.Minute, + } + for id, want := range tests { + if got := updateSlot(id); got != want { + t.Errorf("updateSlot(%d) = %v, want %v", id, got, want) + } + } +} + +func TestNextCheck(t *testing.T) { + day := time.Date(2026, 9, 23, 0, 0, 0, 0, time.UTC) + slot := 17 * time.Minute + tests := []struct { + name string + last time.Time + want time.Time + }{ + {"never checked", time.Time{}, time.Time{}}, + {"checked at the slot", day.Add(slot), day.Add(24*time.Hour + slot)}, + {"checked late in the day", day.Add(20 * time.Hour), day.Add(48*time.Hour + slot)}, + {"checked just before the slot", day.Add(slot - time.Minute), day.Add(24*time.Hour + slot)}, + // Exactly 12 hours before a slot is not more than 12 hours. + {"checked 12 hours before the slot", day.Add(slot - 12*time.Hour), day.Add(24*time.Hour + slot)}, + {"a zone that is not UTC", day.Add(slot).In(time.FixedZone("UTC+6", 6*3600)), day.Add(24*time.Hour + slot)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := nextCheck(tt.last, slot); !got.Equal(tt.want) { + t.Errorf("nextCheck(%v) = %v, want %v", tt.last, got, tt.want) + } + }) + } +} + +// fakeReleases replaces GitHub and the signature check for one test. +type fakeReleases struct { + mu sync.Mutex + tag string + latestErr error + signedAt time.Time + signErr error + + checks []time.Time + signed int +} + +func useFakeReleases(t *testing.T, f *fakeReleases) { + t.Helper() + + oldLatest, oldSigned, oldKeys, oldWritable := latestRelease, signedChecksum, releaseKeys, binaryDirWritable + t.Cleanup(func() { + latestRelease, signedChecksum, releaseKeys, binaryDirWritable = oldLatest, oldSigned, oldKeys, oldWritable + }) + + latestRelease = func(context.Context) (*release.GithubRelease, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.checks = append(f.checks, time.Now()) + if f.latestErr != nil { + return nil, f.latestErr + } + return &release.GithubRelease{TagName: f.tag}, nil + } + signedChecksum = func(context.Context, *release.GithubRelease, map[string]ed25519.PublicKey, time.Time) (*release.Signed, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.signed++ + if f.signErr != nil { + return nil, f.signErr + } + return &release.Signed{URL: "https://example.com/fly-linux-amd64.tar.gz", SHA256: "abc123", SignedAt: f.signedAt}, nil + } + releaseKeys = func() (map[string]ed25519.PublicKey, error) { + return map[string]ed25519.PublicKey{"test": make(ed25519.PublicKey, ed25519.PublicKeySize)}, nil + } + binaryDirWritable = func() bool { return true } +} + +func (f *fakeReleases) checkTimes() []time.Time { + f.mu.Lock() + defer f.mu.Unlock() + return f.checks +} + +// setReportInterval saves the report interval n in dir, as a reply would. +func setReportInterval(t *testing.T, dir string, n int) { + t.Helper() + if err := statefile.Write(filepath.Join(dir, "state.json"), state{ReportInterval: n}); err != nil { + t.Fatal(err) + } +} + +// runAuto runs an agent with server id 17 (slot 00:17 UTC) and auto-update +// on, until it exits or until d passes. It returns true when it exited. +func runAuto(t *testing.T, d time.Duration, dir string, cp *fakeCP, c Collector) (exited bool, rec *recorder) { + t.Helper() + + rec = &recorder{} + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + cfg := Config{ServerID: 17, StateDir: dir, AutoUpdate: true} + if err := run(ctx, cfg, slog.New(rec), cp, c); err != nil { + t.Fatal(err) + } + return ctx.Err() == nil, rec +} + +func TestAutoUpdateChecksOneTimeEachDay(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + setVersion(t, "v0.2.0") + f := &fakeReleases{tag: "v0.1.1"} + useFakeReleases(t, f) + calls := fakeUpdate(t, nil) + dir := t.TempDir() + setReportInterval(t, dir, 10) + + exited, rec := runAuto(t, 72*time.Hour+time.Minute, dir, &fakeCP{}, nil) + + // The first check at the first tick, which is not a report tick. Then + // one check each day at the slot of the server. + want := []time.Time{ + at(0), + bubbleStart.Add(24*time.Hour + 17*time.Minute + 17*time.Second), + bubbleStart.Add(48*time.Hour + 17*time.Minute + 17*time.Second), + } + if got := f.checkTimes(); !equalTimes(got, want) { + t.Errorf("checks at %v, want %v", got, want) + } + // The release is older: the agent does not fetch its signature. + if exited || f.signed != 0 || len(*calls) != 0 { + t.Errorf("exited %v, %d signature checks, %d installs; want none for an older release", exited, f.signed, len(*calls)) + } + if len(rec.times("checked for a new release: none to install")) != 3 { + t.Error("want one log record for each check") + } + }) +} + +func TestAutoUpdateInstallsAfterTheWait(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + setVersion(t, "v0.2.0") + // Signed one hour before the start: the first check is too early. + f := &fakeReleases{tag: "v0.2.1", signedAt: bubbleStart.Add(-time.Hour)} + useFakeReleases(t, f) + calls := fakeUpdate(t, nil) + dir := t.TempDir() + setReportInterval(t, dir, 10) + + exited, rec := runAuto(t, 72*time.Hour, dir, &fakeCP{}, nil) + + if !exited { + t.Fatal("the agent did not exit after the update") + } + if len(rec.times("a new release installs after its wait")) != 1 { + t.Error("want one \"installs after its wait\" record, at the first check") + } + // The second check, on the second day, installs the release. + checks := f.checkTimes() + if len(checks) != 2 || !checks[1].Equal(bubbleStart.Add(24*time.Hour+17*time.Minute+17*time.Second)) { + t.Errorf("checks at %v, want the install at the second check", checks) + } + want := wire.UpdateArgs{URL: "https://example.com/fly-linux-amd64.tar.gz", Version: "v0.2.1", SHA256: "abc123"} + if len(*calls) != 1 || (*calls)[0] != want { + t.Errorf("installs = %+v, want one install of %+v", *calls, want) + } + }) +} + +func TestAutoUpdateDoesNotInstall(t *testing.T) { + old := bubbleStart.Add(-48 * time.Hour) + tests := []struct { + name string + running string + releases *fakeReleases + wantLog string + wantSign int + }{ + {"the same version", "v0.2.1", &fakeReleases{tag: "v0.2.1", signedAt: old}, "checked for a new release: none to install", 0}, + {"an older release", "v0.3.0", &fakeReleases{tag: "v0.2.1", signedAt: old}, "checked for a new release: none to install", 0}, + {"no signature", "v0.2.0", &fakeReleases{tag: "v0.2.1", signErr: release.ErrNotSigned}, "a new release waits for its signature", 1}, + {"a bad signature", "v0.2.0", &fakeReleases{tag: "v0.2.1", signErr: errors.New("the signature is not correct")}, "not installing a new release", 1}, + {"GitHub is down", "v0.2.0", &fakeReleases{latestErr: errors.New("unexpected response: 502")}, "checking for a new release failed", 0}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + setVersion(t, tt.running) + f := tt.releases + useFakeReleases(t, f) + calls := fakeUpdate(t, nil) + dir := t.TempDir() + setReportInterval(t, dir, 10) + + // Two hours: a failure waits for the next day, not for the + // next tick. + exited, rec := runAuto(t, 2*time.Hour, dir, &fakeCP{}, nil) + + if exited || len(*calls) != 0 { + t.Errorf("exited %v with %d installs, want no install", exited, len(*calls)) + } + if n := len(f.checkTimes()); n != 1 { + t.Errorf("%d checks in two hours, want 1", n) + } + if f.signed != tt.wantSign { + t.Errorf("%d signature checks, want %d", f.signed, tt.wantSign) + } + if len(rec.times(tt.wantLog)) != 1 { + t.Errorf("no %q log record", tt.wantLog) + } + }) + }) + } +} + +func TestAutoUpdateFailedInstallKeepsTheAgent(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + setVersion(t, "v0.2.0") + useFakeReleases(t, &fakeReleases{tag: "v0.2.1", signedAt: bubbleStart.Add(-48 * time.Hour)}) + calls := fakeUpdate(t, errors.New("the sha256 of the download does not agree")) + + exited, rec := runAuto(t, 5*time.Minute, t.TempDir(), &fakeCP{}, nil) + + if exited || len(*calls) != 1 { + t.Errorf("exited %v after %d installs, want one failed install and no exit", exited, len(*calls)) + } + if len(rec.times("the update failed; the old binary continues")) != 1 { + t.Error("no log record for the failed update") + } + }) +} + +func TestAutoUpdateIsOff(t *testing.T) { + tests := []struct { + name string + setup func(t *testing.T, cfg *Config) + wantLog string + }{ + {"by the switch", func(_ *testing.T, cfg *Config) { cfg.AutoUpdate = false }, "auto-update is off"}, + {"for a dev build", func(t *testing.T, _ *Config) { setVersion(t, "dev") }, "auto-update is off: this build has no release version"}, + {"for a build after a tag", func(t *testing.T, _ *Config) { setVersion(t, "v0.2.0-3-gabcdef1") }, "auto-update is off: this build has no release version"}, + {"for a build with changes", func(t *testing.T, _ *Config) { setVersion(t, "v0.2.0-dirty") }, "auto-update is off: this build has no release version"}, + {"without a trusted key", func(*testing.T, *Config) { + releaseKeys = func() (map[string]ed25519.PublicKey, error) { return nil, nil } + }, "auto-update is off: this build trusts no release key"}, + {"with keys that are not valid", func(*testing.T, *Config) { + releaseKeys = func() (map[string]ed25519.PublicKey, error) { return nil, errors.New("bad key") } + }, "auto-update is off: the release keys of this build are not valid"}, + {"when the binary folder is not writable", func(*testing.T, *Config) { + binaryDirWritable = func() bool { return false } + }, "auto-update is off: the agent cannot write the folder of its binary"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + setVersion(t, "v0.2.0") + f := &fakeReleases{tag: "v0.2.1", signedAt: bubbleStart.Add(-48 * time.Hour)} + useFakeReleases(t, f) + cfg := Config{ServerID: 17, StateDir: t.TempDir(), AutoUpdate: true} + tt.setup(t, &cfg) + + rec := &recorder{} + ctx, cancel := context.WithTimeout(context.Background(), 48*time.Hour) + defer cancel() + if err := run(ctx, cfg, slog.New(rec), &fakeCP{}, nil); err != nil { + t.Fatal(err) + } + + if n := len(f.checkTimes()); n != 0 { + t.Errorf("%d checks, want none: the agent must not call GitHub", n) + } + if len(rec.times(tt.wantLog)) != 1 { + t.Errorf("no %q log record", tt.wantLog) + } + }) + }) + } +} + +func TestAutoUpdateCheckTimeSurvivesARestart(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + setVersion(t, "v0.2.0") + f := &fakeReleases{tag: "v0.1.1"} + useFakeReleases(t, f) + dir := t.TempDir() + + runAuto(t, 5*time.Minute, dir, &fakeCP{}, nil) + runAuto(t, 5*time.Minute, dir, &fakeCP{}, nil) + + // The second process knows the check of the first one. + if n := len(f.checkTimes()); n != 1 { + t.Errorf("%d checks in two runs on one day, want 1", n) + } + }) +} + +func TestAutoUpdateCheckTimeInTheFuture(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + setVersion(t, "v0.2.0") + f := &fakeReleases{tag: "v0.1.1"} + useFakeReleases(t, f) + dir := t.TempDir() + // A clock that was years ahead saved this time. + saved := state{ReportInterval: 10, LastUpdateCheck: bubbleStart.AddDate(5, 0, 0)} + if err := statefile.Write(filepath.Join(dir, "state.json"), saved); err != nil { + t.Fatal(err) + } + + runAuto(t, 2*time.Minute, dir, &fakeCP{}, nil) + + if got := f.checkTimes(); len(got) != 1 || !got[0].Equal(at(0)) { + t.Errorf("checks at %v, want one check at the first tick", got) + } + }) +} + +func TestAutoUpdateWhileTheEventsFail(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + setVersion(t, "v0.2.0") + f := &fakeReleases{tag: "v0.1.1"} + useFakeReleases(t, f) + cp := &fakeCP{eventsReply: func(int, *wire.EventsRequest) (*wire.EventsReply, error) { + return nil, errors.New("connection refused") + }} + + // Report interval 1: each tick is a report tick, and each report + // stops before the poll. + runAuto(t, 2*time.Minute, t.TempDir(), cp, nil) + + if got := f.checkTimes(); len(got) != 1 || !got[0].Equal(at(0)) { + t.Errorf("checks at %v, want a check at the first tick", got) + } + }) +} + +func TestNewReportIntervalKeepsTheCheckTime(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + setVersion(t, "v0.2.0") + useFakeReleases(t, &fakeReleases{tag: "v0.1.1"}) + dir := t.TempDir() + // The first reply sets 3, before the check. A later reply sets 5, + // after the check: that write must keep the check time. + cp := &fakeCP{metricsReply: func(call int, req *wire.MetricsRequest) (*wire.MetricsReply, error) { + n := 5 + if call == 0 { + n = 3 + } + return &wire.MetricsReply{Accepted: len(req.Samples), ReportInterval: n}, nil + }} + + runAuto(t, 5*time.Minute, dir, cp, &fakeCollector{}) + + var s state + if err := statefile.Read(filepath.Join(dir, "state.json"), &s); err != nil { + t.Fatal(err) + } + if s.ReportInterval != 5 || !s.LastUpdateCheck.Equal(at(0)) { + t.Errorf("state = %+v, want interval 5 and the check at %v", s, at(0)) + } + }) +} + +func TestConfigWarningsAreLogged(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + rec := &recorder{} + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cfg := Config{ServerID: 17, StateDir: t.TempDir(), Warnings: []string{"FLY_AGENT_AUTO_UPDATE=\"of\" is not on or off; auto-update stays on"}} + if err := run(ctx, cfg, slog.New(rec), &fakeCP{}, nil); err != nil { + t.Fatal(err) + } + + if len(rec.times(cfg.Warnings[0])) != 1 { + t.Error("the warning of the configuration is not in the log") + } + }) +} diff --git a/internal/agent/clean.go b/internal/agent/clean.go index 1268168..6a9702b 100644 --- a/internal/agent/clean.go +++ b/internal/agent/clean.go @@ -39,8 +39,8 @@ func cleanStatus(s wire.Status) wire.Status { s.OS = truncate(s.OS, maxStatusTextLen) s.Kernel = truncate(s.Kernel, maxStatusTextLen) s.Arch = truncate(s.Arch, maxArchLen) - s.UpdatesTotal = clampInt(s.UpdatesTotal) - s.UpdatesSecurity = clampInt(s.UpdatesSecurity) + s.UpdatesTotal = clampIntPtr(s.UpdatesTotal) + s.UpdatesSecurity = clampIntPtr(s.UpdatesSecurity) s.UptimeSeconds = clampInt(s.UptimeSeconds) return s } @@ -70,6 +70,15 @@ func clampInt(v uint64) uint64 { return min(v, math.MaxInt64) } +// clampIntPtr is clampInt for a value that can be "not known" (nil). +func clampIntPtr(v *uint64) *uint64 { + if v == nil { + return nil + } + c := clampInt(*v) + return &c +} + // clamp keeps v between lo and hi. NaN becomes lo. func clamp(v, lo, hi float64) float64 { if math.IsNaN(v) { diff --git a/internal/agent/config.go b/internal/agent/config.go index cdd081f..ed43a7a 100644 --- a/internal/agent/config.go +++ b/internal/agent/config.go @@ -1,6 +1,6 @@ // Package agent is the FlyWP monitoring agent: the long-running mode of fly // that "fly agent run" starts. It follows the FlyWP monitoring agent -// contract v0.2.1. +// contract v0.3.1. package agent import ( @@ -15,12 +15,14 @@ import ( ) // The environment keys that the FlyWP installer writes to /etc/fly/agent.env, -// and the key that systemd sets for StateDirectory=. +// and the key that systemd sets for StateDirectory=. EnvAutoUpdate is +// optional: a person adds it to stop the updates by release on one server. const ( - EnvURL = "FLY_AGENT_URL" - EnvToken = "FLY_AGENT_TOKEN" - EnvServerID = "FLY_AGENT_SERVER_ID" - EnvStateDir = "STATE_DIRECTORY" + EnvURL = "FLY_AGENT_URL" + EnvToken = "FLY_AGENT_TOKEN" + EnvServerID = "FLY_AGENT_SERVER_ID" + EnvStateDir = "STATE_DIRECTORY" + EnvAutoUpdate = "FLY_AGENT_AUTO_UPDATE" ) // Config is the configuration of the agent. @@ -36,6 +38,12 @@ type Config struct { ServerID int64 // StateDir keeps the files that must survive a restart. StateDir string + // AutoUpdate lets the agent install a new signed release by itself, one + // time each day. The command agent.update works in both cases. + AutoUpdate bool + // Warnings are the problems of optional keys. They do not stop the + // agent: the agent writes them to its log at start. + Warnings []string } // Offset is the time after each full minute at which the agent works. It @@ -81,9 +89,29 @@ func ConfigFromEnv(getenv func(string) string) (Config, error) { } cfg.StateDir = dir + // A typo in an optional key must not stop the metrics. + on, ok := parseSwitch(getenv(EnvAutoUpdate)) + if !ok { + cfg.Warnings = append(cfg.Warnings, fmt.Sprintf("%s=%q is not on or off; auto-update stays on", EnvAutoUpdate, getenv(EnvAutoUpdate))) + } + cfg.AutoUpdate = on + return cfg, errors.Join(errs...) } +// parseSwitch reads an optional on or off value. An empty value is on. ok is +// false for a value that is not known; that value is also on. +func parseSwitch(v string) (on, ok bool) { + switch strings.ToLower(strings.TrimSpace(v)) { + case "", "on", "true", "1", "yes": + return true, true + case "off", "false", "0", "no": + return false, true + default: + return true, false + } +} + // parseURL accepts an https URL. It also accepts http for a loopback host, // for tests and local development: plain http never leaves the machine. The // URL is a base URL only: no user, password, query or fragment. An error diff --git a/internal/agent/config_test.go b/internal/agent/config_test.go index 21e9a7d..510bcc6 100644 --- a/internal/agent/config_test.go +++ b/internal/agent/config_test.go @@ -145,3 +145,43 @@ func TestConfigURLWithPasswordDoesNotShowIt(t *testing.T) { } } } + +func TestConfigAutoUpdate(t *testing.T) { + tests := []struct { + value string + want bool + wantWarning bool + }{ + {"", true, false}, + {"on", true, false}, + {"TRUE", true, false}, + {"1", true, false}, + {"yes", true, false}, + {"off", false, false}, + {" Off ", false, false}, + {"false", false, false}, + {"0", false, false}, + {"no", false, false}, + // A typo in an optional key must not stop the metrics. + {"of", true, true}, + {"disabled", true, true}, + } + + for _, tt := range tests { + t.Run(tt.value, func(t *testing.T) { + env := validEnv(t) + env[EnvAutoUpdate] = tt.value + + cfg, err := ConfigFromEnv(getenv(env)) + if err != nil { + t.Fatalf("ConfigFromEnv() error = %v, want the agent to start", err) + } + if cfg.AutoUpdate != tt.want { + t.Errorf("AutoUpdate = %v, want %v", cfg.AutoUpdate, tt.want) + } + if got := len(cfg.Warnings) == 1 && strings.Contains(cfg.Warnings[0], EnvAutoUpdate); got != tt.wantWarning { + t.Errorf("Warnings = %q, want a warning that names %s: %v", cfg.Warnings, EnvAutoUpdate, tt.wantWarning) + } + }) + } +} diff --git a/internal/agent/outbox_test.go b/internal/agent/outbox_test.go index dbba21e..0446e92 100644 --- a/internal/agent/outbox_test.go +++ b/internal/agent/outbox_test.go @@ -1,6 +1,7 @@ package agent import ( + "encoding/json" "fmt" "log/slog" "math" @@ -156,10 +157,21 @@ func TestCleanKeepsIntegersInTheRangeOfPHP(t *testing.T) { t.Errorf("cleanSample() = %d, %d, %d; want %d, %d, 42", s.NetInBytes, s.MemoryTotalBytes, s.DiskUsedBytes, uint64(math.MaxInt64), uint64(math.MaxInt64)) } - st := cleanStatus(wire.Status{UpdatesTotal: math.MaxUint64, UpdatesSecurity: 1 << 63, UptimeSeconds: math.MaxUint64}) - if st.UpdatesTotal != math.MaxInt64 || st.UpdatesSecurity != math.MaxInt64 || st.UptimeSeconds != math.MaxInt64 { + total, security := uint64(math.MaxUint64), uint64(1<<63) + st := cleanStatus(wire.Status{UpdatesTotal: &total, UpdatesSecurity: &security, UptimeSeconds: math.MaxUint64}) + if *st.UpdatesTotal != math.MaxInt64 || *st.UpdatesSecurity != math.MaxInt64 || st.UptimeSeconds != math.MaxInt64 { t.Errorf("cleanStatus() = %+v, want each count at most %d", st, uint64(math.MaxInt64)) } + + // Unknown counts stay unknown, and go on the wire as null. + st = cleanStatus(wire.Status{}) + data, err := json.Marshal(st) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), `"updates_total":null`) || !strings.Contains(string(data), `"updates_security":null`) { + t.Errorf("status JSON = %s, want null update counts", data) + } } func TestCleanEventDropsACommandIDThatIsNotAULID(t *testing.T) { diff --git a/internal/agent/wire/wire.go b/internal/agent/wire/wire.go index c144d49..78c1f2c 100644 --- a/internal/agent/wire/wire.go +++ b/internal/agent/wire/wire.go @@ -18,13 +18,15 @@ type MetricsRequest struct { // Status describes the server now. The control plane replaces the stored // status with it, so the agent always sends all fields. type Status struct { - RebootRequired bool `json:"reboot_required"` - UpdatesTotal uint64 `json:"updates_total"` - UpdatesSecurity uint64 `json:"updates_security"` - OS string `json:"os"` - Kernel string `json:"kernel"` - UptimeSeconds uint64 `json:"uptime_seconds"` - Arch string `json:"arch"` + RebootRequired bool `json:"reboot_required"` + // The waiting updates. nil (JSON null) means "not known", for example + // without apt-check (contract v0.3.0). A 0 is a real 0. + UpdatesTotal *uint64 `json:"updates_total"` + UpdatesSecurity *uint64 `json:"updates_security"` + OS string `json:"os"` + Kernel string `json:"kernel"` + UptimeSeconds uint64 `json:"uptime_seconds"` + Arch string `json:"arch"` } // Sample holds the measurements of one minute. diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 57710b6..d87cb6f 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -257,14 +257,18 @@ func (c *Collector) Status(context.Context) wire.Status { c.log.Warn("reading the uptime", "error", err) } - s.UpdatesTotal, s.UpdatesSecurity = c.updatesTotal, c.updatesSecurity + // Without any count, the counts are not known: null, not a false 0. + if c.updatesKnown { + total, security := c.updatesTotal, c.updatesSecurity + s.UpdatesTotal, s.UpdatesSecurity = &total, &security + } return s } // refreshUpdates counts the waiting updates at the first call and then each -// hour. If apt-check fails, the last counts stay. Without any count, they are -// 0: the contract has no value for "not known" yet. +// hour. If apt-check fails, the last counts stay. Without any count, Status +// sends null (contract v0.3.1). func (c *Collector) refreshUpdates(ctx context.Context) { if !c.updatesAt.IsZero() && time.Since(c.updatesAt) < updatesEvery { return @@ -283,7 +287,7 @@ func (c *Collector) refreshUpdates(ctx context.Context) { if c.updatesKnown { c.log.Warn("cannot count the waiting updates; keeping the last counts", "error", err) } else { - c.log.Warn("cannot count the waiting updates; sending 0", "error", err) + c.log.Warn("cannot count the waiting updates; sending null", "error", err) } return } diff --git a/internal/metrics/metrics_test.go b/internal/metrics/metrics_test.go index 726fc2d..c69630b 100644 --- a/internal/metrics/metrics_test.go +++ b/internal/metrics/metrics_test.go @@ -12,6 +12,8 @@ import ( "strings" "testing" "time" + + "github.com/flywp/server-cli/internal/agent/wire" ) // server is a fake file system root with the files that the collector reads. @@ -276,7 +278,7 @@ func TestStatus(t *testing.T) { t.Fatal(err) } s := c.Status(context.Background()) - if !s.RebootRequired || s.UpdatesTotal != 33 || s.UpdatesSecurity != 6 { + if !s.RebootRequired || !counts(s, 33, 6) { t.Errorf("status = %+v, want a restart and 33 updates with 6 security updates", s) } if s.OS != "Ubuntu 24.04.1 LTS" || s.Kernel != "6.8.0-45-generic" || s.UptimeSeconds != 1892344 || s.Arch != runtime.GOARCH { @@ -310,8 +312,9 @@ func TestStatusWithoutAptCheck(t *testing.T) { t.Fatal(err) } s := c.Status(context.Background()) - if s.UpdatesTotal != 0 || s.UpdatesSecurity != 0 || s.RebootRequired { - t.Errorf("status = %+v, want 0 updates and no restart", s) + // Not known is null, not a false "no updates". + if s.UpdatesTotal != nil || s.UpdatesSecurity != nil || s.RebootRequired { + t.Errorf("status = %+v, want unknown (nil) updates and no restart", s) } if s.OS == "" { t.Error("status has no OS: one missing value must not clear the others") @@ -330,8 +333,8 @@ func TestAptCheckWithWarningsBeforeTheResult(t *testing.T) { if _, err := c.Sample(time.Now()); err != nil { t.Fatal(err) } - if s := c.Status(context.Background()); s.UpdatesTotal != 29 || s.UpdatesSecurity != 26 { - t.Errorf("updates = %d;%d, want 29;26 from the last line", s.UpdatesTotal, s.UpdatesSecurity) + if s := c.Status(context.Background()); !counts(s, 29, 26) { + t.Errorf("updates = %v;%v, want 29;26 from the last line", s.UpdatesTotal, s.UpdatesSecurity) } } @@ -348,11 +351,16 @@ func TestAFailedCountKeepsTheLastCounts(t *testing.T) { if _, err := c.Sample(time.Now()); err != nil { t.Fatal(err) } - if s := c.Status(context.Background()); s.UpdatesTotal != 33 || s.UpdatesSecurity != 6 { - t.Errorf("updates = %d;%d, want the last counts 33;6", s.UpdatesTotal, s.UpdatesSecurity) + if s := c.Status(context.Background()); !counts(s, 33, 6) { + t.Errorf("updates = %v;%v, want the last counts 33;6", s.UpdatesTotal, s.UpdatesSecurity) } } +// counts reports whether s has the known update counts total and security. +func counts(s wire.Status, total, security uint64) bool { + return s.UpdatesTotal != nil && s.UpdatesSecurity != nil && *s.UpdatesTotal == total && *s.UpdatesSecurity == security +} + func TestNoCountedInterfaceIsNotZeroTraffic(t *testing.T) { srv := newServer(t) if err := os.RemoveAll(filepath.Join(srv.root, "sys")); err != nil { diff --git a/internal/release/keys.go b/internal/release/keys.go new file mode 100644 index 0000000..f35c5ca --- /dev/null +++ b/internal/release/keys.go @@ -0,0 +1,19 @@ +package release + +import "crypto/ed25519" + +// trustedKeys are the public keys of the release signatures, in the format of +// ParseKeys. The private keys are kept outside GitHub: see "make release-key" +// and "make sign-release". Two keys can be listed while one replaces the other. +// +// Give each key a comment that names it (the COMMENT of make release-key). +// +// It is a string, not a map, so that a test build can set it with +// -ldflags "-X github.com/flywp/server-cli/internal/release.trustedKeys=...". +var trustedKeys = "21ec14c790e96d62:pyMN2C1s+lG2xXUTYeoXCgYtE1rtCh3rgYlYsBRySGk=" // server-cli release key for flywp + +// TrustedKeys returns the public keys that the agent accepts for a release +// signature. An empty map means that no release can install by itself. +func TrustedKeys() (map[string]ed25519.PublicKey, error) { + return ParseKeys(trustedKeys) +} diff --git a/internal/release/release.go b/internal/release/release.go index 62747ab..6adfb0a 100644 --- a/internal/release/release.go +++ b/internal/release/release.go @@ -6,6 +6,7 @@ import ( "archive/tar" "compress/gzip" "context" + "crypto/ed25519" "encoding/json" "errors" "fmt" @@ -71,6 +72,13 @@ func CheckForUpdates(ctx context.Context) (*Update, error) { // commits since the tag, the commit hash and "-dirty" for local changes. var describeSuffix = regexp.MustCompile(`(-\d+-g[0-9a-f]+)?(-dirty)?$`) +// IsLocalBuild reports whether v comes from git describe on a commit that is +// not a release tag, for example v0.2.0-3-gabcdef1 or v0.2.0-dirty. Such a +// build has code that no release has, so it must not update by itself. +func IsLocalBuild(v string) bool { + return describeSuffix.FindString(v) != "" +} + // isNewer reports whether the release version latest is newer than current. // comparable is false when current is not built from a release tag. func isNewer(latest, current string) (newer, comparable bool) { @@ -90,8 +98,10 @@ func LatestRelease(ctx context.Context) (*GithubRelease, error) { } defer func() { _ = resp.Body.Close() }() + // The agent reads this each day without a person: a huge reply must not + // use all of its memory. var release GithubRelease - if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + if err := json.NewDecoder(io.LimitReader(resp.Body, maxAssetSize)).Decode(&release); err != nil { return nil, fmt.Errorf("reading release information: %w", err) } @@ -179,22 +189,86 @@ func selfUpdate(ctx context.Context, release *GithubRelease, exe, goos, goarch s // checksum returns the sha256 of the release file name from the checksum // file of the release. func checksum(ctx context.Context, release *GithubRelease, name string) (string, error) { - url := asset(release, ChecksumsAsset) + data, err := fetchAsset(ctx, release, ChecksumsAsset) + if err != nil { + return "", err + } + + return parseChecksums(data, release.TagName, name) +} + +// ErrNotSigned is the error of SignedChecksum for a release without a +// signature file: the maintainer did not sign it yet. +var ErrNotSigned = errors.New("the release has no signature") + +// Signed is a release archive whose sha256 comes from a signed checksum file. +type Signed struct { + URL string + SHA256 string + SignedAt time.Time +} + +// SignedChecksum returns the archive of release for goos and goarch, with its +// sha256 from the checksum file, after it checks the signature of that file +// with keys. The sum comes from the same bytes that the signature covers. +func SignedChecksum(ctx context.Context, release *GithubRelease, goos, goarch string, keys map[string]ed25519.PublicKey, now time.Time) (*Signed, error) { + archiveURL := assetURL(release, goos, goarch) + if archiveURL == "" { + return nil, fmt.Errorf("release %s has no binary for %s/%s", release.TagName, goos, goarch) + } + if asset(release, SignatureAsset) == "" { + return nil, ErrNotSigned + } + + sigFile, err := fetchAsset(ctx, release, SignatureAsset) + if err != nil { + return nil, err + } + data, err := fetchAsset(ctx, release, ChecksumsAsset) + if err != nil { + return nil, err + } + + sig, err := Verify(sigFile, data, release.TagName, keys, now) + if err != nil { + return nil, err + } + sum, err := parseChecksums(data, release.TagName, BinaryName(goos, goarch)+".tar.gz") + if err != nil { + return nil, err + } + + return &Signed{URL: archiveURL, SHA256: sum, SignedAt: sig.SignedAt}, nil +} + +// maxAssetSize limits a small release file: the checksum file and its +// signature. +const maxAssetSize = 1 << 20 + +// fetchAsset downloads the small release file name. +func fetchAsset(ctx context.Context, release *GithubRelease, name string) ([]byte, error) { + url := asset(release, name) if url == "" { - return "", fmt.Errorf("release %s has no %s, so its download cannot be checked", release.TagName, ChecksumsAsset) + return nil, fmt.Errorf("release %s has no %s, so its download cannot be checked", release.TagName, name) } resp, err := get(ctx, url) if err != nil { - return "", fmt.Errorf("downloading %s: %w", ChecksumsAsset, err) + return nil, fmt.Errorf("downloading %s: %w", name, err) } defer func() { _ = resp.Body.Close() }() - data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + data, err := io.ReadAll(io.LimitReader(resp.Body, maxAssetSize)) if err != nil { - return "", fmt.Errorf("downloading %s: %w", ChecksumsAsset, err) + return nil, fmt.Errorf("downloading %s: %w", name, err) } + return data, nil +} + +// parseChecksums returns the sha256 of the file name from the checksum file +// data of the release tag. +func parseChecksums(data []byte, tag, name string) (string, error) { // install.sh reads the file with the same rules. Two different sums for // one file make the file not valid: it is not clear which one is correct. var sum string @@ -205,12 +279,12 @@ func checksum(ctx context.Context, release *GithubRelease, name string) (string, continue } if sum != "" && !strings.EqualFold(sum, fields[0]) { - return "", fmt.Errorf("%s of release %s has two different sums for %s", ChecksumsAsset, release.TagName, name) + return "", fmt.Errorf("%s of release %s has two different sums for %s", ChecksumsAsset, tag, name) } sum = fields[0] } if sum == "" { - return "", fmt.Errorf("%s of release %s has no line for %s", ChecksumsAsset, release.TagName, name) + return "", fmt.Errorf("%s of release %s has no line for %s", ChecksumsAsset, tag, name) } return sum, nil diff --git a/internal/release/signature.go b/internal/release/signature.go new file mode 100644 index 0000000..2b03ce3 --- /dev/null +++ b/internal/release/signature.go @@ -0,0 +1,151 @@ +package release + +import ( + "bytes" + "crypto/ed25519" + "crypto/sha256" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "strings" + "time" +) + +// SignatureAsset is the signature of the checksum file of a release. The +// maintainer makes it with a key that is kept outside GitHub, after the +// release workflow publishes the release. The agent installs a release by +// itself only when this signature is valid: the checksum file alone proves +// that a download is complete, not who made the release. +const SignatureAsset = ChecksumsAsset + ".sig" + +// The lines of a signature file, in this order: +// +// fly-release-signature-v1 +// key +// tag +// signed-at +// sig +// +// The signature covers the first four lines, each with its "\n", followed by +// the exact bytes of checksums.txt. The tag stops a signature from being +// moved to an other release; signed-at is the start of the wait before +// agents install the release. +const signatureHeader = "fly-release-signature-v1" + +// maxClockSkew is how far in the future a signature time can be: the clock +// of the maintainer and of the server can differ a little. +const maxClockSkew = 10 * time.Minute + +// Signature is a verified signature file. +type Signature struct { + KeyID string + Tag string + SignedAt time.Time +} + +// KeyID returns the id of a public key: the first 8 bytes of its sha256, in +// hex. A signature names its key with it, so that a new key can replace an +// old one. +func KeyID(pub ed25519.PublicKey) string { + sum := sha256.Sum256(pub) + return hex.EncodeToString(sum[:8]) +} + +// Sign returns the signature file of checksums for the release tag. +func Sign(priv ed25519.PrivateKey, tag string, signedAt time.Time, checksums []byte) []byte { + pub, _ := priv.Public().(ed25519.PublicKey) + head := signedHead(KeyID(pub), tag, signedAt.UTC()) + sig := ed25519.Sign(priv, append([]byte(head), checksums...)) + + return []byte(head + "sig " + base64.StdEncoding.EncodeToString(sig) + "\n") +} + +// Verify checks that sigFile is a valid signature of checksums for the +// release tag, by one of the keys (key id to public key). It refuses a +// signature time more than a few minutes after now. +func Verify(sigFile, checksums []byte, tag string, keys map[string]ed25519.PublicKey, now time.Time) (*Signature, error) { + lines := strings.Split(string(sigFile), "\n") + // Five lines, each with its "\n", so the split gives an empty sixth. + if len(lines) != 6 || lines[5] != "" || lines[0] != signatureHeader { + return nil, errors.New("the signature file is not in the fly-release-signature-v1 format") + } + + keyID, ok1 := strings.CutPrefix(lines[1], "key ") + signedTag, ok2 := strings.CutPrefix(lines[2], "tag ") + at, ok3 := strings.CutPrefix(lines[3], "signed-at ") + encoded, ok4 := strings.CutPrefix(lines[4], "sig ") + if !ok1 || !ok2 || !ok3 || !ok4 { + return nil, errors.New("the signature file is not in the fly-release-signature-v1 format") + } + + pub, ok := keys[keyID] + if !ok { + return nil, fmt.Errorf("the signature uses the key %q, which this binary does not trust", keyID) + } + if signedTag != tag { + return nil, fmt.Errorf("the signature is for the release %s, not %s", signedTag, tag) + } + signedAt, err := time.Parse(time.RFC3339, at) + if err != nil { + return nil, fmt.Errorf("the signature time %q is not valid", at) + } + sig, err := base64.StdEncoding.DecodeString(encoded) + if err != nil || len(sig) != ed25519.SignatureSize { + return nil, errors.New("the signature is not a valid ed25519 signature") + } + + // The head is rebuilt from the lines as they are in the file, so that + // the check covers exactly the bytes that were signed. + head := strings.Join(lines[:4], "\n") + "\n" + if !ed25519.Verify(pub, append([]byte(head), checksums...), sig) { + return nil, fmt.Errorf("the signature of %s for %s is not correct", ChecksumsAsset, tag) + } + + // Checked after the signature: only a valid signature makes the time + // worth an error message. + if signedAt.After(now.Add(maxClockSkew)) { + return nil, fmt.Errorf("the signature time %s is in the future", at) + } + + return &Signature{KeyID: keyID, Tag: signedTag, SignedAt: signedAt}, nil +} + +func signedHead(keyID, tag string, signedAt time.Time) string { + var b bytes.Buffer + fmt.Fprintf(&b, "%s\nkey %s\ntag %s\nsigned-at %s\n", signatureHeader, keyID, tag, signedAt.Format(time.RFC3339)) + return b.String() +} + +// ParseKeys parses a list of public keys: ":", separated +// with commas. Each id must be the id of its key. +func ParseKeys(list string) (map[string]ed25519.PublicKey, error) { + keys := make(map[string]ed25519.PublicKey) + for entry := range strings.SplitSeq(list, ",") { + entry = strings.TrimSpace(entry) + if entry == "" { + continue + } + + id, encoded, ok := strings.Cut(entry, ":") + if !ok { + return nil, fmt.Errorf("the key %q is not in the : format", entry) + } + raw, err := base64.StdEncoding.DecodeString(encoded) + if err != nil || len(raw) != ed25519.PublicKeySize { + return nil, fmt.Errorf("the key %s is not a valid ed25519 public key", id) + } + pub := ed25519.PublicKey(raw) + if KeyID(pub) != id { + return nil, fmt.Errorf("the key id %s does not agree with its key (%s)", id, KeyID(pub)) + } + keys[id] = pub + } + + return keys, nil +} + +// KeyLine returns the entry of pub for the list that ParseKeys reads. +func KeyLine(pub ed25519.PublicKey) string { + return KeyID(pub) + ":" + base64.StdEncoding.EncodeToString(pub) +} diff --git a/internal/release/signature_test.go b/internal/release/signature_test.go new file mode 100644 index 0000000..5bd2094 --- /dev/null +++ b/internal/release/signature_test.go @@ -0,0 +1,257 @@ +package release + +import ( + "context" + "crypto/ed25519" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +var signTime = time.Date(2026, 10, 1, 10, 0, 0, 0, time.UTC) + +func testKey(t *testing.T) (ed25519.PrivateKey, map[string]ed25519.PublicKey) { + t.Helper() + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + t.Fatal(err) + } + return priv, map[string]ed25519.PublicKey{KeyID(pub): pub} +} + +func TestSignThenVerify(t *testing.T) { + priv, keys := testKey(t) + checksums := []byte("abc fly-linux-amd64.tar.gz\n") + + sigFile := Sign(priv, "v0.2.1", signTime, checksums) + + sig, err := Verify(sigFile, checksums, "v0.2.1", keys, signTime.Add(time.Hour)) + if err != nil { + t.Fatalf("Verify() error = %v", err) + } + if sig.Tag != "v0.2.1" || !sig.SignedAt.Equal(signTime) { + t.Errorf("Verify() = %+v, want tag v0.2.1 signed at %v", sig, signTime) + } + if !strings.HasPrefix(string(sigFile), "fly-release-signature-v1\nkey ") { + t.Errorf("signature file = %q, want the v1 format", sigFile) + } +} + +func TestVerifyRefuses(t *testing.T) { + priv, keys := testKey(t) + otherPriv, _ := testKey(t) + checksums := []byte("abc fly-linux-amd64.tar.gz\n") + good := string(Sign(priv, "v0.2.1", signTime, checksums)) + now := signTime.Add(time.Hour) + + tests := []struct { + name string + sigFile string + checksums string + tag string + now time.Time + want string + }{ + {"an unknown key", string(Sign(otherPriv, "v0.2.1", signTime, checksums)), string(checksums), "v0.2.1", now, "does not trust"}, + {"an other tag", good, string(checksums), "v0.2.2", now, "for the release v0.2.1, not v0.2.2"}, + {"a changed checksum file", good, "def fly-linux-amd64.tar.gz\n", "v0.2.1", now, "is not correct"}, + {"a changed signature time", strings.Replace(good, "10:00:00Z", "09:00:00Z", 1), string(checksums), "v0.2.1", now, "is not correct"}, + {"a time in the future", good, string(checksums), "v0.2.1", signTime.Add(-time.Hour), "in the future"}, + {"reordered lines", reorder(good), string(checksums), "v0.2.1", now, "format"}, + {"an extra line", good + "extra\n", string(checksums), "v0.2.1", now, "format"}, + {"no end of line", strings.TrimSuffix(good, "\n"), string(checksums), "v0.2.1", now, "format"}, + {"an other format", strings.Replace(good, "-v1", "-v2", 1), string(checksums), "v0.2.1", now, "format"}, + {"a short signature", good[:strings.Index(good, "sig ")] + "sig AAAA\n", string(checksums), "v0.2.1", now, "not a valid ed25519"}, + {"an empty file", "", string(checksums), "v0.2.1", now, "format"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := Verify([]byte(tt.sigFile), []byte(tt.checksums), tt.tag, keys, tt.now) + if err == nil || !strings.Contains(err.Error(), tt.want) { + t.Errorf("Verify() error = %v, want an error that contains %q", err, tt.want) + } + }) + } +} + +func TestVerifyCoversTheTagAndTheKey(t *testing.T) { + priv, keys := testKey(t) + otherPriv, otherKeys := testKey(t) + for id, pub := range otherKeys { + keys[id] = pub + } + otherPub, _ := otherPriv.Public().(ed25519.PublicKey) + checksums := []byte("abc fly-linux-amd64.tar.gz\n") + good := string(Sign(priv, "v0.2.1", signTime, checksums)) + now := signTime.Add(time.Hour) + + // A signature moved to an other release: the tag line is part of the + // signed bytes, not only a label. + moved := strings.Replace(good, "tag v0.2.1\n", "tag v0.2.2\n", 1) + if _, err := Verify([]byte(moved), checksums, "v0.2.2", keys, now); err == nil || !strings.Contains(err.Error(), "is not correct") { + t.Errorf("Verify() of a signature moved to v0.2.2 = %v, want a signature error", err) + } + + // The key line names an other trusted key. + lines := strings.SplitN(good, "\n", 3) + swapped := lines[0] + "\nkey " + KeyID(otherPub) + "\n" + lines[2] + if _, err := Verify([]byte(swapped), checksums, "v0.2.1", keys, now); err == nil || !strings.Contains(err.Error(), "is not correct") { + t.Errorf("Verify() with an other key line = %v, want a signature error", err) + } +} + +func TestIsLocalBuild(t *testing.T) { + tests := map[string]bool{ + "v0.2.0": false, + "v0.2.0-dev.1a2b3c4": false, + "v0.2.0-rc.1": false, + "dev": false, + "v0.2.0-3-gabcdef1": true, + "v0.2.0-dirty": true, + "v0.2.0-3-gabcdef1-dirty": true, + } + for v, want := range tests { + if got := IsLocalBuild(v); got != want { + t.Errorf("IsLocalBuild(%q) = %v, want %v", v, got, want) + } + } +} + +// reorder swaps the key and the tag lines of a signature file. +func reorder(sigFile string) string { + lines := strings.Split(sigFile, "\n") + lines[1], lines[2] = lines[2], lines[1] + return strings.Join(lines, "\n") +} + +func TestVerifyWithoutKeys(t *testing.T) { + priv, _ := testKey(t) + checksums := []byte("abc fly-linux-amd64.tar.gz\n") + + // A binary without a trusted key installs no release by itself. + if _, err := Verify(Sign(priv, "v0.2.1", signTime, checksums), checksums, "v0.2.1", nil, signTime); err == nil { + t.Error("Verify() with no keys = nil, want an error") + } +} + +func TestParseKeys(t *testing.T) { + pub1, _, _ := ed25519.GenerateKey(nil) + pub2, _, _ := ed25519.GenerateKey(nil) + + keys, err := ParseKeys(KeyLine(pub1) + ", " + KeyLine(pub2)) + if err != nil { + t.Fatal(err) + } + if len(keys) != 2 || !keys[KeyID(pub1)].Equal(pub1) { + t.Errorf("ParseKeys() = %v, want both keys by id", keys) + } + + if keys, err := ParseKeys(""); err != nil || len(keys) != 0 { + t.Errorf("ParseKeys(\"\") = %v, %v, want no keys", keys, err) + } + + for _, bad := range []string{ + "no-colon", + KeyID(pub1) + ":not-base64!", + KeyID(pub1) + ":AAAA", + KeyID(pub2) + ":" + strings.SplitN(KeyLine(pub1), ":", 2)[1], + } { + if _, err := ParseKeys(bad); err == nil { + t.Errorf("ParseKeys(%q) = nil error, want an error", bad) + } + } +} + +func TestTrustedKeysParse(t *testing.T) { + // The built-in list must always parse: a bad list would stop every + // update by itself without a clear reason. An empty list would do the + // same, for every agent of the release. + keys, err := TrustedKeys() + if err != nil { + t.Fatalf("TrustedKeys() error = %v", err) + } + if len(keys) == 0 { + t.Fatal("TrustedKeys() is empty: add the public key line of the release key to keys.go") + } +} + +// signedRelease serves a release with an archive, a checksum file and, when +// sigFile is not nil, a signature file. +func signedRelease(t *testing.T, checksums string, sigFile []byte) *GithubRelease { + t.Helper() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/checksums.txt": + _, _ = w.Write([]byte(checksums)) + case "/checksums.txt.sig": + _, _ = w.Write(sigFile) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(srv.Close) + + rel := &GithubRelease{TagName: "v0.2.1"} + names := []string{"fly-linux-amd64.tar.gz", "checksums.txt"} + if sigFile != nil { + names = append(names, "checksums.txt.sig") + } + for _, name := range names { + rel.Assets = append(rel.Assets, Asset{Name: name, BrowserDownloadURL: srv.URL + "/" + name}) + } + return rel +} + +func TestSignedChecksum(t *testing.T) { + priv, keys := testKey(t) + archiveSum := sum([]byte("archive")) + checksums := archiveSum + " fly-linux-amd64.tar.gz\n" + rel := signedRelease(t, checksums, Sign(priv, "v0.2.1", signTime, []byte(checksums))) + + got, err := SignedChecksum(context.Background(), rel, "linux", "amd64", keys, signTime) + if err != nil { + t.Fatalf("SignedChecksum() error = %v", err) + } + if got.SHA256 != archiveSum || !got.SignedAt.Equal(signTime) || !strings.HasSuffix(got.URL, "/fly-linux-amd64.tar.gz") { + t.Errorf("SignedChecksum() = %+v, want the signed sum, time and archive URL", got) + } +} + +func TestSignedChecksumRefuses(t *testing.T) { + priv, keys := testKey(t) + checksums := sum([]byte("archive")) + " fly-linux-amd64.tar.gz\n" + + t.Run("no signature", func(t *testing.T) { + rel := signedRelease(t, checksums, nil) + if _, err := SignedChecksum(context.Background(), rel, "linux", "amd64", keys, signTime); !errors.Is(err, ErrNotSigned) { + t.Errorf("SignedChecksum() error = %v, want ErrNotSigned", err) + } + }) + + t.Run("a signature of other checksums", func(t *testing.T) { + rel := signedRelease(t, checksums, Sign(priv, "v0.2.1", signTime, []byte("other\n"))) + if _, err := SignedChecksum(context.Background(), rel, "linux", "amd64", keys, signTime); err == nil || !strings.Contains(err.Error(), "is not correct") { + t.Errorf("SignedChecksum() error = %v, want a signature error", err) + } + }) + + t.Run("no line for this arch", func(t *testing.T) { + rel := signedRelease(t, checksums, Sign(priv, "v0.2.1", signTime, []byte(checksums))) + rel.Assets = append(rel.Assets, Asset{Name: "fly-linux-arm64.tar.gz", BrowserDownloadURL: rel.Assets[0].BrowserDownloadURL}) + if _, err := SignedChecksum(context.Background(), rel, "linux", "arm64", keys, signTime); err == nil || !strings.Contains(err.Error(), "has no line for fly-linux-arm64.tar.gz") { + t.Errorf("SignedChecksum() error = %v, want no line for arm64", err) + } + }) + + t.Run("no archive for this system", func(t *testing.T) { + rel := signedRelease(t, checksums, Sign(priv, "v0.2.1", signTime, []byte(checksums))) + if _, err := SignedChecksum(context.Background(), rel, "darwin", "arm64", keys, signTime); err == nil || !strings.Contains(err.Error(), "has no binary") { + t.Errorf("SignedChecksum() error = %v, want no binary for darwin", err) + } + }) +} diff --git a/tools/releasesign/main.go b/tools/releasesign/main.go new file mode 100644 index 0000000..dc18fcd --- /dev/null +++ b/tools/releasesign/main.go @@ -0,0 +1,212 @@ +// Command releasesign makes the release signing key and signs the checksum +// file of a release. The agent installs a release by itself only when the +// signature is valid. It is not part of the fly binary. +// +// go run ./tools/releasesign keygen -out [-comment "server-cli release key for flywp"] +// go run ./tools/releasesign sign -key -tag v0.2.1 checksums.txt > checksums.txt.sig +// go run ./tools/releasesign verify -tag v0.2.1 checksums.txt checksums.txt.sig +// +// Keep the private key outside GitHub, for example in a password manager. +// Never commit it, and never put it in a GitHub secret. +package main + +import ( + "crypto/ed25519" + "crypto/x509" + "encoding/pem" + "errors" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/flywp/server-cli/internal/release" +) + +func main() { + if err := run(os.Args[1:], os.Stdin, os.Stdout); err != nil { + fmt.Fprintln(os.Stderr, "releasesign:", err) + os.Exit(1) + } +} + +func run(args []string, stdin io.Reader, stdout io.Writer) error { + if len(args) == 0 { + return errors.New("usage: releasesign keygen|sign|verify [flags]") + } + + switch args[0] { + case "keygen": + return keygen(args[1:], stdout) + case "sign": + return sign(args[1:], stdin, stdout) + case "verify": + return verify(args[1:], stdout) + default: + return fmt.Errorf("unknown command %q: use keygen, sign or verify", args[0]) + } +} + +// keygen writes a new private key to a file that must not exist, and prints +// the public key line for internal/release/keys.go. It never prints the +// private key. The comment names the key: it goes in the key file as a PEM +// header, and next to the public key line. +func keygen(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("keygen", flag.ContinueOnError) + out := fs.String("out", "", "the file for the private key (it must not exist)") + comment := fs.String("comment", "", "a name for the key, for example \"server-cli release key for flywp\"") + if err := fs.Parse(args); err != nil { + return err + } + if *out == "" { + return errors.New("keygen needs -out ") + } + path, err := expandHome(*out) + if err != nil { + return err + } + if strings.ContainsAny(*comment, "\r\n") { + return errors.New("the comment must be one line") + } + + pub, priv, err := ed25519.GenerateKey(nil) + if err != nil { + return err + } + der, err := x509.MarshalPKCS8PrivateKey(priv) + if err != nil { + return err + } + + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + block := &pem.Block{Type: "PRIVATE KEY", Bytes: der} + if *comment != "" { + block.Headers = map[string]string{"Comment": *comment} + } + if err := pem.Encode(f, block); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + + msg := fmt.Sprintf("Private key: %s (keep it outside GitHub, with a backup)\n", path) + if *comment != "" { + msg += fmt.Sprintf("Comment: %s\n", *comment) + } + msg += fmt.Sprintf("Public key line for internal/release/keys.go:\n%s\n", release.KeyLine(pub)) + _, err = io.WriteString(stdout, msg) + return err +} + +// sign prints the signature file of a checksum file. +func sign(args []string, stdin io.Reader, stdout io.Writer) error { + fs := flag.NewFlagSet("sign", flag.ContinueOnError) + keyPath := fs.String("key", "", "the private key file, or - to read it from stdin") + tag := fs.String("tag", "", "the release tag, for example v0.2.1") + if err := fs.Parse(args); err != nil { + return err + } + if *keyPath == "" || *tag == "" || fs.NArg() != 1 { + return errors.New("usage: releasesign sign -key -tag checksums.txt") + } + + priv, err := readKey(*keyPath, stdin) + if err != nil { + return err + } + checksums, err := os.ReadFile(fs.Arg(0)) + if err != nil { + return err + } + + _, err = stdout.Write(release.Sign(priv, *tag, time.Now().UTC().Truncate(time.Second), checksums)) + return err +} + +// verify checks a signature file with the keys that this build trusts: the +// keys of the agents that are built from the same commit. +func verify(args []string, stdout io.Writer) error { + fs := flag.NewFlagSet("verify", flag.ContinueOnError) + tag := fs.String("tag", "", "the release tag, for example v0.2.1") + if err := fs.Parse(args); err != nil { + return err + } + if *tag == "" || fs.NArg() != 2 { + return errors.New("usage: releasesign verify -tag checksums.txt checksums.txt.sig") + } + + keys, err := release.TrustedKeys() + if err != nil { + return err + } + if len(keys) == 0 { + return errors.New("this build trusts no key: add the public key line to internal/release/keys.go") + } + checksums, err := os.ReadFile(fs.Arg(0)) + if err != nil { + return err + } + sigFile, err := os.ReadFile(fs.Arg(1)) + if err != nil { + return err + } + + sig, err := release.Verify(sigFile, checksums, *tag, keys, time.Now()) + if err != nil { + return err + } + + _, err = fmt.Fprintf(stdout, "The signature of %s is valid (key %s, signed at %s).\n", *tag, sig.KeyID, sig.SignedAt.Format(time.RFC3339)) + return err +} + +// expandHome replaces a leading "~/" with the home directory. A shell does +// not do this in "make release-key KEY=~/key": the "~" is not at the start of +// a word. +func expandHome(path string) (string, error) { + rest, ok := strings.CutPrefix(path, "~/") + if !ok { + return path, nil + } + home, err := os.UserHomeDir() + if err != nil { + return "", err + } + return filepath.Join(home, rest), nil +} + +func readKey(path string, stdin io.Reader) (ed25519.PrivateKey, error) { + var data []byte + var err error + if path == "-" { + data, err = io.ReadAll(io.LimitReader(stdin, 1<<16)) + } else if path, err = expandHome(path); err == nil { + data, err = os.ReadFile(path) + } + if err != nil { + return nil, err + } + + block, _ := pem.Decode(data) + if block == nil || block.Type != "PRIVATE KEY" { + return nil, errors.New("the key is not a PEM PRIVATE KEY") + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("reading the key: %w", err) + } + priv, ok := key.(ed25519.PrivateKey) + if !ok { + return nil, errors.New("the key is not an ed25519 key") + } + + return priv, nil +} diff --git a/tools/releasesign/main_test.go b/tools/releasesign/main_test.go new file mode 100644 index 0000000..08584b2 --- /dev/null +++ b/tools/releasesign/main_test.go @@ -0,0 +1,107 @@ +package main + +import ( + "bytes" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/flywp/server-cli/internal/release" +) + +func TestKeygenThenSign(t *testing.T) { + dir := t.TempDir() + keyPath := filepath.Join(dir, "release.key") + + var out bytes.Buffer + if err := run([]string{"keygen", "-out", keyPath, "-comment", "server-cli release key for flywp"}, nil, &out); err != nil { + t.Fatal(err) + } + + info, err := os.Stat(keyPath) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Errorf("key file mode = %v, want 0600", info.Mode().Perm()) + } + key, _ := os.ReadFile(keyPath) + if strings.Contains(out.String(), "PRIVATE KEY") || strings.Contains(out.String(), string(key)) { + t.Fatal("keygen printed the private key") + } + // The comment names the key in the key file and in the output. + if !strings.Contains(string(key), "Comment: server-cli release key for flywp") || !strings.Contains(out.String(), "Comment: server-cli release key for flywp") { + t.Errorf("the comment is not in the key file and the output:\n%s", out.String()) + } + + // The printed line parses, and it is the key of the private key file. + lines := strings.Split(strings.TrimSpace(out.String()), "\n") + keys, err := release.ParseKeys(lines[len(lines)-1]) + if err != nil || len(keys) != 1 { + t.Fatalf("public key line %q: %v, %v", lines[len(lines)-1], keys, err) + } + + checksums := filepath.Join(dir, "checksums.txt") + if err := os.WriteFile(checksums, []byte("abc fly-linux-amd64.tar.gz\n"), 0o600); err != nil { + t.Fatal(err) + } + + // The key comes from stdin, for example from a password manager. + var sigFile bytes.Buffer + if err := run([]string{"sign", "-key", "-", "-tag", "v0.2.1", checksums}, bytes.NewReader(key), &sigFile); err != nil { + t.Fatal(err) + } + data, _ := os.ReadFile(checksums) + if _, err := release.Verify(sigFile.Bytes(), data, "v0.2.1", keys, time.Now()); err != nil { + t.Errorf("Verify() of the signed file = %v", err) + } +} + +func TestKeygenRefusesAMultiLineComment(t *testing.T) { + keyPath := filepath.Join(t.TempDir(), "release.key") + if err := run([]string{"keygen", "-out", keyPath, "-comment", "a\nProc-Type: 4,ENCRYPTED"}, nil, &bytes.Buffer{}); err == nil { + t.Fatal("keygen with a multi-line comment = nil error, want an error") + } + if _, err := os.Stat(keyPath); err == nil { + t.Error("keygen wrote a key file for a bad comment") + } +} + +func TestKeygenDoesNotOverwrite(t *testing.T) { + keyPath := filepath.Join(t.TempDir(), "release.key") + if err := os.WriteFile(keyPath, []byte("old key"), 0o600); err != nil { + t.Fatal(err) + } + + if err := run([]string{"keygen", "-out", keyPath}, nil, &bytes.Buffer{}); err == nil { + t.Fatal("keygen over an existing file = nil error, want an error") + } + if data, _ := os.ReadFile(keyPath); string(data) != "old key" { + t.Error("keygen changed an existing key file") + } +} + +func TestKeyPathsExpandTheHomeDirectory(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + + // make passes "~/release.key" as it is: the shell does not expand it. + var out bytes.Buffer + if err := run([]string{"keygen", "-out", "~/release.key"}, nil, &out); err != nil { + t.Fatalf("keygen -out ~/release.key: %v", err) + } + if _, err := os.Stat(filepath.Join(home, "release.key")); err != nil { + t.Fatalf("the key is not in the home directory: %v", err) + } + if _, err := readKey("~/release.key", nil); err != nil { + t.Errorf("readKey(~/release.key) = %v", err) + } +} + +func TestReadKeyRefusesText(t *testing.T) { + if _, err := readKey("-", strings.NewReader("not a key")); err == nil { + t.Error("readKey() of text = nil error, want an error") + } +} diff --git a/tools/sign-release.sh b/tools/sign-release.sh new file mode 100755 index 0000000..06993c1 --- /dev/null +++ b/tools/sign-release.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# +# Sign a published release: make sign-release VERSION=v0.2.1 KEY= +# +# The signature says "this release is the code of my tag". So the script +# signs only what it can build again: it builds the release from your local +# tag, with the Go version of the CI build, and compares the binaries with +# the binaries that GitHub serves. A swapped archive or checksums.txt on +# GitHub, or a tag that moved on GitHub, stops the script before it signs. +# +# The binary holds the commit (vcs.revision), so equal binaries also prove +# that CI built the commit of your local tag. Review that commit before you +# sign: the signature is the approval. +# +# UPLOAD=0 does all checks and writes checksums.txt.sig, but does not upload it. +# SIGN_YES=1 skips the typed confirmation (tests only). + +set -euo pipefail + +version=${1:?usage: sign-release.sh } +key=${2:?usage: sign-release.sh } + +root=$(git rev-parse --show-toplevel) +dir="$root/build/sign/$version" + +# make passes "~/key" as it is: the shell does not expand a "~" that is not +# at the start of a word. +case "$key" in "~/"*) key="$HOME/${key#\~/}" ;; esac +if [ "$key" != "-" ]; then + [ -f "$key" ] || { echo "No key file: $key" >&2; exit 1; } + key="$(cd "$(dirname "$key")" && pwd)/$(basename "$key")" +fi + +# The source of trust is your local tag, not the tag on GitHub. +if ! commit=$(git -C "$root" rev-parse --verify --quiet "refs/tags/$version^{commit}"); then + echo "There is no local tag $version. Sign only a tag that you made or reviewed." >&2 + exit 1 +fi + +echo "Release $version is commit $(git -C "$root" log -1 --format='%h %s' "$commit")" +if [ "${SIGN_YES:-}" != "1" ]; then + printf 'Type the tag to sign it: ' > /dev/tty + read -r answer < /dev/tty + [ "$answer" = "$version" ] || { echo "Not signed." >&2; exit 1; } +fi + +rm -rf "$dir" +mkdir -p "$dir/ci" "$dir/ci-bin" +cleanup() { git -C "$root" worktree remove --force "$dir/src" >/dev/null 2>&1 || true; } +trap cleanup EXIT + +echo "Downloading the release from GitHub..." +gh release download "$version" --pattern 'fly-linux-*.tar.gz' --pattern checksums.txt --dir "$dir/ci" + +# The archives agree with checksums.txt. +if command -v sha256sum >/dev/null 2>&1; then + (cd "$dir/ci" && sha256sum -c checksums.txt) +else + (cd "$dir/ci" && shasum -a 256 -c checksums.txt) +fi + +# Build the release again from the local tag, with the Go version of CI. +git -C "$root" worktree add --detach --quiet "$dir/src" "$commit" +for archive in "$dir"/ci/fly-linux-*.tar.gz; do + tar -xzf "$archive" -C "$dir/ci-bin" +done +goversions=$(for bin in "$dir"/ci-bin/*; do go version "$bin" | awk '{ print $NF }'; done | sort -u) +if [ "$(printf '%s\n' "$goversions" | wc -l)" -ne 1 ]; then + echo "The CI binaries were built with different Go versions: $goversions" >&2 + exit 1 +fi +echo "Building $version again with $goversions..." +(cd "$dir/src" && GOTOOLCHAIN="$goversions" make release VERSION="$version" >/dev/null) + +# checksums.txt must name exactly the archives of the release build. +names() { sed 's/\r$//' "$1" | awk '{ sub(/^\*/, "", $2); print $2 }' | sort; } +if [ "$(names "$dir/ci/checksums.txt")" != "$(names "$dir/src/build/checksums.txt")" ]; then + echo "checksums.txt on GitHub does not name the archives of the release build." >&2 + exit 1 +fi + +# The binaries are the same, byte for byte. +for bin in "$dir"/src/build/fly-linux-*; do + case "$bin" in *.tar.gz) continue ;; esac + name=$(basename "$bin") + if ! cmp -s "$bin" "$dir/ci-bin/$name"; then + echo "$name on GitHub is not the build of $version. Do not sign it." >&2 + exit 1 + fi + echo "$name: the same as the build of the local tag" +done + +# Sign and verify with the tool and the trusted keys of the tag: the keys +# that the agents of this release have. +(cd "$dir/src" && go run ./tools/releasesign sign -key "$key" -tag "$version" "$dir/ci/checksums.txt") > "$dir/checksums.txt.sig" +(cd "$dir/src" && go run ./tools/releasesign verify -tag "$version" "$dir/ci/checksums.txt" "$dir/checksums.txt.sig") + +if [ "${UPLOAD:-1}" = "0" ]; then + echo "Not uploaded (UPLOAD=0): $dir/checksums.txt.sig" + exit 0 +fi +gh release upload "$version" "$dir/checksums.txt.sig" --clobber +echo "Signed $version. Agents install it 24 hours after the signature."