feat(agent): install signed releases by itself, one time each day

Makefile

@@ -1,7 +1,7 @@
 # Developer commands for fly. Run "make help" for the list.
 # Keep this file compatible with GNU Make 3.81, the version on macOS.
 
-.PHONY: build test vet lint vuln fmt fmt-check check release dev-version dev-release clean help
+.PHONY: build test vet lint vuln fmt fmt-check check release dev-version dev-release release-key sign-release clean help
 
 BINARY := fly
 PKG := github.com/flywp/server-cli
@@ -10,7 +10,10 @@ PKG := github.com/flywp/server-cli
 #   make release VERSION=v0.2.0
 VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
 COMMIT := $(shell git rev-parse HEAD 2>/dev/null || echo unknown)
-BUILD_DATE := $(shell date -u +%Y-%m-%d)
+# The date of the commit, not of the build: the same commit gives the same
+# binary on any computer, so make sign-release can build a release again and
+# compare it with the one that CI published.
+BUILD_DATE := $(shell TZ=UTC git log -1 --date=format-local:%Y-%m-%d --format=%cd 2>/dev/null || date -u +%Y-%m-%d)
 
 LDFLAGS := -X $(PKG)/internal/version.Version=$(VERSION) \
 	-X $(PKG)/internal/version.CommitHash=$(COMMIT) \
@@ -90,6 +93,27 @@ dev-release: ## Tag HEAD as a dev pre-release and push the tag (CI publishes it)
 	echo "Pushed $(DEV_VERSION). The Release workflow publishes it as a pre-release:"; \
 	echo "  https://github.com/flywp/server-cli/releases/tag/$(DEV_VERSION)"
 
+# The agent installs a release by itself only when checksums.txt has a
+# signature from a key that is kept outside GitHub. Make the key one time, put
+# the printed public key line in internal/release/keys.go, and keep the
+# private key in a password manager, with a backup. Never commit it.
+# COMMENT names the key, in the key file and next to its line in keys.go.
+COMMENT ?= server-cli release key for flywp
+release-key: ## Make the release signing key: make release-key KEY=<file outside the repo> [COMMENT=...]
+	@test -n "$(KEY)" || { echo "Usage: make release-key KEY=<file outside the repo> [COMMENT=\"...\"]"; exit 1; }
+	go run ./tools/releasesign keygen -out "$(KEY)" -comment "$(COMMENT)"
+
+# Run this after the Release workflow publishes VERSION, on your own
+# computer. tools/sign-release.sh builds the release again from your local
+# tag, checks that GitHub serves the same binaries, signs checksums.txt,
+# checks the signature with the keys of the tag, and uploads
+# checksums.txt.sig. Agents install the release 24 hours after the
+# signature. KEY=- reads the key from stdin.
+sign-release: ## Sign a published release: make sign-release VERSION=v0.2.1 KEY=<file or ->
+	@if [ "$(origin VERSION)" != "command line" ] || [ -z "$(KEY)" ]; then \
+		echo "Usage: make sign-release VERSION=v0.2.1 KEY=<file or ->"; exit 1; fi
+	@tools/sign-release.sh "$(VERSION)" "$(KEY)"
+
 clean: ## Remove bin/ and build/
 	rm -rf bin/ build/
 

README.md

@@ -2,7 +2,7 @@
 
 Easy CLI tool for servers managed by FlyWP.
 
-Conforms to the FlyWP monitoring agent contract v0.2.1.
+Conforms to the FlyWP monitoring agent contract v0.3.1.
 
 ## Installation
 
@@ -107,6 +107,14 @@ All arguments after the WP-CLI command (or after the command for `fly exec`) go
 
 It reads `FLY_AGENT_URL` (https), `FLY_AGENT_TOKEN` and `FLY_AGENT_SERVER_ID` from `/etc/fly/agent.env`, and keeps its state in `STATE_DIRECTORY` (`/var/lib/fly-agent`).
 
+The agent also updates itself. Once a day, at a time set by the server id, it checks the latest release on GitHub. It installs the release only when:
+
+- the release is newer than the running version (the agent never downgrades)
+- the release has a valid signature from the FlyWP release key (see [Releasing](#releasing))
+- the signature is more than 24 hours old
+
+To turn this off on one server, add `FLY_AGENT_AUTO_UPDATE=off` to `/etc/fly/agent.env` and restart the agent. Updates that FlyWP sends still work.
+
 ```bash
 systemctl status fly-agent    # is the agent running?
 journalctl -u fly-agent -f    # the agent log
@@ -137,7 +145,7 @@ make release    # static linux/amd64 and linux/arm64 archives + checksums.txt in
 make help       # lists all targets
 ```
 
-`make release VERSION=v0.2.0` stamps a specific version. The release archives must keep the names `fly-linux-<arch>.tar.gz` with the binary `fly-linux-<arch>` inside: installed CLIs look for these names when they run `fly update`.
+`make release VERSION=v0.2.0` stamps a specific version. The build date is the date of the commit, so the same commit and Go version give the same binary on any computer. The release archives must keep the names `fly-linux-<arch>.tar.gz` with the binary `fly-linux-<arch>` inside: installed CLIs look for these names when they run `fly update`.
 
 CI runs `make check` and `make release` on every pull request and on every push to `develop` and `main`.
 
@@ -154,6 +162,23 @@ The Release workflow checks that the tag is on `main`, runs `make check`, builds
 
 `install.sh` and `fly update` install only a release that has `checksums.txt`. Releases before v0.2.0 have none, so push the tag right after the merge into `main`: until the release is published, `install.sh` from `main` stops.
 
+After the workflow publishes the release, sign it on your own computer:
+
+```bash
+make sign-release VERSION=v0.2.0 KEY=<private key file>    # KEY=- reads the key from stdin
+```
+
+The signature says "this release is the code of my tag", so the command signs only what it can build again:
+
+1. It shows the commit of your **local** tag and asks you to type the tag. Review that commit first: the signature is the approval.
+2. It downloads the archives and `checksums.txt`, and checks the archives against the sums.
+3. It builds the release again from your local tag, with the Go version of the CI build, and compares the binaries byte for byte. A binary holds its commit, so this also proves that CI built your tag.
+4. It signs `checksums.txt`, checks the signature with the keys of the tag, and uploads `checksums.txt.sig`.
+
+If GitHub serves a swapped archive, or the tag on GitHub moved, step 2 or 3 stops before the signature. Agents install a release by themselves only when it has a valid signature, and only 24 hours after it was signed. Each server then installs it at its own time of day. To stop a bad release in those 24 hours, mark it as a pre-release on GitHub.
+
+The signing key is kept outside GitHub, so that a push to GitHub alone cannot reach every server. `make release-key KEY=<file>` makes a key and prints its public key line for `internal/release/keys.go`. `COMMENT=` names the key, in the key file and next to its line in `keys.go` (the default is "server-cli release key for flywp"). Keep the private key in a password manager, with a backup. Never commit it, and never put it in a GitHub secret. If the key is lost or leaked, ship a binary with a new key through a FlyWP update: that path does not use the signature.
+
 ### Dev pre-releases
 
 To test a branch on real servers before it merges, publish a dev pre-release of its current commit:

agent_test.go

@@ -8,6 +8,7 @@ import (
 	"archive/tar"
 	"bytes"
 	"compress/gzip"
+	"crypto/ed25519"
 	"crypto/sha256"
 	"encoding/hex"
 	"encoding/json"
@@ -45,6 +46,8 @@ func agentEnv(t *testing.T) []string {
 		"FLY_AGENT_TOKEN=" + testToken,
 		"FLY_AGENT_SERVER_ID=17",
 		"STATE_DIRECTORY=" + t.TempDir(),
+		// A test never asks the real GitHub for a release.
+		"FLY_AGENT_AUTO_UPDATE=off",
 	}
 }
 
@@ -260,6 +263,98 @@ func TestAgentUpdatesItself(t *testing.T) {
 	}
 }
 
+func TestAgentInstallsASignedReleaseByItself(t *testing.T) {
+	// A release has archives for Linux only.
+	if runtime.GOOS != "linux" {
+		t.Skip("the releases have binaries for Linux only")
+	}
+	environ := agentEnv(t)
+
+	pub, priv, err := ed25519.GenerateKey(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// The new release: fly built as v9.9.9, signed 25 hours ago.
+	newBin := filepath.Join(t.TempDir(), "fly")
+	build := exec.Command("go", "build", "-ldflags", "-X github.com/flywp/server-cli/internal/version.Version=v9.9.9", "-o", newBin, ".")
+	if out, err := build.CombinedOutput(); err != nil {
+		t.Fatalf("building the new release: %v\n%s", err, out)
+	}
+	archiveName := release.BinaryName(runtime.GOOS, runtime.GOARCH) + ".tar.gz"
+	tarball := releaseArchive(t, newBin, release.BinaryName(runtime.GOOS, runtime.GOARCH))
+	sum := sha256.Sum256(tarball)
+	checksums := []byte(hex.EncodeToString(sum[:]) + "  " + archiveName + "\n")
+	sigFile := release.Sign(priv, "v9.9.9", time.Now().Add(-25*time.Hour), checksums)
+
+	var github *httptest.Server
+	github = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Path {
+		case "/releases/latest":
+			assets := []map[string]string{}
+			for _, name := range []string{archiveName, release.ChecksumsAsset, release.SignatureAsset} {
+				assets = append(assets, map[string]string{"name": name, "browser_download_url": github.URL + "/download/" + name})
+			}
+			_ = json.NewEncoder(w).Encode(map[string]any{"tag_name": "v9.9.9", "assets": assets})
+		case "/download/" + archiveName:
+			_, _ = w.Write(tarball)
+		case "/download/" + release.ChecksumsAsset:
+			_, _ = w.Write(checksums)
+		case "/download/" + release.SignatureAsset:
+			_, _ = w.Write(sigFile)
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer github.Close()
+
+	// The running agent: an older release that asks this GitHub and trusts
+	// the test key.
+	dir := t.TempDir()
+	exe := filepath.Join(dir, "fly")
+	ldflags := strings.Join([]string{
+		"-X github.com/flywp/server-cli/internal/version.Version=v0.0.1",
+		"-X github.com/flywp/server-cli/internal/release.GithubAPI=" + github.URL + "/releases/latest",
+		"-X github.com/flywp/server-cli/internal/release.trustedKeys=" + release.KeyLine(pub),
+	}, " ")
+	if out, err := exec.Command("go", "build", "-ldflags", ldflags, "-o", exe, ".").CombinedOutput(); err != nil {
+		t.Fatalf("building the old release: %v\n%s", err, out)
+	}
+
+	srv := httptest.NewServer(&updateServer{closed: true})
+	defer srv.Close()
+
+	// Work 3 seconds from now. The agent never checked, so it checks at its
+	// first tick.
+	environ = append(environ, "FLY_AGENT_URL="+srv.URL, fmt.Sprintf("FLY_AGENT_SERVER_ID=%d", (time.Now().Second()+3)%60), "FLY_AGENT_AUTO_UPDATE=on")
+
+	agent := exec.Command(exe, "agent", "run")
+	agent.Env = environ
+	stderr := &lockedBuffer{}
+	agent.Stderr = stderr
+	if err := agent.Start(); err != nil {
+		t.Fatal(err)
+	}
+	done := make(chan error, 1)
+	go func() { done <- agent.Wait() }()
+	select {
+	case err := <-done:
+		if err != nil {
+			t.Fatalf("the agent exit after the update: %v, want 0. stderr:\n%s", err, stderr)
+		}
+	case <-time.After(70 * time.Second):
+		_ = agent.Process.Kill()
+		t.Fatalf("the agent did not install the release within 70s. stderr:\n%s", stderr)
+	}
+
+	if out := runFlyAt(t, exe, "version"); !strings.Contains(out, "v9.9.9") {
+		t.Fatalf("fly version = %q, want the new release v9.9.9 on disk. stderr:\n%s", out, stderr)
+	}
+	if !strings.Contains(stderr.String(), "installed the new release") {
+		t.Errorf("stderr does not log the install:\n%s", stderr)
+	}
+}
+
 // runFlyAt runs the fly binary at exe and returns its stdout.
 func runFlyAt(t *testing.T, exe string, args ...string) string {
 	t.Helper()

cmd/agent.go

@@ -23,7 +23,10 @@ var agentRunCmd = &cobra.Command{
 	Short: "Run the monitoring agent until it is stopped",
 	Long: `Run the FlyWP monitoring agent until it is stopped. systemd starts this
 command (fly-agent.service). The agent reads FLY_AGENT_URL, FLY_AGENT_TOKEN,
-FLY_AGENT_SERVER_ID and STATE_DIRECTORY from the environment.`,
+FLY_AGENT_SERVER_ID and STATE_DIRECTORY from the environment.
+
+Once a day the agent installs a newer signed release by itself.
+FLY_AGENT_AUTO_UPDATE=off stops this.`,
 	Args: cobra.NoArgs,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		cfg, err := agent.ConfigFromEnv(os.Getenv)