Agent updates

Makefile

@@ -1,14 +1,23 @@
-.PHONY: proto web clean build-all build-collector build-agent build-alertprocessor build-client pack-all pack-collector pack-agent pack-alertprocessor pack-client
+.PHONY: proto web signing-key clean build-all build-collector build-agent build-alertprocessor build-client pack-all pack-collector pack-agent pack-alertprocessor pack-client
 
 # the version -version and the dashboard show, like v3.0.0-16-g2519821
 VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
-LDFLAGS := -ldflags "-X github.com/dhamith93/SyMon/internal/version.Version=$(VERSION)"
+# agents only install updates from the dashboard when they are signed with
+# this key, which stays where the builds are made and never on the servers.
+# Its public key is built into agents, and is empty until make signing-key.
+SIGNING_KEY ?= $(HOME)/.config/symon/agent-signing.key
+UPDATE_KEY = $(shell cat $(SIGNING_KEY).pub 2>/dev/null)
+# recursive, so a key made earlier in the same run is picked up
+LDFLAGS = -ldflags "-X github.com/dhamith93/SyMon/internal/version.Version=$(VERSION) -X github.com/dhamith93/SyMon/internal/update.PublicKey=$(UPDATE_KEY)"
 
 proto:
 	cd internal && protoc --go_out=. --go_opt=paths=source_relative \
 		--go-grpc_out=. --go-grpc_opt=paths=source_relative \
 		api/api.proto alertapi/alertapi.proto
 
+signing-key:
+	go run ./tools/sign keygen -key $(SIGNING_KEY)
+
 web:
 	cd client/web && npm ci && npm run build
 
@@ -51,7 +60,7 @@ pack-collector: build-collector
 	cd release/ && tar -cvf collector_linux_x86_64.tar.gz collector_linux_x86_64
 	rm -rf release/collector_linux_x86_64
 
-pack-agent: build-agent
+pack-agent: signing-key build-agent
 	mkdir -p release/agent_linux_x86_64
 	cp agent/agent_linux_x86_64 release/agent_linux_x86_64
 	cp agent/.env-example release/agent_linux_x86_64
@@ -67,12 +76,13 @@ pack-alertprocessor: build-alertprocessor
 
 # the agent builds are what new hosts download from the dashboard.
 # GOARM=6 also runs on ARMv7, so one arm build covers every Raspberry Pi.
-pack-client: build-client
+pack-client: signing-key build-client
 	mkdir -p release/client_linux_x86_64/downloads
 	cp client/client_linux_x86_64 release/client_linux_x86_64
 	cd agent && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(LDFLAGS) -o ../release/client_linux_x86_64/downloads/agent-linux-amd64
 	cd agent && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build $(LDFLAGS) -o ../release/client_linux_x86_64/downloads/agent-linux-arm64
 	cd agent && GOOS=linux GOARCH=arm GOARM=6 CGO_ENABLED=0 go build $(LDFLAGS) -o ../release/client_linux_x86_64/downloads/agent-linux-arm
+	go run ./tools/sign sign -key $(SIGNING_KEY) release/client_linux_x86_64/downloads/agent-linux-amd64 release/client_linux_x86_64/downloads/agent-linux-arm64 release/client_linux_x86_64/downloads/agent-linux-arm
 	cp client/.env-example release/client_linux_x86_64
 	cp client/Dockerfile release/client_linux_x86_64
 	cd release/ && tar -cvf client_linux_x86_64.tar.gz client_linux_x86_64

README.MD

@@ -45,6 +45,7 @@ SyMon is a self-hosted monitoring tool for Linux servers, home labs and Raspberr
 - Each host has its own key, and components can talk over TLS
 - A Prometheus endpoint, for Grafana or a Prometheus you already run
 - Every part reports its version: `-version` on each binary, the dashboard footer, and each host's page for its agent
+- Agents update themselves from the dashboard when an admin asks, and only install builds signed with your key
 
 ## Screenshots
 
@@ -88,6 +89,7 @@ Optional. The Collector sends it alerts as they open, change and resolve, and it
 - **Shared key.** The Collector, Client and Alert processor use a shared key from `collector -init`. Each call carries a short-lived token signed with it.
 - **TLS.** Traffic between components can be encrypted. See the `*_TLS_*` and `*_CERT_PATH` settings in each component's `.env-example`.
 - **Dashboard login.** The dashboard stays locked until a user exists. `collector -add-user <name>` creates one and prints its password, `-reset-password`, `-remove-user` and `-list-users` manage them, and users change their own password on the dashboard. A login lasts 30 days. After 10 wrong passwords a user name is locked for 15 minutes. The install script and agent downloads stay public, so new hosts can enroll.
+- **Agent updates.** Agent builds are signed with a key that stays where you build them (`~/.config/symon/agent-signing.key`). Agents only install updates signed with it, so a tampered download or a spoofed update request cannot run code on your hosts.
 - **Roles.** Admins can change alert rules, viewers can only look. `-add-user <name> -role viewer` creates a viewer, `-set-role <name> -role admin` changes it.
 - **HTTPS.** Put a reverse proxy like Caddy or nginx in front of the dashboard, so passwords and the session cookie are encrypted.
 
@@ -132,6 +134,8 @@ The Client exposes a JSON API under `/api/v1`. Times are unix seconds. Errors re
     * Names of the host's custom metrics
 * `GET /api/v1/alerts?host=&open=1&from=&to=`
     * Alerts, newest first. `open=1` leaves out resolved ones. Endpoint alerts have an empty `host`
+* `POST /api/v1/agents/update` with `{"hosts": ["..."]}` as JSON
+    * For admins only. Asks those hosts' agents to update to the dashboard's version, and answers how many were asked. Agents from before updates are left out
 * `GET /api/v1/rules`
     * The alert rules, each with `id`, `enabled` and `rule`, the rule in the alerts.json format
 * `POST /api/v1/rules` and `PUT /api/v1/rules/{id}` with `{"enabled": true, "rule": {...}}` as JSON, `DELETE /api/v1/rules/{id}`

agent/main.go

@@ -10,6 +10,7 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -86,6 +87,10 @@ func main() {
 	}
 
 	logger.Log("info", "agent "+version.String()+" started for "+config.ServerId)
+	// right away, so the dashboard sees this agent's version, and an update
+	// that restarted it is confirmed
+	updates := newUpdater()
+	sendPing(client, &config, updates)
 	interval := time.Duration(config.MonitorIntervalSeconds) * time.Second
 	collector := monitor.NewCollector(&config)
 	ticker := time.NewTicker(interval)
@@ -118,7 +123,7 @@ func main() {
 		for {
 			select {
 			case <-tickerForPing.C:
-				sendPing(client, &config)
+				sendPing(client, &config, updates)
 			case <-quitForPing:
 				ticker.Stop()
 				return
@@ -193,13 +198,22 @@ func initAgent(client api.MonitorDataServiceClient, config *config.Agent) {
 	fmt.Printf("%s \n", response.Body)
 }
 
-func sendPing(client api.MonitorDataServiceClient, config *config.Agent) {
+// sendPing tells the collector the host is alive, and which agent it runs.
+// The reply may carry an update an admin asked for.
+func sendPing(client api.MonitorDataServiceClient, config *config.Agent, updates *updater) {
 	ctx, cancel := transport.Context()
 	defer cancel()
-	_, err := client.HandlePing(ctx, &api.ServerInfo{ServerName: config.ServerId})
+	response, err := client.HandlePing(ctx, &api.ServerInfo{
+		ServerName:   config.ServerId,
+		AgentVersion: version.String(),
+		Arch:         runtime.GOARCH,
+		UpdateError:  updates.err(),
+	})
 	if err != nil {
 		logger.Log("error", "error sending ping: "+err.Error())
+		return
 	}
+	updates.handle(response.Update)
 }
 
 func sendMonitorData(client api.MonitorDataServiceClient, monitorData string, config *config.Agent) {

agent/update.go

@@ -0,0 +1,164 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/dhamith93/SyMon/internal/api"
+	"github.com/dhamith93/SyMon/internal/logger"
+	"github.com/dhamith93/SyMon/internal/update"
+	"github.com/dhamith93/SyMon/internal/version"
+)
+
+// An admin can ask an agent to update itself on the dashboard. The
+// collector passes the request on in its reply to a ping. The agent
+// downloads the build the dashboard hands out, and installs it only when it
+// is signed with the key built into this agent, runs on this machine, and
+// is the version asked for.
+
+const (
+	updateTimeout = 5 * time.Minute
+	// no agent build comes close to this
+	maxBuildSize = 200 << 20
+)
+
+// updater tries each request once, so a failed update is not retried every
+// minute, and keeps the error for the next ping to report
+type updater struct {
+	mu        sync.Mutex
+	tried     int64
+	lastError string
+	// install is selfUpdate, replaced in tests. It returns only on failure.
+	install func(*api.AgentUpdate) error
+}
+
+func newUpdater() *updater {
+	return &updater{install: selfUpdate}
+}
+
+func (u *updater) handle(request *api.AgentUpdate) {
+	if request == nil || request.Version == version.String() {
+		return
+	}
+	u.mu.Lock()
+	if request.RequestedAt == u.tried {
+		u.mu.Unlock()
+		return
+	}
+	u.tried = request.RequestedAt
+	u.mu.Unlock()
+
+	logger.Log("info", "updating to "+request.Version+" as asked on the dashboard")
+	err := u.install(request)
+	logger.Log("error", "cannot update to "+request.Version+": "+err.Error())
+	u.mu.Lock()
+	u.lastError = err.Error()
+	u.mu.Unlock()
+}
+
+func (u *updater) err() string {
+	u.mu.Lock()
+	defer u.mu.Unlock()
+	return u.lastError
+}
+
+// selfUpdate replaces this agent with the build asked for and restarts into
+// it. It returns only when that failed.
+func selfUpdate(request *api.AgentUpdate) error {
+	exe, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	if exe, err = filepath.EvalSymlinks(exe); err != nil {
+		return err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), updateTimeout)
+	defer cancel()
+	build, signature, err := fetchBuild(ctx, request.DownloadUrl)
+	if err != nil {
+		return err
+	}
+	if err := installBuild(exe, build, signature, request.Version); err != nil {
+		return err
+	}
+	logger.Log("info", "updated to "+request.Version+", restarting")
+	return syscall.Exec(exe, os.Args, os.Environ())
+}
+
+// fetchBuild downloads this machine's build and its signature from the
+// dashboard
+func fetchBuild(ctx context.Context, dashboard string) ([]byte, []byte, error) {
+	url := strings.TrimRight(dashboard, "/") + "/downloads/agent-linux-" + runtime.GOARCH
+	build, err := download(ctx, url)
+	if err != nil {
+		return nil, nil, err
+	}
+	signature, err := download(ctx, url+".sig")
+	if err != nil {
+		return nil, nil, err
+	}
+	return build, signature, nil
+}
+
+func download(ctx context.Context, url string) ([]byte, error) {
+	request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+	response, err := http.DefaultClient.Do(request)
+	if err != nil {
+		return nil, err
+	}
+	defer response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("%s answered %s", url, response.Status)
+	}
+	data, err := io.ReadAll(io.LimitReader(response.Body, maxBuildSize+1))
+	if err != nil {
+		return nil, err
+	}
+	if len(data) > maxBuildSize {
+		return nil, fmt.Errorf("%s is too big for an agent build", url)
+	}
+	return data, nil
+}
+
+// installBuild checks a build and puts it in place of exe
+func installBuild(exe string, build []byte, signature []byte, want string) error {
+	if err := update.Verify(build, signature); err != nil {
+		return err
+	}
+	next := exe + ".new"
+	if err := os.WriteFile(next, build, 0755); err != nil {
+		return err
+	}
+	if err := os.Chmod(next, 0755); err != nil {
+		os.Remove(next)
+		return err
+	}
+	// the build has to run here and be the version asked for
+	out, err := exec.Command(next, "-version").Output()
+	if err != nil {
+		os.Remove(next)
+		return fmt.Errorf("the new build does not run here: %w", err)
+	}
+	if got := strings.TrimSpace(string(out)); got != "SyMon agent "+want {
+		os.Remove(next)
+		return fmt.Errorf("the new build says %q, not %s", got, want)
+	}
+	if err := os.Rename(next, exe); err != nil {
+		os.Remove(next)
+		return err
+	}
+	return nil
+}