diff --git a/Makefile b/Makefile index 6f7b563..127439d 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,23 @@ -.PHONY: proto web clean build-all build-collector build-agent build-alertprocessor build-client pack-all pack-collector pack-agent pack-alertprocessor pack-client +.PHONY: proto web signing-key clean build-all build-collector build-agent build-alertprocessor build-client pack-all pack-collector pack-agent pack-alertprocessor pack-client # the version -version and the dashboard show, like v3.0.0-16-g2519821 VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) -LDFLAGS := -ldflags "-X github.com/dhamith93/SyMon/internal/version.Version=$(VERSION)" +# agents only install updates from the dashboard when they are signed with +# this key, which stays where the builds are made and never on the servers. +# Its public key is built into agents, and is empty until make signing-key. +SIGNING_KEY ?= $(HOME)/.config/symon/agent-signing.key +UPDATE_KEY = $(shell cat $(SIGNING_KEY).pub 2>/dev/null) +# recursive, so a key made earlier in the same run is picked up +LDFLAGS = -ldflags "-X github.com/dhamith93/SyMon/internal/version.Version=$(VERSION) -X github.com/dhamith93/SyMon/internal/update.PublicKey=$(UPDATE_KEY)" proto: cd internal && protoc --go_out=. --go_opt=paths=source_relative \ --go-grpc_out=. --go-grpc_opt=paths=source_relative \ api/api.proto alertapi/alertapi.proto +signing-key: + go run ./tools/sign keygen -key $(SIGNING_KEY) + web: cd client/web && npm ci && npm run build @@ -51,7 +60,7 @@ pack-collector: build-collector cd release/ && tar -cvf collector_linux_x86_64.tar.gz collector_linux_x86_64 rm -rf release/collector_linux_x86_64 -pack-agent: build-agent +pack-agent: signing-key build-agent mkdir -p release/agent_linux_x86_64 cp agent/agent_linux_x86_64 release/agent_linux_x86_64 cp agent/.env-example release/agent_linux_x86_64 @@ -67,12 +76,13 @@ pack-alertprocessor: build-alertprocessor # the agent builds are what new hosts download from the dashboard. # GOARM=6 also runs on ARMv7, so one arm build covers every Raspberry Pi. -pack-client: build-client +pack-client: signing-key build-client mkdir -p release/client_linux_x86_64/downloads cp client/client_linux_x86_64 release/client_linux_x86_64 cd agent && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(LDFLAGS) -o ../release/client_linux_x86_64/downloads/agent-linux-amd64 cd agent && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build $(LDFLAGS) -o ../release/client_linux_x86_64/downloads/agent-linux-arm64 cd agent && GOOS=linux GOARCH=arm GOARM=6 CGO_ENABLED=0 go build $(LDFLAGS) -o ../release/client_linux_x86_64/downloads/agent-linux-arm + go run ./tools/sign sign -key $(SIGNING_KEY) release/client_linux_x86_64/downloads/agent-linux-amd64 release/client_linux_x86_64/downloads/agent-linux-arm64 release/client_linux_x86_64/downloads/agent-linux-arm cp client/.env-example release/client_linux_x86_64 cp client/Dockerfile release/client_linux_x86_64 cd release/ && tar -cvf client_linux_x86_64.tar.gz client_linux_x86_64 diff --git a/README.MD b/README.MD index 65da2ba..1b5413e 100644 --- a/README.MD +++ b/README.MD @@ -45,6 +45,7 @@ SyMon is a self-hosted monitoring tool for Linux servers, home labs and Raspberr - Each host has its own key, and components can talk over TLS - A Prometheus endpoint, for Grafana or a Prometheus you already run - Every part reports its version: `-version` on each binary, the dashboard footer, and each host's page for its agent +- Agents update themselves from the dashboard when an admin asks, and only install builds signed with your key ## Screenshots @@ -88,6 +89,7 @@ Optional. The Collector sends it alerts as they open, change and resolve, and it - **Shared key.** The Collector, Client and Alert processor use a shared key from `collector -init`. Each call carries a short-lived token signed with it. - **TLS.** Traffic between components can be encrypted. See the `*_TLS_*` and `*_CERT_PATH` settings in each component's `.env-example`. - **Dashboard login.** The dashboard stays locked until a user exists. `collector -add-user ` creates one and prints its password, `-reset-password`, `-remove-user` and `-list-users` manage them, and users change their own password on the dashboard. A login lasts 30 days. After 10 wrong passwords a user name is locked for 15 minutes. The install script and agent downloads stay public, so new hosts can enroll. +- **Agent updates.** Agent builds are signed with a key that stays where you build them (`~/.config/symon/agent-signing.key`). Agents only install updates signed with it, so a tampered download or a spoofed update request cannot run code on your hosts. - **Roles.** Admins can change alert rules, viewers can only look. `-add-user -role viewer` creates a viewer, `-set-role -role admin` changes it. - **HTTPS.** Put a reverse proxy like Caddy or nginx in front of the dashboard, so passwords and the session cookie are encrypted. @@ -132,6 +134,8 @@ The Client exposes a JSON API under `/api/v1`. Times are unix seconds. Errors re * Names of the host's custom metrics * `GET /api/v1/alerts?host=&open=1&from=&to=` * Alerts, newest first. `open=1` leaves out resolved ones. Endpoint alerts have an empty `host` +* `POST /api/v1/agents/update` with `{"hosts": ["..."]}` as JSON + * For admins only. Asks those hosts' agents to update to the dashboard's version, and answers how many were asked. Agents from before updates are left out * `GET /api/v1/rules` * The alert rules, each with `id`, `enabled` and `rule`, the rule in the alerts.json format * `POST /api/v1/rules` and `PUT /api/v1/rules/{id}` with `{"enabled": true, "rule": {...}}` as JSON, `DELETE /api/v1/rules/{id}` diff --git a/agent/main.go b/agent/main.go index 18166ac..800fecf 100644 --- a/agent/main.go +++ b/agent/main.go @@ -10,6 +10,7 @@ import ( "log" "os" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -86,6 +87,10 @@ func main() { } logger.Log("info", "agent "+version.String()+" started for "+config.ServerId) + // right away, so the dashboard sees this agent's version, and an update + // that restarted it is confirmed + updates := newUpdater() + sendPing(client, &config, updates) interval := time.Duration(config.MonitorIntervalSeconds) * time.Second collector := monitor.NewCollector(&config) ticker := time.NewTicker(interval) @@ -118,7 +123,7 @@ func main() { for { select { case <-tickerForPing.C: - sendPing(client, &config) + sendPing(client, &config, updates) case <-quitForPing: ticker.Stop() return @@ -193,13 +198,22 @@ func initAgent(client api.MonitorDataServiceClient, config *config.Agent) { fmt.Printf("%s \n", response.Body) } -func sendPing(client api.MonitorDataServiceClient, config *config.Agent) { +// sendPing tells the collector the host is alive, and which agent it runs. +// The reply may carry an update an admin asked for. +func sendPing(client api.MonitorDataServiceClient, config *config.Agent, updates *updater) { ctx, cancel := transport.Context() defer cancel() - _, err := client.HandlePing(ctx, &api.ServerInfo{ServerName: config.ServerId}) + response, err := client.HandlePing(ctx, &api.ServerInfo{ + ServerName: config.ServerId, + AgentVersion: version.String(), + Arch: runtime.GOARCH, + UpdateError: updates.err(), + }) if err != nil { logger.Log("error", "error sending ping: "+err.Error()) + return } + updates.handle(response.Update) } func sendMonitorData(client api.MonitorDataServiceClient, monitorData string, config *config.Agent) { diff --git a/agent/update.go b/agent/update.go new file mode 100644 index 0000000..f1251a5 --- /dev/null +++ b/agent/update.go @@ -0,0 +1,164 @@ +package main + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "syscall" + "time" + + "github.com/dhamith93/SyMon/internal/api" + "github.com/dhamith93/SyMon/internal/logger" + "github.com/dhamith93/SyMon/internal/update" + "github.com/dhamith93/SyMon/internal/version" +) + +// An admin can ask an agent to update itself on the dashboard. The +// collector passes the request on in its reply to a ping. The agent +// downloads the build the dashboard hands out, and installs it only when it +// is signed with the key built into this agent, runs on this machine, and +// is the version asked for. + +const ( + updateTimeout = 5 * time.Minute + // no agent build comes close to this + maxBuildSize = 200 << 20 +) + +// updater tries each request once, so a failed update is not retried every +// minute, and keeps the error for the next ping to report +type updater struct { + mu sync.Mutex + tried int64 + lastError string + // install is selfUpdate, replaced in tests. It returns only on failure. + install func(*api.AgentUpdate) error +} + +func newUpdater() *updater { + return &updater{install: selfUpdate} +} + +func (u *updater) handle(request *api.AgentUpdate) { + if request == nil || request.Version == version.String() { + return + } + u.mu.Lock() + if request.RequestedAt == u.tried { + u.mu.Unlock() + return + } + u.tried = request.RequestedAt + u.mu.Unlock() + + logger.Log("info", "updating to "+request.Version+" as asked on the dashboard") + err := u.install(request) + logger.Log("error", "cannot update to "+request.Version+": "+err.Error()) + u.mu.Lock() + u.lastError = err.Error() + u.mu.Unlock() +} + +func (u *updater) err() string { + u.mu.Lock() + defer u.mu.Unlock() + return u.lastError +} + +// selfUpdate replaces this agent with the build asked for and restarts into +// it. It returns only when that failed. +func selfUpdate(request *api.AgentUpdate) error { + exe, err := os.Executable() + if err != nil { + return err + } + if exe, err = filepath.EvalSymlinks(exe); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), updateTimeout) + defer cancel() + build, signature, err := fetchBuild(ctx, request.DownloadUrl) + if err != nil { + return err + } + if err := installBuild(exe, build, signature, request.Version); err != nil { + return err + } + logger.Log("info", "updated to "+request.Version+", restarting") + return syscall.Exec(exe, os.Args, os.Environ()) +} + +// fetchBuild downloads this machine's build and its signature from the +// dashboard +func fetchBuild(ctx context.Context, dashboard string) ([]byte, []byte, error) { + url := strings.TrimRight(dashboard, "/") + "/downloads/agent-linux-" + runtime.GOARCH + build, err := download(ctx, url) + if err != nil { + return nil, nil, err + } + signature, err := download(ctx, url+".sig") + if err != nil { + return nil, nil, err + } + return build, signature, nil +} + +func download(ctx context.Context, url string) ([]byte, error) { + request, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + response, err := http.DefaultClient.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s answered %s", url, response.Status) + } + data, err := io.ReadAll(io.LimitReader(response.Body, maxBuildSize+1)) + if err != nil { + return nil, err + } + if len(data) > maxBuildSize { + return nil, fmt.Errorf("%s is too big for an agent build", url) + } + return data, nil +} + +// installBuild checks a build and puts it in place of exe +func installBuild(exe string, build []byte, signature []byte, want string) error { + if err := update.Verify(build, signature); err != nil { + return err + } + next := exe + ".new" + if err := os.WriteFile(next, build, 0755); err != nil { + return err + } + if err := os.Chmod(next, 0755); err != nil { + os.Remove(next) + return err + } + // the build has to run here and be the version asked for + out, err := exec.Command(next, "-version").Output() + if err != nil { + os.Remove(next) + return fmt.Errorf("the new build does not run here: %w", err) + } + if got := strings.TrimSpace(string(out)); got != "SyMon agent "+want { + os.Remove(next) + return fmt.Errorf("the new build says %q, not %s", got, want) + } + if err := os.Rename(next, exe); err != nil { + os.Remove(next) + return err + } + return nil +} diff --git a/agent/update_test.go b/agent/update_test.go new file mode 100644 index 0000000..c33bfdb --- /dev/null +++ b/agent/update_test.go @@ -0,0 +1,129 @@ +package main + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/dhamith93/SyMon/internal/api" + "github.com/dhamith93/SyMon/internal/update" +) + +// fakeBuild is a script that answers -version like an agent would +func fakeBuild(version string) []byte { + return []byte("#!/bin/sh\necho 'SyMon agent " + version + "'\n") +} + +// signWithTestKey makes this test's agent trust a new key, and signs with it +func signWithTestKey(t *testing.T) func([]byte) []byte { + t.Helper() + private, public, err := update.NewKey() + if err != nil { + t.Fatal(err) + } + update.PublicKey = public + t.Cleanup(func() { update.PublicKey = "" }) + key, _ := update.ParsePrivateKey(private) + return func(build []byte) []byte { return []byte(update.Sign(key, build)) } +} + +func TestInstallBuild(t *testing.T) { + sign := signWithTestKey(t) + exe := filepath.Join(t.TempDir(), "agent") + if err := os.WriteFile(exe, fakeBuild("v1.0.0"), 0755); err != nil { + t.Fatal(err) + } + unchanged := func() { + t.Helper() + if data, _ := os.ReadFile(exe); string(data) != string(fakeBuild("v1.0.0")) { + t.Fatal("expected the agent to be left alone") + } + if _, err := os.Stat(exe + ".new"); !errors.Is(err, os.ErrNotExist) { + t.Error("expected no leftover .new file") + } + } + + good := fakeBuild("v2.0.0") + if err := installBuild(exe, good, sign([]byte("something else")), "v2.0.0"); err == nil || !strings.Contains(err.Error(), "not signed") { + t.Errorf("expected a signature for another build to fail, got %v", err) + } + unchanged() + if err := installBuild(exe, good, sign(good), "v3.0.0"); err == nil || !strings.Contains(err.Error(), `says "SyMon agent v2.0.0"`) { + t.Errorf("expected a build of another version to fail, got %v", err) + } + unchanged() + broken := []byte("not a program") + if err := installBuild(exe, broken, sign(broken), "v2.0.0"); err == nil || !strings.Contains(err.Error(), "does not run here") { + t.Errorf("expected a build that cannot run to fail, got %v", err) + } + unchanged() + + if err := installBuild(exe, good, sign(good), "v2.0.0"); err != nil { + t.Fatal(err) + } + if data, _ := os.ReadFile(exe); string(data) != string(good) { + t.Error("expected the new build in place") + } +} + +func TestInstallBuildNeedsAKey(t *testing.T) { + update.PublicKey = "" + exe := filepath.Join(t.TempDir(), "agent") + if err := installBuild(exe, fakeBuild("v2.0.0"), []byte("x"), "v2.0.0"); !errors.Is(err, update.ErrNoKey) { + t.Errorf("expected ErrNoKey, got %v", err) + } +} + +func TestFetchBuild(t *testing.T) { + name := "/downloads/agent-linux-" + runtime.GOARCH + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case name: + w.Write([]byte("build")) + case name + ".sig": + w.Write([]byte("signature")) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + build, signature, err := fetchBuild(context.Background(), server.URL+"/") + if err != nil || string(build) != "build" || string(signature) != "signature" { + t.Errorf("expected the build and its signature, got %q %q %v", build, signature, err) + } + if _, _, err := fetchBuild(context.Background(), server.URL+"/elsewhere"); err == nil || !strings.Contains(err.Error(), "404") { + t.Errorf("expected a missing build to fail, got %v", err) + } +} + +func TestUpdaterTriesEachRequestOnce(t *testing.T) { + tries := 0 + updates := &updater{install: func(*api.AgentUpdate) error { + tries++ + return errors.New("the build is not signed with this agent's update key") + }} + + updates.handle(nil) + updates.handle(&api.AgentUpdate{Version: "dev", RequestedAt: 1}) + if tries != 0 { + t.Fatalf("expected no try without a request or for this version, got %d", tries) + } + request := &api.AgentUpdate{Version: "v9.0.0", DownloadUrl: "https://symon.example.com", RequestedAt: 1700000000} + updates.handle(request) + updates.handle(request) + if tries != 1 || updates.err() == "" { + t.Errorf("expected one try and its error kept, got %d %q", tries, updates.err()) + } + // asking again on the dashboard is a new request + updates.handle(&api.AgentUpdate{Version: "v9.0.0", RequestedAt: 1700000100}) + if tries != 2 { + t.Errorf("expected a new request to be tried, got %d", tries) + } +} diff --git a/client/internal/server/agents.go b/client/internal/server/agents.go new file mode 100644 index 0000000..e6e036f --- /dev/null +++ b/client/internal/server/agents.go @@ -0,0 +1,60 @@ +package server + +import ( + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + + "github.com/dhamith93/SyMon/internal/api" + "github.com/dhamith93/SyMon/internal/version" +) + +// agentArchs are the builds the downloads folder has, one per CPU +var agentArchs = []string{"amd64", "arm64", "arm"} + +// postAgentUpdate asks agents to update themselves to the build the +// dashboard hands out, which has the dashboard's own version. Agents fetch +// it from the dashboard the way the browser reached it, like the install +// script does, and install it only when it is signed. +func (s *server) postAgentUpdate(w http.ResponseWriter, r *http.Request) { + if !jsonBody(r) { + writeError(w, http.StatusUnsupportedMediaType, "send the hosts as JSON") + return + } + var body struct { + Hosts []string `json:"hosts"` + } + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&body); err != nil || len(body.Hosts) == 0 { + writeError(w, http.StatusBadRequest, "send the hosts to update") + return + } + if !safeHost.MatchString(r.Host) { + writeError(w, http.StatusBadRequest, "unexpected Host header") + return + } + for _, arch := range agentArchs { + if _, err := os.Stat(filepath.Join(s.downloadsDir, "agent-linux-"+arch+".sig")); errors.Is(err, os.ErrNotExist) { + writeError(w, http.StatusConflict, "the agent downloads are not signed, so agents would refuse them. Build them with make pack-client") + return + } + } + + scheme := "http" + if isHTTPS(r) { + scheme = "https" + } + session, _ := s.sessionOf(r) + result, err := s.collector.RequestAgentUpdate(r.Context(), &api.AgentUpdateRequest{ + Hosts: body.Hosts, + Version: version.String(), + DownloadUrl: scheme + "://" + r.Host, + By: session.user, + }) + if err != nil { + writeGRPCError(w, "agent update", err) + return + } + writeJSON(w, map[string]any{"requested": result.Requested, "version": version.String()}) +} diff --git a/client/internal/server/agents_test.go b/client/internal/server/agents_test.go new file mode 100644 index 0000000..3c5e4ff --- /dev/null +++ b/client/internal/server/agents_test.go @@ -0,0 +1,66 @@ +package server + +import ( + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/dhamith93/SyMon/internal/api" + "github.com/dhamith93/SyMon/internal/version" +) + +func TestAgentUpdate(t *testing.T) { + s, fake := newTestServer(t, nil) + version.Version = "v3.2.0" + t.Cleanup(func() { version.Version = "" }) + s.downloadsDir = t.TempDir() + body := `{"hosts":["web1","db1"]}` + + // agents would refuse builds that are not signed + if rec := call(s, "POST", "/api/v1/agents/update", body, testSession, asJSON); rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "not signed") { + t.Errorf("expected 409 without signatures, got %d %s", rec.Code, rec.Body) + } + for _, arch := range agentArchs { + if err := os.WriteFile(filepath.Join(s.downloadsDir, "agent-linux-"+arch+".sig"), []byte("signature\n"), 0644); err != nil { + t.Fatal(err) + } + } + + rec := call(s, "POST", "/api/v1/agents/update", body, testSession, func(r *http.Request) { + asJSON(r) + r.Host = "symon.example.com" + r.Header.Set("X-Forwarded-Proto", "https") + }) + if rec.Code != 200 || strings.TrimSpace(rec.Body.String()) != `{"requested":2,"version":"v3.2.0"}` { + t.Fatalf("unexpected response %d %s", rec.Code, rec.Body) + } + request := fake.lastAgentUpdate.Load().(*api.AgentUpdateRequest) + if request.Version != "v3.2.0" || request.DownloadUrl != "https://symon.example.com" || request.By != "tester" || len(request.Hosts) != 2 { + t.Errorf("unexpected request %+v", request) + } + + tests := []struct { + body string + cookie string + prepare func(*http.Request) + code int + }{ + {body, viewerSession, asJSON, http.StatusForbidden}, + {`{"hosts":[]}`, testSession, asJSON, http.StatusBadRequest}, + {body, testSession, nil, http.StatusUnsupportedMediaType}, + {body, "", asJSON, http.StatusUnauthorized}, + } + for _, tt := range tests { + if rec := call(s, "POST", "/api/v1/agents/update", tt.body, tt.cookie, tt.prepare); rec.Code != tt.code { + t.Errorf("%s as %q: got %d %s, want %d", tt.body, tt.cookie, rec.Code, rec.Body, tt.code) + } + } + + // the signatures are downloads too, without a login + rec = call(s, "GET", "/downloads/agent-linux-arm64.sig", "", "", nil) + if rec.Code != 200 || rec.Body.String() != "signature\n" || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") { + t.Errorf("unexpected signature download %d %q %q", rec.Code, rec.Body, rec.Header().Get("Content-Type")) + } +} diff --git a/client/internal/server/install.go b/client/internal/server/install.go index d3f149d..88f845c 100644 --- a/client/internal/server/install.go +++ b/client/internal/server/install.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "regexp" + "strings" "text/template" ) @@ -18,8 +19,9 @@ var installTemplate = template.Must(template.New("install.sh").Parse(installScri // both values end up inside a shell script, so they are checked first var ( - safeHost = regexp.MustCompile(`^[A-Za-z0-9.\-]+(:[0-9]+)?$|^\[[0-9A-Fa-f:.]+\](:[0-9]+)?$`) - agentFile = regexp.MustCompile(`^agent-linux-(amd64|arm64|arm)$`) + safeHost = regexp.MustCompile(`^[A-Za-z0-9.\-]+(:[0-9]+)?$|^\[[0-9A-Fa-f:.]+\](:[0-9]+)?$`) + // a build, or the signature agents check it with before updating + agentFile = regexp.MustCompile(`^agent-linux-(amd64|arm64|arm)(\.sig)?$`) ) // getInstallScript serves the script that installs and enrolls an agent, @@ -80,5 +82,8 @@ func (s *server) getDownload(w http.ResponseWriter, r *http.Request) { return } w.Header().Set("Content-Type", "application/octet-stream") + if strings.HasSuffix(name, ".sig") { + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + } http.ServeFile(w, r, path) } diff --git a/client/internal/server/server.go b/client/internal/server/server.go index f83b319..38f12b0 100644 --- a/client/internal/server/server.go +++ b/client/internal/server/server.go @@ -87,6 +87,7 @@ func (s *server) routes() http.Handler { data.HandleFunc("POST /api/v1/rules", s.requireAdmin(s.postRule)) data.HandleFunc("PUT /api/v1/rules/{id}", s.requireAdmin(s.putRule)) data.HandleFunc("DELETE /api/v1/rules/{id}", s.requireAdmin(s.deleteRule)) + data.HandleFunc("POST /api/v1/agents/update", s.requireAdmin(s.postAgentUpdate)) data.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusNotFound, "no such endpoint") }) @@ -136,6 +137,12 @@ type hostSummary struct { DiskFullDays *float64 `json:"diskFullDays"` // empty from agents older than versions AgentVersion string `json:"agentVersion"` + // the agent updates itself when asked + CanUpdate bool `json:"canUpdate"` + // an update asked for and not done yet, empty for none + UpdateVersion string `json:"updateVersion"` + UpdateRequestedAt int64 `json:"updateRequestedAt"` + UpdateError string `json:"updateError"` } func (s *server) getFleet(w http.ResponseWriter, r *http.Request) { @@ -147,23 +154,27 @@ func (s *server) getFleet(w http.ResponseWriter, r *http.Request) { hosts := make([]hostSummary, 0, len(fleet.Hosts)) for _, h := range fleet.Hosts { hosts = append(hosts, hostSummary{ - Name: h.Name, - Up: h.Up, - LastSeen: h.LastSeen, - Time: h.Time, - OS: h.Os, - UptimeSeconds: h.UptimeSeconds, - CPUPct: h.CpuPct, - MemUsedPct: h.MemUsedPct, - SwapUsedPct: h.SwapUsedPct, - DiskUsedPct: h.DiskUsedPct, - RxBps: h.RxBps, - TxBps: h.TxBps, - ActiveAlerts: h.ActiveAlerts, - WorstSeverity: h.WorstSeverity, - Containers: h.Containers, - DiskFullDays: h.DiskFullDays, - AgentVersion: h.AgentVersion, + Name: h.Name, + Up: h.Up, + LastSeen: h.LastSeen, + Time: h.Time, + OS: h.Os, + UptimeSeconds: h.UptimeSeconds, + CPUPct: h.CpuPct, + MemUsedPct: h.MemUsedPct, + SwapUsedPct: h.SwapUsedPct, + DiskUsedPct: h.DiskUsedPct, + RxBps: h.RxBps, + TxBps: h.TxBps, + ActiveAlerts: h.ActiveAlerts, + WorstSeverity: h.WorstSeverity, + Containers: h.Containers, + DiskFullDays: h.DiskFullDays, + AgentVersion: h.AgentVersion, + CanUpdate: h.CanUpdate, + UpdateVersion: h.UpdateVersion, + UpdateRequestedAt: h.UpdateRequestedAt, + UpdateError: h.UpdateError, }) } writeJSON(w, map[string]any{"hosts": hosts}) diff --git a/client/internal/server/server_test.go b/client/internal/server/server_test.go index 6bdd051..f482de2 100644 --- a/client/internal/server/server_test.go +++ b/client/internal/server/server_test.go @@ -30,11 +30,12 @@ type fakeCollector struct { // login: testSession is valid, alice's password is "correct horse // battery", and the user "locked" has failed too often - noUsers atomic.Bool - sessionChecks atomic.Int32 - passwordChecks atomic.Int32 - loggedOut atomic.Value - lastRuleBy atomic.Value + noUsers atomic.Bool + sessionChecks atomic.Int32 + passwordChecks atomic.Int32 + loggedOut atomic.Value + lastRuleBy atomic.Value + lastAgentUpdate atomic.Value } const ( @@ -191,6 +192,12 @@ func (f *fakeCollector) DeleteRule(ctx context.Context, in *api.RuleRequest) (*a return &api.Message{Body: "ok"}, nil } +// RequestAgentUpdate asks every host it is given +func (f *fakeCollector) RequestAgentUpdate(ctx context.Context, in *api.AgentUpdateRequest) (*api.AgentUpdateResult, error) { + f.lastAgentUpdate.Store(in) + return &api.AgentUpdateResult{Requested: int32(len(in.Hosts))}, nil +} + func floatPtr(v float64) *float64 { return &v } diff --git a/client/web/src/lib/api.ts b/client/web/src/lib/api.ts index c41cb84..077bab9 100644 --- a/client/web/src/lib/api.ts +++ b/client/web/src/lib/api.ts @@ -22,6 +22,13 @@ export interface HostSummary { diskFullDays: number | null; // empty from agents older than versions agentVersion: string; + // the agent updates itself when asked + canUpdate: boolean; + // an update asked for and not done yet, empty for none + updateVersion: string; + updateRequestedAt: number; + // why the agent's last try at the update failed + updateError: string; } // An endpoint's newest check up to the end of a range, and how it did over it @@ -303,6 +310,7 @@ export const api = { logout: () => post('/api/v1/logout', {}), changePassword: (current: string, next: string) => post('/api/v1/password', { current, new: next }), rules: () => get<{ rules: AlertRule[] }>('/api/v1/rules'), + updateAgents: (hosts: string[]) => post<{ requested: number; version: string }>('/api/v1/agents/update', { hosts }), createRule: (enabled: boolean, rule: RuleConfig) => post<{ id: number }>('/api/v1/rules', { enabled, rule }), updateRule: (id: number, enabled: boolean, rule: RuleConfig) => send<{ id: number }>('PUT', `/api/v1/rules/${id}`, { enabled, rule }), deleteRule: (id: number) => send<{ id: number }>('DELETE', `/api/v1/rules/${id}`), diff --git a/client/web/src/lib/versions.test.ts b/client/web/src/lib/versions.test.ts index 0429013..89f6763 100644 --- a/client/web/src/lib/versions.test.ts +++ b/client/web/src/lib/versions.test.ts @@ -1,5 +1,6 @@ import { describe, expect, it } from 'vitest'; -import { agentOutdated } from './versions'; +import type { HostSummary } from './api'; +import { agentOutdated, agentState } from './versions'; describe('agentOutdated', () => { it('compares with the build the dashboard hands out', () => { @@ -13,3 +14,19 @@ describe('agentOutdated', () => { expect(agentOutdated('', '')).toBe(false); }); }); + +describe('agentState', () => { + const host = (fields: Partial) => + ({ agentVersion: 'v3.1.0', canUpdate: true, updateVersion: '', updateError: '', ...fields }) as HostSummary; + + it('follows the update from asked to done', () => { + expect(agentState(host({ agentVersion: 'v3.2.0' }), 'v3.2.0')).toBe('current'); + expect(agentState(host({}), 'v3.2.0')).toBe('available'); + expect(agentState(host({ updateVersion: 'v3.2.0' }), 'v3.2.0')).toBe('requested'); + expect(agentState(host({ updateVersion: 'v3.2.0', updateError: 'not signed' }), 'v3.2.0')).toBe('failed'); + }); + + it('sends agents from before updates to the install command', () => { + expect(agentState(host({ agentVersion: '', canUpdate: false }), 'v3.2.0')).toBe('manual'); + }); +}); diff --git a/client/web/src/lib/versions.ts b/client/web/src/lib/versions.ts index 9f08b41..28e0cdc 100644 --- a/client/web/src/lib/versions.ts +++ b/client/web/src/lib/versions.ts @@ -1,6 +1,24 @@ +import type { HostSummary } from './api'; + // An agent is outdated when it is not the build the dashboard hands out, // which is what the install script upgrades it to. Agents from before // versions send none. export function agentOutdated(agentVersion: string | undefined, dashboardVersion: string): boolean { return dashboardVersion !== '' && agentVersion !== dashboardVersion; } + +// Where a host's agent stands: +// current runs the build the dashboard hands out +// available can be updated from the dashboard +// requested an update was asked for and the agent has not picked it up +// failed the agent tried the update and it did not work +// manual too old to update itself, needs the install command once +export type AgentState = 'current' | 'available' | 'requested' | 'failed' | 'manual'; + +export function agentState(host: HostSummary, dashboardVersion: string): AgentState { + if (!agentOutdated(host.agentVersion, dashboardVersion)) return 'current'; + if (!host.canUpdate) return 'manual'; + if (host.updateVersion && host.updateError) return 'failed'; + if (host.updateVersion) return 'requested'; + return 'available'; +} diff --git a/client/web/src/pages/Fleet.svelte b/client/web/src/pages/Fleet.svelte index dc2f174..d6ceda4 100644 --- a/client/web/src/pages/Fleet.svelte +++ b/client/web/src/pages/Fleet.svelte @@ -4,7 +4,8 @@ import { appConfig } from '../lib/config.svelte'; import { formatAgo, formatDays, formatDuration, formatRate } from '../lib/format'; import { poll } from '../lib/poll'; - import { agentOutdated } from '../lib/versions'; + import { isAdmin } from '../lib/auth.svelte'; + import { agentOutdated, agentState } from '../lib/versions'; import { hostPath } from '../lib/router.svelte'; import Meter from '../components/Meter.svelte'; import Sparkline from '../components/Sparkline.svelte'; @@ -64,6 +65,23 @@ const up = $derived(hosts.filter((h) => h.up).length); const outdatedAgents = $derived(hosts.filter(outdated).length); + // agents that update themselves when asked, including ones whose last try failed + const updatable = $derived(hosts.filter((h) => outdated(h) && ['available', 'failed'].includes(agentState(h, appConfig.version)))); + let updateMessage = $state(''); + let updating = $state(false); + + async function updateAgents() { + updating = true; + try { + const result = await api.updateAgents(updatable.map((h) => h.name)); + updateMessage = `Asked ${result.requested} agent${result.requested === 1 ? '' : 's'} to update to ${result.version}. They pick it up within a minute.`; + await loadFleet(); + } catch (e) { + updateMessage = `Could not ask the agents to update: ${(e as Error).message}`; + } finally { + updating = false; + } + } const openAlerts = $derived(hosts.reduce((sum, h) => sum + h.activeAlerts, 0)); const visible = $derived.by(() => { @@ -103,11 +121,18 @@ {#if outdatedAgents > 0} {#snippet extra()} - Not on {appConfig.version} + {#if isAdmin() && updatable.length > 0} + + {:else} + Not on {appConfig.version} + {/if} {/snippet} {/if} + {#if updateMessage}

{updateMessage}

{/if}
@@ -177,7 +202,14 @@ {/if} {#if outdated(host)} - + {@const state = agentState(host, appConfig.version)} + {#if state === 'requested'} + + {:else if state === 'failed'} + + {:else} + + {/if} {/if}
{/if} @@ -267,6 +299,16 @@ font-size: 12px; } + .update { + height: 26px; + font-size: 12px; + } + + .update-message { + margin: -6px 0 16px; + font-size: 13px; + } + .alerts { display: flex; flex-wrap: wrap; diff --git a/client/web/src/pages/Host.svelte b/client/web/src/pages/Host.svelte index 5eb7c8d..4af0e56 100644 --- a/client/web/src/pages/Host.svelte +++ b/client/web/src/pages/Host.svelte @@ -1,12 +1,13 @@