Metrics and endpoint checks

Dockerfile

@@ -10,18 +10,23 @@ COPY client/web ./
 RUN npm run build
 
 FROM golang:1.26-alpine AS build
+# the version the binaries report, for example
+#   docker build --build-arg VERSION=$(git describe --tags --always --dirty) ...
+# .git is not copied in, so without it they report "dev"
+ARG VERSION=
+ENV LDFLAGS="-X github.com/dhamith93/SyMon/internal/version.Version=${VERSION}"
 WORKDIR /src
 COPY go.mod go.sum ./
 RUN go mod download
 COPY . .
 COPY --from=web /web/dist ./client/web/dist
 RUN for c in agent collector alertprocessor client; do \
-        CGO_ENABLED=0 go build -o /out/$c ./$c || exit 1; \
+        CGO_ENABLED=0 go build -ldflags "$LDFLAGS" -o /out/$c ./$c || exit 1; \
     done
 # agent builds that new hosts download from the dashboard
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/downloads/agent-linux-amd64 ./agent \
-    && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -o /out/downloads/agent-linux-arm64 ./agent \
-    && CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -o /out/downloads/agent-linux-arm ./agent
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "$LDFLAGS" -o /out/downloads/agent-linux-amd64 ./agent \
+    && CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags "$LDFLAGS" -o /out/downloads/agent-linux-arm64 ./agent \
+    && CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags "$LDFLAGS" -o /out/downloads/agent-linux-arm ./agent
 
 FROM alpine:3.22 AS collector
 RUN adduser -D -H symon

Makefile

@@ -1,5 +1,9 @@
 .PHONY: proto web clean build-all build-collector build-agent build-alertprocessor build-client pack-all pack-collector pack-agent pack-alertprocessor pack-client
 
+# the version -version and the dashboard show, like v3.0.0-16-g2519821
+VERSION ?= $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
+LDFLAGS := -ldflags "-X github.com/dhamith93/SyMon/internal/version.Version=$(VERSION)"
+
 proto:
 	cd internal && protoc --go_out=. --go_opt=paths=source_relative \
 		--go-grpc_out=. --go-grpc_opt=paths=source_relative \
@@ -26,16 +30,16 @@ clean:
 build-all: build-collector build-agent build-alertprocessor build-client
 
 build-collector:
-	cd collector && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o collector_linux_x86_64
+	cd collector && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(LDFLAGS) -o collector_linux_x86_64
 
 build-agent:
-	cd agent && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o agent_linux_x86_64
+	cd agent && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(LDFLAGS) -o agent_linux_x86_64
 
 build-alertprocessor:
-	cd alertprocessor && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o alertprocessor_linux_x86_64
+	cd alertprocessor && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(LDFLAGS) -o alertprocessor_linux_x86_64
 
 build-client: web
-	cd client && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o client_linux_x86_64
+	cd client && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(LDFLAGS) -o client_linux_x86_64
 
 pack-all: pack-collector pack-agent pack-alertprocessor pack-client
 
@@ -66,9 +70,9 @@ pack-alertprocessor: build-alertprocessor
 pack-client: build-client
 	mkdir -p release/client_linux_x86_64/downloads
 	cp client/client_linux_x86_64 release/client_linux_x86_64
-	cd agent && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o ../release/client_linux_x86_64/downloads/agent-linux-amd64
-	cd agent && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -o ../release/client_linux_x86_64/downloads/agent-linux-arm64
-	cd agent && GOOS=linux GOARCH=arm GOARM=6 CGO_ENABLED=0 go build -o ../release/client_linux_x86_64/downloads/agent-linux-arm
+	cd agent && GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build $(LDFLAGS) -o ../release/client_linux_x86_64/downloads/agent-linux-amd64
+	cd agent && GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build $(LDFLAGS) -o ../release/client_linux_x86_64/downloads/agent-linux-arm64
+	cd agent && GOOS=linux GOARCH=arm GOARM=6 CGO_ENABLED=0 go build $(LDFLAGS) -o ../release/client_linux_x86_64/downloads/agent-linux-arm
 	cp client/.env-example release/client_linux_x86_64
 	cp client/Dockerfile release/client_linux_x86_64
 	cd release/ && tar -cvf client_linux_x86_64.tar.gz client_linux_x86_64

README.MD

@@ -11,7 +11,7 @@ SyMon is a self-hosted monitoring tool for Linux servers, home labs and Raspberr
 **Hosts**
 - CPU, overall and per core, load average, memory and swap
 - Disk space, disk IO and busy time, network traffic, TCP connections
-- When each disk will be full, at the rate it grew over the last week
+- When each disk will be full, at the rate it grew over the last week, with a chart of where it is headed
 - Pressure stall information and temperature sensors
 - The top processes by CPU and memory at any point in time, and the programs that used the most over any range
 - Whether chosen systemd services are running
@@ -20,6 +20,9 @@ SyMon is a self-hosted monitoring tool for Linux servers, home labs and Raspberr
 - Every running container on a host: Docker, Podman, containerd, LXC and systemd-nspawn
 - CPU, memory, network and disk IO per container, grouped by Compose project
 
+**Endpoints**
+- HTTP checks of any URL from the server, with response time and uptime history
+
 **Custom metrics**
 - Send any number from a script or cron job and get a chart for it
 
@@ -37,6 +40,7 @@ SyMon is a self-hosted monitoring tool for Linux servers, home labs and Raspberr
 - Raw data kept for 7 days, 1 minute averages for 30 days and 1 hour averages for a year, all adjustable
 - Each host has its own key, and components can talk over TLS
 - A Prometheus endpoint, for Grafana or a Prometheus you already run
+- Every part reports its version: `-version` on each binary, the dashboard footer, and each host's page for its agent
 
 ## Screenshots
 
@@ -113,6 +117,10 @@ The Client exposes a JSON API under `/api/v1`. Times are unix seconds. Errors re
 * `GET /api/v1/hosts/{host}/custom-metrics`
     * Names of the host's custom metrics
 * `GET /api/v1/alerts?host=&open=1&from=&to=`
-    * Alerts, newest first. `open=1` leaves out resolved ones
+    * Alerts, newest first. `open=1` leaves out resolved ones. Endpoint alerts have an empty `host`
+* `GET /api/v1/endpoints?from=&to=`
+    * Every endpoint checked within the range: its newest check, the number of checks, the share that passed and the average response time
+* `GET /api/v1/endpoints/series?name=&metric=latency&from=&to=`
+    * An endpoint's response time in milliseconds (`latency`) or share of passing checks (`availability`) over time, in the same shape as a host's series
 
-The Client also serves `GET /metrics`, every host's latest values in the Prometheus text format. See [Prometheus and Grafana](docs/install.md#prometheus-and-grafana).
+The Client also serves `GET /metrics`, every host's latest values in the Prometheus text format, unless `SYMON_CLIENT_METRICS_ENABLED=false`. See [Prometheus and Grafana](docs/install.md#prometheus-and-grafana).

agent/main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/dhamith93/SyMon/internal/logger"
 	"github.com/dhamith93/SyMon/internal/monitor"
 	"github.com/dhamith93/SyMon/internal/transport"
+	"github.com/dhamith93/SyMon/internal/version"
 	grpccreds "google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/status"
 )
@@ -36,7 +37,12 @@ func main() {
 	token := flag.String("token", "", "With -enroll: the enrollment token")
 	host := flag.String("host", "", "With -enroll: the host name to use, the machine's name by default")
 	envFile := flag.String("env", config.DefaultEnvFile("agent"), "Settings file with KEY=value lines, loaded if it exists")
+	versionPtr := flag.Bool("version", false, "Print the version and exit")
 	flag.Parse()
+	if *versionPtr {
+		fmt.Println("SyMon agent " + version.String())
+		return
+	}
 	if err := config.LoadEnvFile(*envFile); err != nil {
 		log.Fatal(err)
 	}
@@ -79,6 +85,7 @@ func main() {
 		return
 	}
 
+	logger.Log("info", "agent "+version.String()+" started for "+config.ServerId)
 	interval := time.Duration(config.MonitorIntervalSeconds) * time.Second
 	collector := monitor.NewCollector(&config)
 	ticker := time.NewTicker(interval)

alertprocessor/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"log"
 	"net"
 	"os"
@@ -10,12 +11,18 @@ import (
 	"github.com/dhamith93/SyMon/internal/config"
 	"github.com/dhamith93/SyMon/internal/logger"
 	"github.com/dhamith93/SyMon/internal/transport"
+	"github.com/dhamith93/SyMon/internal/version"
 	"github.com/dhamith93/SyMon/pkg/memdb"
 )
 
 func main() {
 	envFile := flag.String("env", config.DefaultEnvFile("alertprocessor"), "Settings file with KEY=value lines, loaded if it exists")
+	versionPtr := flag.Bool("version", false, "Print the version and exit")
 	flag.Parse()
+	if *versionPtr {
+		fmt.Println("SyMon alert processor " + version.String())
+		return
+	}
 	if err := config.LoadEnvFile(*envFile); err != nil {
 		log.Fatal(err)
 	}
@@ -64,6 +71,7 @@ func main() {
 	}
 
 	alertapi.RegisterAlertServiceServer(grpcServer, &s)
+	logger.Log("info", "alert processor "+version.String()+" started on :"+config.Port)
 	if err := grpcServer.Serve(lis); err != nil {
 		log.Fatalf("failed to serve: %s", err)
 	}

client/.env-example

@@ -8,6 +8,8 @@ export SYMON_CLIENT_DOWNLOADS_DIR=downloads
 # how agents on other hosts reach the collector, when the dashboard's host
 # name with the collector's port does not work
 export SYMON_CLIENT_AGENT_COLLECTOR_ENDPOINT=
+# /metrics for Prometheus, on unless this is false
+export SYMON_CLIENT_METRICS_ENABLED=
 export SYMON_CLIENT_LOG_FILE_ENABLED=
 export SYMON_CLIENT_LOG_FILE_PATH=
 export SYMON_KEY=''

client/internal/server/metrics.go

@@ -16,14 +16,20 @@ import (
 // getMetrics serves every host's latest values in the Prometheus text
 // format, so Prometheus can scrape the dashboard
 func (s *server) getMetrics(w http.ResponseWriter, r *http.Request) {
+	if !s.metricsEnabled {
+		http.Error(w, "metrics are switched off with SYMON_CLIENT_METRICS_ENABLED=false", http.StatusNotFound)
+		return
+	}
 	response, err := s.collector.Snapshots(r.Context(), &api.Void{})
 	if err != nil {
 		writeGRPCError(w, "snapshots", err)
 		return
 	}
 
 	metrics := newMetricsWriter()
+	up := map[string]bool{}
 	for _, host := range response.Hosts {
+		up[host.Host] = host.Up
 		metrics.gauge("symon_up", "1 when the host reported within the last minute.", boolValue(host.Up), "host", host.Host)
 		if host.LastSeen > 0 {
 			metrics.gauge("symon_last_seen_timestamp_seconds", "When the host was last heard from.", float64(host.LastSeen), "host", host.Host)
@@ -40,6 +46,15 @@ func (s *server) getMetrics(w http.ResponseWriter, r *http.Request) {
 		}
 		addHostMetrics(metrics, host.Host, &data)
 	}
+	for _, custom := range response.CustomMetrics {
+		if !up[custom.Host] {
+			continue
+		}
+		metrics.gauge("symon_custom_metric", "The newest value of a custom metric sent in the last 2 days.", custom.Value,
+			"host", custom.Host, "name", custom.Name, "unit", custom.Unit)
+		metrics.gauge("symon_custom_metric_timestamp_seconds", "When the custom metric was last sent.", float64(custom.Time),
+			"host", custom.Host, "name", custom.Name)
+	}
 
 	w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
 	if err := metrics.writeTo(w); err != nil {

client/internal/server/metrics_test.go

@@ -3,6 +3,7 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"net/http"
 	"net/http/httptest"
 	"strings"
 	"testing"
@@ -30,11 +31,17 @@ func (f *fakeCollector) Snapshots(ctx context.Context, in *api.Void) (*api.Snaps
 	if err != nil {
 		return nil, err
 	}
-	return &api.SnapshotList{Hosts: []*api.HostSnapshot{
-		{Host: "web1", Up: true, LastSeen: 1700000000, Time: 1700000000, SnapshotJson: string(snapshot)},
-		{Host: "db1", LastSeen: 1690000000, Time: 1690000000, SnapshotJson: string(snapshot)},
-		{Host: "new1", Up: true, LastSeen: 1700000000},
-	}}, nil
+	return &api.SnapshotList{
+		Hosts: []*api.HostSnapshot{
+			{Host: "web1", Up: true, LastSeen: 1700000000, Time: 1700000000, SnapshotJson: string(snapshot)},
+			{Host: "db1", LastSeen: 1690000000, Time: 1690000000, SnapshotJson: string(snapshot)},
+			{Host: "new1", Up: true, LastSeen: 1700000000},
+		},
+		CustomMetrics: []*api.CustomValue{
+			{Host: "web1", Name: "queue-length", Unit: "jobs", Value: 42, Time: 1699999000},
+			{Host: "db1", Name: "backup-age", Unit: "hours", Value: 30, Time: 1689999000},
+		},
+	}, nil
 }
 
 func TestMetrics(t *testing.T) {
@@ -55,13 +62,15 @@ func TestMetrics(t *testing.T) {
 		"# TYPE symon_network_receive_bytes_total counter\n" + `symon_network_receive_bytes_total{host="web1",iface="eth0"} 1000`,
 		`symon_service_up{host="web1",service="nginx"} 1`,
 		`symon_container_receive_bytes_per_second{host="web1",container="web",project="shop"} 2048`,
+		`symon_custom_metric{host="web1",name="queue-length",unit="jobs"} 42`,
+		`symon_custom_metric_timestamp_seconds{host="web1",name="queue-length"} 1.699999e+09`,
 	} {
 		if !strings.Contains(body, line+"\n") {
 			t.Errorf("expected %q in:\n%s", line, body)
 		}
 	}
 	// db1 stopped reporting, so its old values are left out
-	if strings.Contains(body, `{host="db1",`) || strings.Contains(body, `symon_cpu_usage_percent{host="db1"}`) {
+	if strings.Contains(body, `{host="db1",`) || strings.Contains(body, `symon_cpu_usage_percent{host="db1"}`) || strings.Contains(body, "backup-age") {
 		t.Errorf("expected only up and last seen for db1:\n%s", body)
 	}
 	// web1 has no transmit rate, as if it were on the host network
@@ -70,6 +79,15 @@ func TestMetrics(t *testing.T) {
 	}
 }
 
+func TestMetricsSwitchedOff(t *testing.T) {
+	s, _ := newTestServer(t, nil)
+	s.metricsEnabled = false
+	code, body, _ := get(t, s, "/metrics")
+	if code != http.StatusNotFound || !strings.Contains(body, "SYMON_CLIENT_METRICS_ENABLED") {
+		t.Errorf("unexpected response %d: %s", code, body)
+	}
+}
+
 func TestMetricsWriter(t *testing.T) {
 	w := newMetricsWriter()
 	w.gauge("a", "First.", 1, "name", `quote " backslash \ newline`+"\n")