diff --git a/installer/CHANGELOG.md b/installer/CHANGELOG.md index 3bd5952..0ea3c71 100644 --- a/installer/CHANGELOG.md +++ b/installer/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## v0.1.7 (2026-09-21) +## v0.1.8 (2026-09-21) ### Features @@ -9,10 +9,15 @@ - **installer:** hide dashboard tabs for deselected optional services (#31) (59f2d40) - **installer:** open the UI in an embedded system WebView (Part C2) (#35) (84bdc7a) - **installer:** add package sources (file, repo build, GitHub) (#42) (f831eab) -- **services:** give every service its own version and changelog (c2e4e2c) - **dashboard:** download the newest release bundle from the redeploy page (#44) (245b277) +- **services:** give every service its own version and changelog (#45) (5bc91b8) +- **installer:** report the bundle upload progress and write concurrently (19df0c0) ### Fixes - **installer:** restart service units on update and record the installed manifest (#43) (0f2aec2) +- **installer:** expand ~ in the repo package path (873bdda) +- **installer:** replace remote files the SSH user cannot open for writing (eca0198) +- **installer:** give step 20 its MQTT arguments from the node on redeploy and repair (bb4cd40) +- **installer:** add texts for every fault code the bootstrap steps emit (23a64c3) diff --git a/installer/VERSION b/installer/VERSION index fad30c5..da14730 100644 --- a/installer/VERSION +++ b/installer/VERSION @@ -1 +1 @@ -v0.1.7 +v0.1.8 diff --git a/installer/internal/bundle/deploy.go b/installer/internal/bundle/deploy.go index f9b85df..b961665 100644 --- a/installer/internal/bundle/deploy.go +++ b/installer/internal/bundle/deploy.go @@ -63,8 +63,14 @@ func stagingTag(remoteDir string) string { // archive, so it cannot by itself catch a bundle whose entire contents, // script included, were forged together. func Deploy(ctx context.Context, client *transport.Client, localArchivePath, remoteDir string) error { + return DeployProgress(ctx, client, localArchivePath, remoteDir, nil) +} + +// DeployProgress is Deploy that reports the archive upload's progress +// (bytes sent, archive size); onProgress may be nil. +func DeployProgress(ctx context.Context, client *transport.Client, localArchivePath, remoteDir string, onProgress func(done, total int64)) error { remoteArchive := fmt.Sprintf("/tmp/energy-node-installer-bundle-%s-%s.tar.gz", stagingTag(remoteDir), randomSuffix()) - if err := client.UploadFile(localArchivePath, remoteArchive, 0o600); err != nil { + if err := client.UploadFileProgress(localArchivePath, remoteArchive, 0o600, onProgress); err != nil { return fmt.Errorf("uploading bundle archive: %w", err) } defer client.RemoveRemote(remoteArchive) diff --git a/installer/internal/bundlesource/repo.go b/installer/internal/bundlesource/repo.go index 531f1b0..bbe0bf4 100644 --- a/installer/internal/bundlesource/repo.go +++ b/installer/internal/bundlesource/repo.go @@ -4,6 +4,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" ) // lookPath is a seam for tests. @@ -46,8 +47,24 @@ func MissingTools() []string { return missing } -// CheckRepo reports why path cannot be built from, or nil if it can. +// ExpandHome resolves a leading "~" or "~/" to the user's home directory and +// cleans the result. The operator types the path into the UI, where no shell +// expands it. Anything else (including "~user") is returned unchanged. +func ExpandHome(path string) string { + if path != "~" && !strings.HasPrefix(path, "~/") { + return path + } + home, err := os.UserHomeDir() + if err != nil { + return path + } + return filepath.Join(home, path[1:]) +} + +// CheckRepo reports why path cannot be built from, or nil if it can. A +// leading "~" is expanded first. func CheckRepo(path string) *Error { + path = ExpandHome(path) if _, err := os.Stat(filepath.Join(path, filepath.FromSlash(makeBundleScript))); err != nil { return &Error{Code: CodeRepoNotACheckout, Detail: path} } diff --git a/installer/internal/bundlesource/repo_test.go b/installer/internal/bundlesource/repo_test.go index 41f42d0..8985a7a 100644 --- a/installer/internal/bundlesource/repo_test.go +++ b/installer/internal/bundlesource/repo_test.go @@ -57,3 +57,32 @@ func TestCheckRepoReportsMissingTools(t *testing.T) { t.Errorf("MissingTools = %v, want [go]", got) } } + +func TestExpandHome(t *testing.T) { + home, err := os.UserHomeDir() + if err != nil { + t.Skip("no home directory") + } + cases := map[string]string{ + "~": home, + "~/dev/energy-node": filepath.Join(home, "dev", "energy-node"), + "~/dev/x/": filepath.Join(home, "dev", "x"), + "/abs/path": "/abs/path", + "rel/path": "rel/path", + "~other/x": "~other/x", + "": "", + } + for in, want := range cases { + if got := ExpandHome(in); got != want { + t.Errorf("ExpandHome(%q) = %q, want %q", in, got, want) + } + } +} + +func TestCheckRepoExpandsAHomePrefix(t *testing.T) { + root := fakeCheckout(t) + t.Setenv("HOME", root) + if err := CheckRepo("~"); err != nil { + t.Fatalf("CheckRepo(~) = %+v, want nil", err) + } +} diff --git a/installer/internal/bundlesource/resolve.go b/installer/internal/bundlesource/resolve.go index 13f2aee..b577622 100644 --- a/installer/internal/bundlesource/resolve.go +++ b/installer/internal/bundlesource/resolve.go @@ -139,6 +139,7 @@ func (r *Resolver) finish(req Request, dir, archive string, strict bool, cleanup } func (r *Resolver) buildFromRepo(ctx context.Context, req Request, log func(string)) (string, error) { + req.Path = ExpandHome(req.Path) if err := CheckRepo(req.Path); err != nil { return "", err } diff --git a/installer/internal/faults/catalog.go b/installer/internal/faults/catalog.go index 204d5f4..3f32330 100644 --- a/installer/internal/faults/catalog.go +++ b/installer/internal/faults/catalog.go @@ -21,38 +21,100 @@ type Code string // Die bekannten Codes. Quelle: die "Fehlercodes"-Zeilen der Plaene A-I/A-II // und die fuenf Bundle-Codes aus Plan B-I. const ( - CodeBundleManifestMissing Code = "BUNDLE_MANIFEST_MISSING" - CodeBundleSignatureInvalid Code = "BUNDLE_SIGNATURE_INVALID" - CodeBundleHashMismatch Code = "BUNDLE_HASH_MISMATCH" - CodeArchMismatch Code = "ARCH_MISMATCH" - CodePythonABIMismatch Code = "PYTHON_ABI_MISMATCH" - CodeAptFailed Code = "APT_FAILED" - CodeMosquittoConfigInvalid Code = "MOSQUITTO_CONFIG_INVALID" - CodeUFWMissing Code = "UFW_MISSING" - CodePipExternallyManaged Code = "PIP_EXTERNALLY_MANAGED" - CodeWheelMissing Code = "WHEEL_MISSING" - CodeTailscaleFlagInvalid Code = "TAILSCALE_FLAG_INVALID" - CodeCaddyValidateFailed Code = "CADDY_VALIDATE_FAILED" - CodeUnitStartFailed Code = "UNIT_START_FAILED" - CodeConfigExists Code = "CONFIG_EXISTS" - CodeSudoRequired Code = "SUDO_REQUIRED" + CodeBundleManifestMissing Code = "BUNDLE_MANIFEST_MISSING" + CodeBundleSignatureInvalid Code = "BUNDLE_SIGNATURE_INVALID" + CodeBundleHashMismatch Code = "BUNDLE_HASH_MISMATCH" + CodeArchMismatch Code = "ARCH_MISMATCH" + CodePythonABIMismatch Code = "PYTHON_ABI_MISMATCH" + CodeAptFailed Code = "APT_FAILED" + CodeMosquittoArgsMissing Code = "MOSQUITTO_ARGS_MISSING" + CodeMosquittoConfigInvalid Code = "MOSQUITTO_CONFIG_INVALID" + CodeMQTTConfigUnreadable Code = "MQTT_CONFIG_UNREADABLE" + CodeUFWMissing Code = "UFW_MISSING" + CodePipExternallyManaged Code = "PIP_EXTERNALLY_MANAGED" + CodeWheelMissing Code = "WHEEL_MISSING" + CodeTailscaleFlagInvalid Code = "TAILSCALE_FLAG_INVALID" + CodeCaddyValidateFailed Code = "CADDY_VALIDATE_FAILED" + CodeUnitStartFailed Code = "UNIT_START_FAILED" + CodeConfigExists Code = "CONFIG_EXISTS" + CodeSudoRequired Code = "SUDO_REQUIRED" + CodeAptInstallFailed Code = "APT_INSTALL_FAILED" + CodeAptUpdateFailed Code = "APT_UPDATE_FAILED" + CodeBundleIncomplete Code = "BUNDLE_INCOMPLETE" + CodeCaddyBinaryMissing Code = "CADDY_BINARY_MISSING" + CodeCaddyConfigInvalid Code = "CADDY_CONFIG_INVALID" + CodeCaddyStartFailed Code = "CADDY_START_FAILED" + CodeConfigJsonMissing Code = "CONFIG_JSON_MISSING" + CodeConfigTemplateMissing Code = "CONFIG_TEMPLATE_MISSING" + CodeConfigWriteFailed Code = "CONFIG_WRITE_FAILED" + CodeDashboardBinaryMissing Code = "DASHBOARD_BINARY_MISSING" + CodeDashboardStartFailed Code = "DASHBOARD_START_FAILED" + CodeManifestsMissing Code = "MANIFESTS_MISSING" + CodeManifestMissing Code = "MANIFEST_MISSING" + CodeManifestParseFailed Code = "MANIFEST_PARSE_FAILED" + CodeMosquittoConfForeign Code = "MOSQUITTO_CONF_FOREIGN" + CodeMosquittoPasswdFailed Code = "MOSQUITTO_PASSWD_FAILED" + CodePipInstallFailed Code = "PIP_INSTALL_FAILED" + CodeSecretFileMissing Code = "SECRET_FILE_MISSING" + CodeSelectionUnreadable Code = "SELECTION_UNREADABLE" + CodeServiceSourceMissing Code = "SERVICE_SOURCE_MISSING" + CodeServiceStartFailed Code = "SERVICE_START_FAILED" + CodeServiceUnitFailed Code = "SERVICE_UNIT_FAILED" + CodeSudoersInvalid Code = "SUDOERS_INVALID" + CodeTailscaleInstallFailed Code = "TAILSCALE_INSTALL_FAILED" + CodeTailscaleTarballMissing Code = "TAILSCALE_TARBALL_MISSING" + CodeTargetInvalid Code = "TARGET_INVALID" + CodeUfwFailed Code = "UFW_FAILED" + CodeUpdaterPathStartFailed Code = "UPDATER_PATH_START_FAILED" + CodeWheelsMissing Code = "WHEELS_MISSING" ) var allCodes = []Code{ - CodeArchMismatch, CodeAptFailed, + CodeAptInstallFailed, + CodeAptUpdateFailed, + CodeArchMismatch, CodeBundleHashMismatch, + CodeBundleIncomplete, CodeBundleManifestMissing, CodeBundleSignatureInvalid, + CodeCaddyBinaryMissing, + CodeCaddyConfigInvalid, + CodeCaddyStartFailed, CodeCaddyValidateFailed, CodeConfigExists, + CodeConfigJsonMissing, + CodeConfigTemplateMissing, + CodeConfigWriteFailed, + CodeDashboardBinaryMissing, + CodeDashboardStartFailed, + CodeManifestsMissing, + CodeManifestMissing, + CodeManifestParseFailed, + CodeMosquittoArgsMissing, CodeMosquittoConfigInvalid, + CodeMosquittoConfForeign, + CodeMosquittoPasswdFailed, + CodeMQTTConfigUnreadable, CodePipExternallyManaged, + CodePipInstallFailed, CodePythonABIMismatch, + CodeSecretFileMissing, + CodeSelectionUnreadable, + CodeServiceSourceMissing, + CodeServiceStartFailed, + CodeServiceUnitFailed, + CodeSudoersInvalid, CodeSudoRequired, CodeTailscaleFlagInvalid, + CodeTailscaleInstallFailed, + CodeTailscaleTarballMissing, + CodeTargetInvalid, + CodeUfwFailed, CodeUFWMissing, CodeUnitStartFailed, + CodeUpdaterPathStartFailed, + CodeWheelsMissing, CodeWheelMissing, } diff --git a/installer/internal/faults/catalog_test.go b/installer/internal/faults/catalog_test.go index f5b1610..f45097b 100644 --- a/installer/internal/faults/catalog_test.go +++ b/installer/internal/faults/catalog_test.go @@ -66,20 +66,51 @@ func TestEveryCodeHasBothFields(t *testing.T) { // step actually failed on a real node. func TestCatalogCoversTheStableCodeInventory(t *testing.T) { want := []string{ - "ARCH_MISMATCH", "APT_FAILED", + "APT_INSTALL_FAILED", + "APT_UPDATE_FAILED", + "ARCH_MISMATCH", "BUNDLE_HASH_MISMATCH", + "BUNDLE_INCOMPLETE", "BUNDLE_MANIFEST_MISSING", "BUNDLE_SIGNATURE_INVALID", + "CADDY_BINARY_MISSING", + "CADDY_CONFIG_INVALID", + "CADDY_START_FAILED", "CADDY_VALIDATE_FAILED", "CONFIG_EXISTS", + "CONFIG_JSON_MISSING", + "CONFIG_TEMPLATE_MISSING", + "CONFIG_WRITE_FAILED", + "DASHBOARD_BINARY_MISSING", + "DASHBOARD_START_FAILED", + "MANIFESTS_MISSING", + "MANIFEST_MISSING", + "MANIFEST_PARSE_FAILED", + "MOSQUITTO_ARGS_MISSING", "MOSQUITTO_CONFIG_INVALID", + "MOSQUITTO_CONF_FOREIGN", + "MOSQUITTO_PASSWD_FAILED", + "MQTT_CONFIG_UNREADABLE", "PIP_EXTERNALLY_MANAGED", + "PIP_INSTALL_FAILED", "PYTHON_ABI_MISMATCH", + "SECRET_FILE_MISSING", + "SELECTION_UNREADABLE", + "SERVICE_SOURCE_MISSING", + "SERVICE_START_FAILED", + "SERVICE_UNIT_FAILED", + "SUDOERS_INVALID", "SUDO_REQUIRED", "TAILSCALE_FLAG_INVALID", + "TAILSCALE_INSTALL_FAILED", + "TAILSCALE_TARBALL_MISSING", + "TARGET_INVALID", + "UFW_FAILED", "UFW_MISSING", "UNIT_START_FAILED", + "UPDATER_PATH_START_FAILED", + "WHEELS_MISSING", "WHEEL_MISSING", } for _, code := range want { diff --git a/installer/internal/host/package.go b/installer/internal/host/package.go index b584689..f046620 100644 --- a/installer/internal/host/package.go +++ b/installer/internal/host/package.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "runtime" + "strconv" "strings" "github.com/Developer-Simon/energy-node-installer/internal/bundle" @@ -20,12 +21,35 @@ import ( // Die Nahtstellen zum Node; Tests ersetzen sie, weil sie SSH brauchen. var ( detectMachine = defaultDetectMachine - stageBundle = func(ctx context.Context, c *transport.Client, archive, remoteDir string) error { - return bundle.Deploy(ctx, c, archive, remoteDir) + stageBundle = func(ctx context.Context, c *transport.Client, archive, remoteDir string, onProgress func(done, total int64)) error { + return bundle.DeployProgress(ctx, c, archive, remoteDir, onProgress) } verifyStaged = defaultVerifyStaged ) +// uploadProgress turns byte counts into one note per 5 % step, so the UI +// shows movement without a message for every 32 KiB packet. +func uploadProgress(notef func(string, map[string]string)) func(done, total int64) { + lastStep := 0 + return func(done, total int64) { + if total <= 0 { + return + } + step := int(done * 20 / total) // 0..20, one per 5 % + if step <= lastStep { + return + } + lastStep = step + notef("package.log.upload_progress", map[string]string{ + "percent": strconv.Itoa(step * 5), + "done": megabytes(done), + "total": megabytes(total), + }) + } +} + +func megabytes(n int64) string { return fmt.Sprintf("%.1f", float64(n)/(1<<20)) } + func defaultDetectMachine(ctx context.Context, c *transport.Client) (string, error) { var stdout, stderr strings.Builder if err := c.Run(ctx, "uname -m", &stdout, &stderr); err != nil { @@ -78,7 +102,7 @@ func (h *Host) SelectPackage(ctx context.Context, sel hostapi.PackageSelection) if err := bundlesource.CheckRepo(sel.Path); err != nil { return &hostapi.Error{Code: err.Code, Detail: err.Detail, Status: http.StatusBadRequest} } - req.Path = sel.Path + req.Path = bundlesource.ExpandHome(sel.Path) default: return &hostapi.Error{Code: "BAD_REQUEST", Detail: "unbekannte Paketquelle: " + sel.Kind, Status: http.StatusBadRequest} } @@ -223,7 +247,7 @@ func (h *Host) doPrepare(ctx context.Context, client *transport.Client, logf fun return &hostapi.Error{Code: "PACKAGE_STAGE_FAILED", Detail: err.Error()} } logf("Paket auf das Geraet uebertragen") - if err := stageBundle(ctx, client, archive, h.cfg.RemoteBundleDir); err != nil { + if err := stageBundle(ctx, client, archive, h.cfg.RemoteBundleDir, uploadProgress(notef)); err != nil { return &hostapi.Error{Code: "PACKAGE_STAGE_FAILED", Detail: err.Error()} } logf("Paket auf dem Geraet pruefen") diff --git a/installer/internal/host/package_test.go b/installer/internal/host/package_test.go index 50dcddd..efa7fe0 100644 --- a/installer/internal/host/package_test.go +++ b/installer/internal/host/package_test.go @@ -41,8 +41,10 @@ func stubSeams(t *testing.T, machine string) *staged { rec := &staged{} detectMachine = func(context.Context, *transport.Client) (string, error) { return machine, nil } provisionRemoteStateDir = func(context.Context, *transport.Client, string) error { return nil } - stageBundle = func(_ context.Context, _ *transport.Client, archive, remoteDir string) error { + stageBundle = func(_ context.Context, _ *transport.Client, archive, remoteDir string, onProgress func(done, total int64)) error { rec.archive, rec.remoteDir = archive, remoteDir + onProgress(50, 100) + onProgress(100, 100) if _, err := os.Stat(archive); err != nil { return err } @@ -215,3 +217,43 @@ func TestPrepareBundledEmitsTranslatedNoteKeys(t *testing.T) { } } } + +func TestUploadProgressNotesEveryFivePercentOnceAndEndsAtOneHundred(t *testing.T) { + var got []map[string]string + report := uploadProgress(func(key string, args map[string]string) { + if key != "package.log.upload_progress" { + t.Errorf("key = %s", key) + } + got = append(got, args) + }) + const total = 10 << 20 + for done := int64(0); done <= total; done += 64 << 10 { // 64 KiB reads + report(done, total) + } + report(total, total) // a repeated final call must not repeat the note + if len(got) != 20 { + t.Fatalf("got %d notes, want 20 (5%% steps): %v", len(got), got) + } + last := got[len(got)-1] + if last["percent"] != "100" || last["done"] != "10.0" || last["total"] != "10.0" { + t.Errorf("last note = %v, want 100 %% of 10.0 MB", last) + } + if got[0]["percent"] != "5" { + t.Errorf("first note = %v, want 5 %%", got[0]) + } +} + +func TestPrepareReportsTheUploadProgress(t *testing.T) { + stubSeams(t, "armv6l") + sink := &recordingSink{} + if err := hostWithBundled(t).Run(context.Background(), hostapi.RunRequest{Mode: hostapi.ModePrepare}, sink); err != nil { + t.Fatalf("Run: %v", err) + } + found := false + for _, n := range sink.notes { + found = found || n == "package:package.log.upload_progress" + } + if !found { + t.Errorf("notes = %v, want an upload_progress note", sink.notes) + } +} diff --git a/installer/internal/steps/mosquitto_node_config_test.go b/installer/internal/steps/mosquitto_node_config_test.go new file mode 100644 index 0000000..7a19fb2 --- /dev/null +++ b/installer/internal/steps/mosquitto_node_config_test.go @@ -0,0 +1,83 @@ +package steps_test + +import ( + "context" + "errors" + "path" + "testing" + + "github.com/Developer-Simon/energy-node-installer/internal/bundle" + "github.com/Developer-Simon/energy-node-installer/internal/steps" + "github.com/Developer-Simon/energy-node-installer/internal/transport/transporttest" +) + +// Fails the way 20-mosquitto.sh does when it gets no usable arguments. +const mosquittoArgsScript = `#!/bin/sh +printf '##STEP 20 begin\n' +user=""; file="" +while [ $# -gt 0 ]; do + case "$1" in + --user) user="$2"; shift 2 ;; + --password-file) file="$2"; shift 2 ;; + *) shift ;; + esac +done +if [ -z "$user" ] || [ -z "$file" ] || [ ! -f "$file" ]; then + printf '##STEP 20 fail MOSQUITTO_ARGS_MISSING\n' + exit 0 +fi +printf 'user=%s file=%s\n' "$user" "$file" +printf '##STEP 20 ok\n' +` + +// A redeploy or repair carries no credentials; step 20 must then be handed +// the user and password file the node's own config already names, like the +// dashboard's updater does. +func TestRunTakesTheMosquittoArgumentsFromTheNodeWhenNoneAreGiven(t *testing.T) { + requireSFTPServerForSteps(t) + sshd := transporttest.Start(t) + client := dialForStepsTest(t, sshd) + bundleDir, stateDir := deployBootstrapScripts(t, client, map[string]string{"20-mosquitto.sh": mosquittoArgsScript}) + + dir := path.Dir(stateDir) + pwFile := dir + "/mqtt.pw" + config := dir + "/config.json" + if err := client.UploadBytes([]byte("geheim"), pwFile, 0o600); err != nil { + t.Fatal(err) + } + if err := client.UploadBytes([]byte(`{"mqtt": {"username": "energynode", "password_file": "`+pwFile+`"}}`), config, 0o644); err != nil { + t.Fatal(err) + } + + var logs []string + err := steps.Run(context.Background(), steps.RunOptions{ + Client: client, RemoteBundleDir: bundleDir, RemoteStateDir: stateDir, BundleVersion: "v0.1.0", + Steps: []bundle.StepEntry{{ID: "20"}}, + NodeConfigPath: config, + OnLog: func(_, line string) { logs = append(logs, line) }, + }) + if err != nil { + t.Fatalf("Run: %v", err) + } + want := "user=energynode file=" + pwFile + if len(logs) != 1 || logs[0] != want { + t.Fatalf("logs = %q, want [%q]", logs, want) + } +} + +func TestRunReportsAMissingNodeConfigAsAnUnreadableMqttConfig(t *testing.T) { + requireSFTPServerForSteps(t) + sshd := transporttest.Start(t) + client := dialForStepsTest(t, sshd) + bundleDir, stateDir := deployBootstrapScripts(t, client, map[string]string{"20-mosquitto.sh": mosquittoArgsScript}) + + err := steps.Run(context.Background(), steps.RunOptions{ + Client: client, RemoteBundleDir: bundleDir, RemoteStateDir: stateDir, BundleVersion: "v0.1.0", + Steps: []bundle.StepEntry{{ID: "20"}}, + NodeConfigPath: path.Dir(stateDir) + "/gibt-es-nicht.json", + }) + var failure *steps.StepFailure + if !errors.As(err, &failure) || failure.StepID != "20" || failure.Code != "MQTT_CONFIG_UNREADABLE" { + t.Fatalf("err = %v, want step 20 MQTT_CONFIG_UNREADABLE", err) + } +} diff --git a/installer/internal/steps/runner.go b/installer/internal/steps/runner.go index 2bec1be..49b3b59 100644 --- a/installer/internal/steps/runner.go +++ b/installer/internal/steps/runner.go @@ -29,6 +29,7 @@ type RunOptions struct { Steps []bundle.StepEntry Selection *selection.Selection // optional; uploaded before the first step Secrets *Secrets // optional; only step 60 ever receives it + NodeConfigPath string // optional; where step 20 finds the MQTT user without credentials, default DefaultNodeConfigPath OnMarker func(Marker) OnLog func(stepID, line string) } @@ -108,6 +109,18 @@ func runOneStep(ctx context.Context, opts RunOptions, step bundle.StepEntry) (Ma return Marker{}, err } defer cleanupStep20() + if extraArgs == "" { + // A redeploy or repair collects no credentials; the node + // already has them, and the script checks its arguments + // before it checks whether there is anything to do. + extraArgs, err = nodeMosquittoArgs(ctx, opts) + if err != nil { + if opts.OnMarker != nil { + opts.OnMarker(Marker{StepID: step.ID, Kind: Fail, Detail: mqttConfigUnreadable}) + } + return Marker{}, &StepFailure{StepID: step.ID, Code: mqttConfigUnreadable} + } + } scriptCommand += extraArgs } if step.ID == dashboardStepID && opts.Secrets != nil { diff --git a/installer/internal/steps/secrets.go b/installer/internal/steps/secrets.go index 2e72784..b6dceaa 100644 --- a/installer/internal/steps/secrets.go +++ b/installer/internal/steps/secrets.go @@ -1,9 +1,14 @@ package steps import ( + "bytes" + "context" "crypto/rand" "encoding/hex" "fmt" + "io" + "path" + "strings" "github.com/Developer-Simon/energy-node-installer/internal/transport" ) @@ -36,6 +41,52 @@ func mosquittoArgs(user, passwordPath string) string { return " --user " + transport.ShellQuote(user) + " --password-file " + transport.ShellQuote(passwordPath) } +// DefaultNodeConfigPath is the dashboard's config on the node; it names the +// broker user and the 0600 file that holds its password. +const DefaultNodeConfigPath = "/etc/energy-node/config.json" + +// mqttConfigUnreadable is the fault code for a run that has no credentials +// and cannot find them on the node either (same code as the updater's). +const mqttConfigUnreadable = "MQTT_CONFIG_UNREADABLE" + +// readMqttConfigPy prints the broker user and its password file. The bundle's +// template lacks password_file on some versions, hence the default. +const readMqttConfigPy = `import json, sys +mqtt = json.load(open(sys.argv[1], encoding="utf-8")).get("mqtt", {}) +print(mqtt.get("username", "")) +print(mqtt.get("password_file", "/etc/energy-node/mqtt.pw"))` + +// nodeMosquittoArgs builds step 20's arguments from what is already on the +// node, as dashboard/energy-node-updater.sh does: the user and password +// file from the running config, or -- on a first install -- from the +// bundle's template. The password never leaves the node. +func nodeMosquittoArgs(ctx context.Context, opts RunOptions) (string, error) { + configPath := opts.NodeConfigPath + if configPath == "" { + configPath = DefaultNodeConfigPath + } + candidates := []string{configPath, path.Join(opts.RemoteBundleDir, "config", "config.json")} + var quoted []string + for _, c := range candidates { + quoted = append(quoted, transport.ShellQuote(c)) + } + command := fmt.Sprintf(`for f in %s; do [ -f "$f" ] && python3 -c %s "$f" && exit 0; done; exit 1`, + strings.Join(quoted, " "), transport.ShellQuote(readMqttConfigPy)) + var stdout, stderr bytes.Buffer + if err := opts.Client.Run(ctx, command, &stdout, &stderr); err != nil { + return "", fmt.Errorf("reading the MQTT config: %w (stderr: %s)", err, stderr.String()) + } + lines := strings.Split(strings.TrimSpace(stdout.String()), "\n") + if len(lines) != 2 || lines[0] == "" || lines[1] == "" { + return "", fmt.Errorf("the MQTT config names no user or password file") + } + user, passwordFile := lines[0], lines[1] + if err := opts.Client.Run(ctx, "test -f "+transport.ShellQuote(passwordFile), io.Discard, io.Discard); err != nil { + return "", fmt.Errorf("the MQTT password file %s does not exist", passwordFile) + } + return mosquittoArgs(user, passwordFile), nil +} + // stageSecretsForStep20 uploads the MQTT password to a 0600 temp file and // returns step 20's arguments plus a cleanup that removes the file again. func stageSecretsForStep20(client *transport.Client, user string, secrets *Secrets) (string, func(), error) { diff --git a/installer/internal/transport/sftp.go b/installer/internal/transport/sftp.go index d80c040..4819dbc 100644 --- a/installer/internal/transport/sftp.go +++ b/installer/internal/transport/sftp.go @@ -2,6 +2,8 @@ package transport import ( "bytes" + "crypto/rand" + "encoding/hex" "fmt" "io" "os" @@ -16,12 +18,48 @@ import ( // explicitly -- SFTP's default create mode depends on the server's umask, // which must not be trusted for files that will hold secrets. func (c *Client) UploadFile(localPath, remotePath string, mode os.FileMode) error { + return c.UploadFileProgress(localPath, remotePath, mode, nil) +} + +// UploadFileProgress is UploadFile that reports how many bytes have been +// written so far and the file's total size. onProgress may be nil; it is +// called from the copy loop, so it must return quickly. With concurrent +// writes the callback can be invoked from several goroutines, but done only +// ever grows. +func (c *Client) UploadFileProgress(localPath, remotePath string, mode os.FileMode, onProgress func(done, total int64)) error { local, err := os.Open(localPath) if err != nil { return fmt.Errorf("opening %s: %w", localPath, err) } defer local.Close() - return c.uploadReader(local, remotePath, mode) + var reader io.Reader = local + if onProgress != nil { + info, err := local.Stat() + if err != nil { + return fmt.Errorf("reading size of %s: %w", localPath, err) + } + reader = &progressReader{r: local, total: info.Size(), onProgress: onProgress} + } + return c.uploadReader(reader, remotePath, mode) +} + +// progressReader counts the bytes read through it. Reads come from one +// goroutine (sftp's concurrent writer reads sequentially and fans the chunks +// out), so no locking is needed. +type progressReader struct { + r io.Reader + total int64 + done int64 + onProgress func(done, total int64) +} + +func (p *progressReader) Read(b []byte) (int, error) { + n, err := p.r.Read(b) + if n > 0 { + p.done += int64(n) + p.onProgress(p.done, p.total) + } + return n, err } // UploadBytes writes data to remotePath -- for content that only exists in @@ -31,7 +69,9 @@ func (c *Client) UploadBytes(data []byte, remotePath string, mode os.FileMode) e } func (c *Client) uploadReader(r io.Reader, remotePath string, mode os.FileMode) error { - client, err := sftp.NewClient(c.conn) + // Without concurrent writes every 32 KiB packet waits for its + // acknowledgement, so a high-latency link (Tailscale, Wi-Fi) crawls. + client, err := sftp.NewClient(c.conn, sftp.UseConcurrentWrites(true)) if err != nil { return fmt.Errorf("opening SFTP session: %w", err) } @@ -43,23 +83,49 @@ func (c *Client) uploadReader(r io.Reader, remotePath string, mode os.FileMode) } } - remote, err := client.OpenFile(remotePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC) - if err != nil { - return fmt.Errorf("creating %s: %w", remotePath, err) + // Write next to the target and rename over it. Opening an existing + // remotePath with O_TRUNC fails when the SSH user may not write that + // file -- a manual install leaves a root-owned selection.json in a + // directory the user owns -- while replacing it only needs the + // directory. The rename also means a reader never sees a half-written + // file. + suffix := make([]byte, 6) + if _, err := rand.Read(suffix); err != nil { + return fmt.Errorf("choosing a temporary name for %s: %w", remotePath, err) } - defer remote.Close() + tmpPath := remotePath + ".tmp-" + hex.EncodeToString(suffix) - // Chmod before writing any content: the server just created remotePath + remote, err := client.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL) + if err != nil { + return fmt.Errorf("creating %s: %w", tmpPath, err) + } + // Chmod before writing any content: the server just created the file // at its own default mode (mode & ^umask, typically 0644), and a caller // staging a secret must never leave a window where those bytes sit on // disk at a world-readable mode. if err := remote.Chmod(mode); err != nil { - return fmt.Errorf("setting mode of %s: %w", remotePath, err) + remote.Close() + client.Remove(tmpPath) + return fmt.Errorf("setting mode of %s: %w", tmpPath, err) } - if _, err := io.Copy(remote, r); err != nil { + remote.Close() + client.Remove(tmpPath) return fmt.Errorf("writing %s: %w", remotePath, err) } + if err := remote.Close(); err != nil { + client.Remove(tmpPath) + return fmt.Errorf("writing %s: %w", remotePath, err) + } + if err := client.PosixRename(tmpPath, remotePath); err != nil { + // Servers without the posix-rename extension refuse to rename over + // an existing file; remove the target first. + client.Remove(remotePath) + if err := client.Rename(tmpPath, remotePath); err != nil { + client.Remove(tmpPath) + return fmt.Errorf("replacing %s: %w", remotePath, err) + } + } return nil } diff --git a/installer/internal/transport/sftp_test.go b/installer/internal/transport/sftp_test.go index 66804fc..e8cee72 100644 --- a/installer/internal/transport/sftp_test.go +++ b/installer/internal/transport/sftp_test.go @@ -131,3 +131,77 @@ func TestDownloadFileReportsAMissingRemoteFile(t *testing.T) { t.Fatalf("expected an error for a missing remote file") } } + +func TestUploadFileProgressReportsEveryByteAndKeepsContent(t *testing.T) { + requireSFTPServer(t) + sshd := transporttest.Start(t) + client := dialTestSSHD(t, sshd) + + payload := bytes.Repeat([]byte("0123456789abcdef"), 512*1024) // 8 MiB, several SFTP packets + local := filepath.Join(t.TempDir(), "bundle.tar.gz") + if err := os.WriteFile(local, payload, 0o600); err != nil { + t.Fatalf("writing local fixture: %v", err) + } + remotePath := "/tmp/energy-node-installer-test/progress.bin" + t.Cleanup(func() { _ = client.RemoveRemote(remotePath) }) + + var last, total int64 + calls := 0 + err := client.UploadFileProgress(local, remotePath, 0o600, func(done, size int64) { + if done < last { + t.Errorf("progress went backwards: %d after %d", done, last) + } + last, total = done, size + calls++ + }) + if err != nil { + t.Fatalf("UploadFileProgress: %v", err) + } + if last != int64(len(payload)) || total != int64(len(payload)) || calls < 2 { + t.Fatalf("progress ended at %d/%d after %d calls, want %d/%d over several calls", last, total, calls, len(payload), len(payload)) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + var sum bytes.Buffer + if err := client.Run(ctx, "wc -c < "+remotePath, &sum, &bytes.Buffer{}); err != nil { + t.Fatalf("wc: %v", err) + } + if got := trimNewline(sum.String()); got != "8388608" { + t.Fatalf("remote size = %s, want 8388608", got) + } +} + +// A manual install can leave a file in the state directory that the SSH user +// may not open for writing (root-owned selection.json in a directory the user +// owns). The user may still replace it, so the upload must. +func TestUploadBytesReplacesAnExistingFileTheUserCannotWrite(t *testing.T) { + requireSFTPServer(t) + if os.Geteuid() == 0 { + t.Skip("root can write any file, the scenario cannot be reproduced") + } + sshd := transporttest.Start(t) + client := dialTestSSHD(t, sshd) + + dir := t.TempDir() + remotePath := filepath.Join(dir, "selection.json") + if err := os.WriteFile(remotePath, []byte("alt"), 0o444); err != nil { + t.Fatalf("writing fixture: %v", err) + } + + if err := client.UploadBytes([]byte("neu"), remotePath, 0o644); err != nil { + t.Fatalf("UploadBytes over a read-only file: %v", err) + } + got, err := os.ReadFile(remotePath) + if err != nil || string(got) != "neu" { + t.Fatalf("content = %q, %v, want neu", got, err) + } + info, _ := os.Stat(remotePath) + if info.Mode().Perm() != 0o644 { + t.Errorf("mode = %v, want 0644", info.Mode().Perm()) + } + leftovers, _ := filepath.Glob(filepath.Join(dir, "selection.json.*")) + if len(leftovers) != 0 { + t.Errorf("temporary files left behind: %v", leftovers) + } +} diff --git a/installer/webui/CHANGELOG.md b/installer/webui/CHANGELOG.md index e6f3df3..2bd4faa 100644 --- a/installer/webui/CHANGELOG.md +++ b/installer/webui/CHANGELOG.md @@ -1,11 +1,18 @@ # Changelog -## v0.1.5 (2026-09-21) +## v0.1.6 (2026-09-21) ### Features - **installer:** add package sources (file, repo build, GitHub) (#42) (f831eab) - **dashboard:** download the newest release bundle from the redeploy page (#44) (245b277) +- **installer:** report the bundle upload progress and write concurrently (19df0c0) + +### Fixes + +- **webui:** show the error detail of a failed run (5017432) +- **installer:** give step 20 its MQTT arguments from the node on redeploy and repair (bb4cd40) +- **installer:** add texts for every fault code the bootstrap steps emit (23a64c3) ## v0.1.3 (2026-09-16) diff --git a/installer/webui/VERSION b/installer/webui/VERSION index 027a383..82942c3 100644 --- a/installer/webui/VERSION +++ b/installer/webui/VERSION @@ -1 +1 @@ -v0.1.5 +v0.1.6 diff --git a/installer/webui/catalogs/de.json b/installer/webui/catalogs/de.json index 822387c..776e432 100644 --- a/installer/webui/catalogs/de.json +++ b/installer/webui/catalogs/de.json @@ -118,6 +118,7 @@ "package.log.detect_arch": "Prozessorarchitektur des Geräts ermitteln", "package.log.download": "Lade {name}", "package.log.github_search": "Suche das neueste Release für {arch}", + "package.log.upload_progress": "Übertrage auf das Gerät: {percent} % ({done} von {total} MB)", "prepare.done": "Das Paket liegt auf dem Gerät und ist geprüft.", "prepare.failed": "Das Paket konnte nicht vorbereitet werden.", "prepare.heading": "Paket vorbereiten", @@ -172,32 +173,94 @@ "error.unknown": "Unbekannter Fehler: {code}", "fault.APT_FAILED.message": "apt konnte die benoetigten Systempakete nicht installieren.", "fault.APT_FAILED.remediation": "Internetzugang des Node pruefen und `sudo apt update` von Hand ausfuehren, um den eigentlichen apt-Fehler zu sehen.", + "fault.APT_INSTALL_FAILED.message": "Das Installieren der apt-Pakete ist fehlgeschlagen.", + "fault.APT_INSTALL_FAILED.remediation": "Das Schrittprotokoll zeigt den apt-Fehler. Der Node braucht dafür Zugriff auf die Debian-Spiegel.", + "fault.APT_UPDATE_FAILED.message": "apt-get update ist fehlgeschlagen.", + "fault.APT_UPDATE_FAILED.remediation": "Prüfe den Netzzugang des Nodes und seine apt-Quellen.", "fault.ARCH_MISMATCH.message": "Das Bundle wurde fuer eine andere Prozessorarchitektur gebaut, als dieser Node meldet.", "fault.ARCH_MISMATCH.remediation": "Bundle fuer die Architektur bauen oder laden, die `uname -m` auf dem Node meldet (armv6, arm64 oder amd64).", "fault.BUNDLE_HASH_MISMATCH.message": "Mindestens eine Datei im Bundle passt nicht zu ihrer SHA-256-Summe im Manifest.", "fault.BUNDLE_HASH_MISMATCH.remediation": "Bundle erneut uebertragen. Schlaegt es wieder fehl, neu bauen.", + "fault.BUNDLE_INCOMPLETE.message": "Das Paket auf dem Node ist unvollständig: verify_bundle.sh fehlt.", + "fault.BUNDLE_INCOMPLETE.remediation": "Bereite das Paket erneut vor.", "fault.BUNDLE_MANIFEST_MISSING.message": "Das Bundle enthaelt keine lesbare manifest.json.", "fault.BUNDLE_MANIFEST_MISSING.remediation": "Bundle mit scripts/build/make_bundle.sh neu bauen; ohne Manifest ist es nicht pruefbar.", "fault.BUNDLE_SIGNATURE_INVALID.message": "Die Signatur ueber manifest.json passt nicht zum oeffentlichen Schluessel.", "fault.BUNDLE_SIGNATURE_INVALID.remediation": "Bundle erneut aus vertrauenswuerdiger Quelle holen. Nicht installieren: eine falsche Signatur heisst, die Bytes sind nicht die veroeffentlichten.", + "fault.CADDY_BINARY_MISSING.message": "Das Bundle enthält kein Caddy-Beipack, und auf dem Node gibt es weder Caddy noch eine Caddyfile.", + "fault.CADDY_BINARY_MISSING.remediation": "Baue das Bundle mit --caddy-binary , oder installiere Caddy auf dem Node (apt install caddy) und starte erneut.", + "fault.CADDY_CONFIG_INVALID.message": "Caddy hat /etc/caddy/Caddyfile abgelehnt.", + "fault.CADDY_CONFIG_INVALID.remediation": "Führe auf dem Node „caddy validate --config /etc/caddy/Caddyfile“ aus und korrigiere die genannte Zeile. Der Installer überschreibt eine vorhandene Caddyfile nie.", + "fault.CADDY_START_FAILED.message": "Der Caddy-Dienst ließ sich nicht aktivieren oder starten.", + "fault.CADDY_START_FAILED.remediation": "Prüfe auf dem Node „systemctl status caddy“ und „journalctl -u caddy“.", "fault.CADDY_VALIDATE_FAILED.message": "caddy validate hat das erzeugte Caddyfile abgelehnt.", "fault.CADDY_VALIDATE_FAILED.remediation": "Im Schritt-Log die beanstandete Direktive suchen; das Caddyfile entsteht aus der Bundle-Vorlage und dem gewaehlten Hostnamen.", "fault.CONFIG_EXISTS.message": "Eine vorhandene Konfigurationsdatei wurde gefunden und nicht angetastet.", "fault.CONFIG_EXISTS.remediation": "Das ist kein Schaden: der Installer ueberschreibt Konfiguration nie stillschweigend. Mit --force-config erneut laufen lassen, wenn die Vorlage aus dem Bundle gewollt ist.", + "fault.CONFIG_JSON_MISSING.message": "Die Dashboard-Konfiguration /etc/energy-node/config.json fehlt.", + "fault.CONFIG_JSON_MISSING.remediation": "Führe zuerst den Dashboard-Schritt (60) aus oder repariere ihn.", + "fault.CONFIG_TEMPLATE_MISSING.message": "Das Bundle enthält keine Vorlage für die Dashboard-Konfiguration.", + "fault.CONFIG_TEMPLATE_MISSING.remediation": "Bereite das Paket erneut vor. Bleibt es dabei, ist der Bundle-Bau defekt.", + "fault.CONFIG_WRITE_FAILED.message": "Die Dashboard-Konfiguration ließ sich nicht schreiben.", + "fault.CONFIG_WRITE_FAILED.remediation": "Prüfe das Schrittprotokoll und die Rechte von /etc/energy-node.", + "fault.DASHBOARD_BINARY_MISSING.message": "Das Bundle enthält kein Dashboard-Programm.", + "fault.DASHBOARD_BINARY_MISSING.remediation": "Bereite ein Paket vor, das es enthält.", + "fault.DASHBOARD_START_FAILED.message": "Der Dashboard-Dienst ist nicht gestartet.", + "fault.DASHBOARD_START_FAILED.remediation": "Prüfe auf dem Node „systemctl status energy-node-dashboard“ und sein Journal.", + "fault.MANIFESTS_MISSING.message": "Das Bundle enthält keine Dashboard-Manifeste.", + "fault.MANIFESTS_MISSING.remediation": "Bereite das Paket erneut vor. Bleibt es dabei, ist der Bundle-Bau defekt.", + "fault.MANIFEST_MISSING.message": "Eine Manifest-Datei des Dashboards fehlt.", + "fault.MANIFEST_MISSING.remediation": "Bereite das Paket erneut vor.", + "fault.MANIFEST_PARSE_FAILED.message": "Ein Dashboard-Manifest ließ sich nicht lesen.", + "fault.MANIFEST_PARSE_FAILED.remediation": "Das Schrittprotokoll nennt die Datei. Bereite das Paket erneut vor.", + "fault.MOSQUITTO_ARGS_MISSING.message": "Schritt 20 hat weder MQTT-Benutzer noch Passwortdatei bekommen.", + "fault.MOSQUITTO_ARGS_MISSING.remediation": "Ein Lauf des Installers liefert beides selbst. Wer den Schritt von Hand startet, übergibt --user und --password-file .", "fault.MOSQUITTO_CONFIG_INVALID.message": "Mosquitto hat die erzeugte Konfiguration abgelehnt.", "fault.MOSQUITTO_CONFIG_INVALID.remediation": "/etc/mosquitto/conf.d/default.conf und das Schritt-Log ansehen; eine von Hand gepflegte Broker-Konfiguration kann dagegenstehen.", + "fault.MOSQUITTO_CONF_FOREIGN.message": "/etc/mosquitto/conf.d/default.conf stammt nicht vom Installer.", + "fault.MOSQUITTO_CONF_FOREIGN.remediation": "Der Installer überschreibt sie nicht. Verschiebe sie oder übernimm deine Einstellungen, dann starte erneut.", + "fault.MOSQUITTO_PASSWD_FAILED.message": "Der Passworteintrag für den Broker ließ sich nicht erzeugen.", + "fault.MOSQUITTO_PASSWD_FAILED.remediation": "Der Schritt liest die Passwortdatei als der verbundene Benutzer: sie muss für ihn lesbar sein (Besitzer, Modus 0600), und python3 muss vorhanden sein.", + "fault.MQTT_CONFIG_UNREADABLE.message": "Vom Node ließen sich weder MQTT-Benutzer noch Passwortdatei lesen.", + "fault.MQTT_CONFIG_UNREADABLE.remediation": "Prüfe, dass /etc/energy-node/config.json mqtt.username und mqtt.password_file nennt und die Passwortdatei existiert. Bei einer Erstinstallation „Installieren“ wählen, das fragt nach dem Passwort.", "fault.PIP_EXTERNALLY_MANAGED.message": "pip hat die Installation in das System-Python verweigert (PEP 668, extern verwaltete Umgebung).", "fault.PIP_EXTERNALLY_MANAGED.remediation": "Der Schritt wiederholt mit --break-system-packages; fehlt der Schalter auf diesem Image, die Wheels stattdessen in eine virtuelle Umgebung installieren.", + "fault.PIP_INSTALL_FAILED.message": "pip konnte die Wheels aus dem Bundle nicht installieren.", + "fault.PIP_INSTALL_FAILED.remediation": "Das Protokoll zeigt die letzten pip-Zeilen. Fehlt eine Abhängigkeit, wurde das Bundle ohne sie gebaut: Baue es neu.", "fault.PYTHON_ABI_MISMATCH.message": "Die Wheels im Bundle passen zu einer anderen Python-Nebenversion, als der Node ausfuehrt.", "fault.PYTHON_ABI_MISMATCH.remediation": "Bundle mit --python-minor und --abi passend zu `python3 --version` auf dem Node neu bauen.", + "fault.SECRET_FILE_MISSING.message": "Eine Passwortdatei, die der Schritt erwartet, fehlt.", + "fault.SECRET_FILE_MISSING.remediation": "Führe eine vollständige Installation aus, damit der Installer die Passwörter bereitstellt.", + "fault.SELECTION_UNREADABLE.message": "selection.json auf dem Node ist ungültig.", + "fault.SELECTION_UNREADABLE.remediation": "Korrigiere oder lösche /var/lib/energy-node-installer/selection.json; der Installer schreibt sie neu.", + "fault.SERVICE_SOURCE_MISSING.message": "Dem Bundle fehlen die Dateien eines gewählten Dienstes.", + "fault.SERVICE_SOURCE_MISSING.remediation": "Bereite das Paket erneut vor oder wähle den Dienst ab.", + "fault.SERVICE_START_FAILED.message": "Ein Dienst ist nicht gestartet.", + "fault.SERVICE_START_FAILED.remediation": "Prüfe Status und Journal auf dem Node („systemctl status “).", + "fault.SERVICE_UNIT_FAILED.message": "Die systemd-Unit eines Dienstes ließ sich nicht installieren.", + "fault.SERVICE_UNIT_FAILED.remediation": "Prüfe das Schrittprotokoll und die Rechte von /etc/systemd/system.", + "fault.SUDOERS_INVALID.message": "Die erzeugte sudoers-Regel hat die Prüfung nicht bestanden und wurde nicht installiert.", + "fault.SUDOERS_INVALID.remediation": "Das Schrittprotokoll zeigt die Meldung von visudo. Die Regel hat auf dem Node nichts verändert.", "fault.SUDO_REQUIRED.message": "Der Schritt braucht Root-Rechte, aber sudo hat nach einem Passwort gefragt.", "fault.SUDO_REQUIRED.remediation": "Dem Deploy-Benutzer passwortloses sudo geben oder den Schritt in einer SSH-Sitzung mit diesem Recht laufen lassen.", "fault.TAILSCALE_FLAG_INVALID.message": "/etc/default/tailscaled enthaelt Schalter, die tailscaled daran hindern, tailscale0 anzulegen.", "fault.TAILSCALE_FLAG_INVALID.remediation": "FLAGS=\"\" in /etc/default/tailscaled setzen. Besonders --tun=userspace-networking nimmt dem Node das Routen zu anderen Peers.", + "fault.TAILSCALE_INSTALL_FAILED.message": "Tailscale ließ sich nicht aus dem Bundle installieren.", + "fault.TAILSCALE_INSTALL_FAILED.remediation": "Prüfe das Schrittprotokoll und den freien Platz auf dem Node.", + "fault.TAILSCALE_TARBALL_MISSING.message": "Das Bundle enthält nicht genau ein Tailscale-Archiv.", + "fault.TAILSCALE_TARBALL_MISSING.remediation": "Bereite das Paket erneut vor. Bleibt es dabei, ist der Bundle-Bau defekt.", + "fault.TARGET_INVALID.message": "Der Zielbenutzer oder das Basisverzeichnis ist ungültig.", + "fault.TARGET_INVALID.remediation": "Prüfe beides auf dem Konfigurationsbildschirm.", + "fault.UFW_FAILED.message": "Die Firewall (ufw) ließ sich nicht aktivieren.", + "fault.UFW_FAILED.remediation": "Prüfe das Schrittprotokoll. Ohne ufw-Paket oder ohne Firewall-Unterstützung im Kernel geht es nicht.", "fault.UFW_MISSING.message": "Die Firewall-Regeln konnten nicht gesetzt werden, weil ufw fehlt.", "fault.UFW_MISSING.remediation": "ufw installieren (`sudo apt install ufw`) und den Schritt erneut laufen lassen.", "fault.UNIT_START_FAILED.message": "Eine systemd-Unit ist nach der Installation nicht hochgekommen.", "fault.UNIT_START_FAILED.remediation": "Auf dem Node `systemctl status ` und `journalctl -u -n 50` ausfuehren; dort steht meist ein Konfigurationsproblem.", + "fault.UPDATER_PATH_START_FAILED.message": "Die Path-Unit des Updaters ließ sich nicht starten.", + "fault.UPDATER_PATH_START_FAILED.remediation": "Prüfe auf dem Node „systemctl status energy-node-updater.path“.", + "fault.WHEELS_MISSING.message": "Das Bundle enthält keine Python-Wheels.", + "fault.WHEELS_MISSING.remediation": "Baue das Bundle ohne --skip-wheels neu.", "fault.WHEEL_MISSING.message": "Ein benoetigtes Wheel fehlt im Bundle.", "fault.WHEEL_MISSING.remediation": "Bundle neu bauen: make_bundle.sh bricht bei einem fehlenden Wheel ab - ein unvollstaendiges Bundle entstand von Hand oder beim Uebertragen.", "fault.unknown.message": "Unbekannter Fehlercode {code}.", @@ -297,6 +360,7 @@ "result.fail.card": "Was schiefging", "result.fail.heading": "Der Lauf ist abgebrochen", "result.fail.summary": "Bei „{step}“ · nach {duration}", + "result.fail.summary_nostep": "Nach {duration}", "result.lines.heading": "Letzte Protokollzeilen", "result.lines.text": "Aus dem Schritt, der fehlgeschlagen ist.", "result.ok.heading": "Der Node läuft", diff --git a/installer/webui/catalogs/en.json b/installer/webui/catalogs/en.json index 2db7f4a..99ef75b 100644 --- a/installer/webui/catalogs/en.json +++ b/installer/webui/catalogs/en.json @@ -118,6 +118,7 @@ "package.log.detect_arch": "Detecting the device's processor architecture", "package.log.download": "Downloading {name}", "package.log.github_search": "Looking for the newest release for {arch}", + "package.log.upload_progress": "Uploading to the device: {percent} % ({done} of {total} MB)", "prepare.done": "The package is on the device and has been verified.", "prepare.failed": "The package could not be prepared.", "prepare.heading": "Prepare package", @@ -172,32 +173,94 @@ "error.unknown": "Unknown error: {code}", "fault.APT_FAILED.message": "apt could not install the required system packages.", "fault.APT_FAILED.remediation": "Check the node's internet access and run `sudo apt update` by hand to see the underlying apt error.", + "fault.APT_INSTALL_FAILED.message": "Installing the apt packages failed.", + "fault.APT_INSTALL_FAILED.remediation": "The step log shows apt's error. The node needs access to the Debian mirrors for this step.", + "fault.APT_UPDATE_FAILED.message": "apt-get update failed.", + "fault.APT_UPDATE_FAILED.remediation": "Check the node's network access and its apt sources.", "fault.ARCH_MISMATCH.message": "The bundle was built for a different CPU architecture than this node reports.", "fault.ARCH_MISMATCH.remediation": "Build or download the bundle for the architecture the node's `uname -m` reports (armv6, arm64 or amd64).", "fault.BUNDLE_HASH_MISMATCH.message": "At least one file in the bundle does not match its SHA-256 in the manifest.", "fault.BUNDLE_HASH_MISMATCH.remediation": "Transfer the bundle again. If it fails a second time, rebuild it.", + "fault.BUNDLE_INCOMPLETE.message": "The package on the node is incomplete: verify_bundle.sh is missing.", + "fault.BUNDLE_INCOMPLETE.remediation": "Prepare the package again.", "fault.BUNDLE_MANIFEST_MISSING.message": "The bundle carries no readable manifest.json.", "fault.BUNDLE_MANIFEST_MISSING.remediation": "Rebuild the bundle with scripts/build/make_bundle.sh; a bundle without a manifest cannot be verified.", "fault.BUNDLE_SIGNATURE_INVALID.message": "The signature over manifest.json does not match the public key.", "fault.BUNDLE_SIGNATURE_INVALID.remediation": "Fetch the bundle again from a trusted source. Do not install it: a wrong signature means the bytes are not the ones that were released.", + "fault.CADDY_BINARY_MISSING.message": "The bundle has no Caddy pack, and the node has neither Caddy nor a Caddyfile to fall back on.", + "fault.CADDY_BINARY_MISSING.remediation": "Build the bundle with --caddy-binary , or install Caddy on the node (apt install caddy) and run again.", + "fault.CADDY_CONFIG_INVALID.message": "Caddy rejected /etc/caddy/Caddyfile.", + "fault.CADDY_CONFIG_INVALID.remediation": "Run “caddy validate --config /etc/caddy/Caddyfile” on the node and fix the line it names. The installer never overwrites an existing Caddyfile.", + "fault.CADDY_START_FAILED.message": "The Caddy service could not be enabled or started.", + "fault.CADDY_START_FAILED.remediation": "Check “systemctl status caddy” and “journalctl -u caddy” on the node.", "fault.CADDY_VALIDATE_FAILED.message": "caddy validate rejected the generated Caddyfile.", "fault.CADDY_VALIDATE_FAILED.remediation": "Read the step log for the offending directive; the Caddyfile is rendered from the bundle template and the chosen host name.", "fault.CONFIG_EXISTS.message": "An existing configuration file was found and left untouched.", "fault.CONFIG_EXISTS.remediation": "This is not damage: the installer never overwrites a configuration silently. Re-run with --force-config if you want the bundle's template.", + "fault.CONFIG_JSON_MISSING.message": "The dashboard config /etc/energy-node/config.json does not exist.", + "fault.CONFIG_JSON_MISSING.remediation": "Run the dashboard step (60) first, or repair it.", + "fault.CONFIG_TEMPLATE_MISSING.message": "The bundle has no dashboard config template.", + "fault.CONFIG_TEMPLATE_MISSING.remediation": "Prepare the package again; if it stays, the bundle build is broken.", + "fault.CONFIG_WRITE_FAILED.message": "The dashboard config could not be written.", + "fault.CONFIG_WRITE_FAILED.remediation": "Check the step log and the permissions of /etc/energy-node.", + "fault.DASHBOARD_BINARY_MISSING.message": "The bundle has no dashboard binary.", + "fault.DASHBOARD_BINARY_MISSING.remediation": "Prepare a package that contains it; if the bundle was built from the repo, check that the dashboard build succeeded.", + "fault.DASHBOARD_START_FAILED.message": "The dashboard service did not start.", + "fault.DASHBOARD_START_FAILED.remediation": "Check “systemctl status energy-node-dashboard” and its journal on the node.", + "fault.MANIFESTS_MISSING.message": "The bundle has no dashboard manifests.", + "fault.MANIFESTS_MISSING.remediation": "Prepare the package again; if it stays, the bundle build is broken.", + "fault.MANIFEST_MISSING.message": "A dashboard manifest file is missing.", + "fault.MANIFEST_MISSING.remediation": "Prepare the package again.", + "fault.MANIFEST_PARSE_FAILED.message": "A dashboard manifest could not be read.", + "fault.MANIFEST_PARSE_FAILED.remediation": "The step log shows which file. Prepare the package again.", + "fault.MOSQUITTO_ARGS_MISSING.message": "Step 20 got no MQTT user or password file.", + "fault.MOSQUITTO_ARGS_MISSING.remediation": "A run from the installer supplies them itself. If you started the step by hand, pass --user and --password-file .", "fault.MOSQUITTO_CONFIG_INVALID.message": "Mosquitto rejected the generated configuration.", "fault.MOSQUITTO_CONFIG_INVALID.remediation": "Inspect /etc/mosquitto/conf.d/default.conf and the step log; a hand-edited broker configuration may conflict with it.", + "fault.MOSQUITTO_CONF_FOREIGN.message": "/etc/mosquitto/conf.d/default.conf was not written by the installer.", + "fault.MOSQUITTO_CONF_FOREIGN.remediation": "The installer does not overwrite it. Move it away or merge your settings into the installer's version, then run again.", + "fault.MOSQUITTO_PASSWD_FAILED.message": "The broker's password entry could not be created.", + "fault.MOSQUITTO_PASSWD_FAILED.remediation": "The step reads the password file as the connecting user: it must be readable for that user (owner, mode 0600) and python3 must exist.", + "fault.MQTT_CONFIG_UNREADABLE.message": "No MQTT user or password file could be read from the node.", + "fault.MQTT_CONFIG_UNREADABLE.remediation": "Check that /etc/energy-node/config.json names mqtt.username and mqtt.password_file and that the password file exists. On a first install use “Install”, which asks for the password.", "fault.PIP_EXTERNALLY_MANAGED.message": "pip refused to install into the system Python (PEP 668, externally managed environment).", "fault.PIP_EXTERNALLY_MANAGED.remediation": "The step retries with --break-system-packages; if that is unavailable on this image, install the wheels into a virtual environment instead.", + "fault.PIP_INSTALL_FAILED.message": "pip could not install the wheels from the bundle.", + "fault.PIP_INSTALL_FAILED.remediation": "The log shows pip's last lines. A missing dependency means the bundle was built without it: rebuild the bundle.", "fault.PYTHON_ABI_MISMATCH.message": "The wheels in the bundle were built for a different Python minor version than the node runs.", "fault.PYTHON_ABI_MISMATCH.remediation": "Rebuild the bundle with --python-minor and --abi matching `python3 --version` on the node.", + "fault.SECRET_FILE_MISSING.message": "A password file the step expects is missing.", + "fault.SECRET_FILE_MISSING.remediation": "Run a full install so the installer stages the passwords.", + "fault.SELECTION_UNREADABLE.message": "selection.json on the node is not valid.", + "fault.SELECTION_UNREADABLE.remediation": "Fix or delete /var/lib/energy-node-installer/selection.json; the installer writes it again.", + "fault.SERVICE_SOURCE_MISSING.message": "The bundle lacks the files of a selected service.", + "fault.SERVICE_SOURCE_MISSING.remediation": "Prepare the package again, or deselect the service.", + "fault.SERVICE_START_FAILED.message": "A service did not start.", + "fault.SERVICE_START_FAILED.remediation": "Check its status and journal on the node (“systemctl status ”).", + "fault.SERVICE_UNIT_FAILED.message": "A service's systemd unit could not be installed.", + "fault.SERVICE_UNIT_FAILED.remediation": "Check the step log and the permissions of /etc/systemd/system.", + "fault.SUDOERS_INVALID.message": "The generated sudoers rule failed the check; it was not installed.", + "fault.SUDOERS_INVALID.remediation": "The step log shows visudo's message. Nothing on the node was changed by the rule.", "fault.SUDO_REQUIRED.message": "The step needs root rights but sudo asked for a password.", "fault.SUDO_REQUIRED.remediation": "Give the deploy user passwordless sudo, or run the step in an SSH session that has it.", "fault.TAILSCALE_FLAG_INVALID.message": "/etc/default/tailscaled carries flags that keep tailscaled from creating the tailscale0 interface.", "fault.TAILSCALE_FLAG_INVALID.remediation": "Set FLAGS=\"\" in /etc/default/tailscaled. In particular --tun=userspace-networking stops the node from routing to other peers.", + "fault.TAILSCALE_INSTALL_FAILED.message": "Tailscale could not be installed from the bundle.", + "fault.TAILSCALE_INSTALL_FAILED.remediation": "Check the step log and the free space on the node.", + "fault.TAILSCALE_TARBALL_MISSING.message": "The bundle has not exactly one Tailscale archive.", + "fault.TAILSCALE_TARBALL_MISSING.remediation": "Prepare the package again; if it stays, the bundle build is broken.", + "fault.TARGET_INVALID.message": "The target user or base directory is not valid.", + "fault.TARGET_INVALID.remediation": "Check both on the configure screen.", + "fault.UFW_FAILED.message": "The firewall (ufw) could not be enabled.", + "fault.UFW_FAILED.remediation": "Check the step log; a node without the ufw package or without kernel firewall support cannot enable it.", "fault.UFW_MISSING.message": "The firewall rules could not be applied because ufw is not installed.", "fault.UFW_MISSING.remediation": "Install ufw (`sudo apt install ufw`) and run the step again.", "fault.UNIT_START_FAILED.message": "A systemd unit did not come up after installation.", "fault.UNIT_START_FAILED.remediation": "Run `systemctl status ` and `journalctl -u -n 50` on the node; the service usually reports a configuration problem there.", + "fault.UPDATER_PATH_START_FAILED.message": "The updater's path unit could not be started.", + "fault.UPDATER_PATH_START_FAILED.remediation": "Check “systemctl status energy-node-updater.path” on the node.", + "fault.WHEELS_MISSING.message": "The bundle contains no Python wheels.", + "fault.WHEELS_MISSING.remediation": "Rebuild the bundle without --skip-wheels.", "fault.WHEEL_MISSING.message": "A required wheel is not part of the bundle.", "fault.WHEEL_MISSING.remediation": "Rebuild the bundle: make_bundle.sh aborts when a wheel is missing, so an incomplete bundle was built by hand or truncated in transit.", "fault.unknown.message": "Unknown error code {code}.", @@ -297,6 +360,7 @@ "result.fail.card": "What went wrong", "result.fail.heading": "The run stopped", "result.fail.summary": "At “{step}” · after {duration}", + "result.fail.summary_nostep": "After {duration}", "result.lines.heading": "Last log lines", "result.lines.text": "From the step that failed.", "result.ok.heading": "The node is running", diff --git a/installer/webui/hostapi/run.go b/installer/webui/hostapi/run.go index fe4682a..80ca3f9 100644 --- a/installer/webui/hostapi/run.go +++ b/installer/webui/hostapi/run.go @@ -119,16 +119,20 @@ func runFinishPayload(bus *Bus, id string, err error, redactor *Redactor) map[st if err == nil { return payload } + detail := err.Error() var typed *Error switch { case errors.As(err, &typed): payload["code"] = typed.Code payload["step_id"] = lastStepID(bus) + detail = typed.Detail case errors.Is(err, context.Canceled): payload["code"] = "RUN_CANCELLED" + detail = "" default: payload["code"] = "BACKEND_ERROR" - detail := err.Error() + } + if detail != "" { if redactor != nil { detail = redactor.Line(detail) } diff --git a/installer/webui/hostapi/run_payload_test.go b/installer/webui/hostapi/run_payload_test.go new file mode 100644 index 0000000..5d32ee0 --- /dev/null +++ b/installer/webui/hostapi/run_payload_test.go @@ -0,0 +1,25 @@ +package hostapi + +import ( + "errors" + "testing" +) + +func TestRunFinishPayloadCarriesTheDetailOfEveryFailure(t *testing.T) { + redactor := NewRedactor("hunter2-secret") + cases := map[string]struct { + err error + wantCode string + wantDetail string + }{ + "untyped": {errors.New("no bootstrap script found for step 10"), "BACKEND_ERROR", "no bootstrap script found for step 10"}, + "typed": {&Error{Code: "PACKAGE_STAGE_FAILED", Detail: "scp: broken pipe"}, "PACKAGE_STAGE_FAILED", "scp: broken pipe"}, + "secret": {errors.New("login with hunter2-secret failed"), "BACKEND_ERROR", "login with " + redactor.Line("hunter2-secret") + " failed"}, + } + for name, tc := range cases { + payload := runFinishPayload(NewBus(10), "run-1", tc.err, redactor) + if payload["code"] != tc.wantCode || payload["detail"] != tc.wantDetail { + t.Errorf("%s: payload = %v, want code %s detail %q", name, payload, tc.wantCode, tc.wantDetail) + } + } +} diff --git a/installer/webui/static/js/run-model.js b/installer/webui/static/js/run-model.js index 806badf..1e3edd8 100644 --- a/installer/webui/static/js/run-model.js +++ b/installer/webui/static/js/run-model.js @@ -17,7 +17,7 @@ function create(runId) { return { - runId: runId, started: false, finished: false, ok: null, code: '', failedStep: '', + runId: runId, started: false, finished: false, ok: null, code: '', detail: '', failedStep: '', mode: '', only: '', startedAt: 0, finishedAt: 0, lastAt: 0, steps: {}, log: [], logCount: 0, loginUrl: '', loginStep: '', loginPending: false, }; @@ -96,6 +96,7 @@ model.finished = true; model.ok = !!data.ok; model.code = data.code || ''; + model.detail = data.detail || ''; model.failedStep = data.step_id || ''; model.finishedAt = at; return true; @@ -216,7 +217,7 @@ function outcome(model, groups) { return { - ok: model.ok, code: model.code, stepId: model.failedStep, mode: model.mode, only: model.only, + ok: model.ok, code: model.code, detail: model.detail, stepId: model.failedStep, mode: model.mode, only: model.only, startedAt: model.startedAt, finishedAt: model.finishedAt, loginUrl: model.loginUrl, loginPending: model.loginPending, steps: JSON.parse(JSON.stringify(model.steps)), groups: groups, diff --git a/installer/webui/static/js/screen-result.js b/installer/webui/static/js/screen-result.js index 90e910e..9b1f1ad 100644 --- a/installer/webui/static/js/screen-result.js +++ b/installer/webui/static/js/screen-result.js @@ -21,7 +21,7 @@ get outcome() { return this.shell.shared.lastRun || - { ok: false, code: '', mode: '', only: '', steps: {}, groups: [], lastLines: [], logText: '', startedAt: 0, finishedAt: 0 }; + { ok: false, code: '', detail: '', mode: '', only: '', steps: {}, groups: [], lastLines: [], logText: '', startedAt: 0, finishedAt: 0 }; }, async init() { @@ -74,6 +74,9 @@ if (this.cancelled) { return shell.t('result.cancelled.summary', { duration: this.duration }); } + if (!outcome.ok && !outcome.stepId) { + return shell.t('result.fail.summary_nostep', { duration: this.duration }); + } if (!outcome.ok) { return shell.t('result.fail.summary', { step: this.label(outcome.stepId), duration: this.duration }); } @@ -174,6 +177,10 @@ return { message: window.RunModel.faultText(code, t), remediation: remediation === key ? '' : remediation }; }, + get detail() { + return this.outcome.detail || ''; + }, + get lines() { return this.outcome.lastLines || []; }, diff --git a/installer/webui/templates/screen-result.html b/installer/webui/templates/screen-result.html index cfcf647..5cec9f8 100644 --- a/installer/webui/templates/screen-result.html +++ b/installer/webui/templates/screen-result.html @@ -52,6 +52,7 @@

+ diff --git a/installer/webui/test/run-model.test.mjs b/installer/webui/test/run-model.test.mjs index 1742930..dd7de66 100644 --- a/installer/webui/test/run-model.test.mjs +++ b/installer/webui/test/run-model.test.mjs @@ -138,3 +138,15 @@ test('das Protokoll haelt hoechstens 2000 Eintraege', () => { assert.equal(model.log.length, 2000); assert.equal(model.log[0].text, 'z100'); }); + +test('ein Abbruch ohne Schritt traegt sein detail in outcome', () => { + const { M, groups } = setup(); + const model = M.create('run-1'); + play(M, model, [ + ['run-started', { run_id: 'run-1', mode: 'install' }, 0], + ['run-finished', { run_id: 'run-1', ok: false, code: 'BACKEND_ERROR', detail: 'no bootstrap script found for step 10' }, 1], + ]); + const outcome = M.outcome(model, groups); + assert.equal(outcome.stepId, ''); + assert.equal(outcome.detail, 'no bootstrap script found for step 10'); +}); diff --git a/libs/energy_node_common/CHANGELOG.md b/libs/energy_node_common/CHANGELOG.md index ae92252..534baab 100644 --- a/libs/energy_node_common/CHANGELOG.md +++ b/libs/energy_node_common/CHANGELOG.md @@ -6,6 +6,7 @@ - **installer:** build the node-half bootstrap chain and signed bundle pipeline (#25) (f11e962) - **webui:** add the installer's layer-3 web UI, browser tests and CI (#28) (e819b5e) +- **services:** give every service its own version and changelog (#45) (5bc91b8) ### Tests diff --git a/scripts/bootstrap/40-tailscale.sh b/scripts/bootstrap/40-tailscale.sh index d7d5989..f052b7a 100755 --- a/scripts/bootstrap/40-tailscale.sh +++ b/scripts/bootstrap/40-tailscale.sh @@ -11,6 +11,12 @@ source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/lib/step.sh" FORBIDDEN_FLAG='--tun=userspace-networking' +# Nur eine wirksame Zeile zaehlt: Debians Standarddatei und die Vorlage im +# Tarball fuehren das Beispiel auskommentiert (#FLAGS="--tun=...") auf. +has_forbidden_flag() { + sed 's/#.*//' "$1" | grep -q -- "${FORBIDDEN_FLAG}" +} + step_begin 40 if ! step_selected 40; then step_skip "nicht ausgewaehlt" @@ -31,7 +37,7 @@ defaults="${EN_ROOT}/etc/default/tailscaled" # Vor jeder Aenderung: eine vorhandene Datei mit dem verbotenen Schalter ist # ein Abbruchgrund, kein Reparaturfall. Der Schalter verhindert, dass # tailscaled tailscale0 anlegt - der Node kann dann zu keinem Peer routen. -if [[ -f "${defaults}" ]] && grep -q -- "${FORBIDDEN_FLAG}" "${defaults}"; then +if [[ -f "${defaults}" ]] && has_forbidden_flag "${defaults}"; then step_log "In ${defaults} steht ${FORBIDDEN_FLAG}. Bitte entfernen (INSTALLATION.md 3.1)." step_fail TAILSCALE_FLAG_INVALID fi @@ -80,7 +86,7 @@ if [[ "${keep_installed}" == false ]]; then fi # Gegenprobe nach dem Schreiben: auch die mitgelieferte Vorlage darf den # Schalter nicht enthalten. - if grep -q -- "${FORBIDDEN_FLAG}" "${defaults}"; then + if has_forbidden_flag "${defaults}"; then step_log "Die installierte ${defaults} enthaelt ${FORBIDDEN_FLAG}." step_fail TAILSCALE_FLAG_INVALID fi diff --git a/scripts/bootstrap/70-caddy.sh b/scripts/bootstrap/70-caddy.sh index a252c09..e0280f2 100755 --- a/scripts/bootstrap/70-caddy.sh +++ b/scripts/bootstrap/70-caddy.sh @@ -21,22 +21,41 @@ fi binary="${EN_BUNDLE_DIR}/caddy/caddy" template="${EN_BUNDLE_DIR}/dashboard/Caddyfile" -[[ -f "${binary}" && -f "${template}" ]] || step_fail CADDY_BINARY_MISSING +caddy_bin="${EN_ROOT}/usr/bin/caddy" +caddyfile="${EN_ROOT}/etc/caddy/Caddyfile" + +# Das Beipack (E13) ist nur dabei, wenn das Bundle mit --caddy-binary gebaut +# wurde. Gebraucht wird es erst, wenn dieser Schritt ein Binary installieren +# oder eine Caddyfile anlegen soll; ein Node mit Caddy und Konfiguration +# kommt ohne aus. +have_installed=false +[[ -x "${caddy_bin}" ]] && have_installed=true +if [[ ! -f "${binary}" && "${have_installed}" == false ]]; then + step_fail CADDY_BINARY_MISSING +fi +if [[ ! -f "${template}" && ! -f "${caddyfile}" ]]; then + step_fail CADDY_BINARY_MISSING +fi # Ein vorhandenes Caddy-Binary bleibt stehen, wenn es dem Paketmanager gehoert -# (ein spaeteres apt upgrade tauschte die Datei ohnehin wieder aus) oder -# nicht aelter ist als das im Bundle. Ersetzt wird nur ein eigenes, aelteres. -caddy_bin="${EN_ROOT}/usr/bin/caddy" +# (ein spaeteres apt upgrade tauschte die Datei ohnehin wieder aus), nicht +# aelter ist als das im Bundle oder das Bundle keines mitbringt. Ersetzt wird +# nur ein eigenes, aelteres. install_binary=true -if [[ -x "${caddy_bin}" ]]; then +if [[ "${have_installed}" == true ]]; then installed_version="$("${caddy_bin}" version 2>/dev/null | head -n 1 | cut -d' ' -f1 || true)" - bundled_version="$("${binary}" version 2>/dev/null | head -n 1 | cut -d' ' -f1 || true)" if dpkg -S "${caddy_bin}" >/dev/null 2>&1; then install_binary=false step_log "Das vorhandene Caddy ${installed_version} gehoert dem Paketmanager und bleibt unangetastet." - elif step_version_ge "${installed_version}" "${bundled_version}"; then + elif [[ ! -f "${binary}" ]]; then install_binary=false - step_log "Das vorhandene Caddy ${installed_version} ist nicht aelter als das im Bundle (${bundled_version}) und bleibt unangetastet." + step_log "Das Bundle bringt kein Caddy mit; das vorhandene ${installed_version} bleibt unangetastet." + else + bundled_version="$("${binary}" version 2>/dev/null | head -n 1 | cut -d' ' -f1 || true)" + if step_version_ge "${installed_version}" "${bundled_version}"; then + install_binary=false + step_log "Das vorhandene Caddy ${installed_version} ist nicht aelter als das im Bundle (${bundled_version}) und bleibt unangetastet." + fi fi fi @@ -47,9 +66,8 @@ fi # Eine vorhandene Caddyfile gehoert dem Betreiber: INSTALLATION.md 8 sagt # ausdruecklich, dass ein Deploy die Caddy-Konfiguration nicht anfasst. -caddyfile="${EN_ROOT}/etc/caddy/Caddyfile" if [[ -f "${caddyfile}" ]]; then - if ! cmp -s "${template}" "${caddyfile}"; then + if [[ -f "${template}" ]] && ! cmp -s "${template}" "${caddyfile}"; then step_log "Die vorhandene ${caddyfile} weicht ab und bleibt unangetastet." fi else diff --git a/scripts/bootstrap/CHANGELOG.md b/scripts/bootstrap/CHANGELOG.md index 6293668..70121ed 100644 --- a/scripts/bootstrap/CHANGELOG.md +++ b/scripts/bootstrap/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## v0.1.7 (2026-09-21) +## v0.1.8 (2026-09-21) ### Features @@ -15,4 +15,6 @@ - **bootstrap:** make steps 20, 40, 65, 70 and the diagnosis work on a real node (#37) (11eef9d) - **installer:** restart service units on update and record the installed manifest (#43) (0f2aec2) +- **bootstrap:** ignore a commented-out userspace-networking flag in step 40 (a1aeee1) +- **bootstrap:** let step 70 keep an installed Caddy when the bundle has no Caddy pack (f8fbc8d) diff --git a/scripts/bootstrap/VERSION b/scripts/bootstrap/VERSION index fad30c5..da14730 100644 --- a/scripts/bootstrap/VERSION +++ b/scripts/bootstrap/VERSION @@ -1 +1 @@ -v0.1.7 +v0.1.8 diff --git a/scripts/build/lib/missing_requirements.py b/scripts/build/lib/missing_requirements.py new file mode 100644 index 0000000..4cc7434 --- /dev/null +++ b/scripts/build/lib/missing_requirements.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Print the requirements the wheels in a directory need but do not provide. + +Usage: missing_requirements.py + +Markers are evaluated for the *target* (given Python minor, Linux, given +machine), not for the interpreter running this script. `pip download +--python-version` evaluates markers such as python_version < "3.13" with the +build machine's own Python, so a build on 3.14 silently leaves out +typing_extensions that the node's 3.11 needs. One requirement per line, in +the form pip accepts. +""" +import email +import glob +import os +import sys +import zipfile + +try: + from packaging.markers import default_environment + from packaging.requirements import Requirement + from packaging.utils import canonicalize_name + from packaging.version import Version +except ImportError: # pip vendors it + from pip._vendor.packaging.markers import default_environment + from pip._vendor.packaging.requirements import Requirement + from pip._vendor.packaging.utils import canonicalize_name + from pip._vendor.packaging.version import Version + + +def target_environment(minor, machine): + env = dict(default_environment()) + env.update({ + "python_version": minor, + "python_full_version": minor + ".0", + "implementation_name": "cpython", + "platform_python_implementation": "CPython", + "sys_platform": "linux", + "platform_system": "Linux", + "os_name": "posix", + "platform_machine": machine, + "extra": "", + }) + return env + + +def read_metadata(path): + with zipfile.ZipFile(path) as wheel: + name = next(n for n in wheel.namelist() if n.endswith(".dist-info/METADATA")) + return email.message_from_bytes(wheel.read(name)) + + +def main(argv): + wheel_dir, minor, machine = argv[1:4] + env = target_environment(minor, machine) + provided = {} + requires = [] + for path in sorted(glob.glob(os.path.join(wheel_dir, "*.whl"))): + meta = read_metadata(path) + provided[canonicalize_name(meta["Name"])] = Version(meta["Version"]) + requires.extend(meta.get_all("Requires-Dist") or []) + + missing = {} + for text in requires: + req = Requirement(text) + if req.marker is not None and not req.marker.evaluate(env): + continue + have = provided.get(canonicalize_name(req.name)) + if have is not None and req.specifier.contains(have, prereleases=True): + continue + missing[str(canonicalize_name(req.name)) + str(req.specifier)] = None + print("\n".join(missing)) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/scripts/build/lib/wheels.sh b/scripts/build/lib/wheels.sh index 61971c4..c788beb 100644 --- a/scripts/build/lib/wheels.sh +++ b/scripts/build/lib/wheels.sh @@ -72,23 +72,45 @@ fetch_thirdparty_wheels() { args+=(--platform "${tag}") done - if ! "${WHEELS_PIP[@]}" download \ - --only-binary=:all: \ - --index-url https://www.piwheels.org/simple \ - --extra-index-url https://pypi.org/simple \ - "${args[@]}" \ - --python-version "${minor}" \ - --implementation cp \ - --abi "${abi}" \ - -d "${target}" \ - "${THIRDPARTY_PACKAGES[@]}"; then - echo "Fuer mindestens eines der Pakete (${THIRDPARTY_PACKAGES[*]}) gibt es" >&2 - echo "kein Wheel fuer ${arch}/${abi}. Der Bundle-Bau bricht ab - ein Node" >&2 - echo "darf Abhaengigkeiten nicht selbst aufloesen (E11)." >&2 - return 1 - fi + # pip wertet Umgebungsmarker (python_version < "3.13" ...) mit dem Python + # des Bau-Rechners aus, nicht mit dem Ziel. Nach dem Download prueft + # missing_requirements.py die Abhaengigkeiten aller Wheels gegen die + # Zielmarker und holt nach, was fehlt - bis nichts mehr fehlt. + local machine missing=() + machine="$(arch_uname_machines "${arch}" | head -n 1)" + for _ in 1 2 3 4 5 6; do + if ! "${WHEELS_PIP[@]}" download \ + --only-binary=:all: \ + --index-url https://www.piwheels.org/simple \ + --extra-index-url https://pypi.org/simple \ + "${args[@]}" \ + --python-version "${minor}" \ + --implementation cp \ + --abi "${abi}" \ + -d "${target}" \ + "${THIRDPARTY_PACKAGES[@]}" "${missing[@]}"; then + echo "Fuer mindestens eines der Pakete (${THIRDPARTY_PACKAGES[*]} ${missing[*]}) gibt es" >&2 + echo "kein Wheel fuer ${arch}/${abi}. Der Bundle-Bau bricht ab - ein Node" >&2 + echo "darf Abhaengigkeiten nicht selbst aufloesen (E11)." >&2 + return 1 + fi + missing=() + while IFS= read -r req; do + if [[ -n "${req}" ]]; then + missing+=("${req}") + fi + done < <(python3 "$(dirname "${BASH_SOURCE[0]}")/missing_requirements.py" \ + "${target}" "${minor}" "${machine}") || return 1 + if [[ "${#missing[@]}" -eq 0 ]]; then + return 0 + fi + echo "Zusaetzlich fuer ${minor}/${machine} noetig: ${missing[*]}" >&2 + done + echo "Die Abhaengigkeiten der Wheels lassen sich nicht schliessen (fehlt: ${missing[*]})." >&2 + return 1 } + # build_local_wheels # # Baut die beiden eigenen Pakete. Ein im dist/-Verzeichnis liegendes Rad diff --git a/scripts/tests/test_bootstrap_40_tailscale.sh b/scripts/tests/test_bootstrap_40_tailscale.sh index 25d8440..61654ba 100755 --- a/scripts/tests/test_bootstrap_40_tailscale.sh +++ b/scripts/tests/test_bootstrap_40_tailscale.sh @@ -93,6 +93,26 @@ grep -qx 'FLAGS="--tun=userspace-networking"' "$defaults" \ || fail "defaults trotz Abbruch veraendert" "$(cat "$defaults")" rm -f "$defaults" +# --- der Schalter nur als Kommentar ist kein Abbruchgrund ------------------ +# Debians Standarddatei (und die Vorlage im echten Tarball) fuehrt das Beispiel +# auskommentiert auf; das ist keine Konfiguration. +rm -rf "$tmp/state" +printf '# Extra flags.\n#FLAGS="--tun=userspace-networking"\nPORT="41641"\n' > "$defaults" +out="$(TS_STATUS_RC=0 bash "$script")" || fail "auskommentierter Schalter fuehrte zum Abbruch" "$out" +grep -q '^##STEP 40 ok$' <<<"$out" || fail "auskommentierter Schalter: kein ok" "$out" +rm -f "$defaults" + +# ... auch nicht in der mitgelieferten Vorlage (frischer Node, Gegenprobe) +rm -rf "$tmp/state" "$tmp/root" +printf '#FLAGS="--tun=userspace-networking"\nPORT="41641"\n' \ + > "$tmp/pack/tailscale_1.62.0_arm/systemd/tailscaled.defaults" +tar -czf "$bundle/tailscale/tailscale_1.62.0_arm.tgz" -C "$tmp/pack" tailscale_1.62.0_arm +out="$(TS_STATUS_RC=0 bash "$script")" || fail "auskommentierter Schalter in der Vorlage fuehrte zum Abbruch" "$out" +grep -q '^##STEP 40 ok$' <<<"$out" || fail "auskommentierter Schalter in der Vorlage: kein ok" "$out" +printf 'FLAGS=""\n' > "$tmp/pack/tailscale_1.62.0_arm/systemd/tailscaled.defaults" +tar -czf "$bundle/tailscale/tailscale_1.62.0_arm.tgz" -C "$tmp/pack" tailscale_1.62.0_arm +rm -rf "$tmp/state" "$tmp/root" + # --- neuere Version vorhanden: nichts wird angefasst ---------------------- # Ein Node, der Tailscale per "tailscale update" hochgezogen hat (oder von # Hand eine neuere Fassung installierte), darf nicht auf die Fassung im diff --git a/scripts/tests/test_bootstrap_70_caddy.sh b/scripts/tests/test_bootstrap_70_caddy.sh index c54a756..965d6bd 100755 --- a/scripts/tests/test_bootstrap_70_caddy.sh +++ b/scripts/tests/test_bootstrap_70_caddy.sh @@ -121,8 +121,26 @@ set -e [ "$rc" -eq 1 ] || fail "ungueltige Konfiguration nicht abgelehnt" "$rc" grep -q '^##STEP 70 fail CADDY_CONFIG_INVALID$' <<<"$out" || fail "falscher Code" "$out" +# --- Bundle ohne Caddy-Beipack, aber Caddy ist schon da -------------------- +# Ein aus dem Repo gebautes Bundle traegt das Beipack nur mit --caddy-binary. +# Ein Node, dessen Caddy vom Paketmanager stammt, braucht es nicht: der Schritt +# darf dann nicht an dem fehlenden Binary scheitern, sondern bewahrt das +# vorhandene und prueft nur die Konfiguration. +rm -rf "$bundle/caddy" "$bundle/dashboard/Caddyfile" +installed_caddy "v2.6.2 h1:debian" +: > "$CADDY_LOG"; : > "$SYSTEMCTL_LOG" +out="$(DPKG_RC=0 bash "$script")" || fail "vorhandenes Caddy ohne Beipack: Abbruch" "$out" +grep -q '^##STEP 70 ok$' <<<"$out" || fail "vorhandenes Caddy ohne Beipack: kein ok-Marker" "$out" +grep -q '# vorhandenes caddy' "$tmp/root/usr/bin/caddy" || fail "vorhandenes Caddy wurde angefasst" +grep -q 'caddy validate' "$CADDY_LOG" || fail "ohne Beipack keine Validierung" "$(cat "$CADDY_LOG")" +grep -q 'systemctl enable --now caddy' "$SYSTEMCTL_LOG" || fail "ohne Beipack kein enable --now" +grep -qx ':443 {' "$caddyfile" || fail "vorhandene Caddyfile veraendert" +mkdir -p "$bundle/caddy" "$bundle/dashboard" +printf ':443 {\n tls internal\n}\n' > "$bundle/dashboard/Caddyfile" + # --- fehlendes Beipack ---------------------------------------------------- -rm -rf "$tmp/state" "$bundle/caddy" +# Ohne vorhandenes Caddy hat der Node nichts, was den Schritt tragen koennte. +rm -rf "$tmp/state" "$tmp/root" "$bundle/caddy" set +e out="$(bash "$script")" set -e diff --git a/scripts/tests/test_build_wheels.sh b/scripts/tests/test_build_wheels.sh index 302952b..354641d 100755 --- a/scripts/tests/test_build_wheels.sh +++ b/scripts/tests/test_build_wheels.sh @@ -52,6 +52,39 @@ for pkg in paho-mqtt apsystems-ez1 tinytuya requests; do done [ -d "$tmp/wheels" ] || fail "Zielverzeichnis nicht angelegt" +# --- fehlende Abhaengigkeiten unter den Zielmarkern ------------------------ +# pip wertet Marker wie python_version < "3.13" mit dem Interpreter des +# Bau-Rechners aus, nicht mit dem Ziel: Auf Python 3.14 fehlt sonst +# typing_extensions im Bundle, das aiohttp auf dem Node (3.11) braucht. +missing_py="$here/../build/lib/missing_requirements.py" +mk_wheel() { # ... + local dir="$1" nv="$2"; shift 2 + local name="${nv%%-*}" version="${nv#*-}" meta + meta="$(mktemp -d)" + mkdir -p "$meta/${name}-${version}.dist-info" + { + printf 'Metadata-Version: 2.1\nName: %s\nVersion: %s\n' "$name" "$version" + for req in "$@"; do printf 'Requires-Dist: %s\n' "$req"; done + } > "$meta/${name}-${version}.dist-info/METADATA" + mkdir -p "$dir" + (cd "$meta" && python3 -c 'import sys,zipfile,glob; z=zipfile.ZipFile(sys.argv[1],"w"); [z.write(f) for f in glob.glob("*/*")]' \ + "$dir/${name}-${version}-py3-none-any.whl") + rm -rf "$meta" +} +w="$tmp/closure" +mk_wheel "$w" aiohttp-3.14.3 'typing_extensions>=4.4; python_version < "3.13"' 'attrs>=17' \ + 'old_thing; python_version < "3.9"' 'speedups_only; extra == "speedups"' \ + 'arm_only; platform_machine == "armv6l"' 'other_arch; platform_machine == "x86_64"' +mk_wheel "$w" attrs-24.1.0 +mk_wheel "$w" new_enough-1.0 'attrs>=99' +out="$(python3 "$missing_py" "$w" 3.11 armv6l)" || fail "missing_requirements.py schlug fehl" "$out" +[ "$(sort <<<"$out" | tr '\n' ' ')" = "arm-only attrs>=99 typing-extensions>=4.4 " ] \ + || fail "falsche fehlende Abhaengigkeiten" "$out" +out="$(python3 "$missing_py" "$w" 3.14 x86_64)" || fail "missing_requirements.py (3.14) schlug fehl" +grep -q typing-extensions <<<"$out" && fail "typing_extensions unter 3.14 nicht noetig" "$out" +grep -qx other-arch <<<"$out" || fail "x86_64-Marker nicht ausgewertet" "$out" +[ -z "$(python3 "$missing_py" "$tmp/leer" 3.11 armv6l)" ] || fail "leeres Verzeichnis meldet etwas" + # --- arm64 reicht beide Platform-Tags durch ------------------------------- : > "$PIP_LOG" run fetch_thirdparty_wheels "$tmp/wheels64" arm64 3.11 cp311 >/dev/null diff --git a/services/apsystems_ez1/CHANGELOG.md b/services/apsystems_ez1/CHANGELOG.md index 2f213f9..bd26a83 100644 --- a/services/apsystems_ez1/CHANGELOG.md +++ b/services/apsystems_ez1/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- **services:** give every service its own version and changelog (c2e4e2c) +- **services:** give every service its own version and changelog (#45) (5bc91b8) ## v0.3.4 (2026-09-20) diff --git a/services/automation/CHANGELOG.md b/services/automation/CHANGELOG.md index dd087fd..3d7c996 100644 --- a/services/automation/CHANGELOG.md +++ b/services/automation/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- **services:** give every service its own version and changelog (c2e4e2c) +- **services:** give every service its own version and changelog (#45) (5bc91b8) ## v0.3.2 (2026-09-15) diff --git a/services/battery_soc/CHANGELOG.md b/services/battery_soc/CHANGELOG.md index dd087fd..3d7c996 100644 --- a/services/battery_soc/CHANGELOG.md +++ b/services/battery_soc/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- **services:** give every service its own version and changelog (c2e4e2c) +- **services:** give every service its own version and changelog (#45) (5bc91b8) ## v0.3.2 (2026-09-15) diff --git a/services/shelly/CHANGELOG.md b/services/shelly/CHANGELOG.md index dd087fd..3d7c996 100644 --- a/services/shelly/CHANGELOG.md +++ b/services/shelly/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- **services:** give every service its own version and changelog (c2e4e2c) +- **services:** give every service its own version and changelog (#45) (5bc91b8) ## v0.3.2 (2026-09-15) diff --git a/services/trucki/CHANGELOG.md b/services/trucki/CHANGELOG.md index dd087fd..3d7c996 100644 --- a/services/trucki/CHANGELOG.md +++ b/services/trucki/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- **services:** give every service its own version and changelog (c2e4e2c) +- **services:** give every service its own version and changelog (#45) (5bc91b8) ## v0.3.2 (2026-09-15) diff --git a/services/tuya_mqtt/CHANGELOG.md b/services/tuya_mqtt/CHANGELOG.md index dd087fd..3d7c996 100644 --- a/services/tuya_mqtt/CHANGELOG.md +++ b/services/tuya_mqtt/CHANGELOG.md @@ -4,7 +4,7 @@ ### Features -- **services:** give every service its own version and changelog (c2e4e2c) +- **services:** give every service its own version and changelog (#45) (5bc91b8) ## v0.3.2 (2026-09-15)