From dbb0b96765fa1a9245cbedf34c6b8d8a6b7ba246 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 16:07:15 +0200 Subject: [PATCH 01/35] docs: add implementation plan for the --resume flag Co-Authored-By: Claude Opus 5 (1M context) --- .../plans/2026-08-26-resume-flag.md | 1330 +++++++++++++++++ 1 file changed, 1330 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-26-resume-flag.md diff --git a/docs/superpowers/plans/2026-08-26-resume-flag.md b/docs/superpowers/plans/2026-08-26-resume-flag.md new file mode 100644 index 0000000..e107892 --- /dev/null +++ b/docs/superpowers/plans/2026-08-26-resume-flag.md @@ -0,0 +1,1330 @@ +# Resume Flag Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `export --resume ` so an interrupted export continues from the end of an existing `.ndjson`, `.gz`, or `.gzip` file instead of restarting from the contract's start block. + +**Architecture:** A new `pkg/resume` package reads the tail of a previous export and returns a `Cursor` (last block number + log index + whether the file is gzip). `cmd/export.go` uses that cursor as the start block and appends new logs to the same file — plain files via `O_APPEND`, gzip files by appending a second gzip member (concatenated members are a valid gzip stream, verified against both `gzcat` and Go's `gzip.Reader`). Logs already present in the boundary block are filtered out during the write. + +**Tech Stack:** Go 1.25, cobra, `github.com/ethereum/go-ethereum` v1.15.11 (`core/types.Log`, `common/hexutil`), `github.com/ethersphere/bee/v2` v2.7.0 (`pkg/log`), stdlib `compress/gzip`. + +**Spec:** No separate spec file — this was a bounded change designed and approved in-session. The approved design is reproduced in full under "Design Summary" below; executors should treat that section as the spec. + +## Design Summary + +1. **Flag:** `--resume` / `-r`, a path to a previous export. When set it overrides `--start` and `--output`. +2. **Format detection:** by magic bytes (`0x1f 0x8b`), never by file extension. The repo's own archives use `.gzip` while the request also mentions `.gz`; magic bytes make the extension irrelevant. +3. **Cursor:** the last line of the file that parses as JSON *and* carries both `blockNumber` and `logIndex`. Lines that fail either check are skipped, which discards a truncated trailing line left by a hard kill. +4. **Resume point:** `startBlock = cursor.BlockNumber` **inclusive**, because that block may have been only partially written. While writing, any log with `blockNumber < cursor.BlockNumber`, or `blockNumber == cursor.BlockNumber && logIndex <= cursor.LogIndex`, is skipped. No gaps, no duplicates. +5. **Writing:** the resume file is opened `O_APPEND`. Gzip files get a fresh `gzip.NewWriter` (a new member); plain files get the JSON encoder directly. +6. **`--compress` interaction:** a no-op when resuming an already-gzipped file; unchanged behavior when resuming a plain `.ndjson`. + +## Global Constraints + +- Go version: `go 1.25` (per `go.mod`). Do not raise or lower it. +- Do not add dependencies. Everything needed is already in `go.mod`: stdlib, `go-ethereum`, `bee/v2`, `cobra`. +- Lint config (`.golangci.yml`) enables `copyloopvar`, `errname`, `errorlint`, `goconst`, `misspell`, `nilerr`, `unconvert`, plus `gofmt` and `gofumpt` formatters. `errorlint` means: always wrap with `%w`, always compare with `errors.Is`/`errors.As`. +- Every exported identifier gets a doc comment starting with its own name. +- Error strings are lowercase and unpunctuated, matching the existing `pkg/` style (`"error creating file: %w"`). +- Commit messages use Conventional Commits (`feat:`, `fix:`, `test:`, `docs:`, `refactor:`), matching this repo's history. +- Tests run via `make test`, which is `go test -v ./pkg/...`. The repo currently has **zero** test files; these will be the first. +- Do not reformat or restructure code unrelated to this feature. + +## File Structure + +| File | Status | Responsibility | +|---|---|---| +| `pkg/resume/resume.go` | **Create** | Detect gzip vs plain, find the last complete log line, expose `Cursor` + `Cursor.Skip`. All tail-reading edge cases live here and nowhere else. | +| `pkg/resume/resume_test.go` | **Create** | Table tests for every tail-reading edge case, plus an append→re-read round trip. | +| `pkg/gzipstore/gzipstore.go` | **Modify** | Add `AppendWriter` returning an `io.WriteCloser` that appends a new gzip member. Existing `CompressFile` untouched. | +| `pkg/gzipstore/gzipstore_test.go` | **Create** | Verify an appended member reads back as one continuous stream. | +| `pkg/filestore/filestore.go` | **Modify** | Extract the write loop into an unexported `writeLogs`; add `AppendWriter` and `AppendLogsAsync(ctx, logChan, w, skip)`. `SaveLogsAsync` keeps its current signature. | +| `pkg/filestore/filestore_test.go` | **Create** | Verify truncate-vs-append semantics and that `skip` filters correctly. | +| `cmd/export.go` | **Modify** | Register `--resume`, read the cursor before dialing RPC, pick the writer, warn on overridden flags. Also fix the missing `wg.Wait()` on the cancellation path. | +| `README.md` | **Modify** | Document the flag in the Features list and the flag table. | + +**Why `pkg/resume` is its own package:** backward chunked reads, truncated-line recovery, and multi-member gzip handling are the only genuinely tricky logic in this feature. Isolating them behind `Read(path) (*Cursor, error)` keeps `cmd/export.go` readable and makes the edge cases testable without an RPC endpoint. + +--- + +### Task 1: `pkg/resume` — read the cursor from a previous export + +**Files:** +- Create: `pkg/resume/resume.go` +- Test: `pkg/resume/resume_test.go` + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: + - `type Cursor struct { BlockNumber uint64; LogIndex uint; Compressed bool }` + - `func Read(path string) (*Cursor, error)` + - `func (c *Cursor) Skip(l types.Log) bool` + - `var ErrNoLogs error` + +- [ ] **Step 1: Write the failing tests** + +Create `pkg/resume/resume_test.go`. Note the package is `resume_test` (external) — later tasks add a round-trip test here that imports `filestore` and `gzipstore`, and an external test package keeps that free of import cycles. + +```go +package resume_test + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/batch-export/pkg/resume" +) + +// testLog builds a log shaped like the ones the exporter writes. types.Log +// always marshals blockNumber and logIndex, even at zero. +func testLog(blockNumber uint64, logIndex uint) types.Log { + return types.Log{ + Address: common.HexToAddress("0x45a1502382541cd610cc9068e88727426b696293"), + Topics: []common.Hash{common.HexToHash("0xae46785019700e30375a5d7b4f91e32f8060ef085111f896ebf889450aa2ab5a")}, + Data: bytes.Repeat([]byte{0xab}, 32), + BlockNumber: blockNumber, + TxHash: common.HexToHash("0xb08f07656eaafa8efc458e2aa90773648d95ec8119873d212b4377dea5190cc0"), + TxIndex: 9, + BlockHash: common.HexToHash("0x86dc5f9da5fcba5191f6b3d2ba995bd75532ef369a7baa3970b3fb292ae91324"), + Index: logIndex, + Removed: false, + } +} + +// ndjson renders logs as newline-delimited JSON, the exporter's output format. +func ndjson(t *testing.T, logs ...types.Log) []byte { + t.Helper() + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + for _, l := range logs { + if err := enc.Encode(l); err != nil { + t.Fatalf("encode log: %v", err) + } + } + return buf.Bytes() +} + +// gz compresses b into a single gzip member. +func gz(t *testing.T, b []byte) []byte { + t.Helper() + + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := w.Write(b); err != nil { + t.Fatalf("gzip write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + return buf.Bytes() +} + +// write puts content in a temp file and returns its path. +func write(t *testing.T, name string, content []byte) string { + t.Helper() + + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, content, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + return path +} + +func TestReadCursor(t *testing.T) { + t.Parallel() + + threeLogs := ndjson(t, testLog(100, 0), testLog(101, 1), testLog(102, 7)) + + // many spans several 64 KiB backward-read windows. + manyLogs := make([]types.Log, 0, 2000) + for i := range 2000 { + manyLogs = append(manyLogs, testLog(uint64(1000+i), uint(i%16))) + } + many := ndjson(t, manyLogs...) + + // garbageLines are newline-delimited but unparseable, as if a different + // file had been concatenated onto a good export. + garbageLines := bytes.Repeat([]byte("not-json\n"), 12*1024) + + tests := []struct { + name string + content []byte + wantBlock uint64 + wantIndex uint + wantCompressed bool + }{ + { + name: "plain ndjson", + content: threeLogs, + wantBlock: 102, + wantIndex: 7, + }, + { + name: "single line", + content: ndjson(t, testLog(55, 3)), + wantBlock: 55, + wantIndex: 3, + }, + { + name: "truncated trailing line is discarded", + content: append(append([]byte{}, threeLogs...), []byte(`{"address":"0x45a15","topics":["0xae`)...), + wantBlock: 102, + wantIndex: 7, + }, + { + name: "trailing line without newline is still read", + content: bytes.TrimSuffix(threeLogs, []byte("\n")), + wantBlock: 102, + wantIndex: 7, + }, + { + name: "line missing blockNumber is skipped", + content: append(append([]byte{}, threeLogs...), []byte("{\"address\":\"0x1\",\"topics\":[],\"data\":\"0x\"}\n")...), + wantBlock: 102, + wantIndex: 7, + }, + { + name: "spans multiple backward read windows", + content: many, + wantBlock: 2999, + wantIndex: 15, + }, + { + name: "walks back across windows of garbage lines", + content: append(append([]byte{}, threeLogs...), garbageLines...), + wantBlock: 102, + wantIndex: 7, + }, + { + name: "gzip", + content: gz(t, threeLogs), + wantBlock: 102, + wantIndex: 7, + wantCompressed: true, + }, + { + name: "gzip spanning many logs", + content: gz(t, many), + wantBlock: 2999, + wantIndex: 15, + wantCompressed: true, + }, + { + name: "multi member gzip reads through to the last member", + content: append(gz(t, threeLogs), gz(t, ndjson(t, testLog(200, 2)))...), + wantBlock: 200, + wantIndex: 2, + wantCompressed: true, + }, + { + // A resume interrupted before the second member was flushed: the + // header is present but carries no decodable data. + name: "multi member gzip with unflushed final member", + content: append(gz(t, threeLogs), []byte{0x1f, 0x8b, 0x08, 0, 0, 0, 0, 0, 0, 0xff}...), + wantBlock: 102, + wantIndex: 7, + wantCompressed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := resume.Read(write(t, "export.ndjson", tt.content)) + if err != nil { + t.Fatalf("Read() error = %v, want nil", err) + } + if got.BlockNumber != tt.wantBlock { + t.Errorf("BlockNumber = %d, want %d", got.BlockNumber, tt.wantBlock) + } + if got.LogIndex != tt.wantIndex { + t.Errorf("LogIndex = %d, want %d", got.LogIndex, tt.wantIndex) + } + if got.Compressed != tt.wantCompressed { + t.Errorf("Compressed = %t, want %t", got.Compressed, tt.wantCompressed) + } + }) + } +} + +func TestReadCursorErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + content []byte + }{ + {name: "empty file", content: []byte{}}, + {name: "only a newline", content: []byte("\n")}, + {name: "only garbage lines", content: bytes.Repeat([]byte("not-json\n"), 10)}, + {name: "single unterminated line larger than the cap", content: bytes.Repeat([]byte("x"), 2<<20)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := resume.Read(write(t, "export.ndjson", tt.content)) + if !errors.Is(err, resume.ErrNoLogs) { + t.Fatalf("Read() error = %v, want ErrNoLogs", err) + } + }) + } +} + +func TestReadMissingFile(t *testing.T) { + t.Parallel() + + _, err := resume.Read(filepath.Join(t.TempDir(), "does-not-exist.ndjson")) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("Read() error = %v, want os.ErrNotExist", err) + } +} + +func TestCursorSkip(t *testing.T) { + t.Parallel() + + cursor := &resume.Cursor{BlockNumber: 100, LogIndex: 5} + + tests := []struct { + name string + log types.Log + want bool + }{ + {name: "earlier block", log: testLog(99, 0), want: true}, + {name: "same block earlier index", log: testLog(100, 4), want: true}, + {name: "same block same index", log: testLog(100, 5), want: true}, + {name: "same block later index", log: testLog(100, 6), want: false}, + {name: "later block index zero", log: testLog(101, 0), want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := cursor.Skip(tt.log); got != tt.want { + t.Errorf("Skip() = %t, want %t", got, tt.want) + } + }) + } +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +```bash +go test ./pkg/resume/... +``` + +Expected: FAIL — `no required module provides package github.com/ethersphere/batch-export/pkg/resume` (the package does not exist yet). + +- [ ] **Step 3: Write the implementation** + +Create `pkg/resume/resume.go`: + +```go +// Package resume locates the point at which a previous export stopped so that +// a new run can continue from there. +package resume + +import ( + "bufio" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" +) + +const ( + // windowSize is how much of a plain NDJSON file is read at a time when + // walking backwards from the end. + windowSize = 64 * 1024 + // maxLineBytes caps how long a single line may be. Exported log lines run + // to a few hundred bytes, so anything beyond this is corruption, and the + // cap keeps a file without newlines from being read into memory whole. + maxLineBytes = 1 << 20 +) + +// ErrNoLogs indicates that a file holds no complete log entry to resume from. +var ErrNoLogs = errors.New("no complete log entry found") + +// Cursor marks the last log entry saved by a previous export. +type Cursor struct { + // BlockNumber is the block of the last saved log. A resumed export + // re-queries this block, because an interrupted run may have saved only + // some of its logs. + BlockNumber uint64 + // LogIndex is the index of the last saved log within BlockNumber. + LogIndex uint + // Compressed reports whether the file holds gzip data rather than plain + // NDJSON. + Compressed bool +} + +// Skip reports whether l was already written to the file the cursor came from. +func (c *Cursor) Skip(l types.Log) bool { + if l.BlockNumber != c.BlockNumber { + return l.BlockNumber < c.BlockNumber + } + return l.Index <= c.LogIndex +} + +// cursorLine is the part of an exported log line a cursor is built from. Both +// fields are pointers so that a line missing either one can be rejected. +type cursorLine struct { + BlockNumber *hexutil.Uint64 `json:"blockNumber"` + LogIndex *hexutil.Uint `json:"logIndex"` +} + +// Read returns a cursor for the last complete log entry in the file at path. +// The file may be plain NDJSON or gzip; the format is detected from its +// leading bytes rather than its extension. Lines that do not parse are +// skipped, which discards a partial line left behind by an interrupted write. +func Read(path string) (*Cursor, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening resume file: %w", err) + } + defer file.Close() + + compressed, err := isGzip(file) + if err != nil { + return nil, fmt.Errorf("error reading resume file: %w", err) + } + + var cursor *Cursor + if compressed { + cursor, err = lastCursorGzip(file) + } else { + cursor, err = lastCursorPlain(file) + } + if err != nil { + return nil, err + } + + cursor.Compressed = compressed + + return cursor, nil +} + +// isGzip reports whether file starts with the gzip magic bytes. +func isGzip(file *os.File) (bool, error) { + var magic [2]byte + + n, err := file.ReadAt(magic[:], 0) + if err != nil && !errors.Is(err, io.EOF) { + return false, err + } + if n < len(magic) { + return false, nil + } + + return magic[0] == 0x1f && magic[1] == 0x8b, nil +} + +// lastCursorPlain walks a plain NDJSON file backwards a window at a time and +// returns a cursor for the last line that parses. +func lastCursorPlain(file *os.File) (*Cursor, error) { + offset, err := file.Seek(0, io.SeekEnd) + if err != nil { + return nil, fmt.Errorf("error seeking resume file: %w", err) + } + + // carry holds bytes from the window just read that precede its first + // newline. They belong to a line whose start lies in the next window back. + var carry []byte + + for offset > 0 { + size := int64(windowSize) + if offset < size { + size = offset + } + offset -= size + + window := make([]byte, size) + if _, err := file.ReadAt(window, offset); err != nil { + return nil, fmt.Errorf("error reading resume file: %w", err) + } + window = append(window, carry...) + + for { + i := bytes.LastIndexByte(window, '\n') + if i < 0 { + break + } + if cursor, err := parseCursor(window[i+1:]); err == nil { + return cursor, nil + } + window = window[:i] + } + + carry = window + if len(carry) > maxLineBytes { + return nil, ErrNoLogs + } + } + + return parseCursor(carry) +} + +// lastCursorGzip returns a cursor for the last line that parses in a gzip +// file. Gzip cannot be seeked, so the whole stream is decompressed. A stream +// truncated by an interrupted run still yields the last line it did decode. +func lastCursorGzip(file *os.File) (*Cursor, error) { + reader, err := gzip.NewReader(file) + if err != nil { + return nil, fmt.Errorf("error opening gzip resume file: %w", err) + } + defer reader.Close() + + // Multistream is on by default, so concatenated members read as one stream. + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, bufio.MaxScanTokenSize), maxLineBytes) + + var last *Cursor + for scanner.Scan() { + if cursor, err := parseCursor(scanner.Bytes()); err == nil { + last = cursor + } + } + + if last != nil { + return last, nil + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading gzip resume file: %w", err) + } + + return nil, ErrNoLogs +} + +// parseCursor builds a cursor from a single NDJSON line. It rejects blank and +// truncated lines, and logs that carry no block number. +func parseCursor(line []byte) (*Cursor, error) { + line = bytes.TrimSpace(line) + if len(line) == 0 { + return nil, ErrNoLogs + } + + var parsed cursorLine + if err := json.Unmarshal(line, &parsed); err != nil { + return nil, ErrNoLogs + } + if parsed.BlockNumber == nil || parsed.LogIndex == nil { + return nil, ErrNoLogs + } + + return &Cursor{ + BlockNumber: uint64(*parsed.BlockNumber), + LogIndex: uint(*parsed.LogIndex), + }, nil +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +```bash +go test ./pkg/resume/... && gofumpt -l pkg/resume && go vet ./pkg/resume/... +``` + +Expected: `ok github.com/ethersphere/batch-export/pkg/resume`, no files listed by `gofumpt`, no vet output. + +If `gofumpt` is not installed, run `go run mvdan.cc/gofumpt@latest -l pkg/resume` instead. + +- [ ] **Step 5: Commit** + +```bash +git add pkg/resume/resume.go pkg/resume/resume_test.go +git commit -m "feat(resume): read the last saved log from a previous export" +``` + +--- + +### Task 2: `pkg/gzipstore` — append a gzip member + +**Files:** +- Modify: `pkg/gzipstore/gzipstore.go` (add to the existing file; leave `CompressFile` as-is) +- Test: `pkg/gzipstore/gzipstore_test.go` + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: `func AppendWriter(filePath string) (io.WriteCloser, error)` + +- [ ] **Step 1: Write the failing test** + +Create `pkg/gzipstore/gzipstore_test.go`: + +```go +package gzipstore_test + +import ( + "bytes" + "compress/gzip" + "io" + "os" + "path/filepath" + "testing" + + "github.com/ethersphere/batch-export/pkg/gzipstore" +) + +// writeGzip creates a gzip file holding content and returns its path. +func writeGzip(t *testing.T, content string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "export.ndjson.gzip") + + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := io.WriteString(w, content); err != nil { + t.Fatalf("gzip write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + + return path +} + +// readGzip decompresses the whole file, following every member. +func readGzip(t *testing.T, path string) string { + t.Helper() + + file, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer file.Close() + + reader, err := gzip.NewReader(file) + if err != nil { + t.Fatalf("gzip reader: %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("read gzip: %v", err) + } + + return string(got) +} + +func TestAppendWriterAddsReadableMember(t *testing.T) { + t.Parallel() + + path := writeGzip(t, "first\nsecond\n") + + w, err := gzipstore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + if _, err := io.WriteString(w, "third\nfourth\n"); err != nil { + t.Fatalf("write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + const want = "first\nsecond\nthird\nfourth\n" + if got := readGzip(t, path); got != want { + t.Errorf("content = %q, want %q", got, want) + } +} + +func TestAppendWriterRepeatedAppends(t *testing.T) { + t.Parallel() + + path := writeGzip(t, "a\n") + + for _, line := range []string{"b\n", "c\n"} { + w, err := gzipstore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + if _, err := io.WriteString(w, line); err != nil { + t.Fatalf("write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + } + + const want = "a\nb\nc\n" + if got := readGzip(t, path); got != want { + t.Errorf("content = %q, want %q", got, want) + } +} + +func TestAppendWriterMissingFile(t *testing.T) { + t.Parallel() + + if _, err := gzipstore.AppendWriter(filepath.Join(t.TempDir(), "nope.gzip")); err == nil { + t.Fatal("AppendWriter() error = nil, want an error") + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +go test ./pkg/gzipstore/... +``` + +Expected: FAIL — `undefined: gzipstore.AppendWriter`. + +- [ ] **Step 3: Write the implementation** + +Add to `pkg/gzipstore/gzipstore.go`. Add `"errors"` to the import block; `io` and `os` are already imported. + +```go +// AppendWriter opens an existing gzip file for appending and returns a writer +// that adds a new gzip member to it. Concatenated members form a valid gzip +// stream, so readers see one continuous file and the existing bytes are never +// rewritten. The caller must close the writer to flush the member. +func AppendWriter(filePath string) (io.WriteCloser, error) { + file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return nil, fmt.Errorf("failed to open gzip file '%s' for appending: %w", filePath, err) + } + + return &memberWriter{file: file, gzip: gzip.NewWriter(file)}, nil +} + +// memberWriter writes one gzip member and owns the file it was opened from. +type memberWriter struct { + file *os.File + gzip *gzip.Writer +} + +func (w *memberWriter) Write(p []byte) (int, error) { + return w.gzip.Write(p) +} + +// Close finishes the gzip member and then closes the file. Both are attempted +// even if the first fails, so the descriptor is never leaked. +func (w *memberWriter) Close() error { + return errors.Join(w.gzip.Close(), w.file.Close()) +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +go test ./pkg/gzipstore/... && go vet ./pkg/gzipstore/... +``` + +Expected: `ok github.com/ethersphere/batch-export/pkg/gzipstore`, no vet output. + +- [ ] **Step 5: Commit** + +```bash +git add pkg/gzipstore/gzipstore.go pkg/gzipstore/gzipstore_test.go +git commit -m "feat(gzipstore): add AppendWriter for appending a gzip member" +``` + +--- + +### Task 3: `pkg/filestore` — append logs with a skip filter + +**Files:** +- Modify: `pkg/filestore/filestore.go` (whole file; see the full replacement below) +- Test: `pkg/filestore/filestore_test.go` + +**Interfaces:** +- Consumes: nothing from earlier tasks. `Cursor.Skip` from Task 1 satisfies the `skip` parameter, but this package must not import `pkg/resume` — it takes a plain function so the dependency runs one way only. +- Produces: + - `func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath string) error` (unchanged signature) + - `func AppendWriter(filePath string) (io.WriteCloser, error)` + - `func AppendLogsAsync(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, skip func(types.Log) bool) error` + +- [ ] **Step 1: Write the failing test** + +Create `pkg/filestore/filestore_test.go`: + +```go +package filestore_test + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/batch-export/pkg/filestore" +) + +// blocksIn returns the block number of every log line in the file at path. +func blocksIn(t *testing.T, path string) []uint64 { + t.Helper() + + file, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer file.Close() + + var blocks []uint64 + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.TrimSpace(scanner.Text()) == "" { + continue + } + var l types.Log + if err := json.Unmarshal(scanner.Bytes(), &l); err != nil { + t.Fatalf("unmarshal %q: %v", scanner.Text(), err) + } + blocks = append(blocks, l.BlockNumber) + } + if err := scanner.Err(); err != nil { + t.Fatalf("scan %s: %v", path, err) + } + + return blocks +} + +// feed returns a closed channel already holding logs for the given blocks. +func feed(blocks ...uint64) <-chan types.Log { + ch := make(chan types.Log, len(blocks)) + for _, b := range blocks { + ch <- types.Log{BlockNumber: b} + } + close(ch) + + return ch +} + +func equal(a, b []uint64) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + + return true +} + +func TestSaveLogsAsyncReplacesExistingFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "export.ndjson") + if err := os.WriteFile(path, []byte("stale\n"), 0o644); err != nil { + t.Fatalf("seed %s: %v", path, err) + } + + if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { + t.Fatalf("SaveLogsAsync() error = %v", err) + } + + want := []uint64{1, 2} + if got := blocksIn(t, path); !equal(got, want) { + t.Errorf("blocks = %v, want %v", got, want) + } +} + +func TestAppendLogsAsyncKeepsExistingContent(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "export.ndjson") + if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { + t.Fatalf("SaveLogsAsync() error = %v", err) + } + + w, err := filestore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + if err := filestore.AppendLogsAsync(t.Context(), feed(3, 4), w, nil); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + want := []uint64{1, 2, 3, 4} + if got := blocksIn(t, path); !equal(got, want) { + t.Errorf("blocks = %v, want %v", got, want) + } +} + +func TestAppendLogsAsyncSkipsFilteredLogs(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "export.ndjson") + if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { + t.Fatalf("SaveLogsAsync() error = %v", err) + } + + w, err := filestore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + skip := func(l types.Log) bool { return l.BlockNumber <= 2 } + if err := filestore.AppendLogsAsync(t.Context(), feed(1, 2, 3, 4), w, skip); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + want := []uint64{1, 2, 3, 4} + if got := blocksIn(t, path); !equal(got, want) { + t.Errorf("blocks = %v, want %v", got, want) + } +} + +func TestAppendLogsAsyncClosesWriterOnCancel(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "export.ndjson") + if err := filestore.SaveLogsAsync(t.Context(), feed(1), path); err != nil { + t.Fatalf("SaveLogsAsync() error = %v", err) + } + + w, err := filestore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + // An open channel that never delivers, so cancellation is the only exit. + if err := filestore.AppendLogsAsync(ctx, make(chan types.Log), w, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("AppendLogsAsync() error = %v, want context.Canceled", err) + } + + // The writer must already be closed; closing it again must fail. + if err := w.Close(); err == nil { + t.Error("writer was left open after cancellation") + } +} +``` + +- [ ] **Step 2: Run the test to verify it fails** + +```bash +go test ./pkg/filestore/... +``` + +Expected: FAIL — `undefined: filestore.AppendWriter` and `undefined: filestore.AppendLogsAsync`. + +- [ ] **Step 3: Write the implementation** + +Replace the whole of `pkg/filestore/filestore.go` with: + +```go +package filestore + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + + "github.com/ethereum/go-ethereum/core/types" +) + +// SaveLogsAsync writes logs to a file asynchronously, replacing any file +// already at filePath. +func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath string) error { + file, err := os.Create(filePath) + if err != nil { + return fmt.Errorf("error creating file: %w", err) + } + defer file.Close() + + return writeLogs(ctx, logChan, file, nil) +} + +// AppendWriter opens an existing NDJSON file for appending. +func AppendWriter(filePath string) (io.WriteCloser, error) { + file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return nil, fmt.Errorf("error opening file for appending: %w", err) + } + + return file, nil +} + +// AppendLogsAsync writes logs to w asynchronously, keeping whatever the +// destination already holds. Logs for which skip reports true are dropped; a +// nil skip writes every log. The writer is closed before returning, including +// when the context is cancelled, so a buffered destination is always flushed. +func AppendLogsAsync(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, skip func(types.Log) bool) error { + defer w.Close() + + return writeLogs(ctx, logChan, w, skip) +} + +// writeLogs encodes logs from logChan to w as NDJSON until the channel is +// closed or the context is cancelled. +func writeLogs(ctx context.Context, logChan <-chan types.Log, w io.Writer, skip func(types.Log) bool) error { + encoder := json.NewEncoder(w) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case logObj, ok := <-logChan: + if !ok { + return nil + } + + if skip != nil && skip(logObj) { + continue + } + + if err := encoder.Encode(logObj); err != nil { + return fmt.Errorf("error encoding log: %w", err) + } + } + } +} +``` + +- [ ] **Step 4: Run the test to verify it passes** + +```bash +go test ./pkg/... && go vet ./pkg/... +``` + +Expected: `ok` for `pkg/filestore`, `pkg/gzipstore`, and `pkg/resume`; no vet output. + +- [ ] **Step 5: Commit** + +```bash +git add pkg/filestore/filestore.go pkg/filestore/filestore_test.go +git commit -m "feat(filestore): add AppendLogsAsync with a skip filter" +``` + +--- + +### Task 4: Wire `--resume` into the export command + +**Files:** +- Modify: `cmd/export.go` + +**Interfaces:** +- Consumes: `resume.Read`, `resume.Cursor`, `Cursor.Skip` (Task 1); `gzipstore.AppendWriter` (Task 2); `filestore.AppendWriter`, `filestore.AppendLogsAsync` (Task 3). +- Produces: the `--resume` / `-r` CLI flag. Nothing consumes this task. + +This task has no unit test: `RunE` needs a live RPC endpoint, and the repo has no HTTP fixture harness to build on. The logic worth testing was pushed into `pkg/` by Tasks 1–3 and is covered there. Step 5 verifies this task end to end against the real files in `dist/`. + +- [ ] **Step 1: Add the flag variable and registration** + +In `initExportCmd`, add `resumeFile` to the `var` block at the top: + +```go + var ( + startBlock uint64 + endBlock uint64 + rpcEndpoint string + maxRequest int + blockRangeLimit uint32 + outputFile string + compress bool + resumeFile string + ) +``` + +And register the flag alongside the others, after the `--compress` line: + +```go + cmd.Flags().StringVarP(&resumeFile, "resume", "r", "", "Resume a previous export file (.ndjson, .gz or .gzip); overrides --start and --output") +``` + +- [ ] **Step 2: Read the cursor at the top of RunE** + +Insert this as the first statement inside `RunE`, immediately after `ctx := cmd.Context()` and **before** `ethclient.NewClient`. Reading the cursor first means a bad path fails instantly instead of after dialing the RPC endpoint. + +```go + var cursor *resume.Cursor + if resumeFile != "" { + cursor, err = resume.Read(resumeFile) + if err != nil { + return fmt.Errorf("failed to read resume file: %w", err) + } + + if cmd.Flags().Changed("start") { + c.log.Warning("--start is ignored when --resume is set", "resumeFile", resumeFile) + } + if cmd.Flags().Changed("output") { + c.log.Warning("--output is ignored when --resume is set, logs are appended to the resume file", "resumeFile", resumeFile) + } + if cursor.Compressed && compress { + c.log.Warning("--compress is ignored when resuming an already compressed file", "resumeFile", resumeFile) + compress = false + } + + outputFile = resumeFile + startBlock = cursor.BlockNumber + + c.log.Info("Resuming export", + "resumeFile", resumeFile, + "startBlock", startBlock, + "lastLogIndex", cursor.LogIndex, + "compressed", cursor.Compressed, + ) + } +``` + +Then add the imports. The `import` block becomes: + +```go +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "time" + + ethclient "github.com/ethersphere/batch-export/pkg/ethclientwrapper" + "github.com/ethersphere/batch-export/pkg/eventfetcher" + "github.com/ethersphere/batch-export/pkg/filestore" + "github.com/ethersphere/batch-export/pkg/gzipstore" + "github.com/ethersphere/batch-export/pkg/resume" + "github.com/ethersphere/bee/v2/pkg/config" + "github.com/ethersphere/bee/v2/pkg/util/abiutil" + "github.com/spf13/cobra" +) +``` + +- [ ] **Step 3: Branch the writer goroutine** + +Replace the existing saver goroutine — the block from `go func() {` through the closing `}()` that currently calls `filestore.SaveLogsAsync` — with: + +```go + go func() { + defer wg.Done() + + if err := saveLogs(ctx, logChan, outputFile, cursor); err != nil { + if errors.Is(err, context.Canceled) { + c.log.Error(err, "context canceled while saving logs") + return + } + c.log.Error(err, "error saving logs") + return + } + c.log.Info("all logs have been saved", "outputFile", outputFile) + }() +``` + +Then add this helper at the end of `cmd/export.go`, after `initExportCmd`: + +```go +// saveLogs writes logs to outputFile. With a nil cursor it replaces the file; +// otherwise it appends to the file the cursor came from, dropping any log that +// was already written to it. +func saveLogs(ctx context.Context, logChan <-chan types.Log, outputFile string, cursor *resume.Cursor) error { + if cursor == nil { + return filestore.SaveLogsAsync(ctx, logChan, outputFile) + } + + var ( + w io.WriteCloser + err error + ) + if cursor.Compressed { + w, err = gzipstore.AppendWriter(outputFile) + } else { + w, err = filestore.AppendWriter(outputFile) + } + if err != nil { + return fmt.Errorf("error opening output file for appending: %w", err) + } + + return filestore.AppendLogsAsync(ctx, logChan, w, cursor.Skip) +} +``` + +`saveLogs` takes `types.Log`, so add one more import to the block from Step 2, in the third-party group above the `batch-export` imports: + +```go + "github.com/ethereum/go-ethereum/core/types" +``` + +- [ ] **Step 4: Wait for the saver before returning on cancellation** + +The existing `<-ctx.Done()` branch logs `"context canceled, waiting for logs to be saved..."` but returns without ever waiting, so the saver goroutine can be killed mid-write. That was survivable when every write was a lone `Encode` call; it is not survivable now, because an unflushed gzip member leaves a trailing fragment that the next `--resume` has to discard. Fix the branch to actually wait: + +```go + case <-ctx.Done(): + c.log.Info("context canceled, waiting for logs to be saved...") + wg.Wait() + if err := compressFunc(); err != nil { + return errors.Join(fmt.Errorf("error compressing file: %w", err), ctx.Err()) + } + return ctx.Err() +``` + +This cannot deadlock: on cancellation `writeLogs` returns from its `ctx.Done()` case immediately, and `AppendLogsAsync`'s deferred `Close` flushes the gzip member on the way out. + +- [ ] **Step 5: Build and verify against the real export files** + +```bash +make binary && go vet ./... && gofumpt -l cmd pkg +``` + +Expected: the binary builds, no vet output, no files listed by `gofumpt`. + +Then confirm the flag is registered and that both file formats are read correctly. `dist/export.ndjson` ends at block `0x2db0697` (47908503), log index `0x18` (24): + +```bash +./dist/batch-export export --help | grep -A1 resume + +# Plain NDJSON: cursor must be block 47908503. +./dist/batch-export export -v debug --resume dist/export.ndjson --end 47908504 2>&1 | head -20 + +# Gzip: same cursor, read through the compressed stream. +./dist/batch-export export -v debug --resume dist/export.ndjson.gzip --end 47908504 2>&1 | head -20 +``` + +Expected: both runs log `"Resuming export"` with `startBlock=47908503` and `lastLogIndex=24`, the second also with `compressed=true`. Both then fetch the one-block range and exit cleanly. + +Verify the appended gzip is still one readable stream, and that no duplicate was written: + +```bash +cp dist/export.ndjson.gzip /tmp/resume-check.gzip +BEFORE=$(gzcat /tmp/resume-check.gzip | wc -l) +./dist/batch-export export --resume /tmp/resume-check.gzip --end 47908504 +gzcat /tmp/resume-check.gzip | wc -l # >= BEFORE, and must not error +gzcat /tmp/resume-check.gzip | tail -n 3 +gzcat /tmp/resume-check.gzip | sort | uniq -d | head # must print nothing +echo "before=$BEFORE" +``` + +Expected: `gzcat` exits 0, the line count is at least `BEFORE`, and the duplicate check prints nothing. + +Finally, confirm the flag-override warnings fire: + +```bash +./dist/batch-export export --resume dist/export.ndjson --start 100 --output other.ndjson --compress --end 47908504 2>&1 | grep -i "ignored" +``` + +Expected: warnings for `--start` and `--output`. `--compress` is not warned about here because `dist/export.ndjson` is not compressed; re-run with `--resume dist/export.ndjson.gzip` to see that third warning. + +- [ ] **Step 6: Commit** + +```bash +git add cmd/export.go +git commit -m "feat(export): add --resume to continue a previous export" +``` + +--- + +### Task 5: Document the flag + +**Files:** +- Modify: `README.md` + +**Interfaces:** +- Consumes: the `--resume` flag from Task 4. +- Produces: nothing. + +- [ ] **Step 1: Add a feature bullet** + +In the `## Features` list, after the `Graceful shutdown on interrupt signals (Ctrl+C).` bullet, add: + +```markdown +- Resume an interrupted export from an existing `.ndjson`, `.gz`, or `.gzip` file. +``` + +- [ ] **Step 2: Add the flag to the flag table** + +In the `## Flags` block, insert the `--resume` line between `--output` and `--start` so the list stays alphabetical: + +```sh + -r, --resume string Resume a previous export file (.ndjson, .gz or .gzip); overrides --start and --output +``` + +- [ ] **Step 3: Document the behavior** + +After the `## Flags` code block and before the `The produced NDJSON is consumed by ...` line, add: + +````markdown +### Resuming an interrupted export + +Point `--resume` at a file a previous run produced. The tool reads its last +complete entry, restarts from that block, and appends to the same file: + +```sh +./dist/batch-export export --resume dist/export.ndjson +``` + +Compressed exports work the same way and are detected by content, not by +extension, so `.gz` and `.gzip` both work: + +```sh +./dist/batch-export export --resume dist/export.ndjson.gzip +``` + +The resumed block is re-queried, because an interrupted run may have saved only +part of it; entries already in the file are skipped, so resuming never +duplicates or drops a log. Appending to a compressed export adds a second gzip +member — standard tools such as `gzcat`, `gunzip`, and Go's `compress/gzip` +read the result as one continuous stream. + +When `--resume` is set it overrides `--start` and `--output`. +```` + +- [ ] **Step 4: Verify the whole suite still passes** + +```bash +make test && make vet && make lint +``` + +Expected: all `pkg/` tests pass, no vet output, no lint findings. + +- [ ] **Step 5: Commit** + +```bash +git add README.md +git commit -m "docs: document the --resume flag" +``` + +--- + +## Self-Review + +**Spec coverage** — every point of the Design Summary maps to a task: + +| Design point | Task | +|---|---| +| 1. `--resume` / `-r` flag, overrides `--start` / `--output` | Task 4 Steps 1–2 | +| 2. Magic-byte format detection | Task 1 (`isGzip`) | +| 3. Last parseable line; truncated tail discarded | Task 1 (`parseCursor`, `lastCursorPlain`, `lastCursorGzip`) | +| 4. Inclusive resume block + skip already-written logs | Task 1 (`Cursor.Skip`), Task 3 (`skip` param), Task 4 Step 2 | +| 5. `O_APPEND`; gzip gets a new member | Task 2, Task 3 (`AppendWriter`), Task 4 Step 3 | +| 6. `--compress` no-op on already-gzipped input | Task 4 Step 2 | + +**Type consistency** — `Cursor.Skip(types.Log) bool` (Task 1) matches the `skip func(types.Log) bool` parameter of `AppendLogsAsync` (Task 3) and the `cursor.Skip` value passed in Task 4. Both `AppendWriter` functions return `(io.WriteCloser, error)`, which is what `saveLogs` assigns into its `io.WriteCloser` variable. `SaveLogsAsync` keeps its existing three-argument signature, so its call site in Task 4 needs no change beyond moving into `saveLogs`. + +**Dependency direction** — `pkg/filestore` takes a bare `func(types.Log) bool` rather than importing `pkg/resume`, so `cmd` depends on all three packages while none of them depend on each other. + +**Known gaps, deliberate:** +- `cmd/export.go` has no unit test (no RPC fixture harness exists in this repo); Task 4 Step 5 covers it manually against the real `dist/` files instead. +- Resume assumes the file was produced by this tool against the same contract and chain. A file from a different chain would resume from a meaningless block. Guarding that would mean writing a header, which changes the output format that `batch-archive` consumes — out of scope here. +- A multi-member gzip is legal and read transparently by standard tooling, but is a format change in spirit. If anything in [batch-archive](https://github.com/ethersphere/batch-archive) parses gzip by hand instead of through a standard library, that is the one place to check. From 8b1c89f00f0b6a30c861ab5d2e7630f35d28b6ee Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 16:10:16 +0200 Subject: [PATCH 02/35] feat(resume): read the last saved log from a previous export --- pkg/resume/resume.go | 202 ++++++++++++++++++++++++++++++ pkg/resume/resume_test.go | 251 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 453 insertions(+) create mode 100644 pkg/resume/resume.go create mode 100644 pkg/resume/resume_test.go diff --git a/pkg/resume/resume.go b/pkg/resume/resume.go new file mode 100644 index 0000000..435e441 --- /dev/null +++ b/pkg/resume/resume.go @@ -0,0 +1,202 @@ +// Package resume locates the point at which a previous export stopped so that +// a new run can continue from there. +package resume + +import ( + "bufio" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" +) + +const ( + // windowSize is how much of a plain NDJSON file is read at a time when + // walking backwards from the end. + windowSize = 64 * 1024 + // maxLineBytes caps how long a single line may be. Exported log lines run + // to a few hundred bytes, so anything beyond this is corruption, and the + // cap keeps a file without newlines from being read into memory whole. + maxLineBytes = 1 << 20 +) + +// ErrNoLogs indicates that a file holds no complete log entry to resume from. +var ErrNoLogs = errors.New("no complete log entry found") + +// Cursor marks the last log entry saved by a previous export. +type Cursor struct { + // BlockNumber is the block of the last saved log. A resumed export + // re-queries this block, because an interrupted run may have saved only + // some of its logs. + BlockNumber uint64 + // LogIndex is the index of the last saved log within BlockNumber. + LogIndex uint + // Compressed reports whether the file holds gzip data rather than plain + // NDJSON. + Compressed bool +} + +// Skip reports whether l was already written to the file the cursor came from. +func (c *Cursor) Skip(l types.Log) bool { + if l.BlockNumber != c.BlockNumber { + return l.BlockNumber < c.BlockNumber + } + return l.Index <= c.LogIndex +} + +// cursorLine is the part of an exported log line a cursor is built from. Both +// fields are pointers so that a line missing either one can be rejected. +type cursorLine struct { + BlockNumber *hexutil.Uint64 `json:"blockNumber"` + LogIndex *hexutil.Uint `json:"logIndex"` +} + +// Read returns a cursor for the last complete log entry in the file at path. +// The file may be plain NDJSON or gzip; the format is detected from its +// leading bytes rather than its extension. Lines that do not parse are +// skipped, which discards a partial line left behind by an interrupted write. +func Read(path string) (*Cursor, error) { + file, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("error opening resume file: %w", err) + } + defer file.Close() + + compressed, err := isGzip(file) + if err != nil { + return nil, fmt.Errorf("error reading resume file: %w", err) + } + + var cursor *Cursor + if compressed { + cursor, err = lastCursorGzip(file) + } else { + cursor, err = lastCursorPlain(file) + } + if err != nil { + return nil, err + } + + cursor.Compressed = compressed + + return cursor, nil +} + +// isGzip reports whether file starts with the gzip magic bytes. +func isGzip(file *os.File) (bool, error) { + var magic [2]byte + + n, err := file.ReadAt(magic[:], 0) + if err != nil && !errors.Is(err, io.EOF) { + return false, err + } + if n < len(magic) { + return false, nil + } + + return magic[0] == 0x1f && magic[1] == 0x8b, nil +} + +// lastCursorPlain walks a plain NDJSON file backwards a window at a time and +// returns a cursor for the last line that parses. +func lastCursorPlain(file *os.File) (*Cursor, error) { + offset, err := file.Seek(0, io.SeekEnd) + if err != nil { + return nil, fmt.Errorf("error seeking resume file: %w", err) + } + + // carry holds bytes from the window just read that precede its first + // newline. They belong to a line whose start lies in the next window back. + var carry []byte + + for offset > 0 { + size := int64(windowSize) + if offset < size { + size = offset + } + offset -= size + + window := make([]byte, size) + if _, err := file.ReadAt(window, offset); err != nil { + return nil, fmt.Errorf("error reading resume file: %w", err) + } + window = append(window, carry...) + + for { + i := bytes.LastIndexByte(window, '\n') + if i < 0 { + break + } + if cursor, err := parseCursor(window[i+1:]); err == nil { + return cursor, nil + } + window = window[:i] + } + + carry = window + if len(carry) > maxLineBytes { + return nil, ErrNoLogs + } + } + + return parseCursor(carry) +} + +// lastCursorGzip returns a cursor for the last line that parses in a gzip +// file. Gzip cannot be seeked, so the whole stream is decompressed. A stream +// truncated by an interrupted run still yields the last line it did decode. +func lastCursorGzip(file *os.File) (*Cursor, error) { + reader, err := gzip.NewReader(file) + if err != nil { + return nil, fmt.Errorf("error opening gzip resume file: %w", err) + } + defer reader.Close() + + // Multistream is on by default, so concatenated members read as one stream. + scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, bufio.MaxScanTokenSize), maxLineBytes) + + var last *Cursor + for scanner.Scan() { + if cursor, err := parseCursor(scanner.Bytes()); err == nil { + last = cursor + } + } + + if last != nil { + return last, nil + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("error reading gzip resume file: %w", err) + } + + return nil, ErrNoLogs +} + +// parseCursor builds a cursor from a single NDJSON line. It rejects blank and +// truncated lines, and logs that carry no block number. +func parseCursor(line []byte) (*Cursor, error) { + line = bytes.TrimSpace(line) + if len(line) == 0 { + return nil, ErrNoLogs + } + + var parsed cursorLine + if err := json.Unmarshal(line, &parsed); err != nil { + return nil, ErrNoLogs + } + if parsed.BlockNumber == nil || parsed.LogIndex == nil { + return nil, ErrNoLogs + } + + return &Cursor{ + BlockNumber: uint64(*parsed.BlockNumber), + LogIndex: uint(*parsed.LogIndex), + }, nil +} diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go new file mode 100644 index 0000000..9801d10 --- /dev/null +++ b/pkg/resume/resume_test.go @@ -0,0 +1,251 @@ +package resume_test + +import ( + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/batch-export/pkg/resume" +) + +// testLog builds a log shaped like the ones the exporter writes. types.Log +// always marshals blockNumber and logIndex, even at zero. +func testLog(blockNumber uint64, logIndex uint) types.Log { + return types.Log{ + Address: common.HexToAddress("0x45a1502382541cd610cc9068e88727426b696293"), + Topics: []common.Hash{common.HexToHash("0xae46785019700e30375a5d7b4f91e32f8060ef085111f896ebf889450aa2ab5a")}, + Data: bytes.Repeat([]byte{0xab}, 32), + BlockNumber: blockNumber, + TxHash: common.HexToHash("0xb08f07656eaafa8efc458e2aa90773648d95ec8119873d212b4377dea5190cc0"), + TxIndex: 9, + BlockHash: common.HexToHash("0x86dc5f9da5fcba5191f6b3d2ba995bd75532ef369a7baa3970b3fb292ae91324"), + Index: logIndex, + Removed: false, + } +} + +// ndjson renders logs as newline-delimited JSON, the exporter's output format. +func ndjson(t *testing.T, logs ...types.Log) []byte { + t.Helper() + + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + for _, l := range logs { + if err := enc.Encode(l); err != nil { + t.Fatalf("encode log: %v", err) + } + } + return buf.Bytes() +} + +// gz compresses b into a single gzip member. +func gz(t *testing.T, b []byte) []byte { + t.Helper() + + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := w.Write(b); err != nil { + t.Fatalf("gzip write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + return buf.Bytes() +} + +// write puts content in a temp file and returns its path. +func write(t *testing.T, name string, content []byte) string { + t.Helper() + + path := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(path, content, 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + return path +} + +func TestReadCursor(t *testing.T) { + t.Parallel() + + threeLogs := ndjson(t, testLog(100, 0), testLog(101, 1), testLog(102, 7)) + + // many spans several 64 KiB backward-read windows. + manyLogs := make([]types.Log, 0, 2000) + for i := range 2000 { + manyLogs = append(manyLogs, testLog(uint64(1000+i), uint(i%16))) + } + many := ndjson(t, manyLogs...) + + // garbageLines are newline-delimited but unparseable, as if a different + // file had been concatenated onto a good export. + garbageLines := bytes.Repeat([]byte("not-json\n"), 12*1024) + + tests := []struct { + name string + content []byte + wantBlock uint64 + wantIndex uint + wantCompressed bool + }{ + { + name: "plain ndjson", + content: threeLogs, + wantBlock: 102, + wantIndex: 7, + }, + { + name: "single line", + content: ndjson(t, testLog(55, 3)), + wantBlock: 55, + wantIndex: 3, + }, + { + name: "truncated trailing line is discarded", + content: append(append([]byte{}, threeLogs...), []byte(`{"address":"0x45a15","topics":["0xae`)...), + wantBlock: 102, + wantIndex: 7, + }, + { + name: "trailing line without newline is still read", + content: bytes.TrimSuffix(threeLogs, []byte("\n")), + wantBlock: 102, + wantIndex: 7, + }, + { + name: "line missing blockNumber is skipped", + content: append(append([]byte{}, threeLogs...), []byte("{\"address\":\"0x1\",\"topics\":[],\"data\":\"0x\"}\n")...), + wantBlock: 102, + wantIndex: 7, + }, + { + name: "spans multiple backward read windows", + content: many, + wantBlock: 2999, + wantIndex: 15, + }, + { + name: "walks back across windows of garbage lines", + content: append(append([]byte{}, threeLogs...), garbageLines...), + wantBlock: 102, + wantIndex: 7, + }, + { + name: "gzip", + content: gz(t, threeLogs), + wantBlock: 102, + wantIndex: 7, + wantCompressed: true, + }, + { + name: "gzip spanning many logs", + content: gz(t, many), + wantBlock: 2999, + wantIndex: 15, + wantCompressed: true, + }, + { + name: "multi member gzip reads through to the last member", + content: append(gz(t, threeLogs), gz(t, ndjson(t, testLog(200, 2)))...), + wantBlock: 200, + wantIndex: 2, + wantCompressed: true, + }, + { + // A resume interrupted before the second member was flushed: the + // header is present but carries no decodable data. + name: "multi member gzip with unflushed final member", + content: append(gz(t, threeLogs), []byte{0x1f, 0x8b, 0x08, 0, 0, 0, 0, 0, 0, 0xff}...), + wantBlock: 102, + wantIndex: 7, + wantCompressed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got, err := resume.Read(write(t, "export.ndjson", tt.content)) + if err != nil { + t.Fatalf("Read() error = %v, want nil", err) + } + if got.BlockNumber != tt.wantBlock { + t.Errorf("BlockNumber = %d, want %d", got.BlockNumber, tt.wantBlock) + } + if got.LogIndex != tt.wantIndex { + t.Errorf("LogIndex = %d, want %d", got.LogIndex, tt.wantIndex) + } + if got.Compressed != tt.wantCompressed { + t.Errorf("Compressed = %t, want %t", got.Compressed, tt.wantCompressed) + } + }) + } +} + +func TestReadCursorErrors(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + content []byte + }{ + {name: "empty file", content: []byte{}}, + {name: "only a newline", content: []byte("\n")}, + {name: "only garbage lines", content: bytes.Repeat([]byte("not-json\n"), 10)}, + {name: "single unterminated line larger than the cap", content: bytes.Repeat([]byte("x"), 2<<20)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := resume.Read(write(t, "export.ndjson", tt.content)) + if !errors.Is(err, resume.ErrNoLogs) { + t.Fatalf("Read() error = %v, want ErrNoLogs", err) + } + }) + } +} + +func TestReadMissingFile(t *testing.T) { + t.Parallel() + + _, err := resume.Read(filepath.Join(t.TempDir(), "does-not-exist.ndjson")) + if !errors.Is(err, os.ErrNotExist) { + t.Fatalf("Read() error = %v, want os.ErrNotExist", err) + } +} + +func TestCursorSkip(t *testing.T) { + t.Parallel() + + cursor := &resume.Cursor{BlockNumber: 100, LogIndex: 5} + + tests := []struct { + name string + log types.Log + want bool + }{ + {name: "earlier block", log: testLog(99, 0), want: true}, + {name: "same block earlier index", log: testLog(100, 4), want: true}, + {name: "same block same index", log: testLog(100, 5), want: true}, + {name: "same block later index", log: testLog(100, 6), want: false}, + {name: "later block index zero", log: testLog(101, 0), want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := cursor.Skip(tt.log); got != tt.want { + t.Errorf("Skip() = %t, want %t", got, tt.want) + } + }) + } +} From 47dfba66c2a4378a14a9851086789304ff75bd74 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 16:24:52 +0200 Subject: [PATCH 03/35] test(resume): cover a valid line straddling a window boundary TestReadCursor never exercised a valid log line split across the 64 KiB backward-read window boundary in lastCursorPlain; every prior case had the target line either wholly inside the last window or was garbage that fails to parse regardless of reconstruction order. Add a case that deterministically positions a real line across the boundary and asserts the straddle before running, so a broken carry concatenation order regresses this case specifically. --- pkg/resume/resume_test.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index 9801d10..21d1ac8 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -14,6 +14,12 @@ import ( "github.com/ethersphere/batch-export/pkg/resume" ) +// windowSize mirrors the unexported constant of the same name in resume.go: +// how much of a plain NDJSON file lastCursorPlain reads at a time when +// walking backwards from EOF. Tests use it to construct cases that straddle +// a window boundary. +const windowSize = 64 * 1024 + // testLog builds a log shaped like the ones the exporter writes. types.Log // always marshals blockNumber and logIndex, even at zero. func testLog(blockNumber uint64, logIndex uint) types.Log { @@ -86,6 +92,33 @@ func TestReadCursor(t *testing.T) { // file had been concatenated onto a good export. garbageLines := bytes.Repeat([]byte("not-json\n"), 12*1024) + // straddle places a single valid log line so that it straddles the + // boundary between the last two 64 KiB windows lastCursorPlain reads + // backwards from EOF: part of the line falls in the final window, part + // falls in the window before it. Reconstructing it requires appending + // each newly (re-)read window before the previously carried tail, in + // file order — swap that order and the line comes out garbled and + // unparseable, which a case where the target line sits wholly inside one + // window can never catch. + straddleTarget := ndjson(t, testLog(4096, 3)) + straddlePrefix := bytes.Repeat([]byte("not-json\n"), 4) + // afterLen is chosen so that windowSize-many bytes from EOF lands + // strictly inside straddleTarget: it is less than windowSize (so the + // boundary is not beyond the end of the line) and more than + // windowSize-len(straddleTarget) (so the boundary is not before the + // line's start). + afterLen := windowSize - len(straddleTarget)/2 + filler := []byte("not-json\n") + straddleAfter := bytes.Repeat(filler, afterLen/len(filler)+2)[:afterLen] + straddle := append(append(append([]byte{}, straddlePrefix...), straddleTarget...), straddleAfter...) + + straddleLineStart := len(straddlePrefix) + straddleLineEnd := len(straddlePrefix) + len(straddleTarget) + straddleBoundary := len(straddle) - windowSize + if straddleBoundary <= straddleLineStart || straddleBoundary >= straddleLineEnd { + t.Fatalf("test setup: boundary %d does not straddle target line [%d, %d)", straddleBoundary, straddleLineStart, straddleLineEnd) + } + tests := []struct { name string content []byte @@ -135,6 +168,12 @@ func TestReadCursor(t *testing.T) { wantBlock: 102, wantIndex: 7, }, + { + name: "valid line straddles a window boundary", + content: straddle, + wantBlock: 4096, + wantIndex: 3, + }, { name: "gzip", content: gz(t, threeLogs), From e216130077d716fe10b418de848d99fec189ec38 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 16:31:38 +0200 Subject: [PATCH 04/35] feat(gzipstore): add AppendWriter for appending a gzip member --- pkg/gzipstore/gzipstore.go | 30 +++++++++ pkg/gzipstore/gzipstore_test.go | 111 ++++++++++++++++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 pkg/gzipstore/gzipstore_test.go diff --git a/pkg/gzipstore/gzipstore.go b/pkg/gzipstore/gzipstore.go index 6763ad0..5aa8b78 100644 --- a/pkg/gzipstore/gzipstore.go +++ b/pkg/gzipstore/gzipstore.go @@ -2,6 +2,7 @@ package gzipstore import ( "compress/gzip" + "errors" "fmt" "io" "os" @@ -35,3 +36,32 @@ func CompressFile(inputFilePath string, outputFilePath string) error { return nil } + +// AppendWriter opens an existing gzip file for appending and returns a writer +// that adds a new gzip member to it. Concatenated members form a valid gzip +// stream, so readers see one continuous file and the existing bytes are never +// rewritten. The caller must close the writer to flush the member. +func AppendWriter(filePath string) (io.WriteCloser, error) { + file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return nil, fmt.Errorf("failed to open gzip file '%s' for appending: %w", filePath, err) + } + + return &memberWriter{file: file, gzip: gzip.NewWriter(file)}, nil +} + +// memberWriter writes one gzip member and owns the file it was opened from. +type memberWriter struct { + file *os.File + gzip *gzip.Writer +} + +func (w *memberWriter) Write(p []byte) (int, error) { + return w.gzip.Write(p) +} + +// Close finishes the gzip member and then closes the file. Both are attempted +// even if the first fails, so the descriptor is never leaked. +func (w *memberWriter) Close() error { + return errors.Join(w.gzip.Close(), w.file.Close()) +} diff --git a/pkg/gzipstore/gzipstore_test.go b/pkg/gzipstore/gzipstore_test.go new file mode 100644 index 0000000..ad63b68 --- /dev/null +++ b/pkg/gzipstore/gzipstore_test.go @@ -0,0 +1,111 @@ +package gzipstore_test + +import ( + "bytes" + "compress/gzip" + "io" + "os" + "path/filepath" + "testing" + + "github.com/ethersphere/batch-export/pkg/gzipstore" +) + +// writeGzip creates a gzip file holding content and returns its path. +func writeGzip(t *testing.T, content string) string { + t.Helper() + + path := filepath.Join(t.TempDir(), "export.ndjson.gzip") + + var buf bytes.Buffer + w := gzip.NewWriter(&buf) + if _, err := io.WriteString(w, content); err != nil { + t.Fatalf("gzip write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("gzip close: %v", err) + } + if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } + + return path +} + +// readGzip decompresses the whole file, following every member. +func readGzip(t *testing.T, path string) string { + t.Helper() + + file, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer file.Close() + + reader, err := gzip.NewReader(file) + if err != nil { + t.Fatalf("gzip reader: %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("read gzip: %v", err) + } + + return string(got) +} + +func TestAppendWriterAddsReadableMember(t *testing.T) { + t.Parallel() + + path := writeGzip(t, "first\nsecond\n") + + w, err := gzipstore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + if _, err := io.WriteString(w, "third\nfourth\n"); err != nil { + t.Fatalf("write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + + const want = "first\nsecond\nthird\nfourth\n" + if got := readGzip(t, path); got != want { + t.Errorf("content = %q, want %q", got, want) + } +} + +func TestAppendWriterRepeatedAppends(t *testing.T) { + t.Parallel() + + path := writeGzip(t, "a\n") + + for _, line := range []string{"b\n", "c\n"} { + w, err := gzipstore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + if _, err := io.WriteString(w, line); err != nil { + t.Fatalf("write: %v", err) + } + if err := w.Close(); err != nil { + t.Fatalf("Close() error = %v", err) + } + } + + const want = "a\nb\nc\n" + if got := readGzip(t, path); got != want { + t.Errorf("content = %q, want %q", got, want) + } +} + +func TestAppendWriterMissingFile(t *testing.T) { + t.Parallel() + + if _, err := gzipstore.AppendWriter(filepath.Join(t.TempDir(), "nope.gzip")); err == nil { + t.Fatal("AppendWriter() error = nil, want an error") + } +} From 532ffb197c894d50be7cf15f8bbabc3f24cbda2e Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 16:41:17 +0200 Subject: [PATCH 05/35] feat(filestore): add AppendLogsAsync with a skip filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds AppendWriter and AppendLogsAsync alongside the existing SaveLogsAsync so a resumed export can append to an existing NDJSON file instead of overwriting it, dropping logs a skip function (e.g. resume.Cursor.Skip) reports as already written. filestore takes a bare func(types.Log) bool rather than importing pkg/resume, keeping the dependency one-directional. Test helper fix beyond the brief: feed() must give each log a non-nil Topics slice — go-ethereum's generated Log.UnmarshalJSON rejects a null "topics" field as missing, so a nil Topics (the brief's literal feed()) fails blocksIn's round trip regardless of the implementation under test. --- pkg/filestore/filestore.go | 36 +++++++- pkg/filestore/filestore_test.go | 151 ++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 pkg/filestore/filestore_test.go diff --git a/pkg/filestore/filestore.go b/pkg/filestore/filestore.go index 9617158..f93f256 100644 --- a/pkg/filestore/filestore.go +++ b/pkg/filestore/filestore.go @@ -4,12 +4,14 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "github.com/ethereum/go-ethereum/core/types" ) -// SaveLogsAsync writes logs to a file asynchronously. +// SaveLogsAsync writes logs to a file asynchronously, replacing any file +// already at filePath. func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath string) error { file, err := os.Create(filePath) if err != nil { @@ -17,7 +19,33 @@ func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath strin } defer file.Close() - encoder := json.NewEncoder(file) + return writeLogs(ctx, logChan, file, nil) +} + +// AppendWriter opens an existing NDJSON file for appending. +func AppendWriter(filePath string) (io.WriteCloser, error) { + file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, 0o644) + if err != nil { + return nil, fmt.Errorf("error opening file for appending: %w", err) + } + + return file, nil +} + +// AppendLogsAsync writes logs to w asynchronously, keeping whatever the +// destination already holds. Logs for which skip reports true are dropped; a +// nil skip writes every log. The writer is closed before returning, including +// when the context is cancelled, so a buffered destination is always flushed. +func AppendLogsAsync(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, skip func(types.Log) bool) error { + defer w.Close() + + return writeLogs(ctx, logChan, w, skip) +} + +// writeLogs encodes logs from logChan to w as NDJSON until the channel is +// closed or the context is cancelled. +func writeLogs(ctx context.Context, logChan <-chan types.Log, w io.Writer, skip func(types.Log) bool) error { + encoder := json.NewEncoder(w) for { select { @@ -28,6 +56,10 @@ func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath strin return nil } + if skip != nil && skip(logObj) { + continue + } + if err := encoder.Encode(logObj); err != nil { return fmt.Errorf("error encoding log: %w", err) } diff --git a/pkg/filestore/filestore_test.go b/pkg/filestore/filestore_test.go new file mode 100644 index 0000000..aa69f22 --- /dev/null +++ b/pkg/filestore/filestore_test.go @@ -0,0 +1,151 @@ +package filestore_test + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/batch-export/pkg/filestore" +) + +// blocksIn returns the block number of every log line in the file at path. +func blocksIn(t *testing.T, path string) []uint64 { + t.Helper() + + file, err := os.Open(path) + if err != nil { + t.Fatalf("open %s: %v", path, err) + } + defer file.Close() + + var blocks []uint64 + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.TrimSpace(scanner.Text()) == "" { + continue + } + var l types.Log + if err := json.Unmarshal(scanner.Bytes(), &l); err != nil { + t.Fatalf("unmarshal %q: %v", scanner.Text(), err) + } + blocks = append(blocks, l.BlockNumber) + } + if err := scanner.Err(); err != nil { + t.Fatalf("scan %s: %v", path, err) + } + + return blocks +} + +// feed returns a closed channel already holding logs for the given blocks. +// +// Topics is set to a non-nil empty slice rather than left nil: go-ethereum's +// generated Log.UnmarshalJSON rejects a null "topics" field as missing, so a +// nil Topics would fail the round trip through blocksIn below. +func feed(blocks ...uint64) <-chan types.Log { + ch := make(chan types.Log, len(blocks)) + for _, b := range blocks { + ch <- types.Log{BlockNumber: b, Topics: []common.Hash{}} + } + close(ch) + + return ch +} + +func TestSaveLogsAsyncReplacesExistingFile(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "export.ndjson") + if err := os.WriteFile(path, []byte("stale\n"), 0o644); err != nil { + t.Fatalf("seed %s: %v", path, err) + } + + if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { + t.Fatalf("SaveLogsAsync() error = %v", err) + } + + want := []uint64{1, 2} + if got := blocksIn(t, path); !slices.Equal(got, want) { + t.Errorf("blocks = %v, want %v", got, want) + } +} + +func TestAppendLogsAsyncKeepsExistingContent(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "export.ndjson") + if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { + t.Fatalf("SaveLogsAsync() error = %v", err) + } + + w, err := filestore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + if err := filestore.AppendLogsAsync(t.Context(), feed(3, 4), w, nil); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + want := []uint64{1, 2, 3, 4} + if got := blocksIn(t, path); !slices.Equal(got, want) { + t.Errorf("blocks = %v, want %v", got, want) + } +} + +func TestAppendLogsAsyncSkipsFilteredLogs(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "export.ndjson") + if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { + t.Fatalf("SaveLogsAsync() error = %v", err) + } + + w, err := filestore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + skip := func(l types.Log) bool { return l.BlockNumber <= 2 } + if err := filestore.AppendLogsAsync(t.Context(), feed(1, 2, 3, 4), w, skip); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + want := []uint64{1, 2, 3, 4} + if got := blocksIn(t, path); !slices.Equal(got, want) { + t.Errorf("blocks = %v, want %v", got, want) + } +} + +func TestAppendLogsAsyncClosesWriterOnCancel(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), "export.ndjson") + if err := filestore.SaveLogsAsync(t.Context(), feed(1), path); err != nil { + t.Fatalf("SaveLogsAsync() error = %v", err) + } + + w, err := filestore.AppendWriter(path) + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + + // An open channel that never delivers, so cancellation is the only exit. + if err := filestore.AppendLogsAsync(ctx, make(chan types.Log), w, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("AppendLogsAsync() error = %v, want context.Canceled", err) + } + + // The writer must already be closed; closing it again must fail. + if err := w.Close(); err == nil { + t.Error("writer was left open after cancellation") + } +} From 35eb9f15270a41f83f8a91dbb98608dfd85637bb Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 16:45:10 +0200 Subject: [PATCH 06/35] test(resume): add append-resume round trip covering filestore and gzipstore Closes a gap in the plan: no task specified the append-then-re-read round trip promised for pkg/resume, even though it needs the filestore.AppendLogsAsync/AppendWriter and gzipstore.AppendWriter this task completes. Adds TestAppendResumeRoundTrip, table-driven over plain NDJSON and gzip, that builds a starting export with a genuinely partial final block, resumes it through resume.Read + the matching append writer + AppendLogsAsync(skip: cursor.Skip), and asserts: - the cursor lands on the last saved log - a boundary-block log at a higher index than the cursor is written, not skipped, since it was never saved - the full sequence after append is exactly original + new logs, with no duplicates and nothing dropped - the original bytes remain an unchanged prefix of the appended file (for gzip, proof a genuine second member was added rather than the archive being decompressed and rewritten) - resume.Read on the appended file advances to the newly written tail, so a resumed export is itself resumable --- pkg/resume/resume_test.go | 155 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index 21d1ac8..d785869 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -5,12 +5,15 @@ import ( "compress/gzip" "encoding/json" "errors" + "io" "os" "path/filepath" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethersphere/batch-export/pkg/filestore" + "github.com/ethersphere/batch-export/pkg/gzipstore" "github.com/ethersphere/batch-export/pkg/resume" ) @@ -288,3 +291,155 @@ func TestCursorSkip(t *testing.T) { }) } } + +// decompressAll reads every byte of a (possibly multi-member) gzip stream. +// Multistream is on by default, so concatenated members read as one stream. +func decompressAll(t *testing.T, b []byte) []byte { + t.Helper() + + reader, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + t.Fatalf("gzip reader: %v", err) + } + defer reader.Close() + + got, err := io.ReadAll(reader) + if err != nil { + t.Fatalf("decompress: %v", err) + } + + return got +} + +// TestAppendResumeRoundTrip exercises resume, filestore, and gzipstore +// together with no RPC involved: it builds an export file, uses resume.Read +// to find where it stopped, replays the boundary block plus new blocks +// through filestore.AppendLogsAsync with the cursor's Skip as the filter +// (via the appropriate append writer for the format), and checks the result +// byte-for-byte against the original file plus exactly the new logs -- then +// resumes a second time to confirm the appended file is itself resumable. +func TestAppendResumeRoundTrip(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + compressed bool + }{ + {name: "plain ndjson"}, + {name: "gzip", compressed: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // original spans three blocks, the last of which carries two + // logs, so resuming must handle a genuinely partial block. + original := []types.Log{ + testLog(100, 0), + testLog(101, 0), + testLog(102, 0), + testLog(102, 1), + } + // boundaryHigher shares the cursor's block but was never saved, + // so it must be written rather than skipped. + boundaryHigher := testLog(102, 2) + newer := []types.Log{ + testLog(103, 0), + testLog(104, 0), + } + + plain := ndjson(t, original...) + + var ( + originalBytes []byte + name string + ) + if tt.compressed { + originalBytes = gz(t, plain) + name = "export.ndjson.gz" + } else { + originalBytes = plain + name = "export.ndjson" + } + path := write(t, name, originalBytes) + + cursor, err := resume.Read(path) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if cursor.BlockNumber != 102 || cursor.LogIndex != 1 { + t.Fatalf("cursor = {%d,%d}, want {102,1}", cursor.BlockNumber, cursor.LogIndex) + } + if cursor.Compressed != tt.compressed { + t.Fatalf("Compressed = %t, want %t", cursor.Compressed, tt.compressed) + } + + var w io.WriteCloser + if cursor.Compressed { + w, err = gzipstore.AppendWriter(path) + } else { + w, err = filestore.AppendWriter(path) + } + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + + // replay mimics a resumed export re-querying the boundary block: + // its already-written logs must be skipped, its never-saved + // higher-index log must not be, and the later blocks are new. + replay := make([]types.Log, 0, 3+len(newer)) + replay = append(replay, testLog(102, 0), testLog(102, 1), boundaryHigher) + replay = append(replay, newer...) + + ch := make(chan types.Log, len(replay)) + for _, l := range replay { + ch <- l + } + close(ch) + + if err := filestore.AppendLogsAsync(t.Context(), ch, w, cursor.Skip); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + gotRaw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + + // The original bytes must survive as an unchanged prefix: + // appending must never rewrite existing content. For gzip this + // also proves a genuine second member was added rather than the + // archive being decompressed and rewritten. + if len(gotRaw) < len(originalBytes) || !bytes.Equal(gotRaw[:len(originalBytes)], originalBytes) { + t.Fatalf("original bytes are not an unchanged prefix of the appended file") + } + + gotPlain := gotRaw + if tt.compressed { + gotPlain = decompressAll(t, gotRaw) + } + + all := make([]types.Log, 0, len(original)+1+len(newer)) + all = append(all, original...) + all = append(all, boundaryHigher) + all = append(all, newer...) + want := ndjson(t, all...) + + if !bytes.Equal(gotPlain, want) { + t.Fatalf("content = %s, want %s", gotPlain, want) + } + + cursor2, err := resume.Read(path) + if err != nil { + t.Fatalf("second Read() error = %v", err) + } + if cursor2.BlockNumber != 104 || cursor2.LogIndex != 0 { + t.Errorf("resumed cursor = {%d,%d}, want {104,0}", cursor2.BlockNumber, cursor2.LogIndex) + } + if cursor2.Compressed != tt.compressed { + t.Errorf("resumed Compressed = %t, want %t", cursor2.Compressed, tt.compressed) + } + }) + } +} From 441a66f9ec1d4f7a1b1fc82f700e975a8859393b Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 16:56:26 +0200 Subject: [PATCH 07/35] feat(export): add --resume to continue a previous export --- cmd/export.go | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 61 insertions(+), 1 deletion(-) diff --git a/cmd/export.go b/cmd/export.go index f39bb50..7451000 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -4,13 +4,16 @@ import ( "context" "errors" "fmt" + "io" "sync" "time" + "github.com/ethereum/go-ethereum/core/types" ethclient "github.com/ethersphere/batch-export/pkg/ethclientwrapper" "github.com/ethersphere/batch-export/pkg/eventfetcher" "github.com/ethersphere/batch-export/pkg/filestore" "github.com/ethersphere/batch-export/pkg/gzipstore" + "github.com/ethersphere/batch-export/pkg/resume" "github.com/ethersphere/bee/v2/pkg/config" "github.com/ethersphere/bee/v2/pkg/util/abiutil" "github.com/spf13/cobra" @@ -25,6 +28,7 @@ func (c *command) initExportCmd() (err error) { blockRangeLimit uint32 outputFile string compress bool + resumeFile string ) cmd := &cobra.Command{ @@ -39,6 +43,35 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save RunE: func(cmd *cobra.Command, args []string) (err error) { ctx := cmd.Context() + var cursor *resume.Cursor + if resumeFile != "" { + cursor, err = resume.Read(resumeFile) + if err != nil { + return fmt.Errorf("failed to read resume file: %w", err) + } + + if cmd.Flags().Changed("start") { + c.log.Warning("--start is ignored when --resume is set", "resumeFile", resumeFile) + } + if cmd.Flags().Changed("output") { + c.log.Warning("--output is ignored when --resume is set, logs are appended to the resume file", "resumeFile", resumeFile) + } + if cursor.Compressed && compress { + c.log.Warning("--compress is ignored when resuming an already compressed file", "resumeFile", resumeFile) + compress = false + } + + outputFile = resumeFile + startBlock = cursor.BlockNumber + + c.log.Info("Resuming export", + "resumeFile", resumeFile, + "startBlock", startBlock, + "lastLogIndex", cursor.LogIndex, + "compressed", cursor.Compressed, + ) + } + ec, err := ethclient.NewClient(ctx, rpcEndpoint, ethclient.WithRateLimit(maxRequest), ethclient.WithLogger(c.log)) if err != nil { return fmt.Errorf("failed to connect to the Ethereum client: %w", err) @@ -79,7 +112,8 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save go func() { defer wg.Done() - if err := filestore.SaveLogsAsync(ctx, logChan, outputFile); err != nil { + + if err := saveLogs(ctx, logChan, outputFile, cursor); err != nil { if errors.Is(err, context.Canceled) { c.log.Error(err, "context canceled while saving logs") return @@ -112,6 +146,7 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save c.log.Info("still retrieving logs...") case <-ctx.Done(): c.log.Info("context canceled, waiting for logs to be saved...") + wg.Wait() if err := compressFunc(); err != nil { return errors.Join(fmt.Errorf("error compressing file: %w", err), ctx.Err()) } @@ -139,8 +174,33 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save cmd.Flags().Uint32VarP(&blockRangeLimit, "block-range-limit", "b", 5, "Max blocks per log query") cmd.Flags().StringVarP(&outputFile, "output", "o", "export.ndjson", "Output file path (NDJSON)") cmd.Flags().BoolVarP(&compress, "compress", "c", false, "Compress to GZIP") + cmd.Flags().StringVarP(&resumeFile, "resume", "r", "", "Resume a previous export file (.ndjson, .gz or .gzip); overrides --start and --output") c.root.AddCommand(cmd) return nil } + +// saveLogs writes logs to outputFile. With a nil cursor it replaces the file; +// otherwise it appends to the file the cursor came from, dropping any log that +// was already written to it. +func saveLogs(ctx context.Context, logChan <-chan types.Log, outputFile string, cursor *resume.Cursor) error { + if cursor == nil { + return filestore.SaveLogsAsync(ctx, logChan, outputFile) + } + + var ( + w io.WriteCloser + err error + ) + if cursor.Compressed { + w, err = gzipstore.AppendWriter(outputFile) + } else { + w, err = filestore.AppendWriter(outputFile) + } + if err != nil { + return fmt.Errorf("error opening output file for appending: %w", err) + } + + return filestore.AppendLogsAsync(ctx, logChan, w, cursor.Skip) +} From a473b3d308763d1b0e4556ed5492d9b9e1b046ee Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 17:06:20 +0200 Subject: [PATCH 08/35] docs: document the --resume flag --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index cad13db..423412e 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ batch-export is a tool to retrieve Ethereum event logs for specific contracts, p - Supports rate limiting for RPC requests. - Saves retrieved logs to a specified output file (default: `export.ndjson`) in NDJSON format. - Graceful shutdown on interrupt signals (Ctrl+C). +- Resume an interrupted export from an existing `.ndjson`, `.gz`, or `.gzip` file. ## Requirements @@ -49,10 +50,35 @@ The primary command is export. -h, --help help for export -m, --max-request int Max RPC requests/sec (default 15) -o, --output string Output file path (NDJSON) (default "export.ndjson") + -r, --resume string Resume a previous export file (.ndjson, .gz or .gzip); overrides --start and --output --start uint Start block (optional, uses contract start block if 0) (default 31306381) -v, --verbosity string Log verbosity (silent, error, warn, info, debug) (default "info") ``` +### Resuming an interrupted export + +Point `--resume` at a file a previous run produced. The tool reads its last +complete entry, restarts from that block, and appends to the same file: + +```sh +./dist/batch-export export --resume dist/export.ndjson +``` + +Compressed exports work the same way and are detected by content, not by +extension, so `.gz` and `.gzip` both work: + +```sh +./dist/batch-export export --resume dist/export.ndjson.gzip +``` + +The resumed block is re-queried, because an interrupted run may have saved only +part of it; entries already in the file are skipped, so resuming never +duplicates or drops a log. Appending to a compressed export adds a second gzip +member — standard tools such as `gzcat`, `gunzip`, and Go's `compress/gzip` +read the result as one continuous stream. + +When `--resume` is set it overrides `--start` and `--output`. + The produced NDJSON is consumed by [batch-archive](https://github.com/ethersphere/batch-archive), which embeds it for use inside Bee. ## Maintainers From b2e8654b07e0943917a3c098f9eb72871da18a3d Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 17:45:04 +0200 Subject: [PATCH 09/35] fix(resume): never report a partially written export as a clean end An interrupted run can leave a tail that no reader can trust: half an NDJSON line, a line that never got its newline, or a gzip member that never got its CRC and length trailer. Read tolerated all three and told its caller nothing, so the write path appended straight onto them and silently destroyed logs. A valid but unterminated last line was the worst case: the cursor pointed at it, so a resumed query skipped it, while the append fused it with the next log into one unparseable line and the entry was gone for good. The cursor now names the last entry that is complete and properly terminated, and reports the offset at which the file's recoverable content ends (CleanSize) together with whether anything follows it (Truncated). Safety does not rest on recovery: where no boundary can be positively identified, Read refuses with ErrNoCleanBoundary instead of guessing one. For gzip, walk the file a member at a time with Multistream(false) plus Reset over an io.ByteReader, so the reader is never wrapped in a bufio of its own and the byte count stays in step with its real position. That gives exact member boundaries with no new dependency. A member that ends mid-line is not treated as a boundary at all, since appending after it would glue the next log onto an unterminated one. lastCursorGzip also stops discarding scanner errors. Returning the last cursor is still right, but a truncated member, a checksum mismatch or an over-long line now leave the walk at the last clean boundary rather than yielding a too-early cursor with a nil error. That was the single line that made the corruption silent and endlessly repeatable: every later resume appended more unreachable data while reporting success. Co-Authored-By: Claude Opus 5 (1M context) --- pkg/resume/resume.go | 240 +++++++++++++++++---- pkg/resume/resume_test.go | 440 +++++++++++++++++++++++++++++++++++--- 2 files changed, 616 insertions(+), 64 deletions(-) diff --git a/pkg/resume/resume.go b/pkg/resume/resume.go index 435e441..8b1678c 100644 --- a/pkg/resume/resume.go +++ b/pkg/resume/resume.go @@ -24,12 +24,25 @@ const ( // to a few hundred bytes, so anything beyond this is corruption, and the // cap keeps a file without newlines from being read into memory whole. maxLineBytes = 1 << 20 + // bufferSize is how much of a gzip file is buffered per read. + bufferSize = 64 * 1024 ) -// ErrNoLogs indicates that a file holds no complete log entry to resume from. -var ErrNoLogs = errors.New("no complete log entry found") +var ( + // ErrNoLogs indicates that a file holds no complete log entry to resume + // from. + ErrNoLogs = errors.New("no complete log entry found") + // ErrNoCleanBoundary indicates that a file holds log entries but no point + // at which its content is known to be complete, so nothing can be appended + // to it without corrupting what is already there. + ErrNoCleanBoundary = errors.New("file was left partially written and has no clean boundary to append at") + // errLineTooLong indicates that a line ran past maxLineBytes, which means + // the data is not the NDJSON an export writes. + errLineTooLong = errors.New("log line exceeds the maximum length") +) -// Cursor marks the last log entry saved by a previous export. +// Cursor marks the last log entry saved by a previous export, together with +// the point up to which that export's file is known to be complete. type Cursor struct { // BlockNumber is the block of the last saved log. A resumed export // re-queries this block, because an interrupted run may have saved only @@ -40,6 +53,14 @@ type Cursor struct { // Compressed reports whether the file holds gzip data rather than plain // NDJSON. Compressed bool + // CleanSize is the byte offset at which the file's recoverable content + // ends. It always falls just past the entry the cursor points at: for + // plain NDJSON just past that line's newline, for gzip just past the + // member the line ends in. Bytes beyond it are a partial write left by an + // interrupted run and must be discarded before anything is appended. + CleanSize int64 + // Truncated reports whether any bytes follow CleanSize. + Truncated bool } // Skip reports whether l was already written to the file the cursor came from. @@ -59,8 +80,19 @@ type cursorLine struct { // Read returns a cursor for the last complete log entry in the file at path. // The file may be plain NDJSON or gzip; the format is detected from its -// leading bytes rather than its extension. Lines that do not parse are -// skipped, which discards a partial line left behind by an interrupted write. +// leading bytes rather than its extension. +// +// Only an entry that is complete and properly terminated counts: a plain line +// must end in a newline, and a compressed line must sit inside a gzip member +// that decoded to a clean end of stream. The cursor therefore also reports +// where the file's recoverable content ends (CleanSize) and whether a partial +// write follows it (Truncated). A caller that appends must discard everything +// past CleanSize first, which is lossless: every entry discarded that way is +// re-fetched by the resumed query. +// +// It returns ErrNoLogs when the file holds no complete entry at all, and +// ErrNoCleanBoundary when it holds entries but no point at which appending is +// safe. func Read(path string) (*Cursor, error) { file, err := os.Open(path) if err != nil { @@ -68,6 +100,11 @@ func Read(path string) (*Cursor, error) { } defer file.Close() + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("error inspecting resume file: %w", err) + } + compressed, err := isGzip(file) if err != nil { return nil, fmt.Errorf("error reading resume file: %w", err) @@ -77,13 +114,14 @@ func Read(path string) (*Cursor, error) { if compressed { cursor, err = lastCursorGzip(file) } else { - cursor, err = lastCursorPlain(file) + cursor, err = lastCursorPlain(file, info.Size()) } if err != nil { return nil, err } cursor.Compressed = compressed + cursor.Truncated = cursor.CleanSize < info.Size() return cursor, nil } @@ -103,26 +141,35 @@ func isGzip(file *os.File) (bool, error) { return magic[0] == 0x1f && magic[1] == 0x8b, nil } -// lastCursorPlain walks a plain NDJSON file backwards a window at a time and -// returns a cursor for the last line that parses. -func lastCursorPlain(file *os.File) (*Cursor, error) { - offset, err := file.Seek(0, io.SeekEnd) - if err != nil { - return nil, fmt.Errorf("error seeking resume file: %w", err) - } - - // carry holds bytes from the window just read that precede its first - // newline. They belong to a line whose start lies in the next window back. - var carry []byte +// lastCursorPlain walks a plain NDJSON file of the given size backwards a +// window at a time and returns a cursor for the last line that both parses and +// is terminated by a newline. json.Encoder writes a value and its newline in a +// single call, so a trailing line without one was cut short by an interrupted +// run: it is passed over, and CleanSize points just past the newline of the +// last line that was written whole. +func lastCursorPlain(file *os.File, size int64) (*Cursor, error) { + var ( + offset = size + // end is the offset one past the last byte of the region currently + // held in window, and terminated reports whether the file byte at end + // is the newline closing that region's last line. Only a terminated + // region can yield a cursor. + end = size + terminated bool + // carry holds bytes from the window just read that precede its first + // newline. They belong to a line whose start lies in the next window + // back. + carry []byte + ) for offset > 0 { - size := int64(windowSize) - if offset < size { - size = offset + n := int64(windowSize) + if offset < n { + n = offset } - offset -= size + offset -= n - window := make([]byte, size) + window := make([]byte, n) if _, err := file.ReadAt(window, offset); err != nil { return nil, fmt.Errorf("error reading resume file: %w", err) } @@ -133,10 +180,14 @@ func lastCursorPlain(file *os.File) (*Cursor, error) { if i < 0 { break } - if cursor, err := parseCursor(window[i+1:]); err == nil { - return cursor, nil + if terminated { + if cursor, err := parseCursor(window[i+1:]); err == nil { + cursor.CleanSize = end + 1 + return cursor, nil + } } window = window[:i] + end, terminated = offset+int64(i), true } carry = window @@ -145,38 +196,147 @@ func lastCursorPlain(file *os.File) (*Cursor, error) { } } - return parseCursor(carry) + if terminated { + if cursor, err := parseCursor(carry); err == nil { + cursor.CleanSize = end + 1 + return cursor, nil + } + } + + return nil, ErrNoLogs } -// lastCursorGzip returns a cursor for the last line that parses in a gzip -// file. Gzip cannot be seeked, so the whole stream is decompressed. A stream -// truncated by an interrupted run still yields the last line it did decode. +// lastCursorGzip walks a gzip file one member at a time and returns a cursor +// for the last log line that lies wholly inside a cleanly terminated member. +// Gzip cannot be seeked, so the whole stream is decompressed. +// +// A member left half-written by an interrupted run cannot decode cleanly: its +// CRC and length trailer are missing, so the reader reports an error instead +// of io.EOF. The walk stops there and CleanSize is the offset just past the +// last member that did end cleanly, which is a member boundary and so a valid +// place to concatenate the next one. Lines are carried across member +// boundaries, so a line split between two members is still recognised, and a +// member that ends mid-line is not treated as a boundary at all. func lastCursorGzip(file *os.File) (*Cursor, error) { - reader, err := gzip.NewReader(file) + counter := &countingReader{reader: bufio.NewReaderSize(file, bufferSize)} + + reader, err := gzip.NewReader(counter) if err != nil { return nil, fmt.Errorf("error opening gzip resume file: %w", err) } defer reader.Close() - // Multistream is on by default, so concatenated members read as one stream. - scanner := bufio.NewScanner(reader) - scanner.Buffer(make([]byte, 0, bufio.MaxScanTokenSize), maxLineBytes) + var ( + // clean is the cursor as of cleanSize, while pending also covers the + // member being read; pending is promoted only once that member is + // known to have ended cleanly and on a line boundary. + clean, pending *Cursor + cleanSize int64 + carry []byte + ) + + for { + // Reset turns multistream back on, so it has to be switched off for + // every member rather than only for the first. + reader.Multistream(false) + + cursor, tail, err := scanLines(reader, carry) + if cursor != nil { + pending = cursor + } + if err != nil { + // The member did not decode to a clean end of stream, so + // everything it holds is part of an interrupted write. + break + } + if len(tail) == 0 { + clean, cleanSize = pending, counter.read + } + carry = tail + + // A clean end of file makes Reset report io.EOF; anything else is a + // member header that was only partly written. + if err := reader.Reset(counter); err != nil { + break + } + } + + if clean == nil { + if pending == nil { + return nil, ErrNoLogs + } + return nil, ErrNoCleanBoundary + } + + clean.CleanSize = cleanSize + + return clean, nil +} + +// scanLines reads NDJSON from r and returns a cursor for the last line that +// both parses and is terminated by a newline. carry is prepended to the first +// line, so a line split across two gzip members is reassembled, and the +// trailing bytes not yet terminated by a newline are returned so the next +// member can complete them. The returned error is nil only when r ended at a +// clean io.EOF; a truncated member, a checksum mismatch, an over-long line and +// a genuine I/O failure are all reported rather than passed over, because each +// of them means the bytes after the last good line cannot be trusted. +func scanLines(r io.Reader, carry []byte) (*Cursor, []byte, error) { + buffered := bufio.NewReaderSize(r, bufferSize) var last *Cursor - for scanner.Scan() { - if cursor, err := parseCursor(scanner.Bytes()); err == nil { + + line := carry + for { + chunk, err := buffered.ReadSlice('\n') + line = append(line, chunk...) + if len(line) > maxLineBytes { + return last, nil, errLineTooLong + } + + switch { + case errors.Is(err, bufio.ErrBufferFull): + continue + case errors.Is(err, io.EOF): + return last, line, nil + case err != nil: + return last, nil, fmt.Errorf("error reading gzip resume file: %w", err) + } + + if cursor, err := parseCursor(line); err == nil { last = cursor } + line = line[:0] } +} - if last != nil { - return last, nil - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading gzip resume file: %w", err) +// countingReader counts the bytes consumed from the reader it wraps. It +// implements io.ByteReader as well as io.Reader so that gzip.Reader reads from +// it directly rather than wrapping it in a bufio.Reader of its own; without +// that the count would run ahead of the reader's real position and no member +// boundary could be observed exactly. +type countingReader struct { + reader *bufio.Reader + // read is the number of bytes handed out so far. + read int64 +} + +// Read implements io.Reader. +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.reader.Read(p) + c.read += int64(n) + + return n, err +} + +// ReadByte implements io.ByteReader. +func (c *countingReader) ReadByte() (byte, error) { + b, err := c.reader.ReadByte() + if err == nil { + c.read++ } - return nil, ErrNoLogs + return b, err } // parseCursor builds a cursor from a single NDJSON line. It rejects blank and diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index d785869..8e5e721 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -5,9 +5,11 @@ import ( "compress/gzip" "encoding/json" "errors" + "fmt" "io" "os" "path/filepath" + "slices" "testing" "github.com/ethereum/go-ethereum/common" @@ -23,6 +25,13 @@ import ( // a window boundary. const windowSize = 64 * 1024 +// Export file names. The format is detected from the file's leading bytes, so +// the name a case uses never decides how it is read. +const ( + plainFile = "export.ndjson" + gzipFile = "export.ndjson.gzip" +) + // testLog builds a log shaped like the ones the exporter writes. types.Log // always marshals blockNumber and logIndex, even at zero. func testLog(blockNumber uint64, logIndex uint) types.Log { @@ -68,6 +77,12 @@ func gz(t *testing.T, b []byte) []byte { return buf.Bytes() } +// truncateLast drops the final n bytes of b, standing in for a write that a +// hard kill cut short. +func truncateLast(b []byte, n int) []byte { + return b[:len(b)-n] +} + // write puts content in a temp file and returns its path. func write(t *testing.T, name string, content []byte) string { t.Helper() @@ -122,12 +137,22 @@ func TestReadCursor(t *testing.T) { t.Fatalf("test setup: boundary %d does not straddle target line [%d, %d)", straddleBoundary, straddleLineStart, straddleLineEnd) } + // twoLogsEnd is the offset just past the newline closing the second of + // threeLogs: the last clean boundary once the third line loses its own. + twoLogsEnd := int64(len(ndjson(t, testLog(100, 0), testLog(101, 1)))) + threeLogsEnd := int64(len(threeLogs)) + tests := []struct { name string content []byte wantBlock uint64 wantIndex uint wantCompressed bool + // wantTruncated says whether bytes follow the last clean boundary. + // When it is false the boundary must be the end of the file, so + // wantCleanSize is only consulted for the truncated cases. + wantTruncated bool + wantCleanSize int64 }{ { name: "plain ndjson", @@ -142,22 +167,35 @@ func TestReadCursor(t *testing.T) { wantIndex: 3, }, { - name: "truncated trailing line is discarded", - content: append(append([]byte{}, threeLogs...), []byte(`{"address":"0x45a15","topics":["0xae`)...), - wantBlock: 102, - wantIndex: 7, + // A line cut short by an interrupted write. The cursor is the + // line before it and the partial bytes are reported, so a caller + // discards them instead of appending onto them. + name: "truncated trailing line is excluded from the clean boundary", + content: append(append([]byte{}, threeLogs...), []byte(`{"address":"0x45a15","topics":["0xae`)...), + wantBlock: 102, + wantIndex: 7, + wantTruncated: true, + wantCleanSize: threeLogsEnd, }, { - name: "trailing line without newline is still read", - content: bytes.TrimSuffix(threeLogs, []byte("\n")), - wantBlock: 102, - wantIndex: 7, + // json.Encoder writes a value and its newline in one call, so a + // last line that parses but has no newline is still a partial + // write: the cursor must fall back to the line before it, or the + // unterminated log would be skipped on resume and lost. + name: "trailing line without a newline is not a clean end", + content: bytes.TrimSuffix(threeLogs, []byte("\n")), + wantBlock: 101, + wantIndex: 1, + wantTruncated: true, + wantCleanSize: twoLogsEnd, }, { - name: "line missing blockNumber is skipped", - content: append(append([]byte{}, threeLogs...), []byte("{\"address\":\"0x1\",\"topics\":[],\"data\":\"0x\"}\n")...), - wantBlock: 102, - wantIndex: 7, + name: "line missing blockNumber is skipped", + content: append(append([]byte{}, threeLogs...), []byte("{\"address\":\"0x1\",\"topics\":[],\"data\":\"0x\"}\n")...), + wantBlock: 102, + wantIndex: 7, + wantTruncated: true, + wantCleanSize: threeLogsEnd, }, { name: "spans multiple backward read windows", @@ -166,16 +204,20 @@ func TestReadCursor(t *testing.T) { wantIndex: 15, }, { - name: "walks back across windows of garbage lines", - content: append(append([]byte{}, threeLogs...), garbageLines...), - wantBlock: 102, - wantIndex: 7, + name: "walks back across windows of garbage lines", + content: append(append([]byte{}, threeLogs...), garbageLines...), + wantBlock: 102, + wantIndex: 7, + wantTruncated: true, + wantCleanSize: threeLogsEnd, }, { - name: "valid line straddles a window boundary", - content: straddle, - wantBlock: 4096, - wantIndex: 3, + name: "valid line straddles a window boundary", + content: straddle, + wantBlock: 4096, + wantIndex: 3, + wantTruncated: true, + wantCleanSize: int64(straddleLineEnd), }, { name: "gzip", @@ -200,12 +242,27 @@ func TestReadCursor(t *testing.T) { }, { // A resume interrupted before the second member was flushed: the - // header is present but carries no decodable data. + // header is present but carries no decodable data. The clean + // boundary is the end of the first member, so the half-written + // header is reported rather than appended onto. name: "multi member gzip with unflushed final member", content: append(gz(t, threeLogs), []byte{0x1f, 0x8b, 0x08, 0, 0, 0, 0, 0, 0, 0xff}...), wantBlock: 102, wantIndex: 7, wantCompressed: true, + wantTruncated: true, + wantCleanSize: int64(len(gz(t, threeLogs))), + }, + { + // Only the last member's trailer is lost. Everything before it is + // still a member boundary, so the file stays recoverable. + name: "multi member gzip with a truncated final member", + content: truncateLast(append(gz(t, threeLogs), gz(t, ndjson(t, testLog(200, 2)))...), 6), + wantBlock: 102, + wantIndex: 7, + wantCompressed: true, + wantTruncated: true, + wantCleanSize: int64(len(gz(t, threeLogs))), }, } @@ -213,7 +270,7 @@ func TestReadCursor(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - got, err := resume.Read(write(t, "export.ndjson", tt.content)) + got, err := resume.Read(write(t, plainFile, tt.content)) if err != nil { t.Fatalf("Read() error = %v, want nil", err) } @@ -226,6 +283,19 @@ func TestReadCursor(t *testing.T) { if got.Compressed != tt.wantCompressed { t.Errorf("Compressed = %t, want %t", got.Compressed, tt.wantCompressed) } + if got.Truncated != tt.wantTruncated { + t.Errorf("Truncated = %t, want %t", got.Truncated, tt.wantTruncated) + } + + // A file reported as untruncated must be clean all the way to its + // end, which is the invariant an appending caller relies on. + wantCleanSize := int64(len(tt.content)) + if tt.wantTruncated { + wantCleanSize = tt.wantCleanSize + } + if got.CleanSize != wantCleanSize { + t.Errorf("CleanSize = %d, want %d", got.CleanSize, wantCleanSize) + } }) } } @@ -247,7 +317,7 @@ func TestReadCursorErrors(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - _, err := resume.Read(write(t, "export.ndjson", tt.content)) + _, err := resume.Read(write(t, plainFile, tt.content)) if !errors.Is(err, resume.ErrNoLogs) { t.Fatalf("Read() error = %v, want ErrNoLogs", err) } @@ -360,7 +430,7 @@ func TestAppendResumeRoundTrip(t *testing.T) { name = "export.ndjson.gz" } else { originalBytes = plain - name = "export.ndjson" + name = plainFile } path := write(t, name, originalBytes) @@ -443,3 +513,325 @@ func TestAppendResumeRoundTrip(t *testing.T) { }) } } + +// logsIn returns every log an export file holds, decompressing it first when +// it is gzip. Anything the file contains that is not a whole, newline +// terminated NDJSON log line fails the test: a file that merely looks +// recovered must not pass, so corruption is caught here rather than by a +// reader downstream. +func logsIn(t *testing.T, path string) []types.Log { + t.Helper() + + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if bytes.HasPrefix(raw, []byte{0x1f, 0x8b}) { + raw = decompressAll(t, raw) + } + if len(raw) == 0 { + return nil + } + if raw[len(raw)-1] != '\n' { + t.Fatalf("file does not end on a line boundary, last 80 bytes: %q", raw[max(0, len(raw)-80):]) + } + + var logs []types.Log + for i, line := range bytes.Split(bytes.TrimSuffix(raw, []byte("\n")), []byte("\n")) { + var l types.Log + if err := json.Unmarshal(line, &l); err != nil { + t.Fatalf("line %d does not parse: %v: %q", i+1, err, line) + } + logs = append(logs, l) + } + + return logs +} + +// ids renders logs as block/index pairs, the identity in which "no log +// duplicated and none lost" is measured. +func ids(logs []types.Log) []string { + out := make([]string, 0, len(logs)) + for _, l := range logs { + out = append(out, fmt.Sprintf("%d/%d", l.BlockNumber, l.Index)) + } + + return out +} + +// appendWriter opens the append writer that matches the cursor's format. +func appendWriter(t *testing.T, path string, cursor *resume.Cursor) io.WriteCloser { + t.Helper() + + var ( + w io.WriteCloser + err error + ) + if cursor.Compressed { + w, err = gzipstore.AppendWriter(path) + } else { + w, err = filestore.AppendWriter(path) + } + if err != nil { + t.Fatalf("AppendWriter() error = %v", err) + } + + return w +} + +// feed returns a closed channel already holding logs. +func feed(logs ...types.Log) <-chan types.Log { + ch := make(chan types.Log, len(logs)) + for _, l := range logs { + ch <- l + } + close(ch) + + return ch +} + +// TestResumeAfterInterruptedWrite covers the three ways a run killed mid-write +// leaves a file that used to be appended onto blindly, corrupting it: a plain +// line cut in half, a plain line that parses but never got its newline, and a +// gzip member that never got its trailer. +// +// Each case walks the whole recovery: read the cursor, discard everything past +// the clean boundary it reports, append the logs a resumed query would return, +// and then require that the file parses end to end and holds exactly the four +// logs, in order, with none duplicated and none lost. Before the clean +// boundary existed, the first case fused two logs into one unparseable line, +// the second did the same and lost the fused log for good because the cursor +// had told the writer to skip it, and the third left the appended member +// unreachable behind a corrupt one -- so each assertion below fails loudly on +// the old behaviour. +func TestResumeAfterInterruptedWrite(t *testing.T) { + t.Parallel() + + // saved is what the interrupted run had managed to write whole. + saved := ndjson(t, testLog(100, 0), testLog(101, 0)) + savedFirst := ndjson(t, testLog(100, 0)) + + // want is the full export: what the file must hold once resumed. + want := []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(103, 0)} + + tests := []struct { + name string + // content is the file the interrupted run left behind. + content []byte + fileName string + compressed bool + // wantBlock and wantIndex are the last entry that is complete and + // properly terminated. + wantBlock uint64 + wantIndex uint + wantCleanSize int64 + // replay is what the resumed query returns from wantBlock onwards. + replay []types.Log + }{ + { + // The reviewer's first scenario: appending onto the half line + // glued the next log onto it, destroying block 102. + name: "plain file with a truncated last line", + content: append(append([]byte{}, saved...), []byte(`{"address":"0x45a1502382541cd610cc9068e88727426b6`)...), + fileName: plainFile, + wantBlock: 101, + wantIndex: 0, + wantCleanSize: int64(len(saved)), + replay: []types.Log{testLog(101, 0), testLog(102, 0), testLog(103, 0)}, + }, + { + // The reviewer's second scenario, and the nastiest: the last + // line parses, so the old cursor pointed at it and the resumed + // query skipped block 101, while the append glued the next log + // onto the line that had no newline. Block 101 was fused into an + // unparseable line and never re-fetched, so it was unrecoverable. + // The cursor must now name block 100 so that 101 is fetched + // again. + name: "plain file with a valid but unterminated last line", + content: bytes.TrimSuffix(saved, []byte("\n")), + fileName: plainFile, + wantBlock: 100, + wantIndex: 0, + wantCleanSize: int64(len(savedFirst)), + replay: []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(103, 0)}, + }, + { + // The reviewer's third scenario: the final member lost its + // trailer, so a new member appended after it sat behind corrupt + // data and could never be read back -- and every later resume + // appended more of the same while reporting success. + name: "gzip file with a truncated final member", + content: append(gz(t, savedFirst), truncateLast(gz(t, ndjson(t, testLog(101, 0))), 6)...), + fileName: gzipFile, + compressed: true, + wantBlock: 100, + wantIndex: 0, + wantCleanSize: int64(len(gz(t, savedFirst))), + replay: []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(103, 0)}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + path := write(t, tt.fileName, tt.content) + + cursor, err := resume.Read(path) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if cursor.BlockNumber != tt.wantBlock || cursor.LogIndex != tt.wantIndex { + t.Fatalf("cursor = {%d,%d}, want {%d,%d}", cursor.BlockNumber, cursor.LogIndex, tt.wantBlock, tt.wantIndex) + } + if cursor.Compressed != tt.compressed { + t.Fatalf("Compressed = %t, want %t", cursor.Compressed, tt.compressed) + } + if !cursor.Truncated { + t.Fatalf("Truncated = false, want true: the partial write went unreported") + } + if cursor.CleanSize != tt.wantCleanSize { + t.Fatalf("CleanSize = %d, want %d", cursor.CleanSize, tt.wantCleanSize) + } + + // Recovery: drop the partial tail, exactly as the export command + // does, and never further than the reported boundary. + if err := os.Truncate(path, cursor.CleanSize); err != nil { + t.Fatalf("truncate %s: %v", path, err) + } + + if err := filestore.AppendLogsAsync(t.Context(), feed(tt.replay...), appendWriter(t, path, cursor), cursor.Skip); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + if got := ids(logsIn(t, path)); !slices.Equal(got, ids(want)) { + t.Fatalf("logs = %v, want %v", got, ids(want)) + } + + // The recovered file must itself resume cleanly, or a second + // interruption would start the corruption over again. + cursor2, err := resume.Read(path) + if err != nil { + t.Fatalf("second Read() error = %v", err) + } + if cursor2.BlockNumber != 103 || cursor2.LogIndex != 0 { + t.Errorf("resumed cursor = {%d,%d}, want {103,0}", cursor2.BlockNumber, cursor2.LogIndex) + } + if cursor2.Truncated { + t.Errorf("Truncated = true, want false: the recovered file is not clean") + } + }) + } +} + +// TestReadRefusesWithoutACleanBoundary covers files whose content cannot be +// appended to at any offset the reader can positively identify. Refusing is +// the safety property that has to hold even where recovery cannot: guessing a +// boundary would corrupt what is already there. +func TestReadRefusesWithoutACleanBoundary(t *testing.T) { + t.Parallel() + + logs := ndjson(t, testLog(100, 0), testLog(101, 0)) + + tests := []struct { + name string + content []byte + wantErr error + }{ + { + // A single member whose trailer never made it to disk. Its logs + // decode, but there is no member boundary to concatenate at and + // the file cannot be rewritten in place, so the run must stop. + name: "gzip with a single truncated member", + content: truncateLast(gz(t, logs), 6), + wantErr: resume.ErrNoCleanBoundary, + }, + { + // The member is intact but its data stops mid-line, so a second + // member would glue its first log onto the unterminated one. + name: "gzip member ending mid line", + content: gz(t, bytes.TrimSuffix(logs, []byte("\n"))), + wantErr: resume.ErrNoCleanBoundary, + }, + { + // The only line has no newline, so no entry is complete. + name: "plain file holding one unterminated line", + content: bytes.TrimSuffix(ndjson(t, testLog(100, 0)), []byte("\n")), + wantErr: resume.ErrNoLogs, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if _, err := resume.Read(write(t, plainFile, tt.content)); !errors.Is(err, tt.wantErr) { + t.Fatalf("Read() error = %v, want %v", err, tt.wantErr) + } + }) + } +} + +// TestReadReportsGzipReadErrors pins Fix 2: a gzip stream that stops early +// must not be reported as a clean read just because some lines decoded. The +// old code returned the last cursor it had seen with a nil error, which let +// every later resume append more data behind the corruption, for ever. +func TestReadReportsGzipReadErrors(t *testing.T) { + t.Parallel() + + clean := gz(t, ndjson(t, testLog(100, 0))) + content := append(append([]byte{}, clean...), truncateLast(gz(t, ndjson(t, testLog(101, 0), testLog(102, 0))), 6)...) + + cursor, err := resume.Read(write(t, gzipFile, content)) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if !cursor.Truncated { + t.Error("Truncated = false, want true: the unclean stream was reported as clean") + } + if cursor.CleanSize != int64(len(clean)) { + t.Errorf("CleanSize = %d, want %d", cursor.CleanSize, len(clean)) + } + // The cursor must not name a log that only the broken member holds: it is + // about to be discarded, and a cursor past it would skip the refetch. + if cursor.BlockNumber != 100 || cursor.LogIndex != 0 { + t.Errorf("cursor = {%d,%d}, want {100,0}", cursor.BlockNumber, cursor.LogIndex) + } +} + +// TestGzipCleanSizeIsAMemberBoundary checks the offset the gzip walk reports +// is exactly a member boundary and not a byte off, which is what makes +// truncating to it safe. Reading the file back after cutting it there must +// yield the members that came before, whole. +func TestGzipCleanSizeIsAMemberBoundary(t *testing.T) { + t.Parallel() + + first := ndjson(t, testLog(100, 0), testLog(101, 0)) + second := ndjson(t, testLog(102, 0)) + + members := append(gz(t, first), gz(t, second)...) + boundary := int64(len(members)) + content := append(append([]byte{}, members...), truncateLast(gz(t, ndjson(t, testLog(103, 0))), 4)...) + + path := write(t, gzipFile, content) + + cursor, err := resume.Read(path) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if cursor.CleanSize != boundary { + t.Fatalf("CleanSize = %d, want %d", cursor.CleanSize, boundary) + } + + if err := os.Truncate(path, cursor.CleanSize); err != nil { + t.Fatalf("truncate %s: %v", path, err) + } + + raw, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + if got, want := decompressAll(t, raw), append(append([]byte{}, first...), second...); !bytes.Equal(got, want) { + t.Fatalf("content = %s, want %s", got, want) + } +} From 8b7c59327054ebaaff9a41860ba4f4db31dcaf2a Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 17:45:04 +0200 Subject: [PATCH 10/35] fix(filestore): report the error from closing the log destination AppendLogsAsync dropped the error from Close, though its doc comment promised a buffered destination is always flushed. On the gzip path Close writes the deflate terminator and the member footer, so a failure there -- a full disk is entirely plausible for a 90 MB export -- left a truncated member behind while the CLI printed "all logs have been saved". Join the close error into the result instead; errors.Join keeps errors.Is working, so a cancelled context still reads as context.Canceled. SaveLogsAsync now opens through CreateWriter and shares the same path, which fixes the same dropped Close for a fresh export and lets a caller open the destination before it starts producing logs. memberWriter.Close names the file it failed on, as CompressFile does; that error is about to become visible. Co-Authored-By: Claude Opus 5 (1M context) --- pkg/filestore/filestore.go | 32 +++++++++++++++++++++++++------- pkg/gzipstore/gzipstore.go | 10 ++++++++-- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/pkg/filestore/filestore.go b/pkg/filestore/filestore.go index f93f256..b7c700f 100644 --- a/pkg/filestore/filestore.go +++ b/pkg/filestore/filestore.go @@ -3,6 +3,7 @@ package filestore import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -10,16 +11,27 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) +// CreateWriter creates the file at filePath, replacing anything already there, +// and returns a writer for it. Opening the destination separately lets a +// caller fail before it starts producing logs it would have nowhere to put. +func CreateWriter(filePath string) (io.WriteCloser, error) { + file, err := os.Create(filePath) + if err != nil { + return nil, fmt.Errorf("error creating file: %w", err) + } + + return file, nil +} + // SaveLogsAsync writes logs to a file asynchronously, replacing any file -// already at filePath. +// already at filePath. The file is closed before returning. func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath string) error { - file, err := os.Create(filePath) + w, err := CreateWriter(filePath) if err != nil { - return fmt.Errorf("error creating file: %w", err) + return err } - defer file.Close() - return writeLogs(ctx, logChan, file, nil) + return AppendLogsAsync(ctx, logChan, w, nil) } // AppendWriter opens an existing NDJSON file for appending. @@ -36,8 +48,14 @@ func AppendWriter(filePath string) (io.WriteCloser, error) { // destination already holds. Logs for which skip reports true are dropped; a // nil skip writes every log. The writer is closed before returning, including // when the context is cancelled, so a buffered destination is always flushed. -func AppendLogsAsync(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, skip func(types.Log) bool) error { - defer w.Close() +// +// The close error is joined into the result rather than discarded: on a +// compressed destination Close writes the terminator and footer that make the +// data readable, so a failure there leaves a truncated member behind and must +// not be reported as a completed save. Joining preserves errors.Is, so a +// cancelled context still reads as context.Canceled. +func AppendLogsAsync(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, skip func(types.Log) bool) (err error) { + defer func() { err = errors.Join(err, w.Close()) }() return writeLogs(ctx, logChan, w, skip) } diff --git a/pkg/gzipstore/gzipstore.go b/pkg/gzipstore/gzipstore.go index 5aa8b78..3091002 100644 --- a/pkg/gzipstore/gzipstore.go +++ b/pkg/gzipstore/gzipstore.go @@ -61,7 +61,13 @@ func (w *memberWriter) Write(p []byte) (int, error) { } // Close finishes the gzip member and then closes the file. Both are attempted -// even if the first fails, so the descriptor is never leaked. +// even if the first fails, so the descriptor is never leaked. Closing the +// member writes the data the file needs to be readable at all, so the error +// names the file it belongs to. func (w *memberWriter) Close() error { - return errors.Join(w.gzip.Close(), w.file.Close()) + if err := errors.Join(w.gzip.Close(), w.file.Close()); err != nil { + return fmt.Errorf("failed to close gzip file '%s': %w", w.file.Name(), err) + } + + return nil } From 894ba47505d08f285216bba21231b0a97d3418fd Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 17:45:32 +0200 Subject: [PATCH 11/35] fix(export): recover a partial write and open the destination up front Discard whatever follows the resume file's clean boundary before the writer is opened, logging the offset and how many bytes went. Nothing recoverable is lost: everything discarded sits at or after the cursor, so the resumed query fetches it again. Only the offset the reader positively identified is ever truncated to, and never past it. This runs after the RPC connection is up, so an unreachable endpoint leaves the file untouched. Open the writer synchronously before GetLogs as well. Opening it inside the saving goroutine meant a failure there logged and returned while fetchLogs kept pushing onto the 100-slot channel; once full it blocked until cancellation, errorChan never closed, and RunE looped for ever printing "still retrieving logs..." while nothing was saved. It now returns the error immediately, and saveLogs is left as a dispatch. Co-Authored-By: Claude Opus 5 (1M context) --- cmd/export.go | 84 ++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 18 deletions(-) diff --git a/cmd/export.go b/cmd/export.go index 7451000..108010e 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "os" "sync" "time" @@ -47,7 +48,7 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save if resumeFile != "" { cursor, err = resume.Read(resumeFile) if err != nil { - return fmt.Errorf("failed to read resume file: %w", err) + return fmt.Errorf("failed to read resume file %q: %w", resumeFile, err) } if cmd.Flags().Changed("start") { @@ -96,6 +97,18 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save startBlock = chainCfg.PostageStampStartBlock } + if err := c.discardPartialWrite(outputFile, cursor); err != nil { + return err + } + + // The destination is opened before the first log is fetched: were + // it opened in the saving goroutine instead, a failure there would + // leave the fetcher pushing into a channel nobody drains. + w, err := openOutput(outputFile, cursor) + if err != nil { + return fmt.Errorf("failed to open output file: %w", err) + } + c.log.Info("Retrieving logs", "startBlock", startBlock, "endBlock", endBlock) logChan, errorChan := client.GetLogs(ctx, &eventfetcher.Request{ @@ -113,7 +126,7 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save go func() { defer wg.Done() - if err := saveLogs(ctx, logChan, outputFile, cursor); err != nil { + if err := saveLogs(ctx, logChan, w, cursor); err != nil { if errors.Is(err, context.Canceled) { c.log.Error(err, "context canceled while saving logs") return @@ -181,26 +194,61 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save return nil } -// saveLogs writes logs to outputFile. With a nil cursor it replaces the file; -// otherwise it appends to the file the cursor came from, dropping any log that -// was already written to it. -func saveLogs(ctx context.Context, logChan <-chan types.Log, outputFile string, cursor *resume.Cursor) error { - if cursor == nil { - return filestore.SaveLogsAsync(ctx, logChan, outputFile) +// discardPartialWrite drops the tail an interrupted run left behind in the +// resume file, so that new logs are appended onto a boundary the reader +// positively identified rather than onto half a line or half a gzip member. +// Nothing recoverable is lost: every entry discarded here falls at or after +// the cursor, so the resumed query fetches it again. +// +// It is a no-op when there is no resume file or the file ends cleanly, and it +// never truncates past the offset the reader reported. +func (c *command) discardPartialWrite(outputFile string, cursor *resume.Cursor) error { + if cursor == nil || !cursor.Truncated { + return nil } - var ( - w io.WriteCloser - err error + info, err := os.Stat(outputFile) + if err != nil { + return fmt.Errorf("failed to inspect resume file: %w", err) + } + if info.Size() <= cursor.CleanSize { + return nil + } + + c.log.Warning("resume file ends with a partial write, discarding it", + "resumeFile", outputFile, + "offset", cursor.CleanSize, + "discardedBytes", info.Size()-cursor.CleanSize, ) - if cursor.Compressed { - w, err = gzipstore.AppendWriter(outputFile) - } else { - w, err = filestore.AppendWriter(outputFile) + + if err := os.Truncate(outputFile, cursor.CleanSize); err != nil { + return fmt.Errorf("failed to truncate resume file: %w", err) } - if err != nil { - return fmt.Errorf("error opening output file for appending: %w", err) + + return nil +} + +// openOutput opens the destination for a run's logs: a fresh file when cursor +// is nil, or a writer that appends to the file the cursor came from. +func openOutput(outputFile string, cursor *resume.Cursor) (io.WriteCloser, error) { + switch { + case cursor == nil: + return filestore.CreateWriter(outputFile) + case cursor.Compressed: + return gzipstore.AppendWriter(outputFile) + default: + return filestore.AppendWriter(outputFile) + } +} + +// saveLogs writes logs to w, dropping any entry a resumed export already +// holds. A nil cursor means the destination starts empty, so every log is +// kept. w is closed before returning, cancellation included. +func saveLogs(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, cursor *resume.Cursor) error { + var skip func(types.Log) bool + if cursor != nil { + skip = cursor.Skip } - return filestore.AppendLogsAsync(ctx, logChan, w, cursor.Skip) + return filestore.AppendLogsAsync(ctx, logChan, w, skip) } From add5d12f5f233f92c31af5a006e00a81debeb920 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 17:45:32 +0200 Subject: [PATCH 12/35] docs: describe what resuming does with a partial write The claim that "entries already in the file are skipped, so resuming never duplicates or drops a log" held only for a file that ends cleanly. Say so, and document what happens when a run killed mid-write leaves a partial entry behind: the tool truncates to the last complete entry and re-fetches from there, or refuses outright when no such point can be identified. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 35 +++++++++++++++++++++++++++++++---- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 423412e..a0a879c 100644 --- a/README.md +++ b/README.md @@ -72,10 +72,37 @@ extension, so `.gz` and `.gzip` both work: ``` The resumed block is re-queried, because an interrupted run may have saved only -part of it; entries already in the file are skipped, so resuming never -duplicates or drops a log. Appending to a compressed export adds a second gzip -member — standard tools such as `gzcat`, `gunzip`, and Go's `compress/gzip` -read the result as one continuous stream. +part of it, and the entries already in the file are skipped. So long as the +file ends cleanly, resuming neither duplicates nor drops a log. Appending to a +compressed export adds a second gzip member — standard tools such as `gzcat`, +`gunzip`, and Go's `compress/gzip` read the result as one continuous stream. + +#### When the previous run was killed mid-write + +A run stopped by `SIGKILL`, a crash, or a full disk can leave a partial entry +at the end of the file: half a line, a line that never got its newline, or a +gzip member that never got its trailer. Appending onto any of those would +corrupt the file, so only an entry that is complete and properly terminated +counts as the resume point. + +The tool finds the last offset at which the file is known to be complete — the +end of the last newline-terminated line, or of the last whole gzip member — +and discards whatever follows it, logging how many bytes it dropped and from +where: + +``` +"level"="warning" "msg"="resume file ends with a partial write, discarding it" "offset"=89649991 "discardedBytes"=317 +``` + +Nothing is lost by that: everything discarded sits at or after the resume +point, so the resumed query fetches it again. Because a line that parses but +has no newline does not count, the resume point in that case is the line +before it, and that entry is re-fetched too. + +If no such offset can be identified — a gzip file whose only member is +truncated, for instance, since there is no member boundary to append at — the +tool refuses to touch the file and exits with an error rather than guess. Fall +back to a fresh export in that case. When `--resume` is set it overrides `--start` and `--output`. From e8376019fd36ed198b76901956a475c8c09d9fe2 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 18:02:44 +0200 Subject: [PATCH 13/35] docs: scope what resuming discards and re-fetches Clarify that the resume boundary is the last newline-terminated line that parses as a log entry, not just any newline-terminated line. And scope the guarantee about re-fetching: for export content (partial writes), discarded data is re-fetched; for foreign data that was never a log entry, it is simply removed. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a0a879c..074f77c 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ corrupt the file, so only an entry that is complete and properly terminated counts as the resume point. The tool finds the last offset at which the file is known to be complete — the -end of the last newline-terminated line, or of the last whole gzip member — +end of the last newline-terminated line that parses as a log entry, or of the last whole gzip member — and discards whatever follows it, logging how many bytes it dropped and from where: @@ -94,8 +94,8 @@ where: "level"="warning" "msg"="resume file ends with a partial write, discarding it" "offset"=89649991 "discardedBytes"=317 ``` -Nothing is lost by that: everything discarded sits at or after the resume -point, so the resumed query fetches it again. Because a line that parses but +For export content, nothing is lost by that: everything discarded sits at or after the resume +point, so the resumed query fetches it again. Anything discarded that was never a log entry is simply removed and not re-fetched. Because a line that parses but has no newline does not count, the resume point in that case is the line before it, and that entry is re-fetched too. From 3119609b62696ec1f77515775dbb982859672c31 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 18:03:26 +0200 Subject: [PATCH 14/35] docs: reflow the partial-write paragraphs to the file's wrap width Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 074f77c..3170397 100644 --- a/README.md +++ b/README.md @@ -86,18 +86,19 @@ corrupt the file, so only an entry that is complete and properly terminated counts as the resume point. The tool finds the last offset at which the file is known to be complete — the -end of the last newline-terminated line that parses as a log entry, or of the last whole gzip member — -and discards whatever follows it, logging how many bytes it dropped and from -where: +end of the last newline-terminated line that parses as a log entry, or of the +last whole gzip member — and discards whatever follows it, logging how many +bytes it dropped and from where: ``` "level"="warning" "msg"="resume file ends with a partial write, discarding it" "offset"=89649991 "discardedBytes"=317 ``` -For export content, nothing is lost by that: everything discarded sits at or after the resume -point, so the resumed query fetches it again. Anything discarded that was never a log entry is simply removed and not re-fetched. Because a line that parses but -has no newline does not count, the resume point in that case is the line -before it, and that entry is re-fetched too. +For export content, nothing is lost by that: everything discarded sits at or +after the resume point, so the resumed query fetches it again. Anything +discarded that was never a log entry is simply removed and not re-fetched. +Because a line that parses but has no newline does not count, the resume point +in that case is the line before it, and that entry is re-fetched too. If no such offset can be identified — a gzip file whose only member is truncated, for instance, since there is no member boundary to append at — the From 91389f1eee864f6261037e29ec4485f133cec063 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 18:20:32 +0200 Subject: [PATCH 15/35] docs: remove the implementation plan from the repo The plan was scaffolding for building the feature, not reference material for using it. The design rationale it carried now lives in the README and in the PR description. Co-Authored-By: Claude Opus 5 (1M context) --- .../plans/2026-08-26-resume-flag.md | 1330 ----------------- 1 file changed, 1330 deletions(-) delete mode 100644 docs/superpowers/plans/2026-08-26-resume-flag.md diff --git a/docs/superpowers/plans/2026-08-26-resume-flag.md b/docs/superpowers/plans/2026-08-26-resume-flag.md deleted file mode 100644 index e107892..0000000 --- a/docs/superpowers/plans/2026-08-26-resume-flag.md +++ /dev/null @@ -1,1330 +0,0 @@ -# Resume Flag Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add `export --resume ` so an interrupted export continues from the end of an existing `.ndjson`, `.gz`, or `.gzip` file instead of restarting from the contract's start block. - -**Architecture:** A new `pkg/resume` package reads the tail of a previous export and returns a `Cursor` (last block number + log index + whether the file is gzip). `cmd/export.go` uses that cursor as the start block and appends new logs to the same file — plain files via `O_APPEND`, gzip files by appending a second gzip member (concatenated members are a valid gzip stream, verified against both `gzcat` and Go's `gzip.Reader`). Logs already present in the boundary block are filtered out during the write. - -**Tech Stack:** Go 1.25, cobra, `github.com/ethereum/go-ethereum` v1.15.11 (`core/types.Log`, `common/hexutil`), `github.com/ethersphere/bee/v2` v2.7.0 (`pkg/log`), stdlib `compress/gzip`. - -**Spec:** No separate spec file — this was a bounded change designed and approved in-session. The approved design is reproduced in full under "Design Summary" below; executors should treat that section as the spec. - -## Design Summary - -1. **Flag:** `--resume` / `-r`, a path to a previous export. When set it overrides `--start` and `--output`. -2. **Format detection:** by magic bytes (`0x1f 0x8b`), never by file extension. The repo's own archives use `.gzip` while the request also mentions `.gz`; magic bytes make the extension irrelevant. -3. **Cursor:** the last line of the file that parses as JSON *and* carries both `blockNumber` and `logIndex`. Lines that fail either check are skipped, which discards a truncated trailing line left by a hard kill. -4. **Resume point:** `startBlock = cursor.BlockNumber` **inclusive**, because that block may have been only partially written. While writing, any log with `blockNumber < cursor.BlockNumber`, or `blockNumber == cursor.BlockNumber && logIndex <= cursor.LogIndex`, is skipped. No gaps, no duplicates. -5. **Writing:** the resume file is opened `O_APPEND`. Gzip files get a fresh `gzip.NewWriter` (a new member); plain files get the JSON encoder directly. -6. **`--compress` interaction:** a no-op when resuming an already-gzipped file; unchanged behavior when resuming a plain `.ndjson`. - -## Global Constraints - -- Go version: `go 1.25` (per `go.mod`). Do not raise or lower it. -- Do not add dependencies. Everything needed is already in `go.mod`: stdlib, `go-ethereum`, `bee/v2`, `cobra`. -- Lint config (`.golangci.yml`) enables `copyloopvar`, `errname`, `errorlint`, `goconst`, `misspell`, `nilerr`, `unconvert`, plus `gofmt` and `gofumpt` formatters. `errorlint` means: always wrap with `%w`, always compare with `errors.Is`/`errors.As`. -- Every exported identifier gets a doc comment starting with its own name. -- Error strings are lowercase and unpunctuated, matching the existing `pkg/` style (`"error creating file: %w"`). -- Commit messages use Conventional Commits (`feat:`, `fix:`, `test:`, `docs:`, `refactor:`), matching this repo's history. -- Tests run via `make test`, which is `go test -v ./pkg/...`. The repo currently has **zero** test files; these will be the first. -- Do not reformat or restructure code unrelated to this feature. - -## File Structure - -| File | Status | Responsibility | -|---|---|---| -| `pkg/resume/resume.go` | **Create** | Detect gzip vs plain, find the last complete log line, expose `Cursor` + `Cursor.Skip`. All tail-reading edge cases live here and nowhere else. | -| `pkg/resume/resume_test.go` | **Create** | Table tests for every tail-reading edge case, plus an append→re-read round trip. | -| `pkg/gzipstore/gzipstore.go` | **Modify** | Add `AppendWriter` returning an `io.WriteCloser` that appends a new gzip member. Existing `CompressFile` untouched. | -| `pkg/gzipstore/gzipstore_test.go` | **Create** | Verify an appended member reads back as one continuous stream. | -| `pkg/filestore/filestore.go` | **Modify** | Extract the write loop into an unexported `writeLogs`; add `AppendWriter` and `AppendLogsAsync(ctx, logChan, w, skip)`. `SaveLogsAsync` keeps its current signature. | -| `pkg/filestore/filestore_test.go` | **Create** | Verify truncate-vs-append semantics and that `skip` filters correctly. | -| `cmd/export.go` | **Modify** | Register `--resume`, read the cursor before dialing RPC, pick the writer, warn on overridden flags. Also fix the missing `wg.Wait()` on the cancellation path. | -| `README.md` | **Modify** | Document the flag in the Features list and the flag table. | - -**Why `pkg/resume` is its own package:** backward chunked reads, truncated-line recovery, and multi-member gzip handling are the only genuinely tricky logic in this feature. Isolating them behind `Read(path) (*Cursor, error)` keeps `cmd/export.go` readable and makes the edge cases testable without an RPC endpoint. - ---- - -### Task 1: `pkg/resume` — read the cursor from a previous export - -**Files:** -- Create: `pkg/resume/resume.go` -- Test: `pkg/resume/resume_test.go` - -**Interfaces:** -- Consumes: nothing from earlier tasks. -- Produces: - - `type Cursor struct { BlockNumber uint64; LogIndex uint; Compressed bool }` - - `func Read(path string) (*Cursor, error)` - - `func (c *Cursor) Skip(l types.Log) bool` - - `var ErrNoLogs error` - -- [ ] **Step 1: Write the failing tests** - -Create `pkg/resume/resume_test.go`. Note the package is `resume_test` (external) — later tasks add a round-trip test here that imports `filestore` and `gzipstore`, and an external test package keeps that free of import cycles. - -```go -package resume_test - -import ( - "bytes" - "compress/gzip" - "encoding/json" - "errors" - "os" - "path/filepath" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethersphere/batch-export/pkg/resume" -) - -// testLog builds a log shaped like the ones the exporter writes. types.Log -// always marshals blockNumber and logIndex, even at zero. -func testLog(blockNumber uint64, logIndex uint) types.Log { - return types.Log{ - Address: common.HexToAddress("0x45a1502382541cd610cc9068e88727426b696293"), - Topics: []common.Hash{common.HexToHash("0xae46785019700e30375a5d7b4f91e32f8060ef085111f896ebf889450aa2ab5a")}, - Data: bytes.Repeat([]byte{0xab}, 32), - BlockNumber: blockNumber, - TxHash: common.HexToHash("0xb08f07656eaafa8efc458e2aa90773648d95ec8119873d212b4377dea5190cc0"), - TxIndex: 9, - BlockHash: common.HexToHash("0x86dc5f9da5fcba5191f6b3d2ba995bd75532ef369a7baa3970b3fb292ae91324"), - Index: logIndex, - Removed: false, - } -} - -// ndjson renders logs as newline-delimited JSON, the exporter's output format. -func ndjson(t *testing.T, logs ...types.Log) []byte { - t.Helper() - - var buf bytes.Buffer - enc := json.NewEncoder(&buf) - for _, l := range logs { - if err := enc.Encode(l); err != nil { - t.Fatalf("encode log: %v", err) - } - } - return buf.Bytes() -} - -// gz compresses b into a single gzip member. -func gz(t *testing.T, b []byte) []byte { - t.Helper() - - var buf bytes.Buffer - w := gzip.NewWriter(&buf) - if _, err := w.Write(b); err != nil { - t.Fatalf("gzip write: %v", err) - } - if err := w.Close(); err != nil { - t.Fatalf("gzip close: %v", err) - } - return buf.Bytes() -} - -// write puts content in a temp file and returns its path. -func write(t *testing.T, name string, content []byte) string { - t.Helper() - - path := filepath.Join(t.TempDir(), name) - if err := os.WriteFile(path, content, 0o644); err != nil { - t.Fatalf("write %s: %v", path, err) - } - return path -} - -func TestReadCursor(t *testing.T) { - t.Parallel() - - threeLogs := ndjson(t, testLog(100, 0), testLog(101, 1), testLog(102, 7)) - - // many spans several 64 KiB backward-read windows. - manyLogs := make([]types.Log, 0, 2000) - for i := range 2000 { - manyLogs = append(manyLogs, testLog(uint64(1000+i), uint(i%16))) - } - many := ndjson(t, manyLogs...) - - // garbageLines are newline-delimited but unparseable, as if a different - // file had been concatenated onto a good export. - garbageLines := bytes.Repeat([]byte("not-json\n"), 12*1024) - - tests := []struct { - name string - content []byte - wantBlock uint64 - wantIndex uint - wantCompressed bool - }{ - { - name: "plain ndjson", - content: threeLogs, - wantBlock: 102, - wantIndex: 7, - }, - { - name: "single line", - content: ndjson(t, testLog(55, 3)), - wantBlock: 55, - wantIndex: 3, - }, - { - name: "truncated trailing line is discarded", - content: append(append([]byte{}, threeLogs...), []byte(`{"address":"0x45a15","topics":["0xae`)...), - wantBlock: 102, - wantIndex: 7, - }, - { - name: "trailing line without newline is still read", - content: bytes.TrimSuffix(threeLogs, []byte("\n")), - wantBlock: 102, - wantIndex: 7, - }, - { - name: "line missing blockNumber is skipped", - content: append(append([]byte{}, threeLogs...), []byte("{\"address\":\"0x1\",\"topics\":[],\"data\":\"0x\"}\n")...), - wantBlock: 102, - wantIndex: 7, - }, - { - name: "spans multiple backward read windows", - content: many, - wantBlock: 2999, - wantIndex: 15, - }, - { - name: "walks back across windows of garbage lines", - content: append(append([]byte{}, threeLogs...), garbageLines...), - wantBlock: 102, - wantIndex: 7, - }, - { - name: "gzip", - content: gz(t, threeLogs), - wantBlock: 102, - wantIndex: 7, - wantCompressed: true, - }, - { - name: "gzip spanning many logs", - content: gz(t, many), - wantBlock: 2999, - wantIndex: 15, - wantCompressed: true, - }, - { - name: "multi member gzip reads through to the last member", - content: append(gz(t, threeLogs), gz(t, ndjson(t, testLog(200, 2)))...), - wantBlock: 200, - wantIndex: 2, - wantCompressed: true, - }, - { - // A resume interrupted before the second member was flushed: the - // header is present but carries no decodable data. - name: "multi member gzip with unflushed final member", - content: append(gz(t, threeLogs), []byte{0x1f, 0x8b, 0x08, 0, 0, 0, 0, 0, 0, 0xff}...), - wantBlock: 102, - wantIndex: 7, - wantCompressed: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - got, err := resume.Read(write(t, "export.ndjson", tt.content)) - if err != nil { - t.Fatalf("Read() error = %v, want nil", err) - } - if got.BlockNumber != tt.wantBlock { - t.Errorf("BlockNumber = %d, want %d", got.BlockNumber, tt.wantBlock) - } - if got.LogIndex != tt.wantIndex { - t.Errorf("LogIndex = %d, want %d", got.LogIndex, tt.wantIndex) - } - if got.Compressed != tt.wantCompressed { - t.Errorf("Compressed = %t, want %t", got.Compressed, tt.wantCompressed) - } - }) - } -} - -func TestReadCursorErrors(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - content []byte - }{ - {name: "empty file", content: []byte{}}, - {name: "only a newline", content: []byte("\n")}, - {name: "only garbage lines", content: bytes.Repeat([]byte("not-json\n"), 10)}, - {name: "single unterminated line larger than the cap", content: bytes.Repeat([]byte("x"), 2<<20)}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - _, err := resume.Read(write(t, "export.ndjson", tt.content)) - if !errors.Is(err, resume.ErrNoLogs) { - t.Fatalf("Read() error = %v, want ErrNoLogs", err) - } - }) - } -} - -func TestReadMissingFile(t *testing.T) { - t.Parallel() - - _, err := resume.Read(filepath.Join(t.TempDir(), "does-not-exist.ndjson")) - if !errors.Is(err, os.ErrNotExist) { - t.Fatalf("Read() error = %v, want os.ErrNotExist", err) - } -} - -func TestCursorSkip(t *testing.T) { - t.Parallel() - - cursor := &resume.Cursor{BlockNumber: 100, LogIndex: 5} - - tests := []struct { - name string - log types.Log - want bool - }{ - {name: "earlier block", log: testLog(99, 0), want: true}, - {name: "same block earlier index", log: testLog(100, 4), want: true}, - {name: "same block same index", log: testLog(100, 5), want: true}, - {name: "same block later index", log: testLog(100, 6), want: false}, - {name: "later block index zero", log: testLog(101, 0), want: false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - if got := cursor.Skip(tt.log); got != tt.want { - t.Errorf("Skip() = %t, want %t", got, tt.want) - } - }) - } -} -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -go test ./pkg/resume/... -``` - -Expected: FAIL — `no required module provides package github.com/ethersphere/batch-export/pkg/resume` (the package does not exist yet). - -- [ ] **Step 3: Write the implementation** - -Create `pkg/resume/resume.go`: - -```go -// Package resume locates the point at which a previous export stopped so that -// a new run can continue from there. -package resume - -import ( - "bufio" - "bytes" - "compress/gzip" - "encoding/json" - "errors" - "fmt" - "io" - "os" - - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/core/types" -) - -const ( - // windowSize is how much of a plain NDJSON file is read at a time when - // walking backwards from the end. - windowSize = 64 * 1024 - // maxLineBytes caps how long a single line may be. Exported log lines run - // to a few hundred bytes, so anything beyond this is corruption, and the - // cap keeps a file without newlines from being read into memory whole. - maxLineBytes = 1 << 20 -) - -// ErrNoLogs indicates that a file holds no complete log entry to resume from. -var ErrNoLogs = errors.New("no complete log entry found") - -// Cursor marks the last log entry saved by a previous export. -type Cursor struct { - // BlockNumber is the block of the last saved log. A resumed export - // re-queries this block, because an interrupted run may have saved only - // some of its logs. - BlockNumber uint64 - // LogIndex is the index of the last saved log within BlockNumber. - LogIndex uint - // Compressed reports whether the file holds gzip data rather than plain - // NDJSON. - Compressed bool -} - -// Skip reports whether l was already written to the file the cursor came from. -func (c *Cursor) Skip(l types.Log) bool { - if l.BlockNumber != c.BlockNumber { - return l.BlockNumber < c.BlockNumber - } - return l.Index <= c.LogIndex -} - -// cursorLine is the part of an exported log line a cursor is built from. Both -// fields are pointers so that a line missing either one can be rejected. -type cursorLine struct { - BlockNumber *hexutil.Uint64 `json:"blockNumber"` - LogIndex *hexutil.Uint `json:"logIndex"` -} - -// Read returns a cursor for the last complete log entry in the file at path. -// The file may be plain NDJSON or gzip; the format is detected from its -// leading bytes rather than its extension. Lines that do not parse are -// skipped, which discards a partial line left behind by an interrupted write. -func Read(path string) (*Cursor, error) { - file, err := os.Open(path) - if err != nil { - return nil, fmt.Errorf("error opening resume file: %w", err) - } - defer file.Close() - - compressed, err := isGzip(file) - if err != nil { - return nil, fmt.Errorf("error reading resume file: %w", err) - } - - var cursor *Cursor - if compressed { - cursor, err = lastCursorGzip(file) - } else { - cursor, err = lastCursorPlain(file) - } - if err != nil { - return nil, err - } - - cursor.Compressed = compressed - - return cursor, nil -} - -// isGzip reports whether file starts with the gzip magic bytes. -func isGzip(file *os.File) (bool, error) { - var magic [2]byte - - n, err := file.ReadAt(magic[:], 0) - if err != nil && !errors.Is(err, io.EOF) { - return false, err - } - if n < len(magic) { - return false, nil - } - - return magic[0] == 0x1f && magic[1] == 0x8b, nil -} - -// lastCursorPlain walks a plain NDJSON file backwards a window at a time and -// returns a cursor for the last line that parses. -func lastCursorPlain(file *os.File) (*Cursor, error) { - offset, err := file.Seek(0, io.SeekEnd) - if err != nil { - return nil, fmt.Errorf("error seeking resume file: %w", err) - } - - // carry holds bytes from the window just read that precede its first - // newline. They belong to a line whose start lies in the next window back. - var carry []byte - - for offset > 0 { - size := int64(windowSize) - if offset < size { - size = offset - } - offset -= size - - window := make([]byte, size) - if _, err := file.ReadAt(window, offset); err != nil { - return nil, fmt.Errorf("error reading resume file: %w", err) - } - window = append(window, carry...) - - for { - i := bytes.LastIndexByte(window, '\n') - if i < 0 { - break - } - if cursor, err := parseCursor(window[i+1:]); err == nil { - return cursor, nil - } - window = window[:i] - } - - carry = window - if len(carry) > maxLineBytes { - return nil, ErrNoLogs - } - } - - return parseCursor(carry) -} - -// lastCursorGzip returns a cursor for the last line that parses in a gzip -// file. Gzip cannot be seeked, so the whole stream is decompressed. A stream -// truncated by an interrupted run still yields the last line it did decode. -func lastCursorGzip(file *os.File) (*Cursor, error) { - reader, err := gzip.NewReader(file) - if err != nil { - return nil, fmt.Errorf("error opening gzip resume file: %w", err) - } - defer reader.Close() - - // Multistream is on by default, so concatenated members read as one stream. - scanner := bufio.NewScanner(reader) - scanner.Buffer(make([]byte, 0, bufio.MaxScanTokenSize), maxLineBytes) - - var last *Cursor - for scanner.Scan() { - if cursor, err := parseCursor(scanner.Bytes()); err == nil { - last = cursor - } - } - - if last != nil { - return last, nil - } - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("error reading gzip resume file: %w", err) - } - - return nil, ErrNoLogs -} - -// parseCursor builds a cursor from a single NDJSON line. It rejects blank and -// truncated lines, and logs that carry no block number. -func parseCursor(line []byte) (*Cursor, error) { - line = bytes.TrimSpace(line) - if len(line) == 0 { - return nil, ErrNoLogs - } - - var parsed cursorLine - if err := json.Unmarshal(line, &parsed); err != nil { - return nil, ErrNoLogs - } - if parsed.BlockNumber == nil || parsed.LogIndex == nil { - return nil, ErrNoLogs - } - - return &Cursor{ - BlockNumber: uint64(*parsed.BlockNumber), - LogIndex: uint(*parsed.LogIndex), - }, nil -} -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -go test ./pkg/resume/... && gofumpt -l pkg/resume && go vet ./pkg/resume/... -``` - -Expected: `ok github.com/ethersphere/batch-export/pkg/resume`, no files listed by `gofumpt`, no vet output. - -If `gofumpt` is not installed, run `go run mvdan.cc/gofumpt@latest -l pkg/resume` instead. - -- [ ] **Step 5: Commit** - -```bash -git add pkg/resume/resume.go pkg/resume/resume_test.go -git commit -m "feat(resume): read the last saved log from a previous export" -``` - ---- - -### Task 2: `pkg/gzipstore` — append a gzip member - -**Files:** -- Modify: `pkg/gzipstore/gzipstore.go` (add to the existing file; leave `CompressFile` as-is) -- Test: `pkg/gzipstore/gzipstore_test.go` - -**Interfaces:** -- Consumes: nothing from earlier tasks. -- Produces: `func AppendWriter(filePath string) (io.WriteCloser, error)` - -- [ ] **Step 1: Write the failing test** - -Create `pkg/gzipstore/gzipstore_test.go`: - -```go -package gzipstore_test - -import ( - "bytes" - "compress/gzip" - "io" - "os" - "path/filepath" - "testing" - - "github.com/ethersphere/batch-export/pkg/gzipstore" -) - -// writeGzip creates a gzip file holding content and returns its path. -func writeGzip(t *testing.T, content string) string { - t.Helper() - - path := filepath.Join(t.TempDir(), "export.ndjson.gzip") - - var buf bytes.Buffer - w := gzip.NewWriter(&buf) - if _, err := io.WriteString(w, content); err != nil { - t.Fatalf("gzip write: %v", err) - } - if err := w.Close(); err != nil { - t.Fatalf("gzip close: %v", err) - } - if err := os.WriteFile(path, buf.Bytes(), 0o644); err != nil { - t.Fatalf("write %s: %v", path, err) - } - - return path -} - -// readGzip decompresses the whole file, following every member. -func readGzip(t *testing.T, path string) string { - t.Helper() - - file, err := os.Open(path) - if err != nil { - t.Fatalf("open %s: %v", path, err) - } - defer file.Close() - - reader, err := gzip.NewReader(file) - if err != nil { - t.Fatalf("gzip reader: %v", err) - } - defer reader.Close() - - got, err := io.ReadAll(reader) - if err != nil { - t.Fatalf("read gzip: %v", err) - } - - return string(got) -} - -func TestAppendWriterAddsReadableMember(t *testing.T) { - t.Parallel() - - path := writeGzip(t, "first\nsecond\n") - - w, err := gzipstore.AppendWriter(path) - if err != nil { - t.Fatalf("AppendWriter() error = %v", err) - } - if _, err := io.WriteString(w, "third\nfourth\n"); err != nil { - t.Fatalf("write: %v", err) - } - if err := w.Close(); err != nil { - t.Fatalf("Close() error = %v", err) - } - - const want = "first\nsecond\nthird\nfourth\n" - if got := readGzip(t, path); got != want { - t.Errorf("content = %q, want %q", got, want) - } -} - -func TestAppendWriterRepeatedAppends(t *testing.T) { - t.Parallel() - - path := writeGzip(t, "a\n") - - for _, line := range []string{"b\n", "c\n"} { - w, err := gzipstore.AppendWriter(path) - if err != nil { - t.Fatalf("AppendWriter() error = %v", err) - } - if _, err := io.WriteString(w, line); err != nil { - t.Fatalf("write: %v", err) - } - if err := w.Close(); err != nil { - t.Fatalf("Close() error = %v", err) - } - } - - const want = "a\nb\nc\n" - if got := readGzip(t, path); got != want { - t.Errorf("content = %q, want %q", got, want) - } -} - -func TestAppendWriterMissingFile(t *testing.T) { - t.Parallel() - - if _, err := gzipstore.AppendWriter(filepath.Join(t.TempDir(), "nope.gzip")); err == nil { - t.Fatal("AppendWriter() error = nil, want an error") - } -} -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -go test ./pkg/gzipstore/... -``` - -Expected: FAIL — `undefined: gzipstore.AppendWriter`. - -- [ ] **Step 3: Write the implementation** - -Add to `pkg/gzipstore/gzipstore.go`. Add `"errors"` to the import block; `io` and `os` are already imported. - -```go -// AppendWriter opens an existing gzip file for appending and returns a writer -// that adds a new gzip member to it. Concatenated members form a valid gzip -// stream, so readers see one continuous file and the existing bytes are never -// rewritten. The caller must close the writer to flush the member. -func AppendWriter(filePath string) (io.WriteCloser, error) { - file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, 0o644) - if err != nil { - return nil, fmt.Errorf("failed to open gzip file '%s' for appending: %w", filePath, err) - } - - return &memberWriter{file: file, gzip: gzip.NewWriter(file)}, nil -} - -// memberWriter writes one gzip member and owns the file it was opened from. -type memberWriter struct { - file *os.File - gzip *gzip.Writer -} - -func (w *memberWriter) Write(p []byte) (int, error) { - return w.gzip.Write(p) -} - -// Close finishes the gzip member and then closes the file. Both are attempted -// even if the first fails, so the descriptor is never leaked. -func (w *memberWriter) Close() error { - return errors.Join(w.gzip.Close(), w.file.Close()) -} -``` - -- [ ] **Step 4: Run the test to verify it passes** - -```bash -go test ./pkg/gzipstore/... && go vet ./pkg/gzipstore/... -``` - -Expected: `ok github.com/ethersphere/batch-export/pkg/gzipstore`, no vet output. - -- [ ] **Step 5: Commit** - -```bash -git add pkg/gzipstore/gzipstore.go pkg/gzipstore/gzipstore_test.go -git commit -m "feat(gzipstore): add AppendWriter for appending a gzip member" -``` - ---- - -### Task 3: `pkg/filestore` — append logs with a skip filter - -**Files:** -- Modify: `pkg/filestore/filestore.go` (whole file; see the full replacement below) -- Test: `pkg/filestore/filestore_test.go` - -**Interfaces:** -- Consumes: nothing from earlier tasks. `Cursor.Skip` from Task 1 satisfies the `skip` parameter, but this package must not import `pkg/resume` — it takes a plain function so the dependency runs one way only. -- Produces: - - `func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath string) error` (unchanged signature) - - `func AppendWriter(filePath string) (io.WriteCloser, error)` - - `func AppendLogsAsync(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, skip func(types.Log) bool) error` - -- [ ] **Step 1: Write the failing test** - -Create `pkg/filestore/filestore_test.go`: - -```go -package filestore_test - -import ( - "bufio" - "context" - "encoding/json" - "errors" - "os" - "path/filepath" - "strings" - "testing" - - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethersphere/batch-export/pkg/filestore" -) - -// blocksIn returns the block number of every log line in the file at path. -func blocksIn(t *testing.T, path string) []uint64 { - t.Helper() - - file, err := os.Open(path) - if err != nil { - t.Fatalf("open %s: %v", path, err) - } - defer file.Close() - - var blocks []uint64 - scanner := bufio.NewScanner(file) - for scanner.Scan() { - if strings.TrimSpace(scanner.Text()) == "" { - continue - } - var l types.Log - if err := json.Unmarshal(scanner.Bytes(), &l); err != nil { - t.Fatalf("unmarshal %q: %v", scanner.Text(), err) - } - blocks = append(blocks, l.BlockNumber) - } - if err := scanner.Err(); err != nil { - t.Fatalf("scan %s: %v", path, err) - } - - return blocks -} - -// feed returns a closed channel already holding logs for the given blocks. -func feed(blocks ...uint64) <-chan types.Log { - ch := make(chan types.Log, len(blocks)) - for _, b := range blocks { - ch <- types.Log{BlockNumber: b} - } - close(ch) - - return ch -} - -func equal(a, b []uint64) bool { - if len(a) != len(b) { - return false - } - for i := range a { - if a[i] != b[i] { - return false - } - } - - return true -} - -func TestSaveLogsAsyncReplacesExistingFile(t *testing.T) { - t.Parallel() - - path := filepath.Join(t.TempDir(), "export.ndjson") - if err := os.WriteFile(path, []byte("stale\n"), 0o644); err != nil { - t.Fatalf("seed %s: %v", path, err) - } - - if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { - t.Fatalf("SaveLogsAsync() error = %v", err) - } - - want := []uint64{1, 2} - if got := blocksIn(t, path); !equal(got, want) { - t.Errorf("blocks = %v, want %v", got, want) - } -} - -func TestAppendLogsAsyncKeepsExistingContent(t *testing.T) { - t.Parallel() - - path := filepath.Join(t.TempDir(), "export.ndjson") - if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { - t.Fatalf("SaveLogsAsync() error = %v", err) - } - - w, err := filestore.AppendWriter(path) - if err != nil { - t.Fatalf("AppendWriter() error = %v", err) - } - if err := filestore.AppendLogsAsync(t.Context(), feed(3, 4), w, nil); err != nil { - t.Fatalf("AppendLogsAsync() error = %v", err) - } - - want := []uint64{1, 2, 3, 4} - if got := blocksIn(t, path); !equal(got, want) { - t.Errorf("blocks = %v, want %v", got, want) - } -} - -func TestAppendLogsAsyncSkipsFilteredLogs(t *testing.T) { - t.Parallel() - - path := filepath.Join(t.TempDir(), "export.ndjson") - if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { - t.Fatalf("SaveLogsAsync() error = %v", err) - } - - w, err := filestore.AppendWriter(path) - if err != nil { - t.Fatalf("AppendWriter() error = %v", err) - } - skip := func(l types.Log) bool { return l.BlockNumber <= 2 } - if err := filestore.AppendLogsAsync(t.Context(), feed(1, 2, 3, 4), w, skip); err != nil { - t.Fatalf("AppendLogsAsync() error = %v", err) - } - - want := []uint64{1, 2, 3, 4} - if got := blocksIn(t, path); !equal(got, want) { - t.Errorf("blocks = %v, want %v", got, want) - } -} - -func TestAppendLogsAsyncClosesWriterOnCancel(t *testing.T) { - t.Parallel() - - path := filepath.Join(t.TempDir(), "export.ndjson") - if err := filestore.SaveLogsAsync(t.Context(), feed(1), path); err != nil { - t.Fatalf("SaveLogsAsync() error = %v", err) - } - - w, err := filestore.AppendWriter(path) - if err != nil { - t.Fatalf("AppendWriter() error = %v", err) - } - - ctx, cancel := context.WithCancel(t.Context()) - cancel() - - // An open channel that never delivers, so cancellation is the only exit. - if err := filestore.AppendLogsAsync(ctx, make(chan types.Log), w, nil); !errors.Is(err, context.Canceled) { - t.Fatalf("AppendLogsAsync() error = %v, want context.Canceled", err) - } - - // The writer must already be closed; closing it again must fail. - if err := w.Close(); err == nil { - t.Error("writer was left open after cancellation") - } -} -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -go test ./pkg/filestore/... -``` - -Expected: FAIL — `undefined: filestore.AppendWriter` and `undefined: filestore.AppendLogsAsync`. - -- [ ] **Step 3: Write the implementation** - -Replace the whole of `pkg/filestore/filestore.go` with: - -```go -package filestore - -import ( - "context" - "encoding/json" - "fmt" - "io" - "os" - - "github.com/ethereum/go-ethereum/core/types" -) - -// SaveLogsAsync writes logs to a file asynchronously, replacing any file -// already at filePath. -func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath string) error { - file, err := os.Create(filePath) - if err != nil { - return fmt.Errorf("error creating file: %w", err) - } - defer file.Close() - - return writeLogs(ctx, logChan, file, nil) -} - -// AppendWriter opens an existing NDJSON file for appending. -func AppendWriter(filePath string) (io.WriteCloser, error) { - file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, 0o644) - if err != nil { - return nil, fmt.Errorf("error opening file for appending: %w", err) - } - - return file, nil -} - -// AppendLogsAsync writes logs to w asynchronously, keeping whatever the -// destination already holds. Logs for which skip reports true are dropped; a -// nil skip writes every log. The writer is closed before returning, including -// when the context is cancelled, so a buffered destination is always flushed. -func AppendLogsAsync(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, skip func(types.Log) bool) error { - defer w.Close() - - return writeLogs(ctx, logChan, w, skip) -} - -// writeLogs encodes logs from logChan to w as NDJSON until the channel is -// closed or the context is cancelled. -func writeLogs(ctx context.Context, logChan <-chan types.Log, w io.Writer, skip func(types.Log) bool) error { - encoder := json.NewEncoder(w) - - for { - select { - case <-ctx.Done(): - return ctx.Err() - case logObj, ok := <-logChan: - if !ok { - return nil - } - - if skip != nil && skip(logObj) { - continue - } - - if err := encoder.Encode(logObj); err != nil { - return fmt.Errorf("error encoding log: %w", err) - } - } - } -} -``` - -- [ ] **Step 4: Run the test to verify it passes** - -```bash -go test ./pkg/... && go vet ./pkg/... -``` - -Expected: `ok` for `pkg/filestore`, `pkg/gzipstore`, and `pkg/resume`; no vet output. - -- [ ] **Step 5: Commit** - -```bash -git add pkg/filestore/filestore.go pkg/filestore/filestore_test.go -git commit -m "feat(filestore): add AppendLogsAsync with a skip filter" -``` - ---- - -### Task 4: Wire `--resume` into the export command - -**Files:** -- Modify: `cmd/export.go` - -**Interfaces:** -- Consumes: `resume.Read`, `resume.Cursor`, `Cursor.Skip` (Task 1); `gzipstore.AppendWriter` (Task 2); `filestore.AppendWriter`, `filestore.AppendLogsAsync` (Task 3). -- Produces: the `--resume` / `-r` CLI flag. Nothing consumes this task. - -This task has no unit test: `RunE` needs a live RPC endpoint, and the repo has no HTTP fixture harness to build on. The logic worth testing was pushed into `pkg/` by Tasks 1–3 and is covered there. Step 5 verifies this task end to end against the real files in `dist/`. - -- [ ] **Step 1: Add the flag variable and registration** - -In `initExportCmd`, add `resumeFile` to the `var` block at the top: - -```go - var ( - startBlock uint64 - endBlock uint64 - rpcEndpoint string - maxRequest int - blockRangeLimit uint32 - outputFile string - compress bool - resumeFile string - ) -``` - -And register the flag alongside the others, after the `--compress` line: - -```go - cmd.Flags().StringVarP(&resumeFile, "resume", "r", "", "Resume a previous export file (.ndjson, .gz or .gzip); overrides --start and --output") -``` - -- [ ] **Step 2: Read the cursor at the top of RunE** - -Insert this as the first statement inside `RunE`, immediately after `ctx := cmd.Context()` and **before** `ethclient.NewClient`. Reading the cursor first means a bad path fails instantly instead of after dialing the RPC endpoint. - -```go - var cursor *resume.Cursor - if resumeFile != "" { - cursor, err = resume.Read(resumeFile) - if err != nil { - return fmt.Errorf("failed to read resume file: %w", err) - } - - if cmd.Flags().Changed("start") { - c.log.Warning("--start is ignored when --resume is set", "resumeFile", resumeFile) - } - if cmd.Flags().Changed("output") { - c.log.Warning("--output is ignored when --resume is set, logs are appended to the resume file", "resumeFile", resumeFile) - } - if cursor.Compressed && compress { - c.log.Warning("--compress is ignored when resuming an already compressed file", "resumeFile", resumeFile) - compress = false - } - - outputFile = resumeFile - startBlock = cursor.BlockNumber - - c.log.Info("Resuming export", - "resumeFile", resumeFile, - "startBlock", startBlock, - "lastLogIndex", cursor.LogIndex, - "compressed", cursor.Compressed, - ) - } -``` - -Then add the imports. The `import` block becomes: - -```go -import ( - "context" - "errors" - "fmt" - "io" - "sync" - "time" - - ethclient "github.com/ethersphere/batch-export/pkg/ethclientwrapper" - "github.com/ethersphere/batch-export/pkg/eventfetcher" - "github.com/ethersphere/batch-export/pkg/filestore" - "github.com/ethersphere/batch-export/pkg/gzipstore" - "github.com/ethersphere/batch-export/pkg/resume" - "github.com/ethersphere/bee/v2/pkg/config" - "github.com/ethersphere/bee/v2/pkg/util/abiutil" - "github.com/spf13/cobra" -) -``` - -- [ ] **Step 3: Branch the writer goroutine** - -Replace the existing saver goroutine — the block from `go func() {` through the closing `}()` that currently calls `filestore.SaveLogsAsync` — with: - -```go - go func() { - defer wg.Done() - - if err := saveLogs(ctx, logChan, outputFile, cursor); err != nil { - if errors.Is(err, context.Canceled) { - c.log.Error(err, "context canceled while saving logs") - return - } - c.log.Error(err, "error saving logs") - return - } - c.log.Info("all logs have been saved", "outputFile", outputFile) - }() -``` - -Then add this helper at the end of `cmd/export.go`, after `initExportCmd`: - -```go -// saveLogs writes logs to outputFile. With a nil cursor it replaces the file; -// otherwise it appends to the file the cursor came from, dropping any log that -// was already written to it. -func saveLogs(ctx context.Context, logChan <-chan types.Log, outputFile string, cursor *resume.Cursor) error { - if cursor == nil { - return filestore.SaveLogsAsync(ctx, logChan, outputFile) - } - - var ( - w io.WriteCloser - err error - ) - if cursor.Compressed { - w, err = gzipstore.AppendWriter(outputFile) - } else { - w, err = filestore.AppendWriter(outputFile) - } - if err != nil { - return fmt.Errorf("error opening output file for appending: %w", err) - } - - return filestore.AppendLogsAsync(ctx, logChan, w, cursor.Skip) -} -``` - -`saveLogs` takes `types.Log`, so add one more import to the block from Step 2, in the third-party group above the `batch-export` imports: - -```go - "github.com/ethereum/go-ethereum/core/types" -``` - -- [ ] **Step 4: Wait for the saver before returning on cancellation** - -The existing `<-ctx.Done()` branch logs `"context canceled, waiting for logs to be saved..."` but returns without ever waiting, so the saver goroutine can be killed mid-write. That was survivable when every write was a lone `Encode` call; it is not survivable now, because an unflushed gzip member leaves a trailing fragment that the next `--resume` has to discard. Fix the branch to actually wait: - -```go - case <-ctx.Done(): - c.log.Info("context canceled, waiting for logs to be saved...") - wg.Wait() - if err := compressFunc(); err != nil { - return errors.Join(fmt.Errorf("error compressing file: %w", err), ctx.Err()) - } - return ctx.Err() -``` - -This cannot deadlock: on cancellation `writeLogs` returns from its `ctx.Done()` case immediately, and `AppendLogsAsync`'s deferred `Close` flushes the gzip member on the way out. - -- [ ] **Step 5: Build and verify against the real export files** - -```bash -make binary && go vet ./... && gofumpt -l cmd pkg -``` - -Expected: the binary builds, no vet output, no files listed by `gofumpt`. - -Then confirm the flag is registered and that both file formats are read correctly. `dist/export.ndjson` ends at block `0x2db0697` (47908503), log index `0x18` (24): - -```bash -./dist/batch-export export --help | grep -A1 resume - -# Plain NDJSON: cursor must be block 47908503. -./dist/batch-export export -v debug --resume dist/export.ndjson --end 47908504 2>&1 | head -20 - -# Gzip: same cursor, read through the compressed stream. -./dist/batch-export export -v debug --resume dist/export.ndjson.gzip --end 47908504 2>&1 | head -20 -``` - -Expected: both runs log `"Resuming export"` with `startBlock=47908503` and `lastLogIndex=24`, the second also with `compressed=true`. Both then fetch the one-block range and exit cleanly. - -Verify the appended gzip is still one readable stream, and that no duplicate was written: - -```bash -cp dist/export.ndjson.gzip /tmp/resume-check.gzip -BEFORE=$(gzcat /tmp/resume-check.gzip | wc -l) -./dist/batch-export export --resume /tmp/resume-check.gzip --end 47908504 -gzcat /tmp/resume-check.gzip | wc -l # >= BEFORE, and must not error -gzcat /tmp/resume-check.gzip | tail -n 3 -gzcat /tmp/resume-check.gzip | sort | uniq -d | head # must print nothing -echo "before=$BEFORE" -``` - -Expected: `gzcat` exits 0, the line count is at least `BEFORE`, and the duplicate check prints nothing. - -Finally, confirm the flag-override warnings fire: - -```bash -./dist/batch-export export --resume dist/export.ndjson --start 100 --output other.ndjson --compress --end 47908504 2>&1 | grep -i "ignored" -``` - -Expected: warnings for `--start` and `--output`. `--compress` is not warned about here because `dist/export.ndjson` is not compressed; re-run with `--resume dist/export.ndjson.gzip` to see that third warning. - -- [ ] **Step 6: Commit** - -```bash -git add cmd/export.go -git commit -m "feat(export): add --resume to continue a previous export" -``` - ---- - -### Task 5: Document the flag - -**Files:** -- Modify: `README.md` - -**Interfaces:** -- Consumes: the `--resume` flag from Task 4. -- Produces: nothing. - -- [ ] **Step 1: Add a feature bullet** - -In the `## Features` list, after the `Graceful shutdown on interrupt signals (Ctrl+C).` bullet, add: - -```markdown -- Resume an interrupted export from an existing `.ndjson`, `.gz`, or `.gzip` file. -``` - -- [ ] **Step 2: Add the flag to the flag table** - -In the `## Flags` block, insert the `--resume` line between `--output` and `--start` so the list stays alphabetical: - -```sh - -r, --resume string Resume a previous export file (.ndjson, .gz or .gzip); overrides --start and --output -``` - -- [ ] **Step 3: Document the behavior** - -After the `## Flags` code block and before the `The produced NDJSON is consumed by ...` line, add: - -````markdown -### Resuming an interrupted export - -Point `--resume` at a file a previous run produced. The tool reads its last -complete entry, restarts from that block, and appends to the same file: - -```sh -./dist/batch-export export --resume dist/export.ndjson -``` - -Compressed exports work the same way and are detected by content, not by -extension, so `.gz` and `.gzip` both work: - -```sh -./dist/batch-export export --resume dist/export.ndjson.gzip -``` - -The resumed block is re-queried, because an interrupted run may have saved only -part of it; entries already in the file are skipped, so resuming never -duplicates or drops a log. Appending to a compressed export adds a second gzip -member — standard tools such as `gzcat`, `gunzip`, and Go's `compress/gzip` -read the result as one continuous stream. - -When `--resume` is set it overrides `--start` and `--output`. -```` - -- [ ] **Step 4: Verify the whole suite still passes** - -```bash -make test && make vet && make lint -``` - -Expected: all `pkg/` tests pass, no vet output, no lint findings. - -- [ ] **Step 5: Commit** - -```bash -git add README.md -git commit -m "docs: document the --resume flag" -``` - ---- - -## Self-Review - -**Spec coverage** — every point of the Design Summary maps to a task: - -| Design point | Task | -|---|---| -| 1. `--resume` / `-r` flag, overrides `--start` / `--output` | Task 4 Steps 1–2 | -| 2. Magic-byte format detection | Task 1 (`isGzip`) | -| 3. Last parseable line; truncated tail discarded | Task 1 (`parseCursor`, `lastCursorPlain`, `lastCursorGzip`) | -| 4. Inclusive resume block + skip already-written logs | Task 1 (`Cursor.Skip`), Task 3 (`skip` param), Task 4 Step 2 | -| 5. `O_APPEND`; gzip gets a new member | Task 2, Task 3 (`AppendWriter`), Task 4 Step 3 | -| 6. `--compress` no-op on already-gzipped input | Task 4 Step 2 | - -**Type consistency** — `Cursor.Skip(types.Log) bool` (Task 1) matches the `skip func(types.Log) bool` parameter of `AppendLogsAsync` (Task 3) and the `cursor.Skip` value passed in Task 4. Both `AppendWriter` functions return `(io.WriteCloser, error)`, which is what `saveLogs` assigns into its `io.WriteCloser` variable. `SaveLogsAsync` keeps its existing three-argument signature, so its call site in Task 4 needs no change beyond moving into `saveLogs`. - -**Dependency direction** — `pkg/filestore` takes a bare `func(types.Log) bool` rather than importing `pkg/resume`, so `cmd` depends on all three packages while none of them depend on each other. - -**Known gaps, deliberate:** -- `cmd/export.go` has no unit test (no RPC fixture harness exists in this repo); Task 4 Step 5 covers it manually against the real `dist/` files instead. -- Resume assumes the file was produced by this tool against the same contract and chain. A file from a different chain would resume from a meaningless block. Guarding that would mean writing a header, which changes the output format that `batch-archive` consumes — out of scope here. -- A multi-member gzip is legal and read transparently by standard tooling, but is a format change in spirit. If anything in [batch-archive](https://github.com/ethersphere/batch-archive) parses gzip by hand instead of through a standard library, that is the one place to check. From b53a9c33d39439b0991f65e16c0e655eb7fb4f38 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Wed, 26 Aug 2026 18:26:07 +0200 Subject: [PATCH 16/35] refactor: condense comments to the non-obvious rationale Comments had grown into design essays restating what the code already says. Keep only the reasoning that stops the code being broken by a well-meaning simplification -- why countingReader must implement io.ByteReader, why Multistream is switched off per member, why a line without a newline means an interrupted write, why the close error is joined -- and drop the narration around it. Also removes the step-by-step narration in CompressFile and stale references to the review that produced the tests. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 2 +- cmd/export.go | 24 +++--- pkg/filestore/filestore.go | 19 ++--- pkg/filestore/filestore_test.go | 6 +- pkg/gzipstore/gzipstore.go | 21 ++--- pkg/resume/resume.go | 142 ++++++++++++++------------------ pkg/resume/resume_test.go | 84 ++++++++----------- 7 files changed, 124 insertions(+), 174 deletions(-) diff --git a/README.md b/README.md index 3170397..6cd9e99 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ end of the last newline-terminated line that parses as a log entry, or of the last whole gzip member — and discards whatever follows it, logging how many bytes it dropped and from where: -``` +```log "level"="warning" "msg"="resume file ends with a partial write, discarding it" "offset"=89649991 "discardedBytes"=317 ``` diff --git a/cmd/export.go b/cmd/export.go index 108010e..be38228 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -101,9 +101,9 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save return err } - // The destination is opened before the first log is fetched: were - // it opened in the saving goroutine instead, a failure there would - // leave the fetcher pushing into a channel nobody drains. + // Opened before the first log is fetched: from inside the saving + // goroutine, a failure here would leave the fetcher pushing into a + // channel nobody drains. w, err := openOutput(outputFile, cursor) if err != nil { return fmt.Errorf("failed to open output file: %w", err) @@ -194,14 +194,13 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save return nil } -// discardPartialWrite drops the tail an interrupted run left behind in the -// resume file, so that new logs are appended onto a boundary the reader -// positively identified rather than onto half a line or half a gzip member. -// Nothing recoverable is lost: every entry discarded here falls at or after -// the cursor, so the resumed query fetches it again. +// discardPartialWrite drops the tail an interrupted run left in the resume +// file, so logs are appended onto a boundary the reader positively identified +// rather than onto half a line or half a gzip member. Nothing recoverable is +// lost: everything discarded falls at or after the cursor and is re-fetched. // -// It is a no-op when there is no resume file or the file ends cleanly, and it -// never truncates past the offset the reader reported. +// It is a no-op without a resume file or when the file ends cleanly, and never +// truncates past the offset the reader reported. func (c *command) discardPartialWrite(outputFile string, cursor *resume.Cursor) error { if cursor == nil || !cursor.Truncated { return nil @@ -241,9 +240,8 @@ func openOutput(outputFile string, cursor *resume.Cursor) (io.WriteCloser, error } } -// saveLogs writes logs to w, dropping any entry a resumed export already -// holds. A nil cursor means the destination starts empty, so every log is -// kept. w is closed before returning, cancellation included. +// saveLogs writes logs to w, dropping any entry a resumed export already holds. +// A nil cursor means the destination starts empty, so every log is kept. func saveLogs(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, cursor *resume.Cursor) error { var skip func(types.Log) bool if cursor != nil { diff --git a/pkg/filestore/filestore.go b/pkg/filestore/filestore.go index b7c700f..8c5a10c 100644 --- a/pkg/filestore/filestore.go +++ b/pkg/filestore/filestore.go @@ -11,9 +11,9 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// CreateWriter creates the file at filePath, replacing anything already there, -// and returns a writer for it. Opening the destination separately lets a -// caller fail before it starts producing logs it would have nowhere to put. +// CreateWriter creates the file at filePath, replacing anything already there. +// It is separate from writing so a caller can fail before it starts producing +// logs it would have nowhere to put. func CreateWriter(filePath string) (io.WriteCloser, error) { file, err := os.Create(filePath) if err != nil { @@ -46,14 +46,13 @@ func AppendWriter(filePath string) (io.WriteCloser, error) { // AppendLogsAsync writes logs to w asynchronously, keeping whatever the // destination already holds. Logs for which skip reports true are dropped; a -// nil skip writes every log. The writer is closed before returning, including -// when the context is cancelled, so a buffered destination is always flushed. +// nil skip writes every log. w is closed before returning, cancellation +// included, so a buffered destination is always flushed. // -// The close error is joined into the result rather than discarded: on a -// compressed destination Close writes the terminator and footer that make the -// data readable, so a failure there leaves a truncated member behind and must -// not be reported as a completed save. Joining preserves errors.Is, so a -// cancelled context still reads as context.Canceled. +// The close error is joined rather than discarded: on a compressed destination +// Close writes the terminator and footer, so a failure there leaves a truncated +// member that must not be reported as a completed save. errors.Join keeps +// errors.Is working, so a cancelled context still reads as context.Canceled. func AppendLogsAsync(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, skip func(types.Log) bool) (err error) { defer func() { err = errors.Join(err, w.Close()) }() diff --git a/pkg/filestore/filestore_test.go b/pkg/filestore/filestore_test.go index aa69f22..e4c96d6 100644 --- a/pkg/filestore/filestore_test.go +++ b/pkg/filestore/filestore_test.go @@ -46,10 +46,8 @@ func blocksIn(t *testing.T, path string) []uint64 { } // feed returns a closed channel already holding logs for the given blocks. -// -// Topics is set to a non-nil empty slice rather than left nil: go-ethereum's -// generated Log.UnmarshalJSON rejects a null "topics" field as missing, so a -// nil Topics would fail the round trip through blocksIn below. +// Topics must stay a non-nil empty slice: go-ethereum's generated +// Log.UnmarshalJSON rejects a null "topics" as a missing required field. func feed(blocks ...uint64) <-chan types.Log { ch := make(chan types.Log, len(blocks)) for _, b := range blocks { diff --git a/pkg/gzipstore/gzipstore.go b/pkg/gzipstore/gzipstore.go index 3091002..39fa6da 100644 --- a/pkg/gzipstore/gzipstore.go +++ b/pkg/gzipstore/gzipstore.go @@ -10,37 +10,32 @@ import ( // CompressFile compresses the specified input file into the specified output gzip file. func CompressFile(inputFilePath string, outputFilePath string) error { - // open the input file for reading inputFile, err := os.Open(inputFilePath) if err != nil { return fmt.Errorf("failed to open input file '%s': %w", inputFilePath, err) } defer inputFile.Close() - // create the output file for writing outputFile, err := os.Create(outputFilePath) if err != nil { return fmt.Errorf("failed to create output file '%s': %w", outputFilePath, err) } defer outputFile.Close() - // create a new gzip writer that writes to the output file gzipWriter := gzip.NewWriter(outputFile) defer gzipWriter.Close() - // copy the contents from the input file to the gzip writer - _, err = io.Copy(gzipWriter, inputFile) - if err != nil { + if _, err := io.Copy(gzipWriter, inputFile); err != nil { return fmt.Errorf("failed to write compressed data to '%s': %w", outputFilePath, err) } return nil } -// AppendWriter opens an existing gzip file for appending and returns a writer -// that adds a new gzip member to it. Concatenated members form a valid gzip -// stream, so readers see one continuous file and the existing bytes are never -// rewritten. The caller must close the writer to flush the member. +// AppendWriter opens an existing gzip file and returns a writer that adds a new +// gzip member to it. Concatenated members form a valid gzip stream, so readers +// see one continuous file and the existing bytes are never rewritten. The +// caller must close the writer to flush the member. func AppendWriter(filePath string) (io.WriteCloser, error) { file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, 0o644) if err != nil { @@ -60,10 +55,8 @@ func (w *memberWriter) Write(p []byte) (int, error) { return w.gzip.Write(p) } -// Close finishes the gzip member and then closes the file. Both are attempted -// even if the first fails, so the descriptor is never leaked. Closing the -// member writes the data the file needs to be readable at all, so the error -// names the file it belongs to. +// Close finishes the gzip member, then closes the file. Both are attempted even +// if the first fails, so the descriptor is never leaked. func (w *memberWriter) Close() error { if err := errors.Join(w.gzip.Close(), w.file.Close()); err != nil { return fmt.Errorf("failed to close gzip file '%s': %w", w.file.Name(), err) diff --git a/pkg/resume/resume.go b/pkg/resume/resume.go index 8b1678c..72d07b2 100644 --- a/pkg/resume/resume.go +++ b/pkg/resume/resume.go @@ -17,27 +17,23 @@ import ( ) const ( - // windowSize is how much of a plain NDJSON file is read at a time when - // walking backwards from the end. + // windowSize is how much of a plain NDJSON file is read per backward step. windowSize = 64 * 1024 - // maxLineBytes caps how long a single line may be. Exported log lines run - // to a few hundred bytes, so anything beyond this is corruption, and the - // cap keeps a file without newlines from being read into memory whole. + // maxLineBytes caps a single line, so a file without newlines is not read + // into memory whole. Exported log lines run to a few hundred bytes. maxLineBytes = 1 << 20 // bufferSize is how much of a gzip file is buffered per read. bufferSize = 64 * 1024 ) var ( - // ErrNoLogs indicates that a file holds no complete log entry to resume - // from. + // ErrNoLogs indicates that a file holds no complete log entry. ErrNoLogs = errors.New("no complete log entry found") - // ErrNoCleanBoundary indicates that a file holds log entries but no point - // at which its content is known to be complete, so nothing can be appended - // to it without corrupting what is already there. + // ErrNoCleanBoundary indicates that a file holds log entries but no offset + // at which appending is safe. ErrNoCleanBoundary = errors.New("file was left partially written and has no clean boundary to append at") - // errLineTooLong indicates that a line ran past maxLineBytes, which means - // the data is not the NDJSON an export writes. + // errLineTooLong indicates a line ran past maxLineBytes, so the data is not + // the NDJSON an export writes. errLineTooLong = errors.New("log line exceeds the maximum length") ) @@ -45,19 +41,15 @@ var ( // the point up to which that export's file is known to be complete. type Cursor struct { // BlockNumber is the block of the last saved log. A resumed export - // re-queries this block, because an interrupted run may have saved only - // some of its logs. + // re-queries it, because an interrupted run may have saved only part of it. BlockNumber uint64 // LogIndex is the index of the last saved log within BlockNumber. LogIndex uint - // Compressed reports whether the file holds gzip data rather than plain - // NDJSON. + // Compressed reports whether the file holds gzip rather than plain NDJSON. Compressed bool - // CleanSize is the byte offset at which the file's recoverable content - // ends. It always falls just past the entry the cursor points at: for - // plain NDJSON just past that line's newline, for gzip just past the - // member the line ends in. Bytes beyond it are a partial write left by an - // interrupted run and must be discarded before anything is appended. + // CleanSize is the offset just past the entry the cursor points at: past + // that line's newline, or past the gzip member it ends in. Bytes beyond it + // are a partial write and must be discarded before anything is appended. CleanSize int64 // Truncated reports whether any bytes follow CleanSize. Truncated bool @@ -72,27 +64,22 @@ func (c *Cursor) Skip(l types.Log) bool { } // cursorLine is the part of an exported log line a cursor is built from. Both -// fields are pointers so that a line missing either one can be rejected. +// fields are pointers so a line missing either one can be rejected. type cursorLine struct { BlockNumber *hexutil.Uint64 `json:"blockNumber"` LogIndex *hexutil.Uint `json:"logIndex"` } -// Read returns a cursor for the last complete log entry in the file at path. -// The file may be plain NDJSON or gzip; the format is detected from its -// leading bytes rather than its extension. +// Read returns a cursor for the last complete log entry in the file at path, +// detecting plain NDJSON or gzip from the leading bytes rather than the +// extension. // -// Only an entry that is complete and properly terminated counts: a plain line -// must end in a newline, and a compressed line must sit inside a gzip member -// that decoded to a clean end of stream. The cursor therefore also reports -// where the file's recoverable content ends (CleanSize) and whether a partial -// write follows it (Truncated). A caller that appends must discard everything -// past CleanSize first, which is lossless: every entry discarded that way is -// re-fetched by the resumed query. +// Only a properly terminated entry counts: a plain line must end in a newline, +// a compressed one must sit in a member that decoded to a clean end of stream. +// A caller that appends must first discard everything past CleanSize. // -// It returns ErrNoLogs when the file holds no complete entry at all, and -// ErrNoCleanBoundary when it holds entries but no point at which appending is -// safe. +// It returns ErrNoLogs when there is no complete entry, and ErrNoCleanBoundary +// when there are entries but no offset at which appending is safe. func Read(path string) (*Cursor, error) { file, err := os.Open(path) if err != nil { @@ -141,24 +128,20 @@ func isGzip(file *os.File) (bool, error) { return magic[0] == 0x1f && magic[1] == 0x8b, nil } -// lastCursorPlain walks a plain NDJSON file of the given size backwards a -// window at a time and returns a cursor for the last line that both parses and -// is terminated by a newline. json.Encoder writes a value and its newline in a -// single call, so a trailing line without one was cut short by an interrupted -// run: it is passed over, and CleanSize points just past the newline of the -// last line that was written whole. +// lastCursorPlain walks a plain NDJSON file backwards a window at a time and +// returns a cursor for the last line that parses and ends in a newline. +// json.Encoder writes a value and its newline in one call, so a trailing line +// without one was cut short by an interrupted run and is passed over. func lastCursorPlain(file *os.File, size int64) (*Cursor, error) { var ( offset = size - // end is the offset one past the last byte of the region currently - // held in window, and terminated reports whether the file byte at end - // is the newline closing that region's last line. Only a terminated - // region can yield a cursor. + // end is one past the last byte held in window; terminated reports + // whether the file byte at end is the newline closing that region's + // last line. Only a terminated region can yield a cursor. end = size terminated bool - // carry holds bytes from the window just read that precede its first - // newline. They belong to a line whose start lies in the next window - // back. + // carry holds the bytes before the window's first newline, belonging to + // a line that starts in the next window back. carry []byte ) @@ -207,16 +190,14 @@ func lastCursorPlain(file *os.File, size int64) (*Cursor, error) { } // lastCursorGzip walks a gzip file one member at a time and returns a cursor -// for the last log line that lies wholly inside a cleanly terminated member. -// Gzip cannot be seeked, so the whole stream is decompressed. +// for the last log line inside a cleanly terminated member. Gzip cannot be +// seeked, so the whole stream is decompressed. // -// A member left half-written by an interrupted run cannot decode cleanly: its -// CRC and length trailer are missing, so the reader reports an error instead -// of io.EOF. The walk stops there and CleanSize is the offset just past the -// last member that did end cleanly, which is a member boundary and so a valid -// place to concatenate the next one. Lines are carried across member -// boundaries, so a line split between two members is still recognised, and a -// member that ends mid-line is not treated as a boundary at all. +// A half-written member has no CRC and length trailer, so it reports an error +// rather than io.EOF; the walk stops there and CleanSize is the end of the last +// member that did terminate cleanly, which is a valid place to concatenate the +// next one. Lines are carried across members, so one split between two members +// is still recognised and a member ending mid-line is not a boundary. func lastCursorGzip(file *os.File) (*Cursor, error) { counter := &countingReader{reader: bufio.NewReaderSize(file, bufferSize)} @@ -227,17 +208,17 @@ func lastCursorGzip(file *os.File) (*Cursor, error) { defer reader.Close() var ( - // clean is the cursor as of cleanSize, while pending also covers the - // member being read; pending is promoted only once that member is - // known to have ended cleanly and on a line boundary. + // clean is the cursor as of cleanSize; pending also covers the member + // being read and is promoted only once that member is known to have + // ended cleanly and on a line boundary. clean, pending *Cursor cleanSize int64 carry []byte ) for { - // Reset turns multistream back on, so it has to be switched off for - // every member rather than only for the first. + // Reset turns multistream back on, so it must be switched off per + // member, not just for the first. reader.Multistream(false) cursor, tail, err := scanLines(reader, carry) @@ -245,8 +226,8 @@ func lastCursorGzip(file *os.File) (*Cursor, error) { pending = cursor } if err != nil { - // The member did not decode to a clean end of stream, so - // everything it holds is part of an interrupted write. + // Member did not reach a clean end of stream, so everything it + // holds belongs to an interrupted write. break } if len(tail) == 0 { @@ -255,7 +236,7 @@ func lastCursorGzip(file *os.File) (*Cursor, error) { carry = tail // A clean end of file makes Reset report io.EOF; anything else is a - // member header that was only partly written. + // partly written member header. if err := reader.Reset(counter); err != nil { break } @@ -274,13 +255,13 @@ func lastCursorGzip(file *os.File) (*Cursor, error) { } // scanLines reads NDJSON from r and returns a cursor for the last line that -// both parses and is terminated by a newline. carry is prepended to the first -// line, so a line split across two gzip members is reassembled, and the -// trailing bytes not yet terminated by a newline are returned so the next -// member can complete them. The returned error is nil only when r ended at a -// clean io.EOF; a truncated member, a checksum mismatch, an over-long line and -// a genuine I/O failure are all reported rather than passed over, because each -// of them means the bytes after the last good line cannot be trusted. +// parses and ends in a newline. carry is prepended to the first line so a line +// split across two gzip members is reassembled, and the unterminated trailing +// bytes are returned for the next member to complete. +// +// The error is nil only when r reached a clean io.EOF. Truncation, a checksum +// mismatch, an over-long line and an I/O failure are all reported, because each +// means the bytes after the last good line cannot be trusted. func scanLines(r io.Reader, carry []byte) (*Cursor, []byte, error) { buffered := bufio.NewReaderSize(r, bufferSize) @@ -310,18 +291,18 @@ func scanLines(r io.Reader, carry []byte) (*Cursor, []byte, error) { } } -// countingReader counts the bytes consumed from the reader it wraps. It -// implements io.ByteReader as well as io.Reader so that gzip.Reader reads from -// it directly rather than wrapping it in a bufio.Reader of its own; without -// that the count would run ahead of the reader's real position and no member -// boundary could be observed exactly. +// countingReader counts the bytes consumed from the reader it wraps. +// +// It must keep implementing io.ByteReader: that is what makes gzip.Reader read +// from it directly instead of wrapping it in a bufio.Reader of its own. Without +// it the count runs ahead of the real position and member boundaries can no +// longer be observed exactly. type countingReader struct { reader *bufio.Reader // read is the number of bytes handed out so far. read int64 } -// Read implements io.Reader. func (c *countingReader) Read(p []byte) (int, error) { n, err := c.reader.Read(p) c.read += int64(n) @@ -329,7 +310,6 @@ func (c *countingReader) Read(p []byte) (int, error) { return n, err } -// ReadByte implements io.ByteReader. func (c *countingReader) ReadByte() (byte, error) { b, err := c.reader.ReadByte() if err == nil { @@ -339,8 +319,8 @@ func (c *countingReader) ReadByte() (byte, error) { return b, err } -// parseCursor builds a cursor from a single NDJSON line. It rejects blank and -// truncated lines, and logs that carry no block number. +// parseCursor builds a cursor from a single NDJSON line, rejecting blank and +// truncated lines and any log missing a block number or log index. func parseCursor(line []byte) (*Cursor, error) { line = bytes.TrimSpace(line) if len(line) == 0 { diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index 8e5e721..b6ff114 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -110,21 +110,15 @@ func TestReadCursor(t *testing.T) { // file had been concatenated onto a good export. garbageLines := bytes.Repeat([]byte("not-json\n"), 12*1024) - // straddle places a single valid log line so that it straddles the - // boundary between the last two 64 KiB windows lastCursorPlain reads - // backwards from EOF: part of the line falls in the final window, part - // falls in the window before it. Reconstructing it requires appending - // each newly (re-)read window before the previously carried tail, in - // file order — swap that order and the line comes out garbled and - // unparseable, which a case where the target line sits wholly inside one - // window can never catch. + // straddle places a valid log line across the boundary between the last + // two windows lastCursorPlain reads: reassembling it requires appending + // each re-read window before the carried tail, in file order. Swap that + // order and the line comes out garbled — which a case whose target sits + // wholly inside one window can never catch. straddleTarget := ndjson(t, testLog(4096, 3)) straddlePrefix := bytes.Repeat([]byte("not-json\n"), 4) - // afterLen is chosen so that windowSize-many bytes from EOF lands - // strictly inside straddleTarget: it is less than windowSize (so the - // boundary is not beyond the end of the line) and more than - // windowSize-len(straddleTarget) (so the boundary is not before the - // line's start). + // afterLen puts the boundary (windowSize bytes from EOF) strictly inside + // straddleTarget, rather than at a round offset that could drift. afterLen := windowSize - len(straddleTarget)/2 filler := []byte("not-json\n") straddleAfter := bytes.Repeat(filler, afterLen/len(filler)+2)[:afterLen] @@ -381,13 +375,11 @@ func decompressAll(t *testing.T, b []byte) []byte { return got } -// TestAppendResumeRoundTrip exercises resume, filestore, and gzipstore -// together with no RPC involved: it builds an export file, uses resume.Read -// to find where it stopped, replays the boundary block plus new blocks -// through filestore.AppendLogsAsync with the cursor's Skip as the filter -// (via the appropriate append writer for the format), and checks the result -// byte-for-byte against the original file plus exactly the new logs -- then -// resumes a second time to confirm the appended file is itself resumable. +// TestAppendResumeRoundTrip exercises resume, filestore and gzipstore together +// with no RPC: read the cursor, replay the boundary block plus new blocks +// through AppendLogsAsync filtered by Skip, check the result byte-for-byte +// against the original plus exactly the new logs, then resume again to confirm +// the appended file is itself resumable. func TestAppendResumeRoundTrip(t *testing.T) { t.Parallel() @@ -514,11 +506,9 @@ func TestAppendResumeRoundTrip(t *testing.T) { } } -// logsIn returns every log an export file holds, decompressing it first when -// it is gzip. Anything the file contains that is not a whole, newline -// terminated NDJSON log line fails the test: a file that merely looks -// recovered must not pass, so corruption is caught here rather than by a -// reader downstream. +// logsIn returns every log an export file holds, decompressing gzip first. +// Anything that is not a whole, newline-terminated NDJSON line fails the test, +// so a file that merely looks recovered is caught here rather than downstream. func logsIn(t *testing.T, path string) []types.Log { t.Helper() @@ -590,20 +580,15 @@ func feed(logs ...types.Log) <-chan types.Log { return ch } -// TestResumeAfterInterruptedWrite covers the three ways a run killed mid-write -// leaves a file that used to be appended onto blindly, corrupting it: a plain -// line cut in half, a plain line that parses but never got its newline, and a -// gzip member that never got its trailer. +// TestResumeAfterInterruptedWrite covers the three shapes a run killed +// mid-write leaves behind: a plain line cut in half, a plain line that parses +// but never got its newline, and a gzip member that never got its trailer. +// Appending onto any of them blindly corrupts the file. // -// Each case walks the whole recovery: read the cursor, discard everything past -// the clean boundary it reports, append the logs a resumed query would return, -// and then require that the file parses end to end and holds exactly the four -// logs, in order, with none duplicated and none lost. Before the clean -// boundary existed, the first case fused two logs into one unparseable line, -// the second did the same and lost the fused log for good because the cursor -// had told the writer to skip it, and the third left the appended member -// unreachable behind a corrupt one -- so each assertion below fails loudly on -// the old behaviour. +// Each case walks the whole recovery — read the cursor, discard past the clean +// boundary, append what a resumed query returns — and requires the file to +// parse end to end holding exactly the four logs, in order, none duplicated +// and none lost. func TestResumeAfterInterruptedWrite(t *testing.T) { t.Parallel() @@ -629,8 +614,8 @@ func TestResumeAfterInterruptedWrite(t *testing.T) { replay []types.Log }{ { - // The reviewer's first scenario: appending onto the half line - // glued the next log onto it, destroying block 102. + // Appending onto the half line glues the next log onto it, + // destroying block 102. name: "plain file with a truncated last line", content: append(append([]byte{}, saved...), []byte(`{"address":"0x45a1502382541cd610cc9068e88727426b6`)...), fileName: plainFile, @@ -640,13 +625,10 @@ func TestResumeAfterInterruptedWrite(t *testing.T) { replay: []types.Log{testLog(101, 0), testLog(102, 0), testLog(103, 0)}, }, { - // The reviewer's second scenario, and the nastiest: the last - // line parses, so the old cursor pointed at it and the resumed - // query skipped block 101, while the append glued the next log - // onto the line that had no newline. Block 101 was fused into an - // unparseable line and never re-fetched, so it was unrecoverable. - // The cursor must now name block 100 so that 101 is fetched - // again. + // The nastiest: the line parses, so a cursor naming it would make + // the resumed query skip block 101 while the append fuses the next + // log onto it — unparseable, and never re-fetched. The cursor must + // name block 100 so that 101 is fetched again. name: "plain file with a valid but unterminated last line", content: bytes.TrimSuffix(saved, []byte("\n")), fileName: plainFile, @@ -656,10 +638,10 @@ func TestResumeAfterInterruptedWrite(t *testing.T) { replay: []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(103, 0)}, }, { - // The reviewer's third scenario: the final member lost its - // trailer, so a new member appended after it sat behind corrupt - // data and could never be read back -- and every later resume - // appended more of the same while reporting success. + // The final member lost its trailer, so a member appended after + // it sits behind corrupt data and can never be read back, while + // every later resume appends more of the same and reports + // success. name: "gzip file with a truncated final member", content: append(gz(t, savedFirst), truncateLast(gz(t, ndjson(t, testLog(101, 0))), 6)...), fileName: gzipFile, From 68361939c841688de69ef82b91e1ee27205c6809 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 11:00:21 +0200 Subject: [PATCH 17/35] docs: spec for resume as incremental snapshots (input/output model) Co-Authored-By: Claude Opus 5 (1M context) --- .../2026-08-27-resume-input-output-design.md | 246 ++++++++++++++++++ 1 file changed, 246 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-27-resume-input-output-design.md diff --git a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md new file mode 100644 index 0000000..ffd7de8 --- /dev/null +++ b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md @@ -0,0 +1,246 @@ +# Resume as Incremental Snapshots + +**Date:** 2026-08-27 +**Status:** Draft, awaiting review +**Scope:** the complete behavior of the `--resume` feature on PR #11. This +document is the source of truth; the implementation is edited until it +conforms. + +## 1. Goal + +`batch-export` produces periodic snapshots of Postage Stamp contract events. +The operator runs an export today, archives the resulting `.gzip`, and later — +typically a month later — produces the next snapshot by *continuing from the +previous one* instead of re-exporting every block from the contract's start. + +`--resume` therefore means: **continue a previous, normally finished export +made by this same tool.** A run that was interrupted mid-write is handled +(§5), but crash recovery is robustness, not the purpose, and does not shape +the interface or the documentation. + +## 2. Non-Goals + +- **Repairing foreign or manipulated files.** A resume input is trusted to be + this tool's own output; anything else is refused, never "fixed" (§5). +- **Validating chain provenance.** An export from the same tool against a + different chain parses identically; detecting the mismatch would need a file + header, which changes the format `batch-archive` consumes. Documented + limitation. +- **Validating the interior of plain files.** Plain-file validation covers the + region the cursor reading must touch (the tail); silent corruption elsewhere + in a 90 MB file is out of scope. (Gzip necessarily decodes the whole stream, + so it validates every line as a side effect — §5.) +- **Retiring post-hoc `CompressFile` for fresh `--compress` runs.** Fresh + exports keep today's behavior. + +## 3. CLI Contract + +No new flags. `--resume` names the **input** (the previous snapshot); +`--output` names the **destination**. They compose: + +| Invocation | Behavior | +|---|---| +| `export` | Fresh export to `--output` (default `export.ndjson`). Unchanged. | +| `export --resume old.gzip` | **In-place**: append to `old.gzip` itself. An unset `--output` does not redirect the result to its default. | +| `export --resume old.gzip --output new.gzip` | **Copy mode**: `old.gzip` is never modified; `new.gzip` = clean content of `old.gzip` + newly fetched entries. | +| `export --resume f --output f` | Identical paths (`filepath.Clean` equality) mean in-place. Distinct spellings of one file (symlinks, hardlinks) are the operator's responsibility. | + +Flag interactions when `--resume` is set: + +- `--start` is ignored with a warning: the cursor decides the start block. +- `--end` works as usual (default 0 = latest block). +- `--compress` is **always ignored with a warning**. Post-hoc compression of a + resumed plain file is the twin-file trap: `CompressFile`'s `os.Create` would + overwrite an independently accumulated `.gzip`. A compressed result comes + from resuming a compressed input; compressing a plain result is `gzip`'s job. +- The output's format always equals the input's format, detected from the + input's leading magic bytes. Extensions are names, nothing more; `.ndjson`, + `.gz` and `.gzip` all work. +- In copy mode an existing file at `--output` is overwritten (`os.Create` + semantics, same as a fresh export). + +Flag help: `Continue a previous export file (.ndjson, .gz or .gzip); combine +with --output to write a new snapshot instead of appending in place`. + +## 4. Modes + +**Copy mode** (input != output) — the recommended monthly workflow: + +```sh +batch-export export --resume snapshots/2026-07.gzip --output snapshots/2026-08.gzip +``` + +The first `CleanSize` bytes (§5) of the input are copied to the output raw — +no decompression, no re-encoding — and new entries are appended to the output. +The input is opened read-only and is never modified, truncated, or deleted. If +the run fails, the output is incomplete but the input is intact: delete the +output and rerun. + +**In-place** (input == output): the space-saving variant. If the input carries +an interrupted final write (§5), the file is first truncated to `CleanSize` +with a warning naming the discarded byte count; appending then proceeds. + +## 5. Reading the Cursor: Strict Tail Validation + +The *cursor* is the last complete log entry of the input: its `blockNumber` +and `logIndex`, plus `Compressed`, `CleanSize` (the byte offset at which the +input's trustworthy content ends) and `Truncated` (whether anything follows +`CleanSize`). + +The contract is strict because the input is by definition this tool's own +output. The tool writes NDJSON via `json.Encoder` — each entry is one line, +value and terminating newline in a single write — either plainly or inside +gzip members, one member per run, each member holding only whole lines. A +member may be empty (a resumed run that fetched nothing new still closes its +member). Consequently the **only** irregularity the tool itself can produce is +an interrupted final write: + +- plain: a single incomplete trailing line (a partial write never ends in a + newline, and these JSON lines contain no raw newlines); +- gzip: a single trailing member without its terminating CRC/length trailer. + +Exactly that irregularity is tolerated: the cursor is the last complete entry +before it, `CleanSize` excludes it, `Truncated` is true, and each mode handles +it per §4 (copy mode does not copy it; in-place truncates it). Nothing else is +tolerated. Any of the following mean the file is not untouched tool output, +and `Read` refuses with `ErrNotAnExport`, naming what was found and at which +offset: + +- a complete line (newline-terminated) that does not parse as a log entry — + including blank lines; +- a trailing newline-free fragment longer than `maxLineBytes` (1 MiB; real + entries run a few hundred bytes); +- a cleanly terminated gzip member that ends mid-line or contains a complete + non-parsing line; +- gzip content after a member that failed to decode (unreachable in practice — + decoding stops there — but stated for completeness: bytes past `CleanSize` + are ignored, never interpreted). + +Refusal is deliberate: for an archival artifact, silently cutting away content +the tool never wrote is worse than stopping. The operator decides what a +manipulated file means. + +**Errors.** Two sentinel errors replace the current three: + +- `ErrNotAnExport` — foreign content, as above. Terminal; the message says + what and where. +- `ErrNoLogs` — the file is consistent with tool output but holds no complete + entry to resume from: empty, or only an interrupted first write. The remedy + is a fresh export. + +(`ErrNoCleanBoundary` is retired: its gzip case — a sole truncated member — is +`ErrNoLogs`; its mid-line-member case is `ErrNotAnExport`.) + +**Mechanics.** Plain: one read of the last `maxLineBytes` bytes suffices — +find the last newline, check the fragment after it, parse the single line +before it. The current multi-window backward walk with carry exists only to +tolerate foreign junk and is deleted. Gzip: cannot seek, so the stream is +decoded member by member with a counting reader (which must implement +`io.ByteReader` so `gzip.Reader` consumes it directly and member boundaries +are observed exactly); every complete line must parse; `CleanSize` is the end +of the last cleanly terminated member. + +## 6. Appending + +- **Plain**: `O_APPEND` on the destination. +- **Gzip**: a new gzip member (`gzip.NewWriter` on a file opened `O_APPEND`). + Concatenated members are one valid stream per RFC 1952; `gzcat`, `gunzip`, + Go and Python read them transparently. Existing bytes are never rewritten. + Measured on the real 90 MB export: 12 members cost +0.021% over a single + member, 120 members +0.224%. Members can be consolidated any time with + `gzcat old.gzip | gzip > fresh.gzip`. +- **Skip filter**: the resumed query starts at the cursor's block + *inclusively* — an interrupted run may have written only part of it, and for + a finished run the re-query is a no-op — and entries at or before the cursor + (`blockNumber < cursor's`, or equal block with `logIndex <=` cursor's) are + dropped before writing. No gaps, no duplicates. +- The destination writer is opened before the first log is fetched, so an open + failure aborts the run instead of leaving the fetcher pushing into a channel + nobody drains. + +## 7. Error Handling and Exit Codes + +The saver goroutine's error must reach `RunE`'s return value: + +- `RunE` derives a cancellable context. The saver records its error and + **cancels the derived context on failure**, which unblocks the fetcher, + which closes `errorChan`, which lets the select loop exit. No hang wherever + the save fails (open, encode, close). +- **Every** exit path out of the select loop runs `wg.Wait()` before + returning — including the `errorChan` error branch, which today returns + immediately and can lose the buffered gzip member. +- After `wg.Wait()`, the saver's error is joined into the returned error: a + failed save always exits non-zero. On a gzip destination the writer's + `Close` finalizes the member, so its error is part of the save result, not + noise. A save error caused only by cancellation is not double-reported. +- The goroutine still logs the error when it happens, so the operator sees it + immediately. + +## 8. Documentation (README) + +1. Section title **"Continuing a previous snapshot"**; lead example is copy + mode with dated files (§4). In-place follows as the variant. +2. The multi-member note with the measured overhead and the consolidation + one-liner. +3. A short "if the previous run was interrupted" subsection: copy mode — + rerun; in-place — the tool truncates the interrupted write and re-fetches, + warning shown. A fusnote, not the headline. +4. The trust rule, stated plainly: resume only files this tool produced; a + file with any other content is refused, and resuming does not detect a + wrong chain — that is the operator's contract. +5. One canonical snapshot file: keep either the plain file or the gzip, not + both; `--compress` is ignored on resume. +6. Feature bullet: "Continue a previous export from where it stopped + (incremental snapshots)." + +## 9. Code Placement + +- `pkg/resume`: gains `PrepareOutput(cursor, inputPath, outputPath)` — the + in-place truncation or clean-prefix copy of §4. It lives beside `Read` + because the package that computes `CleanSize` should also enforce it; a + caller can then not append without the invariant holding, and the mechanics + are testable without an RPC harness. +- `cmd/export.go`: resolves the mode (§3), calls `PrepareOutput`, and picks + create vs append in `openOutput`; the saver-error plumbing of §7 lives in + `RunE`. +- `pkg/resume`: strict validation per §5; `lastCursorPlain` shrinks to the + single-tail read; `lastCursorGzip` keeps the member walk, drops cross-member + line carry (whole-line members are part of the contract), refuses on any + non-parsing complete line; error taxonomy per §5. +- `pkg/filestore`: `SaveLogsAsync` — exported, no production caller — is + deleted with its tests; fresh exports route through `CreateWriter` + + `AppendLogsAsync`. The `AppendLogsAsync` close-error join stays. +- `pkg/gzipstore`: unchanged. + +## 10. Testing + +All tests remain package-level (`./pkg/...`) plus the cross-package round +trips in `resume_test.go`; `cmd` still has no RPC harness, so §7 is verified +at whatever level extraction permits, with manual evidence recorded in the PR +for the rest. + +- **Copy-mode round trip** (both formats): input byte-identical before and + after; output = input's content + exactly the new entries in order; output + itself resumable; output ends clean. +- **Copy mode from an interrupted input** (both formats): input untouched + *including its partial tail*; output holds the clean content plus re-fetched + entries; nothing lost, nothing duplicated. +- **In-place round trip and interrupted-write recovery**: as today. +- **Strict refusal** (`ErrNotAnExport`): plain with a trailing complete + non-log line; plain with a blank line; plain with a >1 MiB newline-free + tail (refused fast, no scan-back); gzip with a junk member; gzip with a + clean member ending mid-line. +- **Tolerated irregularities**: plain partial trailing line; gzip truncated + final member; empty gzip member (valid, contributes nothing). +- **`ErrNoLogs`**: empty file; file holding only a partial first write. +- **Same-path detection**: `--resume f --output f` behaves as in-place. +- Deleted along with the code they exercised: the multi-window walk tests + (garbage windows, boundary straddle). + +## 11. Compatibility + +PR #11 is unmerged; nothing released changes. Within the PR: `--output` +composes with `--resume` instead of being overridden; `--compress` is ignored +on every resume, not only on compressed input; lenient junk tolerance is +replaced by strict refusal. All three are called out in the PR description. +The final merge squashes, so branch history need not tell this story twice. From ee2a64c9b29a9e49fbbcd9ed0db9b723c26cd1a6 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 11:38:45 +0200 Subject: [PATCH 18/35] docs: implementation plan for resume as incremental snapshots Co-Authored-By: Claude Opus 5 (1M context) --- ...2026-08-27-resume-incremental-snapshots.md | 1075 +++++++++++++++++ 1 file changed, 1075 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-27-resume-incremental-snapshots.md diff --git a/docs/superpowers/plans/2026-08-27-resume-incremental-snapshots.md b/docs/superpowers/plans/2026-08-27-resume-incremental-snapshots.md new file mode 100644 index 0000000..56f4706 --- /dev/null +++ b/docs/superpowers/plans/2026-08-27-resume-incremental-snapshots.md @@ -0,0 +1,1075 @@ +# Resume as Incremental Snapshots Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Bring the existing `feat/resume-flag` branch into conformance with the incremental-snapshots spec: strict tail validation, `--resume`/`--output` composition with copy mode, saver errors reaching the exit code, and docs reframed around continuation. + +**Architecture:** This plan **edits the existing branch**, it does not start over. `pkg/resume` keeps its public shape (`Cursor`, `Read`, `Skip`) but its internals simplify: the lenient multi-window backward walk becomes a single tail read that refuses foreign content, and the gzip member walk refuses non-log lines instead of tolerating them. A new `resume.PrepareOutput` enforces the `CleanSize` invariant (in-place truncate or clean-prefix copy) next to where it is computed. `cmd/export.go` gains mode resolution and a saver-error path to `RunE`'s return value, keeping its existing select-loop shape. + +**Tech Stack:** Go 1.25, stdlib (`compress/gzip`, `compress/flate`, `bufio`, `path/filepath`), `github.com/ethereum/go-ethereum` v1.15.11, `github.com/ethersphere/bee/v2` v2.7.0, cobra. + +**Spec:** `docs/superpowers/specs/2026-08-27-resume-input-output-design.md` — the plan argues from it; read both. Section references (§N) below point into it. + +**Baseline:** branch `feat/resume-flag`, code at commit `b53a9c3` (plus the local spec commit). All file:line references are against that state. + +## Global Constraints + +- Go `1.25` per `go.mod`; do not change it. **No new dependencies**; do not run `go get`. +- Lint: `.golangci.yml` enables `copyloopvar`, `errname`, `errorlint`, `goconst`, `misspell`, `nilerr`, `unconvert` + `gofmt`/`gofumpt`. Wrap with `%w`; compare with `errors.Is`/`errors.As`, never `==`. +- Doc comment on every exported identifier, starting with its name. Error strings lowercase, unpunctuated. +- **Review-friendliness is a requirement, not a preference:** keep the diff against `main` as small as the spec allows; extend existing structures instead of restructuring them; keep `RunE`'s select-loop shape; never reformat code you are not changing. Where minimal-diff and clean idiomatic Go conflict, prefer clean Go — but say so in the commit message. +- Dependency direction stays one-way: `cmd` → {`resume`, `filestore`, `gzipstore`}; none of the three import each other. +- Conventional Commits. **Do not push** — local commits only; the branch is squash-merged at the end (§11). +- `dist/` holds the user's real export archives. Never point `--resume` (or any truncating/copying code) at a `dist/` path; test against copies in a temp dir only. +- Tests: `make test` (= `go test -v ./pkg/...`); also run `go test -race ./pkg/...` before each commit that touches concurrency. + +## File Structure + +| File | Change | Responsibility after this plan | +|---|---|---| +| `pkg/resume/resume.go` | Rewrite internals | Strict tail validation (§5): `Read` + two sentinel errors; single-read plain path; member-walk gzip path; `PrepareOutput` (§4/§9). | +| `pkg/resume/resume_test.go` | Overhaul | Validation, refusal, tolerance, and cross-package round-trip tests for both modes. | +| `pkg/filestore/filestore.go` | Shrink | `CreateWriter`, `AppendWriter`, `AppendLogsAsync` only; `SaveLogsAsync` deleted (§9). | +| `pkg/filestore/filestore_test.go` | Retool | Seeding via `CreateWriter`+`AppendLogsAsync`. | +| `cmd/export.go` | Edit | Mode resolution (§3), `PrepareOutput` wiring, saver-error plumbing (§7), flag help. | +| `README.md` | Rewrite section | §8. | +| `pkg/gzipstore/*` | Untouched | — | + +--- + +### Task 1: Strict tail validation in `pkg/resume` + +**Files:** +- Modify: `pkg/resume/resume.go` (all of `lastCursorPlain`, `lastCursorGzip`, `scanLines`, the const/var blocks, `parseCursor`; `Read`, `Cursor`, `Skip`, `isGzip`, `countingReader` keep their shape) +- Test: `pkg/resume/resume_test.go` + +**Interfaces:** +- Consumes: nothing new. +- Produces (Task 2 and 4 rely on these): + - `var ErrNotAnExport error`, `var ErrNoLogs error` (sentinel; `ErrNoCleanBoundary` and `errLineTooLong` are deleted) + - `Read(path string) (*Cursor, error)` — unchanged signature; new contract per §5. + - `Cursor` fields unchanged: `BlockNumber uint64; LogIndex uint; Compressed bool; CleanSize int64; Truncated bool`. + +- [ ] **Step 1: Update the tests to the strict contract** + +In `pkg/resume/resume_test.go`: + +**(a) Delete** the now-obsolete lenient machinery in `TestReadCursor`'s setup: the `garbageLines` variable, the whole `straddle*` block (from the `// straddle places a valid log line…` comment through the `t.Fatalf("test setup: boundary %d…")` guard), and the `windowSize` test constant with its comment near the top of the file. Keep `threeLogs`, `manyLogs`/`many`, `twoLogsEnd`, `threeLogsEnd`, and all helpers (`testLog`, `ndjson`, `gz`, `truncateLast`, `write`). + +**(b) Delete** these `TestReadCursor` cases (their content is foreign under §5 and moves to the refusal table): `"line missing blockNumber is skipped"`, `"walks back across windows of garbage lines"`, `"valid line straddles a window boundary"`. + +**(c) Rename** the case `"spans multiple backward read windows"` to `"many logs"` (the window concept no longer exists; the case stays as a large-file regression). + +**(d) Add** one `TestReadCursor` case — an empty trailing member is the tool's own output (a resume that fetched nothing) and must stay valid: + +```go + { + name: "empty trailing gzip member is valid", + content: append(gz(t, threeLogs), gz(t, nil)...), + wantBlock: 102, + wantIndex: 7, + wantCompressed: true, + }, +``` + +(`gz(t, nil)` compresses zero bytes into a complete member; `bytes.Buffer.Write(nil)` is a no-op, so the existing helper handles it.) + +All remaining `TestReadCursor` cases keep their current expectations — including `"truncated trailing line is excluded from the clean boundary"`, `"trailing line without a newline is not a clean end"` (cursor falls back to `{101,1}`), both truncated-member gzip cases, and the unflushed-header case. They are the §5 tolerated irregularities. + +**(e) Replace** `TestReadCursorErrors` and `TestReadRefusesWithoutACleanBoundary` with one consolidated table (delete both, add this): + +```go +// TestReadRefusals covers §5's strict contract: the only irregularity +// tolerated is the tool's own interrupted final write. Content the tool never +// writes is ErrNotAnExport; a file consistent with tool output but holding no +// complete entry is ErrNoLogs. +func TestReadRefusals(t *testing.T) { + t.Parallel() + + logs := ndjson(t, testLog(100, 0), testLog(101, 0)) + + tests := []struct { + name string + content []byte + wantErr error + }{ + { + name: "empty file", + content: []byte{}, + wantErr: resume.ErrNoLogs, + }, + { + name: "plain file holding one unterminated line", + content: bytes.TrimSuffix(ndjson(t, testLog(100, 0)), []byte("\n")), + wantErr: resume.ErrNoLogs, + }, + { + // The tool never writes a blank line. + name: "only a newline", + content: []byte("\n"), + wantErr: resume.ErrNotAnExport, + }, + { + name: "trailing blank line after valid logs", + content: append(append([]byte{}, logs...), '\n'), + wantErr: resume.ErrNotAnExport, + }, + { + // A complete line that is not a log entry means the file was + // altered after export; refusing beats guessing what to cut. + name: "trailing non-log line after valid logs", + content: append(append([]byte{}, logs...), []byte("not-json\n")...), + wantErr: resume.ErrNotAnExport, + }, + { + name: "trailing log line missing blockNumber", + content: append(append([]byte{}, logs...), []byte("{\"address\":\"0x1\",\"topics\":[],\"data\":\"0x\"}\n")...), + wantErr: resume.ErrNotAnExport, + }, + { + name: "only garbage lines", + content: bytes.Repeat([]byte("not-json\n"), 10), + wantErr: resume.ErrNotAnExport, + }, + { + // Finding #4's shape: refused from a single tail read, no + // backward scan through the junk. + name: "newline-free tail longer than a line can be", + content: append(append([]byte{}, logs...), bytes.Repeat([]byte("x"), 1<<20+1)...), + wantErr: resume.ErrNotAnExport, + }, + { + name: "single unterminated line larger than the cap", + content: bytes.Repeat([]byte("x"), 2<<20), + wantErr: resume.ErrNotAnExport, + }, + { + // A cleanly terminated member whose content stops mid-line + // cannot come from this tool: members hold whole lines, and an + // interrupted write cannot produce a valid trailer. + name: "gzip member ending mid line", + content: gz(t, bytes.TrimSuffix(logs, []byte("\n"))), + wantErr: resume.ErrNotAnExport, + }, + { + // Finding #6's shape: foreign data concatenated as its own valid + // member. Refused rather than treated as a removable tail. + name: "gzip junk member after valid member", + content: append(gz(t, logs), gz(t, []byte("not-json\nalso-not\n"))...), + wantErr: resume.ErrNotAnExport, + }, + { + name: "gzip member holding a non-log line between logs", + content: gz(t, append(append([]byte{}, logs...), []byte("not-json\n")...)), + wantErr: resume.ErrNotAnExport, + }, + { + // A sole member without its trailer is an interrupted first + // write: nothing to resume from, so a fresh export is the remedy. + name: "gzip with a single truncated member", + content: truncateLast(gz(t, logs), 6), + wantErr: resume.ErrNoLogs, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if _, err := resume.Read(write(t, plainFile, tt.content)); !errors.Is(err, tt.wantErr) { + t.Fatalf("Read() error = %v, want %v", err, tt.wantErr) + } + }) + } +} +``` + +**(f) Keep unchanged:** `TestReadMissingFile`, `TestCursorSkip`, `TestReadReportsGzipReadErrors`, `TestGzipCleanSizeIsAMemberBoundary`, `TestAppendResumeRoundTrip`, `TestResumeAfterInterruptedWrite` (Task 2 touches the last one). + +- [ ] **Step 2: Run the tests to verify the new ones fail** + +Run: `go test ./pkg/resume/ -run 'TestReadRefusals|TestReadCursor' -v 2>&1 | tail -30` +Expected: compile error first (`resume.ErrNotAnExport` undefined, `resume.ErrNoCleanBoundary` still referenced only if a stray use remains); after the file compiles, `TestReadRefusals` cases like `"trailing non-log line after valid logs"` FAIL against the lenient implementation (it returns a cursor instead of refusing). + +- [ ] **Step 3: Rewrite the validation internals** + +In `pkg/resume/resume.go`: + +**(a) Constants** — delete `windowSize`; keep `maxLineBytes` and `bufferSize`: + +```go +const ( + // maxLineBytes caps a single line. Exported log lines run to a few + // hundred bytes, so anything longer is not this tool's output. + maxLineBytes = 1 << 20 + // bufferSize is how much of a gzip file is buffered per read. + bufferSize = 64 * 1024 +) +``` + +**(b) Errors** — replace the var block (deletes `ErrNoCleanBoundary` and `errLineTooLong`): + +```go +var ( + // ErrNotAnExport indicates content this tool never writes. The file was + // altered after export, so resuming it is refused rather than repaired. + ErrNotAnExport = errors.New("not an untouched batch-export file") + // ErrNoLogs indicates a file consistent with tool output that holds no + // complete entry to resume from: it is empty, or holds only an + // interrupted first write. The remedy is a fresh export. + ErrNoLogs = errors.New("no complete log entry found") +) +``` + +**(c) `lastCursorPlain`** — replace entirely: + +```go +// lastCursorPlain validates the tail of a plain NDJSON file and returns a +// cursor for its last complete line. The tool writes one entry per line in a +// single call, so only the tail needs examining: the last newline-terminated +// line must parse as a log entry, and the only bytes allowed after it are a +// single interrupted write — a trailing fragment with no newline. +func lastCursorPlain(file *os.File, size int64) (*Cursor, error) { + window := min(size, 2*maxLineBytes) + offset := size - window + + buf := make([]byte, window) + if _, err := file.ReadAt(buf, offset); err != nil { + return nil, fmt.Errorf("error reading resume file: %w", err) + } + + nl := bytes.LastIndexByte(buf, '\n') + if nl < 0 { + // No newline at all: an interrupted first write, unless the file is + // longer than any single line the tool writes. + if size > maxLineBytes { + return nil, fmt.Errorf("%w: %d bytes without a newline", ErrNotAnExport, size) + } + return nil, ErrNoLogs + } + if tail := window - int64(nl) - 1; tail > maxLineBytes { + return nil, fmt.Errorf("%w: %d bytes without a newline after offset %d", ErrNotAnExport, tail, offset+int64(nl)+1) + } + + start := bytes.LastIndexByte(buf[:nl], '\n') + 1 + if start == 0 && offset > 0 { + return nil, fmt.Errorf("%w: final line is over %d bytes long", ErrNotAnExport, nl) + } + cursor, err := parseCursor(buf[start:nl]) + if err != nil { + return nil, fmt.Errorf("%w: final complete line at offset %d is not a log entry: %v", ErrNotAnExport, offset+int64(start), err) + } + cursor.CleanSize = offset + int64(nl) + 1 + + return cursor, nil +} +``` + +**(d) `lastCursorGzip`** — replace entirely (drops cross-member line carry; members hold whole lines by contract): + +```go +// lastCursorGzip walks a gzip file one member at a time and returns a cursor +// for the last entry inside the cleanly terminated prefix. Gzip cannot be +// seeked, so the whole stream is decompressed; every complete line is +// validated on the way. A member cut short by an interrupted write ends the +// walk: CleanSize stays at the last clean member boundary, and the truncated +// member's content — about to be discarded and re-fetched — never advances +// the cursor. +func lastCursorGzip(file *os.File) (*Cursor, error) { + counter := &countingReader{reader: bufio.NewReaderSize(file, bufferSize)} + + reader, err := gzip.NewReader(counter) + if err != nil { + return nil, fmt.Errorf("error opening gzip resume file: %w", err) + } + defer reader.Close() + + var ( + cursor *Cursor + cleanSize int64 + sawClean bool + ) + for { + // Reset turns multistream back on, so it must be switched off per + // member, not just for the first. + reader.Multistream(false) + + last, err := scanMember(reader) + switch { + case errors.Is(err, ErrNotAnExport): + return nil, err + case err != nil && !truncationShaped(err): + return nil, fmt.Errorf("error reading gzip resume file: %w", err) + case err != nil: + // The member never got its trailer: an interrupted final write. + return gzipResult(cursor, cleanSize, sawClean) + } + + if last != nil { + cursor = last + } + cleanSize, sawClean = counter.read, true + + // A clean end of file makes Reset report io.EOF; a partly written + // next member fails to parse as a header and also ends the walk. + if err := reader.Reset(counter); err != nil { + if errors.Is(err, io.EOF) || truncationShaped(err) { + return gzipResult(cursor, cleanSize, sawClean) + } + return nil, fmt.Errorf("error reading gzip resume file: %w", err) + } + } +} + +// gzipResult finalizes the walk: a file with no clean member, or none holding +// an entry, has nothing to resume from. +func gzipResult(cursor *Cursor, cleanSize int64, sawClean bool) (*Cursor, error) { + if !sawClean || cursor == nil { + return nil, ErrNoLogs + } + cursor.CleanSize = cleanSize + + return cursor, nil +} + +// truncationShaped reports whether err is what an interrupted write produces, +// as opposed to a real read failure that must not be mistaken for one. +func truncationShaped(err error) bool { + var corrupt flate.CorruptInputError + + return errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, gzip.ErrHeader) || + errors.Is(err, gzip.ErrChecksum) || + errors.As(err, &corrupt) +} +``` + +Add `"compress/flate"` to the import block. + +**(e) `scanLines`** — replace with `scanMember` (no carry parameter): + +```go +// scanMember reads one gzip member's NDJSON and returns a cursor for its last +// entry, nil when the member is empty. A nil error means the member decoded +// to a clean end of stream on a line boundary. The tool writes members +// holding whole log lines only, so a complete line that does not parse, or a +// clean member ending mid-line, is foreign content (ErrNotAnExport); any +// other read error is returned as-is for the caller to classify. +func scanMember(r io.Reader) (*Cursor, error) { + buffered := bufio.NewReaderSize(r, bufferSize) + + var ( + last *Cursor + line []byte + ) + for { + chunk, err := buffered.ReadSlice('\n') + line = append(line, chunk...) + if len(line) > maxLineBytes { + return nil, fmt.Errorf("%w: line exceeds %d bytes", ErrNotAnExport, maxLineBytes) + } + + switch { + case errors.Is(err, bufio.ErrBufferFull): + continue + case errors.Is(err, io.EOF): + if len(line) > 0 { + return nil, fmt.Errorf("%w: gzip member ends mid-line", ErrNotAnExport) + } + return last, nil + case err != nil: + return last, err + } + + cursor, err := parseCursor(line) + if err != nil { + return nil, fmt.Errorf("%w: line is not a log entry: %v", ErrNotAnExport, err) + } + last = cursor + line = line[:0] + } +} +``` + +**(f) `parseCursor`** — return descriptive plain errors (callers wrap with the sentinel): + +```go +// parseCursor builds a cursor from a single NDJSON line. Callers wrap the +// returned error with the sentinel that fits their context. +func parseCursor(line []byte) (*Cursor, error) { + line = bytes.TrimSpace(line) + if len(line) == 0 { + return nil, errors.New("blank line") + } + + var parsed cursorLine + if err := json.Unmarshal(line, &parsed); err != nil { + return nil, err + } + if parsed.BlockNumber == nil || parsed.LogIndex == nil { + return nil, errors.New("missing blockNumber or logIndex") + } + + return &Cursor{ + BlockNumber: uint64(*parsed.BlockNumber), + LogIndex: uint(*parsed.LogIndex), + }, nil +} +``` + +`Read`, `Cursor`, `Skip`, `isGzip`, `countingReader` stay as they are (`Read` already computes `Truncated = CleanSize < size` centrally). + +- [ ] **Step 4: Run the package tests** + +Run: `go test ./pkg/resume/... && go vet ./pkg/resume/... && gofumpt -l pkg/resume` +Expected: PASS, no vet output, no files listed. If `TestAppendResumeRoundTrip` or the interrupted-write test fails, the strict path broke a tolerated case — fix the implementation, not the test. + +- [ ] **Step 5: Commit** + +```bash +git add pkg/resume/resume.go pkg/resume/resume_test.go +git commit -m "feat(resume): validate strictly, refuse files this tool did not write" +``` + +--- + +### Task 2: `resume.PrepareOutput` — enforce the clean boundary + +**Files:** +- Modify: `pkg/resume/resume.go` (append after `Skip`) +- Test: `pkg/resume/resume_test.go` + +**Interfaces:** +- Consumes: `Cursor` from Task 1. +- Produces (Task 4 relies on this): `func PrepareOutput(c *Cursor, inputPath, outputPath string) (discarded int64, err error)`. + +- [ ] **Step 1: Write the failing tests** + +Add to `pkg/resume/resume_test.go` (imports of `filestore`/`gzipstore` and the helpers `appendWriter`, `feed`, `logsIn`, `ids` already exist): + +```go +// TestCopyResumeRoundTrip is the spec's recommended workflow (§4): resume an +// archived snapshot into a NEW file. The input must come out byte-identical; +// the output must hold the input's content plus exactly the new entries and +// be itself resumable. +func TestCopyResumeRoundTrip(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + compressed bool + }{ + {name: "plain ndjson"}, + {name: "gzip", compressed: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + original := []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(102, 1)} + boundaryHigher := testLog(102, 2) + newer := []types.Log{testLog(103, 0), testLog(104, 0)} + + inputBytes := ndjson(t, original...) + if tt.compressed { + inputBytes = gz(t, inputBytes) + } + dir := t.TempDir() + input := filepath.Join(dir, "prev.snapshot") + output := filepath.Join(dir, "next.snapshot") + if err := os.WriteFile(input, inputBytes, 0o644); err != nil { + t.Fatalf("write input: %v", err) + } + + cursor, err := resume.Read(input) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + + discarded, err := resume.PrepareOutput(cursor, input, output) + if err != nil { + t.Fatalf("PrepareOutput() error = %v", err) + } + if discarded != 0 { + t.Errorf("discarded = %d, want 0 for a clean input", discarded) + } + + w := appendWriter(t, output, cursor) + replay := []types.Log{testLog(102, 0), testLog(102, 1), boundaryHigher} + replay = append(replay, newer...) + if err := filestore.AppendLogsAsync(t.Context(), feed(replay...), w, cursor.Skip); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + // The input is untouched, byte for byte. + gotInput, err := os.ReadFile(input) + if err != nil { + t.Fatalf("read input: %v", err) + } + if !bytes.Equal(gotInput, inputBytes) { + t.Fatal("input file was modified by a copy-mode resume") + } + + // The output begins with the input's exact bytes (raw prefix + // copy, no recompression) and holds the full sequence. + gotOutput, err := os.ReadFile(output) + if err != nil { + t.Fatalf("read output: %v", err) + } + if len(gotOutput) < len(inputBytes) || !bytes.Equal(gotOutput[:len(inputBytes)], inputBytes) { + t.Fatal("input bytes are not an unchanged prefix of the output") + } + all := append(append(append([]types.Log{}, original...), boundaryHigher), newer...) + if got, want := ids(logsIn(t, output)), ids(all); !slices.Equal(got, want) { + t.Fatalf("output logs = %v, want %v", got, want) + } + + cursor2, err := resume.Read(output) + if err != nil { + t.Fatalf("Read(output) error = %v", err) + } + if cursor2.BlockNumber != 104 || cursor2.LogIndex != 0 || cursor2.Truncated { + t.Errorf("output cursor = {%d,%d,truncated=%t}, want {104,0,false}", cursor2.BlockNumber, cursor2.LogIndex, cursor2.Truncated) + } + }) + } +} + +// TestCopyResumeFromInterruptedInput: copy mode never repairs the input — the +// interrupted tail stays in it — while the output gets only clean content +// plus the re-fetched entries. +func TestCopyResumeFromInterruptedInput(t *testing.T) { + t.Parallel() + + saved := ndjson(t, testLog(100, 0), testLog(101, 0)) + + tests := []struct { + name string + content []byte + compressed bool + }{ + { + name: "plain with a truncated last line", + content: append(append([]byte{}, saved...), []byte(`{"address":"0x45a15`)...), + }, + { + name: "gzip with a truncated final member", + content: append(gz(t, saved), truncateLast(gz(t, ndjson(t, testLog(102, 0))), 6)...), + compressed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + input := filepath.Join(dir, "prev.snapshot") + output := filepath.Join(dir, "next.snapshot") + if err := os.WriteFile(input, tt.content, 0o644); err != nil { + t.Fatalf("write input: %v", err) + } + + cursor, err := resume.Read(input) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if !cursor.Truncated { + t.Fatal("test setup: input should be truncated") + } + + discarded, err := resume.PrepareOutput(cursor, input, output) + if err != nil { + t.Fatalf("PrepareOutput() error = %v", err) + } + if want := int64(len(tt.content)) - cursor.CleanSize; discarded != want { + t.Errorf("discarded = %d, want %d", discarded, want) + } + + w := appendWriter(t, output, cursor) + replay := []types.Log{testLog(101, 0), testLog(102, 0), testLog(103, 0)} + if err := filestore.AppendLogsAsync(t.Context(), feed(replay...), w, cursor.Skip); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + gotInput, err := os.ReadFile(input) + if err != nil { + t.Fatalf("read input: %v", err) + } + if !bytes.Equal(gotInput, tt.content) { + t.Fatal("input was modified, interrupted tail included it must stay") + } + + want := []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(103, 0)} + if got := ids(logsIn(t, output)); !slices.Equal(got, ids(want)) { + t.Fatalf("output logs = %v, want %v", got, ids(want)) + } + }) + } +} +``` + +Add `"path/filepath"` and `"slices"` to the test file's imports if not present. + +- [ ] **Step 2: Update `TestResumeAfterInterruptedWrite` to use `PrepareOutput`** + +In that test's execution section, replace the direct `os.Truncate(path, cursor.CleanSize)` call (and any surrounding size check) with: + +```go + if _, err := resume.PrepareOutput(cursor, path, path); err != nil { + t.Fatalf("PrepareOutput() error = %v", err) + } +``` + +so the in-place recovery path exercises the production mechanics. + +- [ ] **Step 3: Run tests to verify they fail** + +Run: `go test ./pkg/resume/ -run 'TestCopyResume|TestResumeAfterInterruptedWrite' -v 2>&1 | tail -15` +Expected: FAIL — `undefined: resume.PrepareOutput`. + +- [ ] **Step 4: Implement `PrepareOutput`** + +Append to `pkg/resume/resume.go` after `Skip` (add `"io"` — already imported — and `"path/filepath"` to imports): + +```go +// PrepareOutput readies outputPath for appending the continuation of the +// export at inputPath. With equal paths the file is prepared in place: an +// interrupted final write, if any, is truncated away. With distinct paths the +// input is never modified: its clean content is copied raw into outputPath, +// replacing whatever was there, and the interrupted tail is simply not +// copied. Either way it returns how many trailing bytes were left out — +// every entry they held falls at or after the cursor, so the resumed query +// fetches it again. +func PrepareOutput(c *Cursor, inputPath, outputPath string) (int64, error) { + if filepath.Clean(inputPath) == filepath.Clean(outputPath) { + return prepareInPlace(c, inputPath) + } + + in, err := os.Open(inputPath) + if err != nil { + return 0, fmt.Errorf("error opening resume file: %w", err) + } + defer in.Close() + + info, err := in.Stat() + if err != nil { + return 0, fmt.Errorf("error inspecting resume file: %w", err) + } + + out, err := os.Create(outputPath) + if err != nil { + return 0, fmt.Errorf("error creating output file: %w", err) + } + + _, err = io.CopyN(out, in, c.CleanSize) + if cerr := out.Close(); err == nil { + err = cerr + } + if err != nil { + return 0, fmt.Errorf("error copying clean content to output file: %w", err) + } + + return info.Size() - c.CleanSize, nil +} + +// prepareInPlace drops the interrupted tail so appending continues from the +// clean boundary. It never truncates past the offset the reader identified. +func prepareInPlace(c *Cursor, path string) (int64, error) { + if !c.Truncated { + return 0, nil + } + + info, err := os.Stat(path) + if err != nil { + return 0, fmt.Errorf("error inspecting resume file: %w", err) + } + if info.Size() <= c.CleanSize { + return 0, nil + } + + if err := os.Truncate(path, c.CleanSize); err != nil { + return 0, fmt.Errorf("error truncating resume file: %w", err) + } + + return info.Size() - c.CleanSize, nil +} +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `go test ./pkg/resume/... && go vet ./pkg/resume/... && gofumpt -l pkg/resume` +Expected: PASS, clean, nothing listed. + +- [ ] **Step 6: Commit** + +```bash +git add pkg/resume/resume.go pkg/resume/resume_test.go +git commit -m "feat(resume): add PrepareOutput for in-place and copy-mode continuation" +``` + +--- + +### Task 3: Delete `filestore.SaveLogsAsync` + +**Files:** +- Modify: `pkg/filestore/filestore.go` +- Test: `pkg/filestore/filestore_test.go` + +**Interfaces:** +- Consumes: nothing. +- Produces: `filestore` exports exactly `CreateWriter`, `AppendWriter`, `AppendLogsAsync`. Task 4's `cmd` already uses only these. + +- [ ] **Step 1: Retool the tests** + +In `pkg/filestore/filestore_test.go`, add a seeding helper and rewrite the four tests to use it: + +```go +// seed writes logs for the given blocks to a fresh file at path. +func seed(t *testing.T, path string, blocks ...uint64) { + t.Helper() + + w, err := filestore.CreateWriter(path) + if err != nil { + t.Fatalf("CreateWriter() error = %v", err) + } + if err := filestore.AppendLogsAsync(t.Context(), feed(blocks...), w, nil); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } +} +``` + +- `TestSaveLogsAsyncReplacesExistingFile` → rename to `TestCreateWriterReplacesExistingFile`; keep the stale-content pre-write, then `seed(t, path, 1, 2)` and the same `blocksIn` assertion (`[]uint64{1, 2}`). +- In `TestAppendLogsAsyncKeepsExistingContent`, `TestAppendLogsAsyncSkipsFilteredLogs`, `TestAppendLogsAsyncClosesWriterOnCancel`: replace each `filestore.SaveLogsAsync(t.Context(), feed(...), path)` seeding call (and its error check) with the matching `seed(t, path, ...)` call. Assertions unchanged. + +- [ ] **Step 2: Delete the function** + +Remove `SaveLogsAsync` (currently `pkg/filestore/filestore.go:26-35`) and nothing else. `CreateWriter`'s doc comment already explains the split. + +- [ ] **Step 3: Verify** + +Run: `go build ./... && go test ./pkg/filestore/... && grep -rn "SaveLogsAsync" --include='*.go' .` +Expected: build OK (proves `cmd` never used it), tests PASS, grep prints nothing. + +- [ ] **Step 4: Commit** + +```bash +git add pkg/filestore/filestore.go pkg/filestore/filestore_test.go +git commit -m "refactor(filestore): drop SaveLogsAsync, callers compose CreateWriter and AppendLogsAsync" +``` + +--- + +### Task 4: Mode resolution in `cmd/export.go` + +**Files:** +- Modify: `cmd/export.go` + +**Interfaces:** +- Consumes: `resume.Read`, `resume.PrepareOutput` (Tasks 1–2); `filestore.CreateWriter`/`AppendWriter`, `gzipstore.AppendWriter` (existing). +- Produces: the §3 CLI contract. Task 5 edits the same file afterwards. + +- [ ] **Step 1: Rework the resume block in `RunE`** + +Replace the current resume block (`cmd/export.go:47-74`) with: + +```go + var cursor *resume.Cursor + if resumeFile != "" { + cursor, err = resume.Read(resumeFile) + if err != nil { + return fmt.Errorf("failed to read resume file %q: %w", resumeFile, err) + } + + if cmd.Flags().Changed("start") { + c.log.Warning("--start is ignored when --resume is set", "resumeFile", resumeFile) + } + if compress { + c.log.Warning("--compress is ignored when resuming; resume a compressed file to get a compressed result", "resumeFile", resumeFile) + compress = false + } + // An unset --output means in-place; so does naming the input. + if !cmd.Flags().Changed("output") || filepath.Clean(outputFile) == filepath.Clean(resumeFile) { + outputFile = resumeFile + } + + startBlock = cursor.BlockNumber + + c.log.Info("Resuming export", + "resumeFile", resumeFile, + "outputFile", outputFile, + "startBlock", startBlock, + "lastLogIndex", cursor.LogIndex, + "compressed", cursor.Compressed, + ) + } +``` + +This removes the old `--output is ignored` warning and the `cursor.Compressed && compress` condition (the warning now fires for every resume with `--compress`). Add `"path/filepath"` to the imports. + +- [ ] **Step 2: Replace `discardPartialWrite` with `PrepareOutput`** + +Replace the call site (`cmd/export.go:100-102`) with: + +```go + if cursor != nil { + discarded, err := resume.PrepareOutput(cursor, resumeFile, outputFile) + if err != nil { + return err + } + if discarded > 0 { + c.log.Warning("previous export ends with an interrupted write, leaving it out", + "resumeFile", resumeFile, + "offset", cursor.CleanSize, + "discardedBytes", discarded, + ) + } + } +``` + +Delete the whole `discardPartialWrite` method (`cmd/export.go:197-229`). If `"os"` is now unused in the file, remove it from the imports. `openOutput` and `saveLogs` stay exactly as they are — in copy mode they receive the already-prepared `outputFile`, and `openOutput`'s `cursor.Compressed` branch matches because the output's format equals the input's. + +- [ ] **Step 3: Update the flag help** + +Replace the `--resume` registration line (`cmd/export.go:190`) with: + +```go + cmd.Flags().StringVarP(&resumeFile, "resume", "r", "", "Continue a previous export file (.ndjson, .gz or .gzip); combine with --output to write a new snapshot instead of appending in place") +``` + +- [ ] **Step 4: Build and verify by hand against copies** + +```bash +make binary && go vet ./... && gofumpt -l cmd pkg +mkdir -p /tmp/resume-verify && cp dist/export.ndjson.gzip /tmp/resume-verify/prev.gzip +# Copy mode: prev untouched, next = prev + nothing new (end pinned at the cursor block). +./dist/batch-export export --resume /tmp/resume-verify/prev.gzip --output /tmp/resume-verify/next.gzip --end 47908504 +cmp /tmp/resume-verify/prev.gzip dist/export.ndjson.gzip && echo "input untouched" +gzcat /tmp/resume-verify/next.gzip | wc -l # equals gzcat prev | wc -l +# In-place still works, and --compress warns: +./dist/batch-export export --resume /tmp/resume-verify/prev.gzip --compress --end 47908504 2>&1 | grep -i "ignored" +``` + +Expected: build clean; copy run logs `Resuming export` with `outputFile=/tmp/resume-verify/next.gzip`; `cmp` silent; line counts equal; the last run warns about `--compress` and appends in place to the copy. **Never point `--resume` or `--output` at `dist/` paths.** + +- [ ] **Step 5: Commit** + +```bash +git add cmd/export.go +git commit -m "feat(export): compose --resume with --output for copy-mode continuation" +``` + +--- + +### Task 5: Saver errors reach the exit code + +**Files:** +- Modify: `cmd/export.go` + +**Interfaces:** +- Consumes: everything already in the file. No new symbols. +- Produces: §7's behavior — a failed save is a non-zero exit, never a hang, never a lost gzip member. + +- [ ] **Step 1: Derive a cancellable context** + +Replace `ctx := cmd.Context()` (top of `RunE`) with: + +```go + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() +``` + +- [ ] **Step 2: Capture the saver's error and cancel on failure** + +Replace the saver goroutine with: + +```go + var saveErr error + go func() { + defer wg.Done() + + if err := saveLogs(ctx, logChan, w, cursor); err != nil { + if errors.Is(err, context.Canceled) { + c.log.Error(err, "context canceled while saving logs") + return + } + c.log.Error(err, "error saving logs") + // Stop the fetcher too: with the saver gone, logChan + // would fill and block it forever. + saveErr = err + cancel() + return + } + c.log.Info("all logs have been saved", "outputFile", outputFile) + }() +``` + +(`saveErr` is written before `wg.Done` and read after `wg.Wait`, so the WaitGroup orders the accesses; `go test -race` confirms.) + +- [ ] **Step 3: Wait and join on every exit path** + +The select loop's `errorChan` branch (`cmd/export.go:152-157`) becomes: + +```go + case err, ok := <-errorChan: + if !ok { + errorChan = nil + } else { + wg.Wait() + return errors.Join(fmt.Errorf("error retrieving logs: %w", err), saveErr) + } +``` + +(no deadlock: the fetcher closes `logChan` on its way out, so the saver finishes and `wg.Wait` returns). The `ctx.Done` branch becomes: + +```go + case <-ctx.Done(): + c.log.Info("context canceled, waiting for logs to be saved...") + wg.Wait() + if saveErr != nil { + return saveErr + } + if err := compressFunc(); err != nil { + return errors.Join(fmt.Errorf("error compressing file: %w", err), ctx.Err()) + } + return ctx.Err() +``` + +And the normal exit (`cmd/export.go:174-179`): + +```go + wg.Wait() + if saveErr != nil { + return saveErr + } + if err := compressFunc(); err != nil { + return fmt.Errorf("error compressing file: %w", err) + } + + return nil +``` + +- [ ] **Step 4: Verify the failure modes by hand** + +```bash +make binary && go vet ./... && go test -race ./pkg/... +cp dist/export.ndjson.gzip /tmp/resume-verify/failing.gzip +# Read-only destination: the run must exit non-zero promptly, not hang. +chmod a-w /tmp/resume-verify/failing.gzip +./dist/batch-export export --resume /tmp/resume-verify/failing.gzip --end 47908504; echo "exit=$?" +chmod u+w /tmp/resume-verify/failing.gzip +# Ctrl+C mid-run on a copy: file must still be readable afterwards. +./dist/batch-export export --resume /tmp/resume-verify/failing.gzip & sleep 3; kill -INT %1; wait +gzcat /tmp/resume-verify/failing.gzip > /dev/null && echo "stream intact" +``` + +Expected: the read-only run prints the open error and `exit=1` immediately (the open happens before fetching); the interrupted run exits, and `gzcat` reads the file end to end (the member was flushed under `wg.Wait`). + +- [ ] **Step 5: Commit** + +```bash +git add cmd/export.go +git commit -m "fix(export): surface save failures in the exit code and never skip the final flush" +``` + +--- + +### Task 6: README, PR description, final sweep + +**Files:** +- Modify: `README.md` + +**Interfaces:** +- Consumes: the finished behavior of Tasks 1–5. +- Produces: §8's documentation. Nothing depends on it. + +- [ ] **Step 1: Update the feature bullet and flag table** + +In `## Features`, replace the resume bullet with: + +```markdown +- Continue a previous export from where it stopped (incremental snapshots). +``` + +In the `## Flags` block, replace the `--resume` line with: + +```sh + -r, --resume string Continue a previous export file (.ndjson, .gz or .gzip); combine with --output to write a new snapshot instead of appending in place +``` + +- [ ] **Step 2: Replace the resume section** + +Replace everything from `### Resuming an interrupted export` up to (not including) the `The produced NDJSON is consumed by ...` line with: + +````markdown +### Continuing a previous snapshot + +Instead of re-exporting every block, point `--resume` at the previous +snapshot and name the new one with `--output`. The previous file is read, +never modified; the new file holds everything the previous one did plus the +blocks exported since: + +```sh +./dist/batch-export export --resume snapshots/2026-07.gzip --output snapshots/2026-08.gzip +``` + +Formats are detected by content, not extension: `.ndjson`, `.gz` and `.gzip` +all work, and the output's format always matches the input's. Omitting +`--output` (or naming the input) appends to the previous file in place — the +space-saving variant: + +```sh +./dist/batch-export export --resume export.ndjson.gzip +``` + +The last exported block is re-queried and entries already present are +skipped, so continuing neither duplicates nor drops a log. Each continuation +of a compressed snapshot adds a gzip member — standard tools (`gzcat`, +`gunzip`, Go, Python) read multi-member files as one stream, a year of +monthly continuations costs about 0.02% in size, and +`gzcat old.gzip | gzip > fresh.gzip` consolidates the members any time. + +Keep one canonical snapshot file. `--compress` is ignored when resuming: +regenerating a `.gzip` from a plain twin is how an independently continued +archive gets overwritten. Resume a compressed file to get a compressed +result. + +Resume only files this tool produced. The file's tail is validated before +anything is written: content the tool never writes — a non-log line, foreign +data, an alien gzip member — is refused rather than repaired. The one +exception is the tool's own interrupted final write (a run killed +mid-export): in copy mode it is simply not copied, in place it is truncated +away with a warning, and its entries are re-fetched. Note that resuming does +not detect a file from a different chain; pairing the snapshot with the +right `--endpoint` is the operator's contract. +```` + +- [ ] **Step 3: Full verification sweep** + +```bash +make test && go test -race ./pkg/... && make vet && make lint && make binary +``` + +Expected: everything green, `0 issues.` from lint. + +- [ ] **Step 4: Commit** + +```bash +git add README.md +git commit -m "docs: reframe resume around incremental snapshots" +``` + +- [ ] **Step 5: Update the PR description (no push without approval)** + +Rewrite the PR #11 body to lead with incremental snapshots and copy mode, and list the three semantic changes from §11 (output composes; `--compress` always ignored on resume; strict refusal replaces lenient tolerance). Hold the `git push` and `gh pr edit` until the human approves pushing this phase. + +--- + +## Self-Review + +**Spec coverage:** §3 CLI table → Task 4 Step 1 (mode resolution) + Step 3 (help text); §4 modes → Task 2 (`PrepareOutput`) + Task 4 Step 2; §5 strict validation, both formats, error taxonomy → Task 1; §6 appending/skip → unchanged code, re-verified by Task 2's round trips; §7 exit codes → Task 5; §8 README → Task 6; §9 placement → Tasks 1–4 as mapped; §10 testing → Tasks 1–3 test steps; §11 compatibility → Task 6 Step 5. No spec requirement without a task. + +**Type consistency:** `PrepareOutput(c *Cursor, inputPath, outputPath string) (int64, error)` is defined in Task 2 and consumed with that exact shape in Task 4 Step 2 and the Task 2 tests. `ErrNotAnExport`/`ErrNoLogs` defined in Task 1, consumed in Task 1's refusal table. `seed(t, path, blocks...)` defined and used only in Task 3. `appendWriter`, `feed`, `logsIn`, `ids`, `truncateLast`, `gz`, `ndjson`, `write` all pre-exist in `resume_test.go`. + +**Known deliberate residue:** `cmd`'s §7 plumbing has no unit test (no RPC harness — §10 sanctions manual evidence, Task 5 Step 4 collects it); `compressFunc` stays for fresh runs only; the gzip walk treats unparseable bytes after a clean member as an interrupted tail (indistinguishable from a partial member header — §5 documents this). From 85e61b656974faf8229866c54b1b59041e8460f0 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 11:47:14 +0200 Subject: [PATCH 19/35] feat(resume): validate strictly, refuse files this tool did not write --- pkg/resume/resume.go | 235 ++++++++++++++++++-------------------- pkg/resume/resume_test.go | 207 +++++++++++++++------------------ 2 files changed, 208 insertions(+), 234 deletions(-) diff --git a/pkg/resume/resume.go b/pkg/resume/resume.go index 72d07b2..96ee6c8 100644 --- a/pkg/resume/resume.go +++ b/pkg/resume/resume.go @@ -5,6 +5,7 @@ package resume import ( "bufio" "bytes" + "compress/flate" "compress/gzip" "encoding/json" "errors" @@ -17,24 +18,21 @@ import ( ) const ( - // windowSize is how much of a plain NDJSON file is read per backward step. - windowSize = 64 * 1024 - // maxLineBytes caps a single line, so a file without newlines is not read - // into memory whole. Exported log lines run to a few hundred bytes. + // maxLineBytes caps a single line. Exported log lines run to a few + // hundred bytes, so anything longer is not this tool's output. maxLineBytes = 1 << 20 // bufferSize is how much of a gzip file is buffered per read. bufferSize = 64 * 1024 ) var ( - // ErrNoLogs indicates that a file holds no complete log entry. + // ErrNotAnExport indicates content this tool never writes. The file was + // altered after export, so resuming it is refused rather than repaired. + ErrNotAnExport = errors.New("not an untouched batch-export file") + // ErrNoLogs indicates a file consistent with tool output that holds no + // complete entry to resume from: it is empty, or holds only an + // interrupted first write. The remedy is a fresh export. ErrNoLogs = errors.New("no complete log entry found") - // ErrNoCleanBoundary indicates that a file holds log entries but no offset - // at which appending is safe. - ErrNoCleanBoundary = errors.New("file was left partially written and has no clean boundary to append at") - // errLineTooLong indicates a line ran past maxLineBytes, so the data is not - // the NDJSON an export writes. - errLineTooLong = errors.New("log line exceeds the maximum length") ) // Cursor marks the last log entry saved by a previous export, together with @@ -78,8 +76,8 @@ type cursorLine struct { // a compressed one must sit in a member that decoded to a clean end of stream. // A caller that appends must first discard everything past CleanSize. // -// It returns ErrNoLogs when there is no complete entry, and ErrNoCleanBoundary -// when there are entries but no offset at which appending is safe. +// It returns ErrNoLogs when there is no complete entry, and ErrNotAnExport +// when the file holds content this tool did not write. func Read(path string) (*Cursor, error) { file, err := os.Open(path) if err != nil { @@ -128,76 +126,53 @@ func isGzip(file *os.File) (bool, error) { return magic[0] == 0x1f && magic[1] == 0x8b, nil } -// lastCursorPlain walks a plain NDJSON file backwards a window at a time and -// returns a cursor for the last line that parses and ends in a newline. -// json.Encoder writes a value and its newline in one call, so a trailing line -// without one was cut short by an interrupted run and is passed over. +// lastCursorPlain validates the tail of a plain NDJSON file and returns a +// cursor for its last complete line. The tool writes one entry per line in a +// single call, so only the tail needs examining: the last newline-terminated +// line must parse as a log entry, and the only bytes allowed after it are a +// single interrupted write — a trailing fragment with no newline. func lastCursorPlain(file *os.File, size int64) (*Cursor, error) { - var ( - offset = size - // end is one past the last byte held in window; terminated reports - // whether the file byte at end is the newline closing that region's - // last line. Only a terminated region can yield a cursor. - end = size - terminated bool - // carry holds the bytes before the window's first newline, belonging to - // a line that starts in the next window back. - carry []byte - ) - - for offset > 0 { - n := int64(windowSize) - if offset < n { - n = offset - } - offset -= n - - window := make([]byte, n) - if _, err := file.ReadAt(window, offset); err != nil { - return nil, fmt.Errorf("error reading resume file: %w", err) - } - window = append(window, carry...) + window := min(size, 2*maxLineBytes) + offset := size - window - for { - i := bytes.LastIndexByte(window, '\n') - if i < 0 { - break - } - if terminated { - if cursor, err := parseCursor(window[i+1:]); err == nil { - cursor.CleanSize = end + 1 - return cursor, nil - } - } - window = window[:i] - end, terminated = offset+int64(i), true - } + buf := make([]byte, window) + if _, err := file.ReadAt(buf, offset); err != nil { + return nil, fmt.Errorf("error reading resume file: %w", err) + } - carry = window - if len(carry) > maxLineBytes { - return nil, ErrNoLogs + nl := bytes.LastIndexByte(buf, '\n') + if nl < 0 { + // No newline at all: an interrupted first write, unless the file is + // longer than any single line the tool writes. + if size > maxLineBytes { + return nil, fmt.Errorf("%w: %d bytes without a newline", ErrNotAnExport, size) } + return nil, ErrNoLogs + } + if tail := window - int64(nl) - 1; tail > maxLineBytes { + return nil, fmt.Errorf("%w: %d bytes without a newline after offset %d", ErrNotAnExport, tail, offset+int64(nl)+1) } - if terminated { - if cursor, err := parseCursor(carry); err == nil { - cursor.CleanSize = end + 1 - return cursor, nil - } + start := bytes.LastIndexByte(buf[:nl], '\n') + 1 + if start == 0 && offset > 0 { + return nil, fmt.Errorf("%w: final line is over %d bytes long", ErrNotAnExport, nl) + } + cursor, err := parseCursor(buf[start:nl]) + if err != nil { + return nil, fmt.Errorf("%w: final complete line at offset %d is not a log entry: %w", ErrNotAnExport, offset+int64(start), err) } + cursor.CleanSize = offset + int64(nl) + 1 - return nil, ErrNoLogs + return cursor, nil } // lastCursorGzip walks a gzip file one member at a time and returns a cursor -// for the last log line inside a cleanly terminated member. Gzip cannot be -// seeked, so the whole stream is decompressed. -// -// A half-written member has no CRC and length trailer, so it reports an error -// rather than io.EOF; the walk stops there and CleanSize is the end of the last -// member that did terminate cleanly, which is a valid place to concatenate the -// next one. Lines are carried across members, so one split between two members -// is still recognised and a member ending mid-line is not a boundary. +// for the last entry inside the cleanly terminated prefix. Gzip cannot be +// seeked, so the whole stream is decompressed; every complete line is +// validated on the way. A member cut short by an interrupted write ends the +// walk: CleanSize stays at the last clean member boundary, and the truncated +// member's content — about to be discarded and re-fetched — never advances +// the cursor. func lastCursorGzip(file *os.File) (*Cursor, error) { counter := &countingReader{reader: bufio.NewReaderSize(file, bufferSize)} @@ -208,85 +183,101 @@ func lastCursorGzip(file *os.File) (*Cursor, error) { defer reader.Close() var ( - // clean is the cursor as of cleanSize; pending also covers the member - // being read and is promoted only once that member is known to have - // ended cleanly and on a line boundary. - clean, pending *Cursor - cleanSize int64 - carry []byte + cursor *Cursor + cleanSize int64 + sawClean bool ) - for { // Reset turns multistream back on, so it must be switched off per // member, not just for the first. reader.Multistream(false) - cursor, tail, err := scanLines(reader, carry) - if cursor != nil { - pending = cursor - } - if err != nil { - // Member did not reach a clean end of stream, so everything it - // holds belongs to an interrupted write. - break + last, err := scanMember(reader) + switch { + case errors.Is(err, ErrNotAnExport): + return nil, err + case err != nil && !truncationShaped(err): + return nil, fmt.Errorf("error reading gzip resume file: %w", err) + case err != nil: + // The member never got its trailer: an interrupted final write. + return gzipResult(cursor, cleanSize, sawClean) } - if len(tail) == 0 { - clean, cleanSize = pending, counter.read + + if last != nil { + cursor = last } - carry = tail + cleanSize, sawClean = counter.read, true - // A clean end of file makes Reset report io.EOF; anything else is a - // partly written member header. + // A clean end of file makes Reset report io.EOF; a partly written + // next member fails to parse as a header and also ends the walk. if err := reader.Reset(counter); err != nil { - break + if errors.Is(err, io.EOF) || truncationShaped(err) { + return gzipResult(cursor, cleanSize, sawClean) + } + return nil, fmt.Errorf("error reading gzip resume file: %w", err) } } +} - if clean == nil { - if pending == nil { - return nil, ErrNoLogs - } - return nil, ErrNoCleanBoundary +// gzipResult finalizes the walk: a file with no clean member, or none holding +// an entry, has nothing to resume from. +func gzipResult(cursor *Cursor, cleanSize int64, sawClean bool) (*Cursor, error) { + if !sawClean || cursor == nil { + return nil, ErrNoLogs } + cursor.CleanSize = cleanSize + + return cursor, nil +} - clean.CleanSize = cleanSize +// truncationShaped reports whether err is what an interrupted write produces, +// as opposed to a real read failure that must not be mistaken for one. +func truncationShaped(err error) bool { + var corrupt flate.CorruptInputError - return clean, nil + return errors.Is(err, io.ErrUnexpectedEOF) || + errors.Is(err, gzip.ErrHeader) || + errors.Is(err, gzip.ErrChecksum) || + errors.As(err, &corrupt) } -// scanLines reads NDJSON from r and returns a cursor for the last line that -// parses and ends in a newline. carry is prepended to the first line so a line -// split across two gzip members is reassembled, and the unterminated trailing -// bytes are returned for the next member to complete. -// -// The error is nil only when r reached a clean io.EOF. Truncation, a checksum -// mismatch, an over-long line and an I/O failure are all reported, because each -// means the bytes after the last good line cannot be trusted. -func scanLines(r io.Reader, carry []byte) (*Cursor, []byte, error) { +// scanMember reads one gzip member's NDJSON and returns a cursor for its last +// entry, nil when the member is empty. A nil error means the member decoded +// to a clean end of stream on a line boundary. The tool writes members +// holding whole log lines only, so a complete line that does not parse, or a +// clean member ending mid-line, is foreign content (ErrNotAnExport); any +// other read error is returned as-is for the caller to classify. +func scanMember(r io.Reader) (*Cursor, error) { buffered := bufio.NewReaderSize(r, bufferSize) - var last *Cursor - - line := carry + var ( + last *Cursor + line []byte + ) for { chunk, err := buffered.ReadSlice('\n') line = append(line, chunk...) if len(line) > maxLineBytes { - return last, nil, errLineTooLong + return nil, fmt.Errorf("%w: line exceeds %d bytes", ErrNotAnExport, maxLineBytes) } switch { case errors.Is(err, bufio.ErrBufferFull): continue case errors.Is(err, io.EOF): - return last, line, nil + if len(line) > 0 { + return nil, fmt.Errorf("%w: gzip member ends mid-line", ErrNotAnExport) + } + return last, nil case err != nil: - return last, nil, fmt.Errorf("error reading gzip resume file: %w", err) + return last, err } - if cursor, err := parseCursor(line); err == nil { - last = cursor + cursor, err := parseCursor(line) + if err != nil { + return nil, fmt.Errorf("%w: line is not a log entry: %w", ErrNotAnExport, err) } + last = cursor line = line[:0] } } @@ -319,20 +310,20 @@ func (c *countingReader) ReadByte() (byte, error) { return b, err } -// parseCursor builds a cursor from a single NDJSON line, rejecting blank and -// truncated lines and any log missing a block number or log index. +// parseCursor builds a cursor from a single NDJSON line. Callers wrap the +// returned error with the sentinel that fits their context. func parseCursor(line []byte) (*Cursor, error) { line = bytes.TrimSpace(line) if len(line) == 0 { - return nil, ErrNoLogs + return nil, errors.New("blank line") } var parsed cursorLine if err := json.Unmarshal(line, &parsed); err != nil { - return nil, ErrNoLogs + return nil, err } if parsed.BlockNumber == nil || parsed.LogIndex == nil { - return nil, ErrNoLogs + return nil, errors.New("missing blockNumber or logIndex") } return &Cursor{ diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index b6ff114..5964237 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -19,12 +19,6 @@ import ( "github.com/ethersphere/batch-export/pkg/resume" ) -// windowSize mirrors the unexported constant of the same name in resume.go: -// how much of a plain NDJSON file lastCursorPlain reads at a time when -// walking backwards from EOF. Tests use it to construct cases that straddle -// a window boundary. -const windowSize = 64 * 1024 - // Export file names. The format is detected from the file's leading bytes, so // the name a case uses never decides how it is read. const ( @@ -106,31 +100,6 @@ func TestReadCursor(t *testing.T) { } many := ndjson(t, manyLogs...) - // garbageLines are newline-delimited but unparseable, as if a different - // file had been concatenated onto a good export. - garbageLines := bytes.Repeat([]byte("not-json\n"), 12*1024) - - // straddle places a valid log line across the boundary between the last - // two windows lastCursorPlain reads: reassembling it requires appending - // each re-read window before the carried tail, in file order. Swap that - // order and the line comes out garbled — which a case whose target sits - // wholly inside one window can never catch. - straddleTarget := ndjson(t, testLog(4096, 3)) - straddlePrefix := bytes.Repeat([]byte("not-json\n"), 4) - // afterLen puts the boundary (windowSize bytes from EOF) strictly inside - // straddleTarget, rather than at a round offset that could drift. - afterLen := windowSize - len(straddleTarget)/2 - filler := []byte("not-json\n") - straddleAfter := bytes.Repeat(filler, afterLen/len(filler)+2)[:afterLen] - straddle := append(append(append([]byte{}, straddlePrefix...), straddleTarget...), straddleAfter...) - - straddleLineStart := len(straddlePrefix) - straddleLineEnd := len(straddlePrefix) + len(straddleTarget) - straddleBoundary := len(straddle) - windowSize - if straddleBoundary <= straddleLineStart || straddleBoundary >= straddleLineEnd { - t.Fatalf("test setup: boundary %d does not straddle target line [%d, %d)", straddleBoundary, straddleLineStart, straddleLineEnd) - } - // twoLogsEnd is the offset just past the newline closing the second of // threeLogs: the last clean boundary once the third line loses its own. twoLogsEnd := int64(len(ndjson(t, testLog(100, 0), testLog(101, 1)))) @@ -184,35 +153,11 @@ func TestReadCursor(t *testing.T) { wantCleanSize: twoLogsEnd, }, { - name: "line missing blockNumber is skipped", - content: append(append([]byte{}, threeLogs...), []byte("{\"address\":\"0x1\",\"topics\":[],\"data\":\"0x\"}\n")...), - wantBlock: 102, - wantIndex: 7, - wantTruncated: true, - wantCleanSize: threeLogsEnd, - }, - { - name: "spans multiple backward read windows", + name: "many logs", content: many, wantBlock: 2999, wantIndex: 15, }, - { - name: "walks back across windows of garbage lines", - content: append(append([]byte{}, threeLogs...), garbageLines...), - wantBlock: 102, - wantIndex: 7, - wantTruncated: true, - wantCleanSize: threeLogsEnd, - }, - { - name: "valid line straddles a window boundary", - content: straddle, - wantBlock: 4096, - wantIndex: 3, - wantTruncated: true, - wantCleanSize: int64(straddleLineEnd), - }, { name: "gzip", content: gz(t, threeLogs), @@ -258,6 +203,13 @@ func TestReadCursor(t *testing.T) { wantTruncated: true, wantCleanSize: int64(len(gz(t, threeLogs))), }, + { + name: "empty trailing gzip member is valid", + content: append(gz(t, threeLogs), gz(t, nil)...), + wantBlock: 102, + wantIndex: 7, + wantCompressed: true, + }, } for _, tt := range tests { @@ -294,26 +246,105 @@ func TestReadCursor(t *testing.T) { } } -func TestReadCursorErrors(t *testing.T) { +// TestReadRefusals covers §5's strict contract: the only irregularity +// tolerated is the tool's own interrupted final write. Content the tool never +// writes is ErrNotAnExport; a file consistent with tool output but holding no +// complete entry is ErrNoLogs. +func TestReadRefusals(t *testing.T) { t.Parallel() + logs := ndjson(t, testLog(100, 0), testLog(101, 0)) + tests := []struct { name string content []byte + wantErr error }{ - {name: "empty file", content: []byte{}}, - {name: "only a newline", content: []byte("\n")}, - {name: "only garbage lines", content: bytes.Repeat([]byte("not-json\n"), 10)}, - {name: "single unterminated line larger than the cap", content: bytes.Repeat([]byte("x"), 2<<20)}, + { + name: "empty file", + content: []byte{}, + wantErr: resume.ErrNoLogs, + }, + { + name: "plain file holding one unterminated line", + content: bytes.TrimSuffix(ndjson(t, testLog(100, 0)), []byte("\n")), + wantErr: resume.ErrNoLogs, + }, + { + // The tool never writes a blank line. + name: "only a newline", + content: []byte("\n"), + wantErr: resume.ErrNotAnExport, + }, + { + name: "trailing blank line after valid logs", + content: append(append([]byte{}, logs...), '\n'), + wantErr: resume.ErrNotAnExport, + }, + { + // A complete line that is not a log entry means the file was + // altered after export; refusing beats guessing what to cut. + name: "trailing non-log line after valid logs", + content: append(append([]byte{}, logs...), []byte("not-json\n")...), + wantErr: resume.ErrNotAnExport, + }, + { + name: "trailing log line missing blockNumber", + content: append(append([]byte{}, logs...), []byte("{\"address\":\"0x1\",\"topics\":[],\"data\":\"0x\"}\n")...), + wantErr: resume.ErrNotAnExport, + }, + { + name: "only garbage lines", + content: bytes.Repeat([]byte("not-json\n"), 10), + wantErr: resume.ErrNotAnExport, + }, + { + // Finding #4's shape: refused from a single tail read, no + // backward scan through the junk. + name: "newline-free tail longer than a line can be", + content: append(append([]byte{}, logs...), bytes.Repeat([]byte("x"), 1<<20+1)...), + wantErr: resume.ErrNotAnExport, + }, + { + name: "single unterminated line larger than the cap", + content: bytes.Repeat([]byte("x"), 2<<20), + wantErr: resume.ErrNotAnExport, + }, + { + // A cleanly terminated member whose content stops mid-line + // cannot come from this tool: members hold whole lines, and an + // interrupted write cannot produce a valid trailer. + name: "gzip member ending mid line", + content: gz(t, bytes.TrimSuffix(logs, []byte("\n"))), + wantErr: resume.ErrNotAnExport, + }, + { + // Finding #6's shape: foreign data concatenated as its own valid + // member. Refused rather than treated as a removable tail. + name: "gzip junk member after valid member", + content: append(gz(t, logs), gz(t, []byte("not-json\nalso-not\n"))...), + wantErr: resume.ErrNotAnExport, + }, + { + name: "gzip member holding a non-log line between logs", + content: gz(t, append(append([]byte{}, logs...), []byte("not-json\n")...)), + wantErr: resume.ErrNotAnExport, + }, + { + // A sole member without its trailer is an interrupted first + // write: nothing to resume from, so a fresh export is the remedy. + name: "gzip with a single truncated member", + content: truncateLast(gz(t, logs), 6), + wantErr: resume.ErrNoLogs, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { t.Parallel() - _, err := resume.Read(write(t, plainFile, tt.content)) - if !errors.Is(err, resume.ErrNoLogs) { - t.Fatalf("Read() error = %v, want ErrNoLogs", err) + if _, err := resume.Read(write(t, plainFile, tt.content)); !errors.Is(err, tt.wantErr) { + t.Fatalf("Read() error = %v, want %v", err, tt.wantErr) } }) } @@ -706,54 +737,6 @@ func TestResumeAfterInterruptedWrite(t *testing.T) { } } -// TestReadRefusesWithoutACleanBoundary covers files whose content cannot be -// appended to at any offset the reader can positively identify. Refusing is -// the safety property that has to hold even where recovery cannot: guessing a -// boundary would corrupt what is already there. -func TestReadRefusesWithoutACleanBoundary(t *testing.T) { - t.Parallel() - - logs := ndjson(t, testLog(100, 0), testLog(101, 0)) - - tests := []struct { - name string - content []byte - wantErr error - }{ - { - // A single member whose trailer never made it to disk. Its logs - // decode, but there is no member boundary to concatenate at and - // the file cannot be rewritten in place, so the run must stop. - name: "gzip with a single truncated member", - content: truncateLast(gz(t, logs), 6), - wantErr: resume.ErrNoCleanBoundary, - }, - { - // The member is intact but its data stops mid-line, so a second - // member would glue its first log onto the unterminated one. - name: "gzip member ending mid line", - content: gz(t, bytes.TrimSuffix(logs, []byte("\n"))), - wantErr: resume.ErrNoCleanBoundary, - }, - { - // The only line has no newline, so no entry is complete. - name: "plain file holding one unterminated line", - content: bytes.TrimSuffix(ndjson(t, testLog(100, 0)), []byte("\n")), - wantErr: resume.ErrNoLogs, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - if _, err := resume.Read(write(t, plainFile, tt.content)); !errors.Is(err, tt.wantErr) { - t.Fatalf("Read() error = %v, want %v", err, tt.wantErr) - } - }) - } -} - // TestReadReportsGzipReadErrors pins Fix 2: a gzip stream that stops early // must not be reported as a clean read just because some lines decoded. The // old code returned the last cursor it had seen with a nil error, which let From a0251e6f5cf1a2e4cda959799e75b943179fc5da Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 12:01:38 +0200 Subject: [PATCH 20/35] feat(resume): add PrepareOutput for in-place and copy-mode continuation --- pkg/resume/resume.go | 63 ++++++++++++++ pkg/resume/resume_test.go | 179 ++++++++++++++++++++++++++++++++++++-- 2 files changed, 236 insertions(+), 6 deletions(-) diff --git a/pkg/resume/resume.go b/pkg/resume/resume.go index 96ee6c8..bfcedc4 100644 --- a/pkg/resume/resume.go +++ b/pkg/resume/resume.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "os" + "path/filepath" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" @@ -61,6 +62,68 @@ func (c *Cursor) Skip(l types.Log) bool { return l.Index <= c.LogIndex } +// PrepareOutput readies outputPath for appending the continuation of the +// export at inputPath. With equal paths the file is prepared in place: an +// interrupted final write, if any, is truncated away. With distinct paths the +// input is never modified: its clean content is copied raw into outputPath, +// replacing whatever was there, and the interrupted tail is simply not +// copied. Either way it returns how many trailing bytes were left out — +// every entry they held falls at or after the cursor, so the resumed query +// fetches it again. +func PrepareOutput(c *Cursor, inputPath, outputPath string) (int64, error) { + if filepath.Clean(inputPath) == filepath.Clean(outputPath) { + return prepareInPlace(c, inputPath) + } + + in, err := os.Open(inputPath) + if err != nil { + return 0, fmt.Errorf("error opening resume file: %w", err) + } + defer in.Close() + + info, err := in.Stat() + if err != nil { + return 0, fmt.Errorf("error inspecting resume file: %w", err) + } + + out, err := os.Create(outputPath) + if err != nil { + return 0, fmt.Errorf("error creating output file: %w", err) + } + + _, err = io.CopyN(out, in, c.CleanSize) + if cerr := out.Close(); err == nil { + err = cerr + } + if err != nil { + return 0, fmt.Errorf("error copying clean content to output file: %w", err) + } + + return info.Size() - c.CleanSize, nil +} + +// prepareInPlace drops the interrupted tail so appending continues from the +// clean boundary. It never truncates past the offset the reader identified. +func prepareInPlace(c *Cursor, path string) (int64, error) { + if !c.Truncated { + return 0, nil + } + + info, err := os.Stat(path) + if err != nil { + return 0, fmt.Errorf("error inspecting resume file: %w", err) + } + if info.Size() <= c.CleanSize { + return 0, nil + } + + if err := os.Truncate(path, c.CleanSize); err != nil { + return 0, fmt.Errorf("error truncating resume file: %w", err) + } + + return info.Size() - c.CleanSize, nil +} + // cursorLine is the part of an exported log line a cursor is built from. Both // fields are pointers so a line missing either one can be rejected. type cursorLine struct { diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index 5964237..5b7f14a 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -26,6 +26,12 @@ const ( gzipFile = "export.ndjson.gzip" ) +// Format names reused across several tables of test cases. +const ( + nameFormatPlain = "plain ndjson" + nameFormatGzip = "gzip" +) + // testLog builds a log shaped like the ones the exporter writes. types.Log // always marshals blockNumber and logIndex, even at zero. func testLog(blockNumber uint64, logIndex uint) types.Log { @@ -118,7 +124,7 @@ func TestReadCursor(t *testing.T) { wantCleanSize int64 }{ { - name: "plain ndjson", + name: nameFormatPlain, content: threeLogs, wantBlock: 102, wantIndex: 7, @@ -159,7 +165,7 @@ func TestReadCursor(t *testing.T) { wantIndex: 15, }, { - name: "gzip", + name: nameFormatGzip, content: gz(t, threeLogs), wantBlock: 102, wantIndex: 7, @@ -418,8 +424,8 @@ func TestAppendResumeRoundTrip(t *testing.T) { name string compressed bool }{ - {name: "plain ndjson"}, - {name: "gzip", compressed: true}, + {name: nameFormatPlain}, + {name: nameFormatGzip, compressed: true}, } for _, tt := range tests { @@ -611,6 +617,167 @@ func feed(logs ...types.Log) <-chan types.Log { return ch } +// TestCopyResumeRoundTrip is the spec's recommended workflow (§4): resume an +// archived snapshot into a NEW file. The input must come out byte-identical; +// the output must hold the input's content plus exactly the new entries and +// be itself resumable. +func TestCopyResumeRoundTrip(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + compressed bool + }{ + {name: nameFormatPlain}, + {name: nameFormatGzip, compressed: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + original := []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(102, 1)} + boundaryHigher := testLog(102, 2) + newer := []types.Log{testLog(103, 0), testLog(104, 0)} + + inputBytes := ndjson(t, original...) + if tt.compressed { + inputBytes = gz(t, inputBytes) + } + dir := t.TempDir() + input := filepath.Join(dir, "prev.snapshot") + output := filepath.Join(dir, "next.snapshot") + if err := os.WriteFile(input, inputBytes, 0o644); err != nil { + t.Fatalf("write input: %v", err) + } + + cursor, err := resume.Read(input) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + + discarded, err := resume.PrepareOutput(cursor, input, output) + if err != nil { + t.Fatalf("PrepareOutput() error = %v", err) + } + if discarded != 0 { + t.Errorf("discarded = %d, want 0 for a clean input", discarded) + } + + w := appendWriter(t, output, cursor) + replay := []types.Log{testLog(102, 0), testLog(102, 1), boundaryHigher} + replay = append(replay, newer...) + if err := filestore.AppendLogsAsync(t.Context(), feed(replay...), w, cursor.Skip); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + // The input is untouched, byte for byte. + gotInput, err := os.ReadFile(input) + if err != nil { + t.Fatalf("read input: %v", err) + } + if !bytes.Equal(gotInput, inputBytes) { + t.Fatal("input file was modified by a copy-mode resume") + } + + // The output begins with the input's exact bytes (raw prefix + // copy, no recompression) and holds the full sequence. + gotOutput, err := os.ReadFile(output) + if err != nil { + t.Fatalf("read output: %v", err) + } + if len(gotOutput) < len(inputBytes) || !bytes.Equal(gotOutput[:len(inputBytes)], inputBytes) { + t.Fatal("input bytes are not an unchanged prefix of the output") + } + all := append(append(append([]types.Log{}, original...), boundaryHigher), newer...) + if got, want := ids(logsIn(t, output)), ids(all); !slices.Equal(got, want) { + t.Fatalf("output logs = %v, want %v", got, want) + } + + cursor2, err := resume.Read(output) + if err != nil { + t.Fatalf("Read(output) error = %v", err) + } + if cursor2.BlockNumber != 104 || cursor2.LogIndex != 0 || cursor2.Truncated { + t.Errorf("output cursor = {%d,%d,truncated=%t}, want {104,0,false}", cursor2.BlockNumber, cursor2.LogIndex, cursor2.Truncated) + } + }) + } +} + +// TestCopyResumeFromInterruptedInput: copy mode never repairs the input — the +// interrupted tail stays in it — while the output gets only clean content +// plus the re-fetched entries. +func TestCopyResumeFromInterruptedInput(t *testing.T) { + t.Parallel() + + saved := ndjson(t, testLog(100, 0), testLog(101, 0)) + + tests := []struct { + name string + content []byte + compressed bool + }{ + { + name: "plain with a truncated last line", + content: append(append([]byte{}, saved...), []byte(`{"address":"0x45a15`)...), + }, + { + name: "gzip with a truncated final member", + content: append(gz(t, saved), truncateLast(gz(t, ndjson(t, testLog(102, 0))), 6)...), + compressed: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + dir := t.TempDir() + input := filepath.Join(dir, "prev.snapshot") + output := filepath.Join(dir, "next.snapshot") + if err := os.WriteFile(input, tt.content, 0o644); err != nil { + t.Fatalf("write input: %v", err) + } + + cursor, err := resume.Read(input) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if !cursor.Truncated { + t.Fatal("test setup: input should be truncated") + } + + discarded, err := resume.PrepareOutput(cursor, input, output) + if err != nil { + t.Fatalf("PrepareOutput() error = %v", err) + } + if want := int64(len(tt.content)) - cursor.CleanSize; discarded != want { + t.Errorf("discarded = %d, want %d", discarded, want) + } + + w := appendWriter(t, output, cursor) + replay := []types.Log{testLog(101, 0), testLog(102, 0), testLog(103, 0)} + if err := filestore.AppendLogsAsync(t.Context(), feed(replay...), w, cursor.Skip); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } + + gotInput, err := os.ReadFile(input) + if err != nil { + t.Fatalf("read input: %v", err) + } + if !bytes.Equal(gotInput, tt.content) { + t.Fatal("input was modified, interrupted tail included it must stay") + } + + want := []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(103, 0)} + if got := ids(logsIn(t, output)); !slices.Equal(got, ids(want)) { + t.Fatalf("output logs = %v, want %v", got, ids(want)) + } + }) + } +} + // TestResumeAfterInterruptedWrite covers the three shapes a run killed // mid-write leaves behind: a plain line cut in half, a plain line that parses // but never got its newline, and a gzip member that never got its trailer. @@ -709,8 +876,8 @@ func TestResumeAfterInterruptedWrite(t *testing.T) { // Recovery: drop the partial tail, exactly as the export command // does, and never further than the reported boundary. - if err := os.Truncate(path, cursor.CleanSize); err != nil { - t.Fatalf("truncate %s: %v", path, err) + if _, err := resume.PrepareOutput(cursor, path, path); err != nil { + t.Fatalf("PrepareOutput() error = %v", err) } if err := filestore.AppendLogsAsync(t.Context(), feed(tt.replay...), appendWriter(t, path, cursor), cursor.Skip); err != nil { From 3413d963e5fc3d3786f621cfba647b054f253080 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 12:09:30 +0200 Subject: [PATCH 21/35] refactor(filestore): drop SaveLogsAsync, callers compose CreateWriter and AppendLogsAsync Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016mxruRkPfuH8dd5nVxgB8S --- pkg/filestore/filestore.go | 11 ----------- pkg/filestore/filestore_test.go | 31 ++++++++++++++++++------------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/pkg/filestore/filestore.go b/pkg/filestore/filestore.go index 8c5a10c..56132de 100644 --- a/pkg/filestore/filestore.go +++ b/pkg/filestore/filestore.go @@ -23,17 +23,6 @@ func CreateWriter(filePath string) (io.WriteCloser, error) { return file, nil } -// SaveLogsAsync writes logs to a file asynchronously, replacing any file -// already at filePath. The file is closed before returning. -func SaveLogsAsync(ctx context.Context, logChan <-chan types.Log, filePath string) error { - w, err := CreateWriter(filePath) - if err != nil { - return err - } - - return AppendLogsAsync(ctx, logChan, w, nil) -} - // AppendWriter opens an existing NDJSON file for appending. func AppendWriter(filePath string) (io.WriteCloser, error) { file, err := os.OpenFile(filePath, os.O_WRONLY|os.O_APPEND, 0o644) diff --git a/pkg/filestore/filestore_test.go b/pkg/filestore/filestore_test.go index e4c96d6..584a478 100644 --- a/pkg/filestore/filestore_test.go +++ b/pkg/filestore/filestore_test.go @@ -45,6 +45,19 @@ func blocksIn(t *testing.T, path string) []uint64 { return blocks } +// seed writes logs for the given blocks to a fresh file at path. +func seed(t *testing.T, path string, blocks ...uint64) { + t.Helper() + + w, err := filestore.CreateWriter(path) + if err != nil { + t.Fatalf("CreateWriter() error = %v", err) + } + if err := filestore.AppendLogsAsync(t.Context(), feed(blocks...), w, nil); err != nil { + t.Fatalf("AppendLogsAsync() error = %v", err) + } +} + // feed returns a closed channel already holding logs for the given blocks. // Topics must stay a non-nil empty slice: go-ethereum's generated // Log.UnmarshalJSON rejects a null "topics" as a missing required field. @@ -58,7 +71,7 @@ func feed(blocks ...uint64) <-chan types.Log { return ch } -func TestSaveLogsAsyncReplacesExistingFile(t *testing.T) { +func TestCreateWriterReplacesExistingFile(t *testing.T) { t.Parallel() path := filepath.Join(t.TempDir(), "export.ndjson") @@ -66,9 +79,7 @@ func TestSaveLogsAsyncReplacesExistingFile(t *testing.T) { t.Fatalf("seed %s: %v", path, err) } - if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { - t.Fatalf("SaveLogsAsync() error = %v", err) - } + seed(t, path, 1, 2) want := []uint64{1, 2} if got := blocksIn(t, path); !slices.Equal(got, want) { @@ -80,9 +91,7 @@ func TestAppendLogsAsyncKeepsExistingContent(t *testing.T) { t.Parallel() path := filepath.Join(t.TempDir(), "export.ndjson") - if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { - t.Fatalf("SaveLogsAsync() error = %v", err) - } + seed(t, path, 1, 2) w, err := filestore.AppendWriter(path) if err != nil { @@ -102,9 +111,7 @@ func TestAppendLogsAsyncSkipsFilteredLogs(t *testing.T) { t.Parallel() path := filepath.Join(t.TempDir(), "export.ndjson") - if err := filestore.SaveLogsAsync(t.Context(), feed(1, 2), path); err != nil { - t.Fatalf("SaveLogsAsync() error = %v", err) - } + seed(t, path, 1, 2) w, err := filestore.AppendWriter(path) if err != nil { @@ -125,9 +132,7 @@ func TestAppendLogsAsyncClosesWriterOnCancel(t *testing.T) { t.Parallel() path := filepath.Join(t.TempDir(), "export.ndjson") - if err := filestore.SaveLogsAsync(t.Context(), feed(1), path); err != nil { - t.Fatalf("SaveLogsAsync() error = %v", err) - } + seed(t, path, 1) w, err := filestore.AppendWriter(path) if err != nil { From a5c3ed34ecb301880a3f3a7cb8622bcea93a96b4 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 12:14:29 +0200 Subject: [PATCH 22/35] feat(export): compose --resume with --output for copy-mode continuation --- cmd/export.go | 64 +++++++++++++++++---------------------------------- 1 file changed, 21 insertions(+), 43 deletions(-) diff --git a/cmd/export.go b/cmd/export.go index be38228..2c22e44 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -5,7 +5,7 @@ import ( "errors" "fmt" "io" - "os" + "path/filepath" "sync" "time" @@ -54,19 +54,20 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save if cmd.Flags().Changed("start") { c.log.Warning("--start is ignored when --resume is set", "resumeFile", resumeFile) } - if cmd.Flags().Changed("output") { - c.log.Warning("--output is ignored when --resume is set, logs are appended to the resume file", "resumeFile", resumeFile) - } - if cursor.Compressed && compress { - c.log.Warning("--compress is ignored when resuming an already compressed file", "resumeFile", resumeFile) + if compress { + c.log.Warning("--compress is ignored when resuming; resume a compressed file to get a compressed result", "resumeFile", resumeFile) compress = false } + // An unset --output means in-place; so does naming the input. + if !cmd.Flags().Changed("output") || filepath.Clean(outputFile) == filepath.Clean(resumeFile) { + outputFile = resumeFile + } - outputFile = resumeFile startBlock = cursor.BlockNumber c.log.Info("Resuming export", "resumeFile", resumeFile, + "outputFile", outputFile, "startBlock", startBlock, "lastLogIndex", cursor.LogIndex, "compressed", cursor.Compressed, @@ -97,8 +98,18 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save startBlock = chainCfg.PostageStampStartBlock } - if err := c.discardPartialWrite(outputFile, cursor); err != nil { - return err + if cursor != nil { + discarded, err := resume.PrepareOutput(cursor, resumeFile, outputFile) + if err != nil { + return err + } + if discarded > 0 { + c.log.Warning("previous export ends with an interrupted write, leaving it out", + "resumeFile", resumeFile, + "offset", cursor.CleanSize, + "discardedBytes", discarded, + ) + } } // Opened before the first log is fetched: from inside the saving @@ -187,46 +198,13 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save cmd.Flags().Uint32VarP(&blockRangeLimit, "block-range-limit", "b", 5, "Max blocks per log query") cmd.Flags().StringVarP(&outputFile, "output", "o", "export.ndjson", "Output file path (NDJSON)") cmd.Flags().BoolVarP(&compress, "compress", "c", false, "Compress to GZIP") - cmd.Flags().StringVarP(&resumeFile, "resume", "r", "", "Resume a previous export file (.ndjson, .gz or .gzip); overrides --start and --output") + cmd.Flags().StringVarP(&resumeFile, "resume", "r", "", "Continue a previous export file (.ndjson, .gz or .gzip); combine with --output to write a new snapshot instead of appending in place") c.root.AddCommand(cmd) return nil } -// discardPartialWrite drops the tail an interrupted run left in the resume -// file, so logs are appended onto a boundary the reader positively identified -// rather than onto half a line or half a gzip member. Nothing recoverable is -// lost: everything discarded falls at or after the cursor and is re-fetched. -// -// It is a no-op without a resume file or when the file ends cleanly, and never -// truncates past the offset the reader reported. -func (c *command) discardPartialWrite(outputFile string, cursor *resume.Cursor) error { - if cursor == nil || !cursor.Truncated { - return nil - } - - info, err := os.Stat(outputFile) - if err != nil { - return fmt.Errorf("failed to inspect resume file: %w", err) - } - if info.Size() <= cursor.CleanSize { - return nil - } - - c.log.Warning("resume file ends with a partial write, discarding it", - "resumeFile", outputFile, - "offset", cursor.CleanSize, - "discardedBytes", info.Size()-cursor.CleanSize, - ) - - if err := os.Truncate(outputFile, cursor.CleanSize); err != nil { - return fmt.Errorf("failed to truncate resume file: %w", err) - } - - return nil -} - // openOutput opens the destination for a run's logs: a fresh file when cursor // is nil, or a writer that appends to the file the cursor came from. func openOutput(outputFile string, cursor *resume.Cursor) (io.WriteCloser, error) { From cc96d16fe3aa3a1ff4ea4a1a5f45895e7213b04b Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 12:21:22 +0200 Subject: [PATCH 23/35] fix(export): surface save failures in the exit code and never skip the final flush --- cmd/export.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/export.go b/cmd/export.go index 2c22e44..7c41e2e 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -42,7 +42,8 @@ and respects RPC rate limits (--max-request). The retrieved logs are saved to the specified output file (default: 'export.ndjson') in NDJSON format. The process can be interrupted at any time (Ctrl+C), and it will attempt to save already retrieved logs before exiting.`, RunE: func(cmd *cobra.Command, args []string) (err error) { - ctx := cmd.Context() + ctx, cancel := context.WithCancel(cmd.Context()) + defer cancel() var cursor *resume.Cursor if resumeFile != "" { @@ -134,6 +135,7 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save ticker := time.NewTicker(15 * time.Second) defer ticker.Stop() + var saveErr error go func() { defer wg.Done() @@ -143,6 +145,10 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save return } c.log.Error(err, "error saving logs") + // Stop the fetcher too: with the saver gone, logChan + // would fill and block it forever. + saveErr = err + cancel() return } c.log.Info("all logs have been saved", "outputFile", outputFile) @@ -164,13 +170,17 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save if !ok { errorChan = nil } else { - return fmt.Errorf("error retrieving logs: %w", err) + wg.Wait() + return errors.Join(fmt.Errorf("error retrieving logs: %w", err), saveErr) } case <-ticker.C: c.log.Info("still retrieving logs...") case <-ctx.Done(): c.log.Info("context canceled, waiting for logs to be saved...") wg.Wait() + if saveErr != nil { + return saveErr + } if err := compressFunc(); err != nil { return errors.Join(fmt.Errorf("error compressing file: %w", err), ctx.Err()) } @@ -183,6 +193,9 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save } wg.Wait() + if saveErr != nil { + return saveErr + } if err := compressFunc(); err != nil { return fmt.Errorf("error compressing file: %w", err) } From 96a43f3e2123f1c376cc9374964898230ce4fe41 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 12:37:17 +0200 Subject: [PATCH 24/35] fix(export): report a saver-caused stop as the save error, not a fetch error --- cmd/export.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/export.go b/cmd/export.go index 7c41e2e..7414d41 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -171,6 +171,9 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save errorChan = nil } else { wg.Wait() + if saveErr != nil && errors.Is(err, context.Canceled) { + return saveErr + } return errors.Join(fmt.Errorf("error retrieving logs: %w", err), saveErr) } case <-ticker.C: From 5c6be38b5179a4fef96ed1505bcadb64bee9bd1b Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 12:41:30 +0200 Subject: [PATCH 25/35] docs: reframe resume around incremental snapshots --- README.md | 77 ++++++++++++++++++++++++------------------------------- 1 file changed, 33 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 6cd9e99..74ee3b7 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ batch-export is a tool to retrieve Ethereum event logs for specific contracts, p - Supports rate limiting for RPC requests. - Saves retrieved logs to a specified output file (default: `export.ndjson`) in NDJSON format. - Graceful shutdown on interrupt signals (Ctrl+C). -- Resume an interrupted export from an existing `.ndjson`, `.gz`, or `.gzip` file. +- Continue a previous export from where it stopped (incremental snapshots). ## Requirements @@ -50,62 +50,51 @@ The primary command is export. -h, --help help for export -m, --max-request int Max RPC requests/sec (default 15) -o, --output string Output file path (NDJSON) (default "export.ndjson") - -r, --resume string Resume a previous export file (.ndjson, .gz or .gzip); overrides --start and --output + -r, --resume string Continue a previous export file (.ndjson, .gz or .gzip); combine with --output to write a new snapshot instead of appending in place --start uint Start block (optional, uses contract start block if 0) (default 31306381) -v, --verbosity string Log verbosity (silent, error, warn, info, debug) (default "info") ``` -### Resuming an interrupted export +### Continuing a previous snapshot -Point `--resume` at a file a previous run produced. The tool reads its last -complete entry, restarts from that block, and appends to the same file: +Instead of re-exporting every block, point `--resume` at the previous +snapshot and name the new one with `--output`. The previous file is read, +never modified; the new file holds everything the previous one did plus the +blocks exported since: ```sh -./dist/batch-export export --resume dist/export.ndjson +./dist/batch-export export --resume snapshots/2026-07.gzip --output snapshots/2026-08.gzip ``` -Compressed exports work the same way and are detected by content, not by -extension, so `.gz` and `.gzip` both work: +Formats are detected by content, not extension: `.ndjson`, `.gz` and `.gzip` +all work, and the output's format always matches the input's. Omitting +`--output` (or naming the input) appends to the previous file in place — the +space-saving variant: ```sh -./dist/batch-export export --resume dist/export.ndjson.gzip +./dist/batch-export export --resume export.ndjson.gzip ``` -The resumed block is re-queried, because an interrupted run may have saved only -part of it, and the entries already in the file are skipped. So long as the -file ends cleanly, resuming neither duplicates nor drops a log. Appending to a -compressed export adds a second gzip member — standard tools such as `gzcat`, -`gunzip`, and Go's `compress/gzip` read the result as one continuous stream. - -#### When the previous run was killed mid-write - -A run stopped by `SIGKILL`, a crash, or a full disk can leave a partial entry -at the end of the file: half a line, a line that never got its newline, or a -gzip member that never got its trailer. Appending onto any of those would -corrupt the file, so only an entry that is complete and properly terminated -counts as the resume point. - -The tool finds the last offset at which the file is known to be complete — the -end of the last newline-terminated line that parses as a log entry, or of the -last whole gzip member — and discards whatever follows it, logging how many -bytes it dropped and from where: - -```log -"level"="warning" "msg"="resume file ends with a partial write, discarding it" "offset"=89649991 "discardedBytes"=317 -``` - -For export content, nothing is lost by that: everything discarded sits at or -after the resume point, so the resumed query fetches it again. Anything -discarded that was never a log entry is simply removed and not re-fetched. -Because a line that parses but has no newline does not count, the resume point -in that case is the line before it, and that entry is re-fetched too. - -If no such offset can be identified — a gzip file whose only member is -truncated, for instance, since there is no member boundary to append at — the -tool refuses to touch the file and exits with an error rather than guess. Fall -back to a fresh export in that case. - -When `--resume` is set it overrides `--start` and `--output`. +The last exported block is re-queried and entries already present are +skipped, so continuing neither duplicates nor drops a log. Each continuation +of a compressed snapshot adds a gzip member — standard tools (`gzcat`, +`gunzip`, Go, Python) read multi-member files as one stream, a year of +monthly continuations costs about 0.02% in size, and +`gzcat old.gzip | gzip > fresh.gzip` consolidates the members any time. + +Keep one canonical snapshot file. `--compress` is ignored when resuming: +regenerating a `.gzip` from a plain twin is how an independently continued +archive gets overwritten. Resume a compressed file to get a compressed +result. + +Resume only files this tool produced. The file's tail is validated before +anything is written: content the tool never writes — a non-log line, foreign +data, an alien gzip member — is refused rather than repaired. The one +exception is the tool's own interrupted final write (a run killed +mid-export): in copy mode it is simply not copied, in place it is truncated +away with a warning, and its entries are re-fetched. Note that resuming does +not detect a file from a different chain; pairing the snapshot with the +right `--endpoint` is the operator's contract. The produced NDJSON is consumed by [batch-archive](https://github.com/ethersphere/batch-archive), which embeds it for use inside Bee. From c134c4c17c92b8dd692662d8f87e6a7cc8231596 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 13:08:12 +0200 Subject: [PATCH 26/35] fix(resume): treat any spelling of the same file as in-place, not copy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PrepareOutput compared filepath.Clean(inputPath) == filepath.Clean(outputPath) to decide copy vs. in-place. A relative path against an absolute one, a symlink or hardlink, or a case-insensitive filesystem folding two names together all pass that check as "different", so os.Create(outputPath) truncated the input to zero bytes before io.CopyN ever got to read from it — destroying the file copy mode promises to leave untouched. Add an os.SameFile (device+inode) check as a second guard, falling back to prepareInPlace whenever the paths name the same file by any route. Add a table test covering an absolute/relative spelling pair and a symlink, over both a clean input and one with an interrupted final write, and amend the spec's §3 table row to describe the os.SameFile detection instead of leaving "distinct spellings" as the operator's problem. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016mxruRkPfuH8dd5nVxgB8S --- .../2026-08-27-resume-input-output-design.md | 2 +- pkg/resume/resume.go | 25 ++++-- pkg/resume/resume_test.go | 89 +++++++++++++++++++ 3 files changed, 108 insertions(+), 8 deletions(-) diff --git a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md index ffd7de8..5c91a1e 100644 --- a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md +++ b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md @@ -43,7 +43,7 @@ No new flags. `--resume` names the **input** (the previous snapshot); | `export` | Fresh export to `--output` (default `export.ndjson`). Unchanged. | | `export --resume old.gzip` | **In-place**: append to `old.gzip` itself. An unset `--output` does not redirect the result to its default. | | `export --resume old.gzip --output new.gzip` | **Copy mode**: `old.gzip` is never modified; `new.gzip` = clean content of `old.gzip` + newly fetched entries. | -| `export --resume f --output f` | Identical paths (`filepath.Clean` equality) mean in-place. Distinct spellings of one file (symlinks, hardlinks) are the operator's responsibility. | +| `export --resume f --output f` | Identical paths (`filepath.Clean` equality) mean in-place. Same file under any other spelling — absolute vs. relative, a symlink, a hardlink, two names a case-insensitive filesystem folds together — is detected via `os.SameFile` (device+inode) and also treated as in-place. | Flag interactions when `--resume` is set: diff --git a/pkg/resume/resume.go b/pkg/resume/resume.go index bfcedc4..a9306da 100644 --- a/pkg/resume/resume.go +++ b/pkg/resume/resume.go @@ -63,13 +63,17 @@ func (c *Cursor) Skip(l types.Log) bool { } // PrepareOutput readies outputPath for appending the continuation of the -// export at inputPath. With equal paths the file is prepared in place: an -// interrupted final write, if any, is truncated away. With distinct paths the -// input is never modified: its clean content is copied raw into outputPath, -// replacing whatever was there, and the interrupted tail is simply not -// copied. Either way it returns how many trailing bytes were left out — -// every entry they held falls at or after the cursor, so the resumed query -// fetches it again. +// export at inputPath. The same file is prepared in place: an interrupted +// final write, if any, is truncated away. "Same file" is not lexical — a +// relative and an absolute spelling of one path, a symlink or hardlink, or +// two names a case-insensitive filesystem folds together all count, detected +// via os.SameFile (device+inode), because os.Create on a distinct-looking +// spelling of the input would truncate it before it could be copied from. +// With genuinely distinct files the input is never modified: its clean +// content is copied raw into outputPath, replacing whatever was there, and +// the interrupted tail is simply not copied. Either way it returns how many +// trailing bytes were left out — every entry they held falls at or after the +// cursor, so the resumed query fetches it again. func PrepareOutput(c *Cursor, inputPath, outputPath string) (int64, error) { if filepath.Clean(inputPath) == filepath.Clean(outputPath) { return prepareInPlace(c, inputPath) @@ -86,6 +90,13 @@ func PrepareOutput(c *Cursor, inputPath, outputPath string) (int64, error) { return 0, fmt.Errorf("error inspecting resume file: %w", err) } + // The same file under two names — relative vs absolute, a symlink or + // hardlink, a case-insensitive filesystem — is in-place, not copy mode: + // os.Create would truncate the input we are about to read. + if outInfo, err := os.Stat(outputPath); err == nil && os.SameFile(info, outInfo) { + return prepareInPlace(c, inputPath) + } + out, err := os.Create(outputPath) if err != nil { return 0, fmt.Errorf("error creating output file: %w", err) diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index 5b7f14a..ff0d2e5 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -778,6 +778,95 @@ func TestCopyResumeFromInterruptedInput(t *testing.T) { } } +// TestPrepareOutputSameFileUnderDifferentSpellings pins Fix 1: two spellings +// of the SAME underlying file — an absolute path against a relative one, or a +// symlink pointed at the input — must be treated as in-place, not copy mode. +// Before the fix, PrepareOutput compared filepath.Clean strings, missed both +// cases, and its os.Create(outputPath) truncated the input to zero bytes +// before io.CopyN ever got to read from it, destroying the file it was +// supposed to leave untouched. +// +// Each case covers both a clean input (must survive byte for byte) and one +// with an interrupted final write (must be truncated to exactly CleanSize, +// the same result prepareInPlace produces for a same-path call). +func TestPrepareOutputSameFileUnderDifferentSpellings(t *testing.T) { + logs := ndjson(t, testLog(100, 0), testLog(101, 0)) + interrupted := append(append([]byte{}, logs...), []byte(`{"address":"0x45a1502382541`)...) + + contents := []struct { + name string + content []byte + }{ + {name: "clean input", content: logs}, + {name: "interrupted write", content: interrupted}, + } + + // spelling produces an outputPath that names the same file as input by + // some route other than an identical string. + spellings := []struct { + name string + outPath func(t *testing.T, dir, input string) string + }{ + { + // t.Chdir affects the whole process, so this case (and therefore + // the whole test) cannot run in parallel with anything else. + name: "absolute input, relative output", + outPath: func(t *testing.T, dir, input string) string { + t.Chdir(dir) + return filepath.Base(input) + }, + }, + { + name: "symlink to the input", + outPath: func(t *testing.T, dir, input string) string { + link := filepath.Join(dir, "alias-of-input") + if err := os.Symlink(input, link); err != nil { + t.Skipf("symlinks unsupported on this filesystem: %v", err) + } + return link + }, + }, + } + + for _, ct := range contents { + for _, sp := range spellings { + t.Run(ct.name+"/"+sp.name, func(t *testing.T) { + dir := t.TempDir() + input := filepath.Join(dir, "snapshot.ndjson") + if err := os.WriteFile(input, ct.content, 0o644); err != nil { + t.Fatalf("write input: %v", err) + } + + cursor, err := resume.Read(input) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + + output := sp.outPath(t, dir, input) + + discarded, err := resume.PrepareOutput(cursor, input, output) + if err != nil { + t.Fatalf("PrepareOutput() error = %v", err) + } + + wantDiscarded := int64(len(ct.content)) - cursor.CleanSize + if discarded != wantDiscarded { + t.Errorf("discarded = %d, want %d", discarded, wantDiscarded) + } + + got, err := os.ReadFile(input) + if err != nil { + t.Fatalf("read input: %v", err) + } + want := ct.content[:cursor.CleanSize] + if !bytes.Equal(got, want) { + t.Fatalf("input holds %d bytes, want the %d-byte clean prefix preserved (copy mode must not have truncated the input)", len(got), len(want)) + } + }) + } + } +} + // TestResumeAfterInterruptedWrite covers the three shapes a run killed // mid-write leaves behind: a plain line cut in half, a plain line that parses // but never got its newline, and a gzip member that never got its trailer. From 3ae7d0dd13453a6ec33a02592dc55683f817e5f7 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 13:08:51 +0200 Subject: [PATCH 27/35] fix(resume): pinpoint gzip refusals and stop misclassifying decode errors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three related diagnostic fixes to gzip resume validation (§5): - Thread a 1-based member index and per-member line number through scanMember so a refusal on a 150k+-line archive names where the offending line is ("member 4, line 812 is not a log entry") instead of nothing. - Drop gzip.ErrChecksum from truncationShaped: an interrupted write cannot produce a full-length member body plus a complete 8-byte trailer holding the wrong CRC, so that shape is corruption or tampering, not truncation. It now falls into the non-truncation-shaped branch, which is reclassified from a bare wrapped error to ErrNotAnExport (documented tradeoff: a genuine I/O error shaped like corruption is now also refused rather than surfaced distinctly, since the two are indistinguishable at this layer and §5 favors refusal). - Classify gzip.NewReader's first-header failure the same way the plain format's equivalent is handled: truncation-shaped (file cut inside the 10-byte header) is ErrNoLogs; anything else is ErrNotAnExport. Previously it returned a bare wrapped error outside the two-sentinel taxonomy. Also reword lastCursorPlain's "N bytes without a newline" refusal, which named the whole file size even when the newline was simply outside the examined window, to name the window and its starting offset instead. New tests: a CRC-flip in a valid member's trailer, and a file cut to 5 bytes inside the gzip header. Spec §5 corrected to say the plain read window is 2*maxLineBytes, not maxLineBytes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016mxruRkPfuH8dd5nVxgB8S --- .../2026-08-27-resume-input-output-design.md | 8 ++-- pkg/resume/resume.go | 46 ++++++++++++++----- pkg/resume/resume_test.go | 34 ++++++++++++++ 3 files changed, 74 insertions(+), 14 deletions(-) diff --git a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md index 5c91a1e..522efd7 100644 --- a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md +++ b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md @@ -131,9 +131,11 @@ manipulated file means. (`ErrNoCleanBoundary` is retired: its gzip case — a sole truncated member — is `ErrNoLogs`; its mid-line-member case is `ErrNotAnExport`.) -**Mechanics.** Plain: one read of the last `maxLineBytes` bytes suffices — -find the last newline, check the fragment after it, parse the single line -before it. The current multi-window backward walk with carry exists only to +**Mechanics.** Plain: one read of the last `2*maxLineBytes` bytes suffices — +enough for the last complete line plus a trailing fragment, up to +`maxLineBytes` (1 MiB) each — to find the last newline, check the fragment +after it, and parse the single line before it. The current multi-window +backward walk with carry exists only to tolerate foreign junk and is deleted. Gzip: cannot seek, so the stream is decoded member by member with a counting reader (which must implement `io.ByteReader` so `gzip.Reader` consumes it directly and member boundaries diff --git a/pkg/resume/resume.go b/pkg/resume/resume.go index a9306da..a444615 100644 --- a/pkg/resume/resume.go +++ b/pkg/resume/resume.go @@ -219,7 +219,7 @@ func lastCursorPlain(file *os.File, size int64) (*Cursor, error) { // No newline at all: an interrupted first write, unless the file is // longer than any single line the tool writes. if size > maxLineBytes { - return nil, fmt.Errorf("%w: %d bytes without a newline", ErrNotAnExport, size) + return nil, fmt.Errorf("%w: no newline in the final %d bytes (from offset %d)", ErrNotAnExport, window, offset) } return nil, ErrNoLogs } @@ -252,7 +252,14 @@ func lastCursorGzip(file *os.File) (*Cursor, error) { reader, err := gzip.NewReader(counter) if err != nil { - return nil, fmt.Errorf("error opening gzip resume file: %w", err) + // isGzip already confirmed the magic bytes, so failure here means the + // file was cut inside the 10-byte header (truncation-shaped, like the + // plain format's interrupted-first-write case) or the header past the + // magic bytes is corrupt (tampering). + if truncationShaped(err) { + return nil, ErrNoLogs + } + return nil, fmt.Errorf("%w: gzip header: %w", ErrNotAnExport, err) } defer reader.Close() @@ -260,18 +267,19 @@ func lastCursorGzip(file *os.File) (*Cursor, error) { cursor *Cursor cleanSize int64 sawClean bool + member = 1 ) for { // Reset turns multistream back on, so it must be switched off per // member, not just for the first. reader.Multistream(false) - last, err := scanMember(reader) + last, err := scanMember(reader, member) switch { case errors.Is(err, ErrNotAnExport): return nil, err case err != nil && !truncationShaped(err): - return nil, fmt.Errorf("error reading gzip resume file: %w", err) + return nil, fmt.Errorf("%w: member %d: %w", ErrNotAnExport, member, err) case err != nil: // The member never got its trailer: an interrupted final write. return gzipResult(cursor, cleanSize, sawClean) @@ -290,6 +298,7 @@ func lastCursorGzip(file *os.File) (*Cursor, error) { } return nil, fmt.Errorf("error reading gzip resume file: %w", err) } + member++ } } @@ -306,12 +315,20 @@ func gzipResult(cursor *Cursor, cleanSize int64, sawClean bool) (*Cursor, error) // truncationShaped reports whether err is what an interrupted write produces, // as opposed to a real read failure that must not be mistaken for one. +// +// gzip.ErrChecksum is deliberately excluded: an interrupted write cannot +// produce a full-length member body plus a complete 8-byte trailer that then +// holds the wrong CRC — that shape is corruption or tampering, not +// truncation, so callers classify it ErrNotAnExport instead. One side effect: +// a genuine I/O error from the underlying file that happens to look like +// corruption is also classified ErrNotAnExport, since it is rare and +// indistinguishable at this layer; §5's strictness favors refusal over +// guessing. func truncationShaped(err error) bool { var corrupt flate.CorruptInputError return errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, gzip.ErrHeader) || - errors.Is(err, gzip.ErrChecksum) || errors.As(err, &corrupt) } @@ -321,18 +338,24 @@ func truncationShaped(err error) bool { // holding whole log lines only, so a complete line that does not parse, or a // clean member ending mid-line, is foreign content (ErrNotAnExport); any // other read error is returned as-is for the caller to classify. -func scanMember(r io.Reader) (*Cursor, error) { +// +// member is this member's 1-based position in the file, threaded through +// purely so a refusal can name where the offending line lives — a file with +// tens of thousands of lines split across gzip members otherwise gives the +// operator nothing to search for. +func scanMember(r io.Reader, member int) (*Cursor, error) { buffered := bufio.NewReaderSize(r, bufferSize) var ( - last *Cursor - line []byte + last *Cursor + line []byte + lineNum = 1 ) for { chunk, err := buffered.ReadSlice('\n') line = append(line, chunk...) if len(line) > maxLineBytes { - return nil, fmt.Errorf("%w: line exceeds %d bytes", ErrNotAnExport, maxLineBytes) + return nil, fmt.Errorf("%w: member %d, line %d exceeds %d bytes", ErrNotAnExport, member, lineNum, maxLineBytes) } switch { @@ -340,7 +363,7 @@ func scanMember(r io.Reader) (*Cursor, error) { continue case errors.Is(err, io.EOF): if len(line) > 0 { - return nil, fmt.Errorf("%w: gzip member ends mid-line", ErrNotAnExport) + return nil, fmt.Errorf("%w: member %d, line %d ends mid-line", ErrNotAnExport, member, lineNum) } return last, nil case err != nil: @@ -349,10 +372,11 @@ func scanMember(r io.Reader) (*Cursor, error) { cursor, err := parseCursor(line) if err != nil { - return nil, fmt.Errorf("%w: line is not a log entry: %w", ErrNotAnExport, err) + return nil, fmt.Errorf("%w: member %d, line %d is not a log entry: %w", ErrNotAnExport, member, lineNum, err) } last = cursor line = line[:0] + lineNum++ } } diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index ff0d2e5..e1b3ff4 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -867,6 +867,40 @@ func TestPrepareOutputSameFileUnderDifferentSpellings(t *testing.T) { } } +// TestReadRejectsCorruptedChecksum pins Fix 3: a member whose body is intact +// but whose trailing CRC has been altered is corruption or tampering, not the +// tool's own interrupted write, and must be refused with ErrNotAnExport +// rather than silently treated as a truncated final member. +func TestReadRejectsCorruptedChecksum(t *testing.T) { + t.Parallel() + + content := gz(t, ndjson(t, testLog(100, 0), testLog(101, 0))) + // The trailer is the last 8 bytes: a 4-byte CRC32 followed by a 4-byte + // ISIZE. Flipping a byte within the first 4 corrupts the CRC alone, + // leaving the member body and its length untouched. + content[len(content)-8] ^= 0xff + + _, err := resume.Read(write(t, gzipFile, content)) + if !errors.Is(err, resume.ErrNotAnExport) { + t.Fatalf("Read() error = %v, want ErrNotAnExport", err) + } +} + +// TestReadGzipCutInsideHeader pins Fix 4: a file cut short before the gzip +// header is even complete is an interrupted first write, exactly like the +// plain format's equivalent, and must report ErrNoLogs rather than a bare +// wrapped error that escapes the two-sentinel taxonomy. +func TestReadGzipCutInsideHeader(t *testing.T) { + t.Parallel() + + full := gz(t, ndjson(t, testLog(100, 0))) + + _, err := resume.Read(write(t, gzipFile, full[:5])) + if !errors.Is(err, resume.ErrNoLogs) { + t.Fatalf("Read() error = %v, want ErrNoLogs", err) + } +} + // TestResumeAfterInterruptedWrite covers the three shapes a run killed // mid-write leaves behind: a plain line cut in half, a plain line that parses // but never got its newline, and a gzip member that never got its trailer. From 4189448ae1e667ddd1b9830921cb97fbeca26903 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 13:09:16 +0200 Subject: [PATCH 28/35] fix(export): do not mask a real save error behind a joined cancellation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The saver goroutine treated any errors.Is(err, context.Canceled) as pure cancellation and returned without recording saveErr. But AppendLogsAsync can return errors.Join(context.Canceled, closeErr) — SIGINT racing a failing gzip-member flush on Close — and errors.Is only asks "does this contain context.Canceled anywhere," which such a joined error satisfies even though it also carries a real failure. saveErr stayed nil, so on a fresh --compress run compressFunc would run over a file whose final write had failed. Add solelyCanceled, which recurses through Join and single-wrap chains and only returns true when cancellation is the ENTIRE error, and use it in place of errors.Is for that check. The errorChan branch's own errors.Is(err, context.Canceled) is untouched — it classifies the fetcher's echo of the same cancellation, not a joined save error. No cmd unit harness exists for this path (per the design doc, §10); verified by build, go vet, and tracing both error shapes through solelyCanceled: a bare context.Canceled returns true, Join(context.Canceled, ENOSPC) returns false. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016mxruRkPfuH8dd5nVxgB8S --- cmd/export.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/cmd/export.go b/cmd/export.go index 7414d41..5b16846 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -140,7 +140,7 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save defer wg.Done() if err := saveLogs(ctx, logChan, w, cursor); err != nil { - if errors.Is(err, context.Canceled) { + if solelyCanceled(err) { c.log.Error(err, "context canceled while saving logs") return } @@ -244,3 +244,26 @@ func saveLogs(ctx context.Context, logChan <-chan types.Log, w io.WriteCloser, c return filestore.AppendLogsAsync(ctx, logChan, w, skip) } + +// solelyCanceled reports whether err contains nothing beyond context +// cancellation, unwrapping joined and wrapped errors along the way. It is +// stricter than errors.Is(err, context.Canceled): AppendLogsAsync joins the +// save error with the destination's Close error, and a SIGINT racing a +// failing gzip-member flush must not be reported as pure cancellation. +func solelyCanceled(err error) bool { + if err == nil { + return false + } + if u, ok := err.(interface{ Unwrap() []error }); ok { + for _, e := range u.Unwrap() { + if !solelyCanceled(e) { + return false + } + } + return true + } + if u := errors.Unwrap(err); u != nil { + return solelyCanceled(u) + } + return errors.Is(err, context.Canceled) +} From b6af15b64202b0c46a16ea71b780f021bab68385 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 13:09:46 +0200 Subject: [PATCH 29/35] docs(resume): fix stale wording in README, spec and a test comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - README: Go 1.24 -> 1.25 (matches go.mod); the --end flag-table line's default text was a stale copy from --start's default and never matched the code (default 0 = latest block, cobra prints no "(default N)" suffix for it); add the copy-mode overwrite behavior, note --start is ignored on resume, and add the recovery one-liner for a failed copy-mode run. - pkg/resume/resume_test.go: reword the stale comment above manyLogs, which described the deleted multi-window backward-read walk; it now earns its keep via the gzip path's per-member buffered scan. Also assert PrepareOutput's discarded-count return exactly in TestResumeAfterInterruptedWrite instead of discarding it with _. - Spec: §8 "A fusnote" -> "A footnote"; §9's "pkg/gzipstore: unchanged" no longer describes this branch, which added AppendWriter to it earlier. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_016mxruRkPfuH8dd5nVxgB8S --- README.md | 24 ++++++++++++------- .../2026-08-27-resume-input-output-design.md | 5 ++-- pkg/resume/resume_test.go | 10 ++++++-- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 74ee3b7..8f5d3b2 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ batch-export is a tool to retrieve Ethereum event logs for specific contracts, p ## Requirements -- Go 1.24 or later +- Go 1.25 or later ## Installation @@ -45,7 +45,7 @@ The primary command is export. ```sh -b, --block-range-limit uint32 Max blocks per log query (default 5) -c, --compress Compress to GZIP - --end uint End block (optional, uses latest block if 0) (default 39810670) + --end uint End block (optional, uses latest block if 0) -e, --endpoint string Ethereum RPC endpoint URL -h, --help help for export -m, --max-request int Max RPC requests/sec (default 15) @@ -66,6 +66,9 @@ blocks exported since: ./dist/batch-export export --resume snapshots/2026-07.gzip --output snapshots/2026-08.gzip ``` +If `--output` already names an existing file, it is overwritten — the same +`os.Create` semantics as a fresh export, so pick a new name for each snapshot. + Formats are detected by content, not extension: `.ndjson`, `.gz` and `.gzip` all work, and the output's format always matches the input's. Omitting `--output` (or naming the input) appends to the previous file in place — the @@ -75,12 +78,14 @@ space-saving variant: ./dist/batch-export export --resume export.ndjson.gzip ``` -The last exported block is re-queried and entries already present are -skipped, so continuing neither duplicates nor drops a log. Each continuation -of a compressed snapshot adds a gzip member — standard tools (`gzcat`, -`gunzip`, Go, Python) read multi-member files as one stream, a year of -monthly continuations costs about 0.02% in size, and -`gzcat old.gzip | gzip > fresh.gzip` consolidates the members any time. +`--start` is ignored (with a warning) when `--resume` is set: the cursor in +the previous file decides where the fetch resumes. The last exported block is +re-queried and entries already present are skipped, so continuing neither +duplicates nor drops a log. Each continuation of a compressed snapshot adds a +gzip member — standard tools (`gzcat`, `gunzip`, Go, Python) read +multi-member files as one stream, a year of monthly continuations costs about +0.02% in size, and `gzcat old.gzip | gzip > fresh.gzip` consolidates the +members any time. Keep one canonical snapshot file. `--compress` is ignored when resuming: regenerating a `.gzip` from a plain twin is how an independently continued @@ -96,6 +101,9 @@ away with a warning, and its entries are re-fetched. Note that resuming does not detect a file from a different chain; pairing the snapshot with the right `--endpoint` is the operator's contract. +If a copy-mode run itself is interrupted or fails, the input was never +touched — delete the incomplete `--output` file and rerun. + The produced NDJSON is consumed by [batch-archive](https://github.com/ethersphere/batch-archive), which embeds it for use inside Bee. ## Maintainers diff --git a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md index 522efd7..067a77b 100644 --- a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md +++ b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md @@ -186,7 +186,7 @@ The saver goroutine's error must reach `RunE`'s return value: one-liner. 3. A short "if the previous run was interrupted" subsection: copy mode — rerun; in-place — the tool truncates the interrupted write and re-fetches, - warning shown. A fusnote, not the headline. + warning shown. A footnote, not the headline. 4. The trust rule, stated plainly: resume only files this tool produced; a file with any other content is refused, and resuming does not detect a wrong chain — that is the operator's contract. @@ -212,7 +212,8 @@ The saver goroutine's error must reach `RunE`'s return value: - `pkg/filestore`: `SaveLogsAsync` — exported, no production caller — is deleted with its tests; fresh exports route through `CreateWriter` + `AppendLogsAsync`. The `AppendLogsAsync` close-error join stays. -- `pkg/gzipstore`: unchanged. +- `pkg/gzipstore`: gained `AppendWriter` earlier on this branch (§6); not + further changed by this design. ## 10. Testing diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index e1b3ff4..18a0f60 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -99,7 +99,9 @@ func TestReadCursor(t *testing.T) { threeLogs := ndjson(t, testLog(100, 0), testLog(101, 1), testLog(102, 7)) - // many spans several 64 KiB backward-read windows. + // many is large enough to span several of the gzip path's 64 KiB member + // buffers, exercising the buffered scan across bufio.ReadSlice refills + // rather than a single in-memory read. manyLogs := make([]types.Log, 0, 2000) for i := range 2000 { manyLogs = append(manyLogs, testLog(uint64(1000+i), uint(i%16))) @@ -999,9 +1001,13 @@ func TestResumeAfterInterruptedWrite(t *testing.T) { // Recovery: drop the partial tail, exactly as the export command // does, and never further than the reported boundary. - if _, err := resume.PrepareOutput(cursor, path, path); err != nil { + discarded, err := resume.PrepareOutput(cursor, path, path) + if err != nil { t.Fatalf("PrepareOutput() error = %v", err) } + if want := int64(len(tt.content)) - cursor.CleanSize; discarded != want { + t.Errorf("discarded = %d, want %d", discarded, want) + } if err := filestore.AppendLogsAsync(t.Context(), feed(tt.replay...), appendWriter(t, path, cursor), cursor.Skip); err != nil { t.Fatalf("AppendLogsAsync() error = %v", err) From b0f54cf93bf294a2e4f723dcac7bc7f9833aec00 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 14:20:48 +0200 Subject: [PATCH 30/35] docs: remove the implementation plan, keep the design spec The spec documents the durable design; the plan was execution scaffolding. Co-Authored-By: Claude Opus 5 (1M context) --- ...2026-08-27-resume-incremental-snapshots.md | 1075 ----------------- 1 file changed, 1075 deletions(-) delete mode 100644 docs/superpowers/plans/2026-08-27-resume-incremental-snapshots.md diff --git a/docs/superpowers/plans/2026-08-27-resume-incremental-snapshots.md b/docs/superpowers/plans/2026-08-27-resume-incremental-snapshots.md deleted file mode 100644 index 56f4706..0000000 --- a/docs/superpowers/plans/2026-08-27-resume-incremental-snapshots.md +++ /dev/null @@ -1,1075 +0,0 @@ -# Resume as Incremental Snapshots Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Bring the existing `feat/resume-flag` branch into conformance with the incremental-snapshots spec: strict tail validation, `--resume`/`--output` composition with copy mode, saver errors reaching the exit code, and docs reframed around continuation. - -**Architecture:** This plan **edits the existing branch**, it does not start over. `pkg/resume` keeps its public shape (`Cursor`, `Read`, `Skip`) but its internals simplify: the lenient multi-window backward walk becomes a single tail read that refuses foreign content, and the gzip member walk refuses non-log lines instead of tolerating them. A new `resume.PrepareOutput` enforces the `CleanSize` invariant (in-place truncate or clean-prefix copy) next to where it is computed. `cmd/export.go` gains mode resolution and a saver-error path to `RunE`'s return value, keeping its existing select-loop shape. - -**Tech Stack:** Go 1.25, stdlib (`compress/gzip`, `compress/flate`, `bufio`, `path/filepath`), `github.com/ethereum/go-ethereum` v1.15.11, `github.com/ethersphere/bee/v2` v2.7.0, cobra. - -**Spec:** `docs/superpowers/specs/2026-08-27-resume-input-output-design.md` — the plan argues from it; read both. Section references (§N) below point into it. - -**Baseline:** branch `feat/resume-flag`, code at commit `b53a9c3` (plus the local spec commit). All file:line references are against that state. - -## Global Constraints - -- Go `1.25` per `go.mod`; do not change it. **No new dependencies**; do not run `go get`. -- Lint: `.golangci.yml` enables `copyloopvar`, `errname`, `errorlint`, `goconst`, `misspell`, `nilerr`, `unconvert` + `gofmt`/`gofumpt`. Wrap with `%w`; compare with `errors.Is`/`errors.As`, never `==`. -- Doc comment on every exported identifier, starting with its name. Error strings lowercase, unpunctuated. -- **Review-friendliness is a requirement, not a preference:** keep the diff against `main` as small as the spec allows; extend existing structures instead of restructuring them; keep `RunE`'s select-loop shape; never reformat code you are not changing. Where minimal-diff and clean idiomatic Go conflict, prefer clean Go — but say so in the commit message. -- Dependency direction stays one-way: `cmd` → {`resume`, `filestore`, `gzipstore`}; none of the three import each other. -- Conventional Commits. **Do not push** — local commits only; the branch is squash-merged at the end (§11). -- `dist/` holds the user's real export archives. Never point `--resume` (or any truncating/copying code) at a `dist/` path; test against copies in a temp dir only. -- Tests: `make test` (= `go test -v ./pkg/...`); also run `go test -race ./pkg/...` before each commit that touches concurrency. - -## File Structure - -| File | Change | Responsibility after this plan | -|---|---|---| -| `pkg/resume/resume.go` | Rewrite internals | Strict tail validation (§5): `Read` + two sentinel errors; single-read plain path; member-walk gzip path; `PrepareOutput` (§4/§9). | -| `pkg/resume/resume_test.go` | Overhaul | Validation, refusal, tolerance, and cross-package round-trip tests for both modes. | -| `pkg/filestore/filestore.go` | Shrink | `CreateWriter`, `AppendWriter`, `AppendLogsAsync` only; `SaveLogsAsync` deleted (§9). | -| `pkg/filestore/filestore_test.go` | Retool | Seeding via `CreateWriter`+`AppendLogsAsync`. | -| `cmd/export.go` | Edit | Mode resolution (§3), `PrepareOutput` wiring, saver-error plumbing (§7), flag help. | -| `README.md` | Rewrite section | §8. | -| `pkg/gzipstore/*` | Untouched | — | - ---- - -### Task 1: Strict tail validation in `pkg/resume` - -**Files:** -- Modify: `pkg/resume/resume.go` (all of `lastCursorPlain`, `lastCursorGzip`, `scanLines`, the const/var blocks, `parseCursor`; `Read`, `Cursor`, `Skip`, `isGzip`, `countingReader` keep their shape) -- Test: `pkg/resume/resume_test.go` - -**Interfaces:** -- Consumes: nothing new. -- Produces (Task 2 and 4 rely on these): - - `var ErrNotAnExport error`, `var ErrNoLogs error` (sentinel; `ErrNoCleanBoundary` and `errLineTooLong` are deleted) - - `Read(path string) (*Cursor, error)` — unchanged signature; new contract per §5. - - `Cursor` fields unchanged: `BlockNumber uint64; LogIndex uint; Compressed bool; CleanSize int64; Truncated bool`. - -- [ ] **Step 1: Update the tests to the strict contract** - -In `pkg/resume/resume_test.go`: - -**(a) Delete** the now-obsolete lenient machinery in `TestReadCursor`'s setup: the `garbageLines` variable, the whole `straddle*` block (from the `// straddle places a valid log line…` comment through the `t.Fatalf("test setup: boundary %d…")` guard), and the `windowSize` test constant with its comment near the top of the file. Keep `threeLogs`, `manyLogs`/`many`, `twoLogsEnd`, `threeLogsEnd`, and all helpers (`testLog`, `ndjson`, `gz`, `truncateLast`, `write`). - -**(b) Delete** these `TestReadCursor` cases (their content is foreign under §5 and moves to the refusal table): `"line missing blockNumber is skipped"`, `"walks back across windows of garbage lines"`, `"valid line straddles a window boundary"`. - -**(c) Rename** the case `"spans multiple backward read windows"` to `"many logs"` (the window concept no longer exists; the case stays as a large-file regression). - -**(d) Add** one `TestReadCursor` case — an empty trailing member is the tool's own output (a resume that fetched nothing) and must stay valid: - -```go - { - name: "empty trailing gzip member is valid", - content: append(gz(t, threeLogs), gz(t, nil)...), - wantBlock: 102, - wantIndex: 7, - wantCompressed: true, - }, -``` - -(`gz(t, nil)` compresses zero bytes into a complete member; `bytes.Buffer.Write(nil)` is a no-op, so the existing helper handles it.) - -All remaining `TestReadCursor` cases keep their current expectations — including `"truncated trailing line is excluded from the clean boundary"`, `"trailing line without a newline is not a clean end"` (cursor falls back to `{101,1}`), both truncated-member gzip cases, and the unflushed-header case. They are the §5 tolerated irregularities. - -**(e) Replace** `TestReadCursorErrors` and `TestReadRefusesWithoutACleanBoundary` with one consolidated table (delete both, add this): - -```go -// TestReadRefusals covers §5's strict contract: the only irregularity -// tolerated is the tool's own interrupted final write. Content the tool never -// writes is ErrNotAnExport; a file consistent with tool output but holding no -// complete entry is ErrNoLogs. -func TestReadRefusals(t *testing.T) { - t.Parallel() - - logs := ndjson(t, testLog(100, 0), testLog(101, 0)) - - tests := []struct { - name string - content []byte - wantErr error - }{ - { - name: "empty file", - content: []byte{}, - wantErr: resume.ErrNoLogs, - }, - { - name: "plain file holding one unterminated line", - content: bytes.TrimSuffix(ndjson(t, testLog(100, 0)), []byte("\n")), - wantErr: resume.ErrNoLogs, - }, - { - // The tool never writes a blank line. - name: "only a newline", - content: []byte("\n"), - wantErr: resume.ErrNotAnExport, - }, - { - name: "trailing blank line after valid logs", - content: append(append([]byte{}, logs...), '\n'), - wantErr: resume.ErrNotAnExport, - }, - { - // A complete line that is not a log entry means the file was - // altered after export; refusing beats guessing what to cut. - name: "trailing non-log line after valid logs", - content: append(append([]byte{}, logs...), []byte("not-json\n")...), - wantErr: resume.ErrNotAnExport, - }, - { - name: "trailing log line missing blockNumber", - content: append(append([]byte{}, logs...), []byte("{\"address\":\"0x1\",\"topics\":[],\"data\":\"0x\"}\n")...), - wantErr: resume.ErrNotAnExport, - }, - { - name: "only garbage lines", - content: bytes.Repeat([]byte("not-json\n"), 10), - wantErr: resume.ErrNotAnExport, - }, - { - // Finding #4's shape: refused from a single tail read, no - // backward scan through the junk. - name: "newline-free tail longer than a line can be", - content: append(append([]byte{}, logs...), bytes.Repeat([]byte("x"), 1<<20+1)...), - wantErr: resume.ErrNotAnExport, - }, - { - name: "single unterminated line larger than the cap", - content: bytes.Repeat([]byte("x"), 2<<20), - wantErr: resume.ErrNotAnExport, - }, - { - // A cleanly terminated member whose content stops mid-line - // cannot come from this tool: members hold whole lines, and an - // interrupted write cannot produce a valid trailer. - name: "gzip member ending mid line", - content: gz(t, bytes.TrimSuffix(logs, []byte("\n"))), - wantErr: resume.ErrNotAnExport, - }, - { - // Finding #6's shape: foreign data concatenated as its own valid - // member. Refused rather than treated as a removable tail. - name: "gzip junk member after valid member", - content: append(gz(t, logs), gz(t, []byte("not-json\nalso-not\n"))...), - wantErr: resume.ErrNotAnExport, - }, - { - name: "gzip member holding a non-log line between logs", - content: gz(t, append(append([]byte{}, logs...), []byte("not-json\n")...)), - wantErr: resume.ErrNotAnExport, - }, - { - // A sole member without its trailer is an interrupted first - // write: nothing to resume from, so a fresh export is the remedy. - name: "gzip with a single truncated member", - content: truncateLast(gz(t, logs), 6), - wantErr: resume.ErrNoLogs, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - if _, err := resume.Read(write(t, plainFile, tt.content)); !errors.Is(err, tt.wantErr) { - t.Fatalf("Read() error = %v, want %v", err, tt.wantErr) - } - }) - } -} -``` - -**(f) Keep unchanged:** `TestReadMissingFile`, `TestCursorSkip`, `TestReadReportsGzipReadErrors`, `TestGzipCleanSizeIsAMemberBoundary`, `TestAppendResumeRoundTrip`, `TestResumeAfterInterruptedWrite` (Task 2 touches the last one). - -- [ ] **Step 2: Run the tests to verify the new ones fail** - -Run: `go test ./pkg/resume/ -run 'TestReadRefusals|TestReadCursor' -v 2>&1 | tail -30` -Expected: compile error first (`resume.ErrNotAnExport` undefined, `resume.ErrNoCleanBoundary` still referenced only if a stray use remains); after the file compiles, `TestReadRefusals` cases like `"trailing non-log line after valid logs"` FAIL against the lenient implementation (it returns a cursor instead of refusing). - -- [ ] **Step 3: Rewrite the validation internals** - -In `pkg/resume/resume.go`: - -**(a) Constants** — delete `windowSize`; keep `maxLineBytes` and `bufferSize`: - -```go -const ( - // maxLineBytes caps a single line. Exported log lines run to a few - // hundred bytes, so anything longer is not this tool's output. - maxLineBytes = 1 << 20 - // bufferSize is how much of a gzip file is buffered per read. - bufferSize = 64 * 1024 -) -``` - -**(b) Errors** — replace the var block (deletes `ErrNoCleanBoundary` and `errLineTooLong`): - -```go -var ( - // ErrNotAnExport indicates content this tool never writes. The file was - // altered after export, so resuming it is refused rather than repaired. - ErrNotAnExport = errors.New("not an untouched batch-export file") - // ErrNoLogs indicates a file consistent with tool output that holds no - // complete entry to resume from: it is empty, or holds only an - // interrupted first write. The remedy is a fresh export. - ErrNoLogs = errors.New("no complete log entry found") -) -``` - -**(c) `lastCursorPlain`** — replace entirely: - -```go -// lastCursorPlain validates the tail of a plain NDJSON file and returns a -// cursor for its last complete line. The tool writes one entry per line in a -// single call, so only the tail needs examining: the last newline-terminated -// line must parse as a log entry, and the only bytes allowed after it are a -// single interrupted write — a trailing fragment with no newline. -func lastCursorPlain(file *os.File, size int64) (*Cursor, error) { - window := min(size, 2*maxLineBytes) - offset := size - window - - buf := make([]byte, window) - if _, err := file.ReadAt(buf, offset); err != nil { - return nil, fmt.Errorf("error reading resume file: %w", err) - } - - nl := bytes.LastIndexByte(buf, '\n') - if nl < 0 { - // No newline at all: an interrupted first write, unless the file is - // longer than any single line the tool writes. - if size > maxLineBytes { - return nil, fmt.Errorf("%w: %d bytes without a newline", ErrNotAnExport, size) - } - return nil, ErrNoLogs - } - if tail := window - int64(nl) - 1; tail > maxLineBytes { - return nil, fmt.Errorf("%w: %d bytes without a newline after offset %d", ErrNotAnExport, tail, offset+int64(nl)+1) - } - - start := bytes.LastIndexByte(buf[:nl], '\n') + 1 - if start == 0 && offset > 0 { - return nil, fmt.Errorf("%w: final line is over %d bytes long", ErrNotAnExport, nl) - } - cursor, err := parseCursor(buf[start:nl]) - if err != nil { - return nil, fmt.Errorf("%w: final complete line at offset %d is not a log entry: %v", ErrNotAnExport, offset+int64(start), err) - } - cursor.CleanSize = offset + int64(nl) + 1 - - return cursor, nil -} -``` - -**(d) `lastCursorGzip`** — replace entirely (drops cross-member line carry; members hold whole lines by contract): - -```go -// lastCursorGzip walks a gzip file one member at a time and returns a cursor -// for the last entry inside the cleanly terminated prefix. Gzip cannot be -// seeked, so the whole stream is decompressed; every complete line is -// validated on the way. A member cut short by an interrupted write ends the -// walk: CleanSize stays at the last clean member boundary, and the truncated -// member's content — about to be discarded and re-fetched — never advances -// the cursor. -func lastCursorGzip(file *os.File) (*Cursor, error) { - counter := &countingReader{reader: bufio.NewReaderSize(file, bufferSize)} - - reader, err := gzip.NewReader(counter) - if err != nil { - return nil, fmt.Errorf("error opening gzip resume file: %w", err) - } - defer reader.Close() - - var ( - cursor *Cursor - cleanSize int64 - sawClean bool - ) - for { - // Reset turns multistream back on, so it must be switched off per - // member, not just for the first. - reader.Multistream(false) - - last, err := scanMember(reader) - switch { - case errors.Is(err, ErrNotAnExport): - return nil, err - case err != nil && !truncationShaped(err): - return nil, fmt.Errorf("error reading gzip resume file: %w", err) - case err != nil: - // The member never got its trailer: an interrupted final write. - return gzipResult(cursor, cleanSize, sawClean) - } - - if last != nil { - cursor = last - } - cleanSize, sawClean = counter.read, true - - // A clean end of file makes Reset report io.EOF; a partly written - // next member fails to parse as a header and also ends the walk. - if err := reader.Reset(counter); err != nil { - if errors.Is(err, io.EOF) || truncationShaped(err) { - return gzipResult(cursor, cleanSize, sawClean) - } - return nil, fmt.Errorf("error reading gzip resume file: %w", err) - } - } -} - -// gzipResult finalizes the walk: a file with no clean member, or none holding -// an entry, has nothing to resume from. -func gzipResult(cursor *Cursor, cleanSize int64, sawClean bool) (*Cursor, error) { - if !sawClean || cursor == nil { - return nil, ErrNoLogs - } - cursor.CleanSize = cleanSize - - return cursor, nil -} - -// truncationShaped reports whether err is what an interrupted write produces, -// as opposed to a real read failure that must not be mistaken for one. -func truncationShaped(err error) bool { - var corrupt flate.CorruptInputError - - return errors.Is(err, io.ErrUnexpectedEOF) || - errors.Is(err, gzip.ErrHeader) || - errors.Is(err, gzip.ErrChecksum) || - errors.As(err, &corrupt) -} -``` - -Add `"compress/flate"` to the import block. - -**(e) `scanLines`** — replace with `scanMember` (no carry parameter): - -```go -// scanMember reads one gzip member's NDJSON and returns a cursor for its last -// entry, nil when the member is empty. A nil error means the member decoded -// to a clean end of stream on a line boundary. The tool writes members -// holding whole log lines only, so a complete line that does not parse, or a -// clean member ending mid-line, is foreign content (ErrNotAnExport); any -// other read error is returned as-is for the caller to classify. -func scanMember(r io.Reader) (*Cursor, error) { - buffered := bufio.NewReaderSize(r, bufferSize) - - var ( - last *Cursor - line []byte - ) - for { - chunk, err := buffered.ReadSlice('\n') - line = append(line, chunk...) - if len(line) > maxLineBytes { - return nil, fmt.Errorf("%w: line exceeds %d bytes", ErrNotAnExport, maxLineBytes) - } - - switch { - case errors.Is(err, bufio.ErrBufferFull): - continue - case errors.Is(err, io.EOF): - if len(line) > 0 { - return nil, fmt.Errorf("%w: gzip member ends mid-line", ErrNotAnExport) - } - return last, nil - case err != nil: - return last, err - } - - cursor, err := parseCursor(line) - if err != nil { - return nil, fmt.Errorf("%w: line is not a log entry: %v", ErrNotAnExport, err) - } - last = cursor - line = line[:0] - } -} -``` - -**(f) `parseCursor`** — return descriptive plain errors (callers wrap with the sentinel): - -```go -// parseCursor builds a cursor from a single NDJSON line. Callers wrap the -// returned error with the sentinel that fits their context. -func parseCursor(line []byte) (*Cursor, error) { - line = bytes.TrimSpace(line) - if len(line) == 0 { - return nil, errors.New("blank line") - } - - var parsed cursorLine - if err := json.Unmarshal(line, &parsed); err != nil { - return nil, err - } - if parsed.BlockNumber == nil || parsed.LogIndex == nil { - return nil, errors.New("missing blockNumber or logIndex") - } - - return &Cursor{ - BlockNumber: uint64(*parsed.BlockNumber), - LogIndex: uint(*parsed.LogIndex), - }, nil -} -``` - -`Read`, `Cursor`, `Skip`, `isGzip`, `countingReader` stay as they are (`Read` already computes `Truncated = CleanSize < size` centrally). - -- [ ] **Step 4: Run the package tests** - -Run: `go test ./pkg/resume/... && go vet ./pkg/resume/... && gofumpt -l pkg/resume` -Expected: PASS, no vet output, no files listed. If `TestAppendResumeRoundTrip` or the interrupted-write test fails, the strict path broke a tolerated case — fix the implementation, not the test. - -- [ ] **Step 5: Commit** - -```bash -git add pkg/resume/resume.go pkg/resume/resume_test.go -git commit -m "feat(resume): validate strictly, refuse files this tool did not write" -``` - ---- - -### Task 2: `resume.PrepareOutput` — enforce the clean boundary - -**Files:** -- Modify: `pkg/resume/resume.go` (append after `Skip`) -- Test: `pkg/resume/resume_test.go` - -**Interfaces:** -- Consumes: `Cursor` from Task 1. -- Produces (Task 4 relies on this): `func PrepareOutput(c *Cursor, inputPath, outputPath string) (discarded int64, err error)`. - -- [ ] **Step 1: Write the failing tests** - -Add to `pkg/resume/resume_test.go` (imports of `filestore`/`gzipstore` and the helpers `appendWriter`, `feed`, `logsIn`, `ids` already exist): - -```go -// TestCopyResumeRoundTrip is the spec's recommended workflow (§4): resume an -// archived snapshot into a NEW file. The input must come out byte-identical; -// the output must hold the input's content plus exactly the new entries and -// be itself resumable. -func TestCopyResumeRoundTrip(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - compressed bool - }{ - {name: "plain ndjson"}, - {name: "gzip", compressed: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - original := []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(102, 1)} - boundaryHigher := testLog(102, 2) - newer := []types.Log{testLog(103, 0), testLog(104, 0)} - - inputBytes := ndjson(t, original...) - if tt.compressed { - inputBytes = gz(t, inputBytes) - } - dir := t.TempDir() - input := filepath.Join(dir, "prev.snapshot") - output := filepath.Join(dir, "next.snapshot") - if err := os.WriteFile(input, inputBytes, 0o644); err != nil { - t.Fatalf("write input: %v", err) - } - - cursor, err := resume.Read(input) - if err != nil { - t.Fatalf("Read() error = %v", err) - } - - discarded, err := resume.PrepareOutput(cursor, input, output) - if err != nil { - t.Fatalf("PrepareOutput() error = %v", err) - } - if discarded != 0 { - t.Errorf("discarded = %d, want 0 for a clean input", discarded) - } - - w := appendWriter(t, output, cursor) - replay := []types.Log{testLog(102, 0), testLog(102, 1), boundaryHigher} - replay = append(replay, newer...) - if err := filestore.AppendLogsAsync(t.Context(), feed(replay...), w, cursor.Skip); err != nil { - t.Fatalf("AppendLogsAsync() error = %v", err) - } - - // The input is untouched, byte for byte. - gotInput, err := os.ReadFile(input) - if err != nil { - t.Fatalf("read input: %v", err) - } - if !bytes.Equal(gotInput, inputBytes) { - t.Fatal("input file was modified by a copy-mode resume") - } - - // The output begins with the input's exact bytes (raw prefix - // copy, no recompression) and holds the full sequence. - gotOutput, err := os.ReadFile(output) - if err != nil { - t.Fatalf("read output: %v", err) - } - if len(gotOutput) < len(inputBytes) || !bytes.Equal(gotOutput[:len(inputBytes)], inputBytes) { - t.Fatal("input bytes are not an unchanged prefix of the output") - } - all := append(append(append([]types.Log{}, original...), boundaryHigher), newer...) - if got, want := ids(logsIn(t, output)), ids(all); !slices.Equal(got, want) { - t.Fatalf("output logs = %v, want %v", got, want) - } - - cursor2, err := resume.Read(output) - if err != nil { - t.Fatalf("Read(output) error = %v", err) - } - if cursor2.BlockNumber != 104 || cursor2.LogIndex != 0 || cursor2.Truncated { - t.Errorf("output cursor = {%d,%d,truncated=%t}, want {104,0,false}", cursor2.BlockNumber, cursor2.LogIndex, cursor2.Truncated) - } - }) - } -} - -// TestCopyResumeFromInterruptedInput: copy mode never repairs the input — the -// interrupted tail stays in it — while the output gets only clean content -// plus the re-fetched entries. -func TestCopyResumeFromInterruptedInput(t *testing.T) { - t.Parallel() - - saved := ndjson(t, testLog(100, 0), testLog(101, 0)) - - tests := []struct { - name string - content []byte - compressed bool - }{ - { - name: "plain with a truncated last line", - content: append(append([]byte{}, saved...), []byte(`{"address":"0x45a15`)...), - }, - { - name: "gzip with a truncated final member", - content: append(gz(t, saved), truncateLast(gz(t, ndjson(t, testLog(102, 0))), 6)...), - compressed: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - dir := t.TempDir() - input := filepath.Join(dir, "prev.snapshot") - output := filepath.Join(dir, "next.snapshot") - if err := os.WriteFile(input, tt.content, 0o644); err != nil { - t.Fatalf("write input: %v", err) - } - - cursor, err := resume.Read(input) - if err != nil { - t.Fatalf("Read() error = %v", err) - } - if !cursor.Truncated { - t.Fatal("test setup: input should be truncated") - } - - discarded, err := resume.PrepareOutput(cursor, input, output) - if err != nil { - t.Fatalf("PrepareOutput() error = %v", err) - } - if want := int64(len(tt.content)) - cursor.CleanSize; discarded != want { - t.Errorf("discarded = %d, want %d", discarded, want) - } - - w := appendWriter(t, output, cursor) - replay := []types.Log{testLog(101, 0), testLog(102, 0), testLog(103, 0)} - if err := filestore.AppendLogsAsync(t.Context(), feed(replay...), w, cursor.Skip); err != nil { - t.Fatalf("AppendLogsAsync() error = %v", err) - } - - gotInput, err := os.ReadFile(input) - if err != nil { - t.Fatalf("read input: %v", err) - } - if !bytes.Equal(gotInput, tt.content) { - t.Fatal("input was modified, interrupted tail included it must stay") - } - - want := []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(103, 0)} - if got := ids(logsIn(t, output)); !slices.Equal(got, ids(want)) { - t.Fatalf("output logs = %v, want %v", got, ids(want)) - } - }) - } -} -``` - -Add `"path/filepath"` and `"slices"` to the test file's imports if not present. - -- [ ] **Step 2: Update `TestResumeAfterInterruptedWrite` to use `PrepareOutput`** - -In that test's execution section, replace the direct `os.Truncate(path, cursor.CleanSize)` call (and any surrounding size check) with: - -```go - if _, err := resume.PrepareOutput(cursor, path, path); err != nil { - t.Fatalf("PrepareOutput() error = %v", err) - } -``` - -so the in-place recovery path exercises the production mechanics. - -- [ ] **Step 3: Run tests to verify they fail** - -Run: `go test ./pkg/resume/ -run 'TestCopyResume|TestResumeAfterInterruptedWrite' -v 2>&1 | tail -15` -Expected: FAIL — `undefined: resume.PrepareOutput`. - -- [ ] **Step 4: Implement `PrepareOutput`** - -Append to `pkg/resume/resume.go` after `Skip` (add `"io"` — already imported — and `"path/filepath"` to imports): - -```go -// PrepareOutput readies outputPath for appending the continuation of the -// export at inputPath. With equal paths the file is prepared in place: an -// interrupted final write, if any, is truncated away. With distinct paths the -// input is never modified: its clean content is copied raw into outputPath, -// replacing whatever was there, and the interrupted tail is simply not -// copied. Either way it returns how many trailing bytes were left out — -// every entry they held falls at or after the cursor, so the resumed query -// fetches it again. -func PrepareOutput(c *Cursor, inputPath, outputPath string) (int64, error) { - if filepath.Clean(inputPath) == filepath.Clean(outputPath) { - return prepareInPlace(c, inputPath) - } - - in, err := os.Open(inputPath) - if err != nil { - return 0, fmt.Errorf("error opening resume file: %w", err) - } - defer in.Close() - - info, err := in.Stat() - if err != nil { - return 0, fmt.Errorf("error inspecting resume file: %w", err) - } - - out, err := os.Create(outputPath) - if err != nil { - return 0, fmt.Errorf("error creating output file: %w", err) - } - - _, err = io.CopyN(out, in, c.CleanSize) - if cerr := out.Close(); err == nil { - err = cerr - } - if err != nil { - return 0, fmt.Errorf("error copying clean content to output file: %w", err) - } - - return info.Size() - c.CleanSize, nil -} - -// prepareInPlace drops the interrupted tail so appending continues from the -// clean boundary. It never truncates past the offset the reader identified. -func prepareInPlace(c *Cursor, path string) (int64, error) { - if !c.Truncated { - return 0, nil - } - - info, err := os.Stat(path) - if err != nil { - return 0, fmt.Errorf("error inspecting resume file: %w", err) - } - if info.Size() <= c.CleanSize { - return 0, nil - } - - if err := os.Truncate(path, c.CleanSize); err != nil { - return 0, fmt.Errorf("error truncating resume file: %w", err) - } - - return info.Size() - c.CleanSize, nil -} -``` - -- [ ] **Step 5: Run tests to verify they pass** - -Run: `go test ./pkg/resume/... && go vet ./pkg/resume/... && gofumpt -l pkg/resume` -Expected: PASS, clean, nothing listed. - -- [ ] **Step 6: Commit** - -```bash -git add pkg/resume/resume.go pkg/resume/resume_test.go -git commit -m "feat(resume): add PrepareOutput for in-place and copy-mode continuation" -``` - ---- - -### Task 3: Delete `filestore.SaveLogsAsync` - -**Files:** -- Modify: `pkg/filestore/filestore.go` -- Test: `pkg/filestore/filestore_test.go` - -**Interfaces:** -- Consumes: nothing. -- Produces: `filestore` exports exactly `CreateWriter`, `AppendWriter`, `AppendLogsAsync`. Task 4's `cmd` already uses only these. - -- [ ] **Step 1: Retool the tests** - -In `pkg/filestore/filestore_test.go`, add a seeding helper and rewrite the four tests to use it: - -```go -// seed writes logs for the given blocks to a fresh file at path. -func seed(t *testing.T, path string, blocks ...uint64) { - t.Helper() - - w, err := filestore.CreateWriter(path) - if err != nil { - t.Fatalf("CreateWriter() error = %v", err) - } - if err := filestore.AppendLogsAsync(t.Context(), feed(blocks...), w, nil); err != nil { - t.Fatalf("AppendLogsAsync() error = %v", err) - } -} -``` - -- `TestSaveLogsAsyncReplacesExistingFile` → rename to `TestCreateWriterReplacesExistingFile`; keep the stale-content pre-write, then `seed(t, path, 1, 2)` and the same `blocksIn` assertion (`[]uint64{1, 2}`). -- In `TestAppendLogsAsyncKeepsExistingContent`, `TestAppendLogsAsyncSkipsFilteredLogs`, `TestAppendLogsAsyncClosesWriterOnCancel`: replace each `filestore.SaveLogsAsync(t.Context(), feed(...), path)` seeding call (and its error check) with the matching `seed(t, path, ...)` call. Assertions unchanged. - -- [ ] **Step 2: Delete the function** - -Remove `SaveLogsAsync` (currently `pkg/filestore/filestore.go:26-35`) and nothing else. `CreateWriter`'s doc comment already explains the split. - -- [ ] **Step 3: Verify** - -Run: `go build ./... && go test ./pkg/filestore/... && grep -rn "SaveLogsAsync" --include='*.go' .` -Expected: build OK (proves `cmd` never used it), tests PASS, grep prints nothing. - -- [ ] **Step 4: Commit** - -```bash -git add pkg/filestore/filestore.go pkg/filestore/filestore_test.go -git commit -m "refactor(filestore): drop SaveLogsAsync, callers compose CreateWriter and AppendLogsAsync" -``` - ---- - -### Task 4: Mode resolution in `cmd/export.go` - -**Files:** -- Modify: `cmd/export.go` - -**Interfaces:** -- Consumes: `resume.Read`, `resume.PrepareOutput` (Tasks 1–2); `filestore.CreateWriter`/`AppendWriter`, `gzipstore.AppendWriter` (existing). -- Produces: the §3 CLI contract. Task 5 edits the same file afterwards. - -- [ ] **Step 1: Rework the resume block in `RunE`** - -Replace the current resume block (`cmd/export.go:47-74`) with: - -```go - var cursor *resume.Cursor - if resumeFile != "" { - cursor, err = resume.Read(resumeFile) - if err != nil { - return fmt.Errorf("failed to read resume file %q: %w", resumeFile, err) - } - - if cmd.Flags().Changed("start") { - c.log.Warning("--start is ignored when --resume is set", "resumeFile", resumeFile) - } - if compress { - c.log.Warning("--compress is ignored when resuming; resume a compressed file to get a compressed result", "resumeFile", resumeFile) - compress = false - } - // An unset --output means in-place; so does naming the input. - if !cmd.Flags().Changed("output") || filepath.Clean(outputFile) == filepath.Clean(resumeFile) { - outputFile = resumeFile - } - - startBlock = cursor.BlockNumber - - c.log.Info("Resuming export", - "resumeFile", resumeFile, - "outputFile", outputFile, - "startBlock", startBlock, - "lastLogIndex", cursor.LogIndex, - "compressed", cursor.Compressed, - ) - } -``` - -This removes the old `--output is ignored` warning and the `cursor.Compressed && compress` condition (the warning now fires for every resume with `--compress`). Add `"path/filepath"` to the imports. - -- [ ] **Step 2: Replace `discardPartialWrite` with `PrepareOutput`** - -Replace the call site (`cmd/export.go:100-102`) with: - -```go - if cursor != nil { - discarded, err := resume.PrepareOutput(cursor, resumeFile, outputFile) - if err != nil { - return err - } - if discarded > 0 { - c.log.Warning("previous export ends with an interrupted write, leaving it out", - "resumeFile", resumeFile, - "offset", cursor.CleanSize, - "discardedBytes", discarded, - ) - } - } -``` - -Delete the whole `discardPartialWrite` method (`cmd/export.go:197-229`). If `"os"` is now unused in the file, remove it from the imports. `openOutput` and `saveLogs` stay exactly as they are — in copy mode they receive the already-prepared `outputFile`, and `openOutput`'s `cursor.Compressed` branch matches because the output's format equals the input's. - -- [ ] **Step 3: Update the flag help** - -Replace the `--resume` registration line (`cmd/export.go:190`) with: - -```go - cmd.Flags().StringVarP(&resumeFile, "resume", "r", "", "Continue a previous export file (.ndjson, .gz or .gzip); combine with --output to write a new snapshot instead of appending in place") -``` - -- [ ] **Step 4: Build and verify by hand against copies** - -```bash -make binary && go vet ./... && gofumpt -l cmd pkg -mkdir -p /tmp/resume-verify && cp dist/export.ndjson.gzip /tmp/resume-verify/prev.gzip -# Copy mode: prev untouched, next = prev + nothing new (end pinned at the cursor block). -./dist/batch-export export --resume /tmp/resume-verify/prev.gzip --output /tmp/resume-verify/next.gzip --end 47908504 -cmp /tmp/resume-verify/prev.gzip dist/export.ndjson.gzip && echo "input untouched" -gzcat /tmp/resume-verify/next.gzip | wc -l # equals gzcat prev | wc -l -# In-place still works, and --compress warns: -./dist/batch-export export --resume /tmp/resume-verify/prev.gzip --compress --end 47908504 2>&1 | grep -i "ignored" -``` - -Expected: build clean; copy run logs `Resuming export` with `outputFile=/tmp/resume-verify/next.gzip`; `cmp` silent; line counts equal; the last run warns about `--compress` and appends in place to the copy. **Never point `--resume` or `--output` at `dist/` paths.** - -- [ ] **Step 5: Commit** - -```bash -git add cmd/export.go -git commit -m "feat(export): compose --resume with --output for copy-mode continuation" -``` - ---- - -### Task 5: Saver errors reach the exit code - -**Files:** -- Modify: `cmd/export.go` - -**Interfaces:** -- Consumes: everything already in the file. No new symbols. -- Produces: §7's behavior — a failed save is a non-zero exit, never a hang, never a lost gzip member. - -- [ ] **Step 1: Derive a cancellable context** - -Replace `ctx := cmd.Context()` (top of `RunE`) with: - -```go - ctx, cancel := context.WithCancel(cmd.Context()) - defer cancel() -``` - -- [ ] **Step 2: Capture the saver's error and cancel on failure** - -Replace the saver goroutine with: - -```go - var saveErr error - go func() { - defer wg.Done() - - if err := saveLogs(ctx, logChan, w, cursor); err != nil { - if errors.Is(err, context.Canceled) { - c.log.Error(err, "context canceled while saving logs") - return - } - c.log.Error(err, "error saving logs") - // Stop the fetcher too: with the saver gone, logChan - // would fill and block it forever. - saveErr = err - cancel() - return - } - c.log.Info("all logs have been saved", "outputFile", outputFile) - }() -``` - -(`saveErr` is written before `wg.Done` and read after `wg.Wait`, so the WaitGroup orders the accesses; `go test -race` confirms.) - -- [ ] **Step 3: Wait and join on every exit path** - -The select loop's `errorChan` branch (`cmd/export.go:152-157`) becomes: - -```go - case err, ok := <-errorChan: - if !ok { - errorChan = nil - } else { - wg.Wait() - return errors.Join(fmt.Errorf("error retrieving logs: %w", err), saveErr) - } -``` - -(no deadlock: the fetcher closes `logChan` on its way out, so the saver finishes and `wg.Wait` returns). The `ctx.Done` branch becomes: - -```go - case <-ctx.Done(): - c.log.Info("context canceled, waiting for logs to be saved...") - wg.Wait() - if saveErr != nil { - return saveErr - } - if err := compressFunc(); err != nil { - return errors.Join(fmt.Errorf("error compressing file: %w", err), ctx.Err()) - } - return ctx.Err() -``` - -And the normal exit (`cmd/export.go:174-179`): - -```go - wg.Wait() - if saveErr != nil { - return saveErr - } - if err := compressFunc(); err != nil { - return fmt.Errorf("error compressing file: %w", err) - } - - return nil -``` - -- [ ] **Step 4: Verify the failure modes by hand** - -```bash -make binary && go vet ./... && go test -race ./pkg/... -cp dist/export.ndjson.gzip /tmp/resume-verify/failing.gzip -# Read-only destination: the run must exit non-zero promptly, not hang. -chmod a-w /tmp/resume-verify/failing.gzip -./dist/batch-export export --resume /tmp/resume-verify/failing.gzip --end 47908504; echo "exit=$?" -chmod u+w /tmp/resume-verify/failing.gzip -# Ctrl+C mid-run on a copy: file must still be readable afterwards. -./dist/batch-export export --resume /tmp/resume-verify/failing.gzip & sleep 3; kill -INT %1; wait -gzcat /tmp/resume-verify/failing.gzip > /dev/null && echo "stream intact" -``` - -Expected: the read-only run prints the open error and `exit=1` immediately (the open happens before fetching); the interrupted run exits, and `gzcat` reads the file end to end (the member was flushed under `wg.Wait`). - -- [ ] **Step 5: Commit** - -```bash -git add cmd/export.go -git commit -m "fix(export): surface save failures in the exit code and never skip the final flush" -``` - ---- - -### Task 6: README, PR description, final sweep - -**Files:** -- Modify: `README.md` - -**Interfaces:** -- Consumes: the finished behavior of Tasks 1–5. -- Produces: §8's documentation. Nothing depends on it. - -- [ ] **Step 1: Update the feature bullet and flag table** - -In `## Features`, replace the resume bullet with: - -```markdown -- Continue a previous export from where it stopped (incremental snapshots). -``` - -In the `## Flags` block, replace the `--resume` line with: - -```sh - -r, --resume string Continue a previous export file (.ndjson, .gz or .gzip); combine with --output to write a new snapshot instead of appending in place -``` - -- [ ] **Step 2: Replace the resume section** - -Replace everything from `### Resuming an interrupted export` up to (not including) the `The produced NDJSON is consumed by ...` line with: - -````markdown -### Continuing a previous snapshot - -Instead of re-exporting every block, point `--resume` at the previous -snapshot and name the new one with `--output`. The previous file is read, -never modified; the new file holds everything the previous one did plus the -blocks exported since: - -```sh -./dist/batch-export export --resume snapshots/2026-07.gzip --output snapshots/2026-08.gzip -``` - -Formats are detected by content, not extension: `.ndjson`, `.gz` and `.gzip` -all work, and the output's format always matches the input's. Omitting -`--output` (or naming the input) appends to the previous file in place — the -space-saving variant: - -```sh -./dist/batch-export export --resume export.ndjson.gzip -``` - -The last exported block is re-queried and entries already present are -skipped, so continuing neither duplicates nor drops a log. Each continuation -of a compressed snapshot adds a gzip member — standard tools (`gzcat`, -`gunzip`, Go, Python) read multi-member files as one stream, a year of -monthly continuations costs about 0.02% in size, and -`gzcat old.gzip | gzip > fresh.gzip` consolidates the members any time. - -Keep one canonical snapshot file. `--compress` is ignored when resuming: -regenerating a `.gzip` from a plain twin is how an independently continued -archive gets overwritten. Resume a compressed file to get a compressed -result. - -Resume only files this tool produced. The file's tail is validated before -anything is written: content the tool never writes — a non-log line, foreign -data, an alien gzip member — is refused rather than repaired. The one -exception is the tool's own interrupted final write (a run killed -mid-export): in copy mode it is simply not copied, in place it is truncated -away with a warning, and its entries are re-fetched. Note that resuming does -not detect a file from a different chain; pairing the snapshot with the -right `--endpoint` is the operator's contract. -```` - -- [ ] **Step 3: Full verification sweep** - -```bash -make test && go test -race ./pkg/... && make vet && make lint && make binary -``` - -Expected: everything green, `0 issues.` from lint. - -- [ ] **Step 4: Commit** - -```bash -git add README.md -git commit -m "docs: reframe resume around incremental snapshots" -``` - -- [ ] **Step 5: Update the PR description (no push without approval)** - -Rewrite the PR #11 body to lead with incremental snapshots and copy mode, and list the three semantic changes from §11 (output composes; `--compress` always ignored on resume; strict refusal replaces lenient tolerance). Hold the `git push` and `gh pr edit` until the human approves pushing this phase. - ---- - -## Self-Review - -**Spec coverage:** §3 CLI table → Task 4 Step 1 (mode resolution) + Step 3 (help text); §4 modes → Task 2 (`PrepareOutput`) + Task 4 Step 2; §5 strict validation, both formats, error taxonomy → Task 1; §6 appending/skip → unchanged code, re-verified by Task 2's round trips; §7 exit codes → Task 5; §8 README → Task 6; §9 placement → Tasks 1–4 as mapped; §10 testing → Tasks 1–3 test steps; §11 compatibility → Task 6 Step 5. No spec requirement without a task. - -**Type consistency:** `PrepareOutput(c *Cursor, inputPath, outputPath string) (int64, error)` is defined in Task 2 and consumed with that exact shape in Task 4 Step 2 and the Task 2 tests. `ErrNotAnExport`/`ErrNoLogs` defined in Task 1, consumed in Task 1's refusal table. `seed(t, path, blocks...)` defined and used only in Task 3. `appendWriter`, `feed`, `logsIn`, `ids`, `truncateLast`, `gz`, `ndjson`, `write` all pre-exist in `resume_test.go`. - -**Known deliberate residue:** `cmd`'s §7 plumbing has no unit test (no RPC harness — §10 sanctions manual evidence, Task 5 Step 4 collects it); `compressFunc` stays for fresh runs only; the gzip walk treats unparseable bytes after a clean member as an interrupted tail (indistinguishable from a partial member header — §5 documents this). From f411eef2134983d0c20caafcb086fbf6745e4ce7 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 15:39:21 +0200 Subject: [PATCH 31/35] docs(spec): record the fix-wave behaviors and the deferred revalidation non-goal Co-Authored-By: Claude Opus 5 (1M context) --- .../2026-08-27-resume-input-output-design.md | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md index 067a77b..4026317 100644 --- a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md +++ b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md @@ -26,6 +26,10 @@ the interface or the documentation. different chain parses identically; detecting the mismatch would need a file header, which changes the format `batch-archive` consumes. Documented limitation. +- **Revalidating the input between the cursor read and the append**, and + running two resumes of one file concurrently. The model is a single + operator; a follow-up may carry the input's size and mtime in the cursor + and refuse in `PrepareOutput` if the file moved. - **Validating the interior of plain files.** Plain-file validation covers the region the cursor reading must touch (the tail); silent corruption elsewhere in a 90 MB file is out of scope. (Gzip necessarily decodes the whole stream, @@ -103,8 +107,8 @@ Exactly that irregularity is tolerated: the cursor is the last complete entry before it, `CleanSize` excludes it, `Truncated` is true, and each mode handles it per §4 (copy mode does not copy it; in-place truncates it). Nothing else is tolerated. Any of the following mean the file is not untouched tool output, -and `Read` refuses with `ErrNotAnExport`, naming what was found and at which -offset: +and `Read` refuses with `ErrNotAnExport`, naming what was found and where — a +byte offset for plain files, a member index and line number for gzip: - a complete line (newline-terminated) that does not parse as a log entry — including blank lines; @@ -112,10 +116,18 @@ offset: entries run a few hundred bytes); - a cleanly terminated gzip member that ends mid-line or contains a complete non-parsing line; +- a gzip member whose body decodes in full but whose CRC or length trailer + does not match — an interrupted write cannot produce a complete trailer + with wrong values, so that shape is corruption, not truncation; - gzip content after a member that failed to decode (unreachable in practice — decoding stops there — but stated for completeness: bytes past `CleanSize` are ignored, never interpreted). +A gzip file cut short inside its very first header is an interrupted first +write and yields `ErrNoLogs`, like its plain equivalent. A header whose bytes +beyond the magic are corrupt is indistinguishable from that and is classified +the same way — a documented, tolerated misclassification. + Refusal is deliberate: for an archival artifact, silently cutting away content the tool never wrote is worse than stopping. The operator decides what a manipulated file means. @@ -174,7 +186,12 @@ The saver goroutine's error must reach `RunE`'s return value: - After `wg.Wait()`, the saver's error is joined into the returned error: a failed save always exits non-zero. On a gzip destination the writer's `Close` finalizes the member, so its error is part of the save result, not - noise. A save error caused only by cancellation is not double-reported. + noise. A save error caused only by cancellation is not double-reported — + and "only" is literal: a cancellation joined with a real close error still + counts as a failure. When the saver itself cancelled the run, the return is + the save error alone, even if the fetcher's echo of that cancellation wins + the select — an internal failure is never dressed up as a user + cancellation. - The goroutine still logs the error when it happens, so the operator sees it immediately. From a837a44c03be37c52922c9ef53be70e083ac8de7 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 16:19:13 +0200 Subject: [PATCH 32/35] docs: recommend the double extension for snapshots and fix flag help Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8f5d3b2..ee21c33 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ The primary command is export. -b, --block-range-limit uint32 Max blocks per log query (default 5) -c, --compress Compress to GZIP --end uint End block (optional, uses latest block if 0) - -e, --endpoint string Ethereum RPC endpoint URL + -e, --endpoint string Ethereum based RPC endpoint URL (default "https://rpc.gnosis.gateway.fm") -h, --help help for export -m, --max-request int Max RPC requests/sec (default 15) -o, --output string Output file path (NDJSON) (default "export.ndjson") @@ -63,14 +63,20 @@ never modified; the new file holds everything the previous one did plus the blocks exported since: ```sh -./dist/batch-export export --resume snapshots/2026-07.gzip --output snapshots/2026-08.gzip +./dist/batch-export export --resume snapshots/2026-07.ndjson.gzip --output snapshots/2026-08.ndjson.gzip ``` -If `--output` already names an existing file, it is overwritten — the same -`os.Create` semantics as a fresh export, so pick a new name for each snapshot. +If `--output` names a different existing file, it is overwritten — the same +`os.Create` semantics as a fresh export — so pick a new name for each +snapshot. (Naming the input itself under another spelling — absolute vs +relative, a symlink, a case difference — is detected and appends in place +instead.) Formats are detected by content, not extension: `.ndjson`, `.gz` and `.gzip` -all work, and the output's format always matches the input's. Omitting +all work, and the output's format always matches the input's. Gzip stores no +filename inside, so decompressing names the result after the archive minus +its extension — name snapshots with the double extension, as above, and +extraction yields a `.ndjson` file. Omitting `--output` (or naming the input) appends to the previous file in place — the space-saving variant: From 012edf0b9434b2f8f71f0db29fa21847cd95bfe9 Mon Sep 17 00:00:00 2001 From: Ljubisa Gacevic Date: Thu, 27 Aug 2026 16:19:13 +0200 Subject: [PATCH 33/35] docs: remove the design spec from the repo The design rationale lives in the PR description and the README's behavior documentation. Co-Authored-By: Claude Opus 5 (1M context) --- .../2026-08-27-resume-input-output-design.md | 266 ------------------ 1 file changed, 266 deletions(-) delete mode 100644 docs/superpowers/specs/2026-08-27-resume-input-output-design.md diff --git a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md b/docs/superpowers/specs/2026-08-27-resume-input-output-design.md deleted file mode 100644 index 4026317..0000000 --- a/docs/superpowers/specs/2026-08-27-resume-input-output-design.md +++ /dev/null @@ -1,266 +0,0 @@ -# Resume as Incremental Snapshots - -**Date:** 2026-08-27 -**Status:** Draft, awaiting review -**Scope:** the complete behavior of the `--resume` feature on PR #11. This -document is the source of truth; the implementation is edited until it -conforms. - -## 1. Goal - -`batch-export` produces periodic snapshots of Postage Stamp contract events. -The operator runs an export today, archives the resulting `.gzip`, and later — -typically a month later — produces the next snapshot by *continuing from the -previous one* instead of re-exporting every block from the contract's start. - -`--resume` therefore means: **continue a previous, normally finished export -made by this same tool.** A run that was interrupted mid-write is handled -(§5), but crash recovery is robustness, not the purpose, and does not shape -the interface or the documentation. - -## 2. Non-Goals - -- **Repairing foreign or manipulated files.** A resume input is trusted to be - this tool's own output; anything else is refused, never "fixed" (§5). -- **Validating chain provenance.** An export from the same tool against a - different chain parses identically; detecting the mismatch would need a file - header, which changes the format `batch-archive` consumes. Documented - limitation. -- **Revalidating the input between the cursor read and the append**, and - running two resumes of one file concurrently. The model is a single - operator; a follow-up may carry the input's size and mtime in the cursor - and refuse in `PrepareOutput` if the file moved. -- **Validating the interior of plain files.** Plain-file validation covers the - region the cursor reading must touch (the tail); silent corruption elsewhere - in a 90 MB file is out of scope. (Gzip necessarily decodes the whole stream, - so it validates every line as a side effect — §5.) -- **Retiring post-hoc `CompressFile` for fresh `--compress` runs.** Fresh - exports keep today's behavior. - -## 3. CLI Contract - -No new flags. `--resume` names the **input** (the previous snapshot); -`--output` names the **destination**. They compose: - -| Invocation | Behavior | -|---|---| -| `export` | Fresh export to `--output` (default `export.ndjson`). Unchanged. | -| `export --resume old.gzip` | **In-place**: append to `old.gzip` itself. An unset `--output` does not redirect the result to its default. | -| `export --resume old.gzip --output new.gzip` | **Copy mode**: `old.gzip` is never modified; `new.gzip` = clean content of `old.gzip` + newly fetched entries. | -| `export --resume f --output f` | Identical paths (`filepath.Clean` equality) mean in-place. Same file under any other spelling — absolute vs. relative, a symlink, a hardlink, two names a case-insensitive filesystem folds together — is detected via `os.SameFile` (device+inode) and also treated as in-place. | - -Flag interactions when `--resume` is set: - -- `--start` is ignored with a warning: the cursor decides the start block. -- `--end` works as usual (default 0 = latest block). -- `--compress` is **always ignored with a warning**. Post-hoc compression of a - resumed plain file is the twin-file trap: `CompressFile`'s `os.Create` would - overwrite an independently accumulated `.gzip`. A compressed result comes - from resuming a compressed input; compressing a plain result is `gzip`'s job. -- The output's format always equals the input's format, detected from the - input's leading magic bytes. Extensions are names, nothing more; `.ndjson`, - `.gz` and `.gzip` all work. -- In copy mode an existing file at `--output` is overwritten (`os.Create` - semantics, same as a fresh export). - -Flag help: `Continue a previous export file (.ndjson, .gz or .gzip); combine -with --output to write a new snapshot instead of appending in place`. - -## 4. Modes - -**Copy mode** (input != output) — the recommended monthly workflow: - -```sh -batch-export export --resume snapshots/2026-07.gzip --output snapshots/2026-08.gzip -``` - -The first `CleanSize` bytes (§5) of the input are copied to the output raw — -no decompression, no re-encoding — and new entries are appended to the output. -The input is opened read-only and is never modified, truncated, or deleted. If -the run fails, the output is incomplete but the input is intact: delete the -output and rerun. - -**In-place** (input == output): the space-saving variant. If the input carries -an interrupted final write (§5), the file is first truncated to `CleanSize` -with a warning naming the discarded byte count; appending then proceeds. - -## 5. Reading the Cursor: Strict Tail Validation - -The *cursor* is the last complete log entry of the input: its `blockNumber` -and `logIndex`, plus `Compressed`, `CleanSize` (the byte offset at which the -input's trustworthy content ends) and `Truncated` (whether anything follows -`CleanSize`). - -The contract is strict because the input is by definition this tool's own -output. The tool writes NDJSON via `json.Encoder` — each entry is one line, -value and terminating newline in a single write — either plainly or inside -gzip members, one member per run, each member holding only whole lines. A -member may be empty (a resumed run that fetched nothing new still closes its -member). Consequently the **only** irregularity the tool itself can produce is -an interrupted final write: - -- plain: a single incomplete trailing line (a partial write never ends in a - newline, and these JSON lines contain no raw newlines); -- gzip: a single trailing member without its terminating CRC/length trailer. - -Exactly that irregularity is tolerated: the cursor is the last complete entry -before it, `CleanSize` excludes it, `Truncated` is true, and each mode handles -it per §4 (copy mode does not copy it; in-place truncates it). Nothing else is -tolerated. Any of the following mean the file is not untouched tool output, -and `Read` refuses with `ErrNotAnExport`, naming what was found and where — a -byte offset for plain files, a member index and line number for gzip: - -- a complete line (newline-terminated) that does not parse as a log entry — - including blank lines; -- a trailing newline-free fragment longer than `maxLineBytes` (1 MiB; real - entries run a few hundred bytes); -- a cleanly terminated gzip member that ends mid-line or contains a complete - non-parsing line; -- a gzip member whose body decodes in full but whose CRC or length trailer - does not match — an interrupted write cannot produce a complete trailer - with wrong values, so that shape is corruption, not truncation; -- gzip content after a member that failed to decode (unreachable in practice — - decoding stops there — but stated for completeness: bytes past `CleanSize` - are ignored, never interpreted). - -A gzip file cut short inside its very first header is an interrupted first -write and yields `ErrNoLogs`, like its plain equivalent. A header whose bytes -beyond the magic are corrupt is indistinguishable from that and is classified -the same way — a documented, tolerated misclassification. - -Refusal is deliberate: for an archival artifact, silently cutting away content -the tool never wrote is worse than stopping. The operator decides what a -manipulated file means. - -**Errors.** Two sentinel errors replace the current three: - -- `ErrNotAnExport` — foreign content, as above. Terminal; the message says - what and where. -- `ErrNoLogs` — the file is consistent with tool output but holds no complete - entry to resume from: empty, or only an interrupted first write. The remedy - is a fresh export. - -(`ErrNoCleanBoundary` is retired: its gzip case — a sole truncated member — is -`ErrNoLogs`; its mid-line-member case is `ErrNotAnExport`.) - -**Mechanics.** Plain: one read of the last `2*maxLineBytes` bytes suffices — -enough for the last complete line plus a trailing fragment, up to -`maxLineBytes` (1 MiB) each — to find the last newline, check the fragment -after it, and parse the single line before it. The current multi-window -backward walk with carry exists only to -tolerate foreign junk and is deleted. Gzip: cannot seek, so the stream is -decoded member by member with a counting reader (which must implement -`io.ByteReader` so `gzip.Reader` consumes it directly and member boundaries -are observed exactly); every complete line must parse; `CleanSize` is the end -of the last cleanly terminated member. - -## 6. Appending - -- **Plain**: `O_APPEND` on the destination. -- **Gzip**: a new gzip member (`gzip.NewWriter` on a file opened `O_APPEND`). - Concatenated members are one valid stream per RFC 1952; `gzcat`, `gunzip`, - Go and Python read them transparently. Existing bytes are never rewritten. - Measured on the real 90 MB export: 12 members cost +0.021% over a single - member, 120 members +0.224%. Members can be consolidated any time with - `gzcat old.gzip | gzip > fresh.gzip`. -- **Skip filter**: the resumed query starts at the cursor's block - *inclusively* — an interrupted run may have written only part of it, and for - a finished run the re-query is a no-op — and entries at or before the cursor - (`blockNumber < cursor's`, or equal block with `logIndex <=` cursor's) are - dropped before writing. No gaps, no duplicates. -- The destination writer is opened before the first log is fetched, so an open - failure aborts the run instead of leaving the fetcher pushing into a channel - nobody drains. - -## 7. Error Handling and Exit Codes - -The saver goroutine's error must reach `RunE`'s return value: - -- `RunE` derives a cancellable context. The saver records its error and - **cancels the derived context on failure**, which unblocks the fetcher, - which closes `errorChan`, which lets the select loop exit. No hang wherever - the save fails (open, encode, close). -- **Every** exit path out of the select loop runs `wg.Wait()` before - returning — including the `errorChan` error branch, which today returns - immediately and can lose the buffered gzip member. -- After `wg.Wait()`, the saver's error is joined into the returned error: a - failed save always exits non-zero. On a gzip destination the writer's - `Close` finalizes the member, so its error is part of the save result, not - noise. A save error caused only by cancellation is not double-reported — - and "only" is literal: a cancellation joined with a real close error still - counts as a failure. When the saver itself cancelled the run, the return is - the save error alone, even if the fetcher's echo of that cancellation wins - the select — an internal failure is never dressed up as a user - cancellation. -- The goroutine still logs the error when it happens, so the operator sees it - immediately. - -## 8. Documentation (README) - -1. Section title **"Continuing a previous snapshot"**; lead example is copy - mode with dated files (§4). In-place follows as the variant. -2. The multi-member note with the measured overhead and the consolidation - one-liner. -3. A short "if the previous run was interrupted" subsection: copy mode — - rerun; in-place — the tool truncates the interrupted write and re-fetches, - warning shown. A footnote, not the headline. -4. The trust rule, stated plainly: resume only files this tool produced; a - file with any other content is refused, and resuming does not detect a - wrong chain — that is the operator's contract. -5. One canonical snapshot file: keep either the plain file or the gzip, not - both; `--compress` is ignored on resume. -6. Feature bullet: "Continue a previous export from where it stopped - (incremental snapshots)." - -## 9. Code Placement - -- `pkg/resume`: gains `PrepareOutput(cursor, inputPath, outputPath)` — the - in-place truncation or clean-prefix copy of §4. It lives beside `Read` - because the package that computes `CleanSize` should also enforce it; a - caller can then not append without the invariant holding, and the mechanics - are testable without an RPC harness. -- `cmd/export.go`: resolves the mode (§3), calls `PrepareOutput`, and picks - create vs append in `openOutput`; the saver-error plumbing of §7 lives in - `RunE`. -- `pkg/resume`: strict validation per §5; `lastCursorPlain` shrinks to the - single-tail read; `lastCursorGzip` keeps the member walk, drops cross-member - line carry (whole-line members are part of the contract), refuses on any - non-parsing complete line; error taxonomy per §5. -- `pkg/filestore`: `SaveLogsAsync` — exported, no production caller — is - deleted with its tests; fresh exports route through `CreateWriter` + - `AppendLogsAsync`. The `AppendLogsAsync` close-error join stays. -- `pkg/gzipstore`: gained `AppendWriter` earlier on this branch (§6); not - further changed by this design. - -## 10. Testing - -All tests remain package-level (`./pkg/...`) plus the cross-package round -trips in `resume_test.go`; `cmd` still has no RPC harness, so §7 is verified -at whatever level extraction permits, with manual evidence recorded in the PR -for the rest. - -- **Copy-mode round trip** (both formats): input byte-identical before and - after; output = input's content + exactly the new entries in order; output - itself resumable; output ends clean. -- **Copy mode from an interrupted input** (both formats): input untouched - *including its partial tail*; output holds the clean content plus re-fetched - entries; nothing lost, nothing duplicated. -- **In-place round trip and interrupted-write recovery**: as today. -- **Strict refusal** (`ErrNotAnExport`): plain with a trailing complete - non-log line; plain with a blank line; plain with a >1 MiB newline-free - tail (refused fast, no scan-back); gzip with a junk member; gzip with a - clean member ending mid-line. -- **Tolerated irregularities**: plain partial trailing line; gzip truncated - final member; empty gzip member (valid, contributes nothing). -- **`ErrNoLogs`**: empty file; file holding only a partial first write. -- **Same-path detection**: `--resume f --output f` behaves as in-place. -- Deleted along with the code they exercised: the multi-window walk tests - (garbage windows, boundary straddle). - -## 11. Compatibility - -PR #11 is unmerged; nothing released changes. Within the PR: `--output` -composes with `--resume` instead of being overridden; `--compress` is ignored -on every resume, not only on compressed input; lenient junk tolerance is -replaced by strict refusal. All three are called out in the PR description. -The final merge squashes, so branch history need not tell this story twice. From 22eaf6c156afcd75b2a702af302f4d8fc1b2fa32 Mon Sep 17 00:00:00 2001 From: Calin Martinconi Date: Fri, 28 Aug 2026 13:12:09 +0300 Subject: [PATCH 34/35] fix(docs): resolve merge conflict in README and add edge case tests --- README.md | 3 - cmd/export_test.go | 75 ++++++++++++++++++ pkg/resume/resume_test.go | 161 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 cmd/export_test.go diff --git a/README.md b/README.md index e850794..74064f0 100644 --- a/README.md +++ b/README.md @@ -51,12 +51,9 @@ The primary command is export. -h, --help help for export -m, --max-request int Max RPC requests/sec (default 15) -o, --output string Output file path (NDJSON) (default "export.ndjson") -<<<<<<< HEAD -r, --resume string Continue a previous export file (.ndjson, .gz or .gzip); combine with --output to write a new snapshot instead of appending in place -======= --retry-delay duration Delay before the first retry, doubling per retry up to 30s (default 1s) --retry-max int Max retries per RPC request on transient network errors (0 disables retrying) (default 5) ->>>>>>> origin --start uint Start block (optional, uses contract start block if 0) (default 31306381) -v, --verbosity string Log verbosity (silent, error, warn, info, debug) (default "info") ``` diff --git a/cmd/export_test.go b/cmd/export_test.go new file mode 100644 index 0000000..f3cf096 --- /dev/null +++ b/cmd/export_test.go @@ -0,0 +1,75 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "testing" +) + +func TestSolelyCanceled(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + err error + want bool + }{ + { + name: "nil error", + err: nil, + want: false, + }, + { + name: "pure context.Canceled", + err: context.Canceled, + want: true, + }, + { + name: "wrapped context.Canceled", + err: fmt.Errorf("wrap: %w", context.Canceled), + want: true, + }, + { + name: "joined context.Canceled with context.Canceled", + err: errors.Join(context.Canceled, context.Canceled), + want: true, + }, + { + name: "joined context.Canceled with wrapped context.Canceled", + err: errors.Join(context.Canceled, fmt.Errorf("wrap: %w", context.Canceled)), + want: true, + }, + { + name: "joined context.Canceled with real io error", + err: errors.Join(context.Canceled, io.ErrUnexpectedEOF), + want: false, + }, + { + name: "joined real error with context.Canceled", + err: errors.Join(errors.New("disk full"), context.Canceled), + want: false, + }, + { + name: "pure io error", + err: io.ErrClosedPipe, + want: false, + }, + { + name: "context deadline exceeded", + err: context.DeadlineExceeded, + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + if got := solelyCanceled(tt.err); got != tt.want { + t.Errorf("solelyCanceled(%v) = %v, want %v", tt.err, got, tt.want) + } + }) + } +} diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index 18a0f60..1bdf734 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -1096,3 +1096,164 @@ func TestGzipCleanSizeIsAMemberBoundary(t *testing.T) { t.Fatalf("content = %s, want %s", got, want) } } + +// TestResumeMultipleEmptyAppends checks that resuming multiple times when no +// new logs exist on chain (e.g. producing empty writes or empty gzip members) +// preserves existing data and allows subsequent appends with new logs. +func TestResumeMultipleEmptyAppends(t *testing.T) { + t.Parallel() + + for _, tc := range []struct { + name string + fileName string + compressed bool + }{ + {name: nameFormatPlain, fileName: plainFile, compressed: false}, + {name: nameFormatGzip, fileName: gzipFile, compressed: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + path := filepath.Join(t.TempDir(), tc.fileName) + + // Initial write of 2 logs + initLogs := []types.Log{testLog(100, 0), testLog(101, 0)} + var initialContent []byte + if tc.compressed { + initialContent = gz(t, ndjson(t, initLogs...)) + } else { + initialContent = ndjson(t, initLogs...) + } + if err := os.WriteFile(path, initialContent, 0o644); err != nil { + t.Fatalf("write initial file: %v", err) + } + + // First empty resume: 0 new logs + cursor, err := resume.Read(path) + if err != nil { + t.Fatalf("first Read() error = %v", err) + } + if _, err := resume.PrepareOutput(cursor, path, path); err != nil { + t.Fatalf("first PrepareOutput() error = %v", err) + } + if err := filestore.AppendLogsAsync(t.Context(), feed(), appendWriter(t, path, cursor), cursor.Skip); err != nil { + t.Fatalf("first AppendLogsAsync() error = %v", err) + } + + // Second empty resume: 0 new logs + cursor, err = resume.Read(path) + if err != nil { + t.Fatalf("second Read() error = %v", err) + } + if _, err := resume.PrepareOutput(cursor, path, path); err != nil { + t.Fatalf("second PrepareOutput() error = %v", err) + } + if err := filestore.AppendLogsAsync(t.Context(), feed(), appendWriter(t, path, cursor), cursor.Skip); err != nil { + t.Fatalf("second AppendLogsAsync() error = %v", err) + } + + // Third resume: 2 new logs + newLogs := []types.Log{testLog(101, 0), testLog(102, 0), testLog(103, 0)} + cursor, err = resume.Read(path) + if err != nil { + t.Fatalf("third Read() error = %v", err) + } + if _, err := resume.PrepareOutput(cursor, path, path); err != nil { + t.Fatalf("third PrepareOutput() error = %v", err) + } + if err := filestore.AppendLogsAsync(t.Context(), feed(newLogs...), appendWriter(t, path, cursor), cursor.Skip); err != nil { + t.Fatalf("third AppendLogsAsync() error = %v", err) + } + + // Verify all 4 logs are present and correctly ordered + want := []types.Log{testLog(100, 0), testLog(101, 0), testLog(102, 0), testLog(103, 0)} + got := logsIn(t, path) + if !slices.Equal(ids(got), ids(want)) { + t.Fatalf("logs = %v, want %v", ids(got), ids(want)) + } + }) + } +} + +// TestCursorSkipWithIndexGaps tests that Cursor.Skip correctly skips +// non-consecutive log indices within the same block and preceding blocks. +func TestCursorSkipWithIndexGaps(t *testing.T) { + t.Parallel() + + cursor := &resume.Cursor{ + BlockNumber: 500, + LogIndex: 7, + } + + tests := []struct { + name string + log types.Log + want bool + }{ + {name: "earlier block", log: testLog(499, 100), want: true}, + {name: "same block lower index", log: testLog(500, 3), want: true}, + {name: "same block exact index", log: testLog(500, 7), want: true}, + {name: "same block higher index", log: testLog(500, 8), want: false}, + {name: "same block much higher index", log: testLog(500, 20), want: false}, + {name: "later block index 0", log: testLog(501, 0), want: false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := cursor.Skip(tt.log); got != tt.want { + t.Errorf("Skip(%+v) = %v, want %v", tt.log, got, tt.want) + } + }) + } +} + +// TestCRLFLineEndings ensures NDJSON files with CRLF (\r\n) line endings +// are parsed and resumed without errors. +func TestCRLFLineEndings(t *testing.T) { + t.Parallel() + + raw := ndjson(t, testLog(100, 0), testLog(101, 5)) + crlf := bytes.ReplaceAll(raw, []byte("\n"), []byte("\r\n")) + + path := write(t, plainFile, crlf) + cursor, err := resume.Read(path) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + if cursor.BlockNumber != 101 || cursor.LogIndex != 5 { + t.Errorf("cursor = {%d,%d}, want {101,5}", cursor.BlockNumber, cursor.LogIndex) + } + if cursor.CleanSize != int64(len(crlf)) { + t.Errorf("CleanSize = %d, want %d", cursor.CleanSize, len(crlf)) + } +} + +// TestPrepareOutputInvalidDirectory asserts that copying to a non-existent +// target directory reports an error and leaves the input file untouched. +func TestPrepareOutputInvalidDirectory(t *testing.T) { + t.Parallel() + + content := ndjson(t, testLog(100, 0)) + input := write(t, plainFile, content) + + cursor, err := resume.Read(input) + if err != nil { + t.Fatalf("Read() error = %v", err) + } + + output := filepath.Join(t.TempDir(), "nonexistent", "nested", "out.ndjson") + if _, err := resume.PrepareOutput(cursor, input, output); err == nil { + t.Fatal("PrepareOutput() succeeded for non-existent dir, want error") + } + + // Assert input remains intact + got, err := os.ReadFile(input) + if err != nil { + t.Fatalf("ReadFile() error = %v", err) + } + if !bytes.Equal(got, content) { + t.Fatal("input was modified on failed PrepareOutput") + } +} + From 385226985a4b7247122635f91b50f761b13bb667 Mon Sep 17 00:00:00 2001 From: Calin Martinconi Date: Fri, 28 Aug 2026 13:34:32 +0300 Subject: [PATCH 35/35] style: fix gofmt formatting in resume_test.go --- pkg/resume/resume_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/resume/resume_test.go b/pkg/resume/resume_test.go index 1bdf734..ad458f4 100644 --- a/pkg/resume/resume_test.go +++ b/pkg/resume/resume_test.go @@ -1256,4 +1256,3 @@ func TestPrepareOutputInvalidDirectory(t *testing.T) { t.Fatal("input was modified on failed PrepareOutput") } } -