Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ batch-export is a tool to retrieve Ethereum event logs for specific contracts, p
- Supports rate limiting for RPC requests.
- Saves retrieved logs to a specified output file (default: `export.ndjson`) in NDJSON format.
- Graceful shutdown on interrupt signals (Ctrl+C).
- By default emits a slim log shape — only the `types.Log` fields Bee consumes (`address`, `topics`, `data`, `blockNumber`, `transactionHash`). Pass `--slim=false` to emit the full geth `types.Log` JSON shape for other consumers. The slim shape is decoder-compatible with geth's `types.Log` (so Bee reads both interchangeably).

## Requirements

Expand Down Expand Up @@ -49,6 +50,7 @@ The primary command is export.
-h, --help help for export
-m, --max-request int Max RPC requests/sec (default 15)
-o, --output string Output file path (NDJSON) (default "export.ndjson")
--slim Emit only the types.Log fields Bee consumes (default true)
--start uint Start block (optional, uses contract start block if 0) (default 31306381)
-v, --verbosity string Log verbosity (silent, error, warn, info, debug) (default "info")
```
Expand Down
4 changes: 3 additions & 1 deletion cmd/export.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ func (c *command) initExportCmd() (err error) {
blockRangeLimit uint32
outputFile string
compress bool
slim bool
)

cmd := &cobra.Command{
Expand Down Expand Up @@ -79,7 +80,7 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save

go func() {
defer wg.Done()
if err := filestore.SaveLogsAsync(ctx, logChan, outputFile); err != nil {
if err := filestore.SaveLogsAsync(ctx, logChan, outputFile, slim); err != nil {
if errors.Is(err, context.Canceled) {
c.log.Error(err, "context canceled while saving logs")
return
Expand Down Expand Up @@ -139,6 +140,7 @@ The process can be interrupted at any time (Ctrl+C), and it will attempt to save
cmd.Flags().Uint32VarP(&blockRangeLimit, "block-range-limit", "b", 5, "Max blocks per log query")
cmd.Flags().StringVarP(&outputFile, "output", "o", "export.ndjson", "Output file path (NDJSON)")
cmd.Flags().BoolVarP(&compress, "compress", "c", false, "Compress to GZIP")
cmd.Flags().BoolVar(&slim, "slim", true, "Emit only the types.Log fields Bee consumes (address, topics, data, blockNumber, transactionHash); set false for full geth shape")

c.root.AddCommand(cmd)

Expand Down
29 changes: 17 additions & 12 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,28 @@ module github.com/ethersphere/batch-export
go 1.25

require (
github.com/ethereum/go-ethereum v1.15.11
github.com/ethereum/go-ethereum v1.17.3
github.com/ethersphere/bee/v2 v2.7.0
github.com/spf13/cobra v1.9.1
golang.org/x/time v0.12.0
)

require (
github.com/Microsoft/go-winio v0.6.2 // indirect
github.com/ProjectZKM/Ziren/crates/go-runtime/zkvm_runtime v0.0.0-20251001021608-1fe7b43fc4d6 // indirect
github.com/StackExchange/wmi v1.2.1 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.20.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/consensys/gnark-crypto v0.18.1 // indirect
github.com/crate-crypto/go-eth-kzg v1.3.0 // indirect
github.com/crate-crypto/go-ipa v0.0.0-20240724233137-53bbb0ceb27a // indirect
github.com/crate-crypto/go-eth-kzg v1.5.0 // indirect
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
github.com/ethereum/c-kzg-4844/v2 v2.1.0 // indirect
github.com/ethereum/go-verkle v0.2.2 // indirect
github.com/ethereum/c-kzg-4844/v2 v2.1.6 // indirect
github.com/ethersphere/go-storage-incentives-abi v0.9.4 // indirect
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/go-logr/logr v1.4.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
Expand All @@ -41,13 +42,17 @@ require (
github.com/prometheus/procfs v0.16.1 // indirect
github.com/shirou/gopsutil v3.21.5+incompatible // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/supranational/blst v0.3.14 // indirect
github.com/supranational/blst v0.3.16 // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
golang.org/x/crypto v0.45.0 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/sync v0.18.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/text v0.31.0 // indirect
google.golang.org/protobuf v1.36.6 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
go.opentelemetry.io/otel v1.40.0 // indirect
go.opentelemetry.io/otel/metric v1.40.0 // indirect
go.opentelemetry.io/otel/trace v1.40.0 // indirect
golang.org/x/crypto v0.47.0 // indirect
golang.org/x/net v0.49.0 // indirect
golang.org/x/sync v0.19.0 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/text v0.33.0 // indirect
google.golang.org/protobuf v1.36.11 // indirect
)
Loading
Loading