Merge pull request #97 from barrydeen/lmdb-migration-and-bootstrap-tuning

Migrate eventstore to LMDB and tune WoT bootstrap fetches

Author: barrydeen

README.md

@@ -244,6 +244,83 @@ Once everything is set up, the relay will be running on `localhost:3334`.
 http://localhost:3334
 ```
 
+## Migrating from Badger to LMDB
+
+Older versions of wot-relay stored events in a Badger database. The relay now
+uses LMDB via `fiatjaf.com/nostr/eventstore/lmdb` and Badger is no longer
+supported. Because the two backends use different on-disk formats, an
+in-place upgrade is not possible — events must be exported to JSONL from the
+old store and re-imported into a fresh LMDB store.
+
+1. Stop the relay.
+2. Build the legacy exporter (separate module, uses the old Badger code):
+
+   ```bash
+   cd tools/export-badger
+   go build -o ../../export-badger .
+   cd ../..
+   ```
+3. Export every event to JSONL:
+
+   ```bash
+   ./export-badger -db ./db > events.jsonl
+   ```
+4. Move the old database aside (keep it until the new one is verified):
+
+   ```bash
+   mv ./db ./db.badger.bak
+   ```
+5. Build the new relay and importer:
+
+   ```bash
+   go build -ldflags "-X main.version=$(git describe --tags --always)"
+   go build -o import-jsonl ./cmd/import-jsonl
+   ```
+6. Populate a fresh LMDB directory:
+
+   ```bash
+   ./import-jsonl -db ./db < events.jsonl
+   ```
+7. Start the relay as normal. `DB_PATH` now points at the LMDB directory.
+
+### Migrating with Docker
+
+If you run wot-relay via Docker Compose, you don't need Go installed on the
+host — the steps above can be run inside the `golang:bookworm` image against
+the same `./db` bind mount your compose file already uses.
+
+1. Stop the relay:
+
+   ```bash
+   docker compose down
+   ```
+2. Export the Badger DB to JSONL:
+
+   ```bash
+   docker run --rm -v "$PWD:/src" -w /src golang:bookworm sh -c \
+       "cd tools/export-badger && go build -o /tmp/export-badger . && /tmp/export-badger -db /src/db > /src/events.jsonl"
+   ```
+3. Move the old database aside:
+
+   ```bash
+   mv db db.badger.bak
+   ```
+4. Import the JSONL into a fresh LMDB directory:
+
+   ```bash
+   docker run --rm -v "$PWD:/src" -w /src golang:bookworm sh -c \
+       "go build -o /tmp/import-jsonl ./cmd/import-jsonl && /tmp/import-jsonl -db /src/db < /src/events.jsonl"
+   ```
+5. Rebuild and start the relay:
+
+   ```bash
+   docker compose up -d --build
+   ```
+
+Files written by the `docker run` steps will be owned by root on the host
+because the container runs as root by default. If that's a problem, `chown`
+them back after migration.
+
 ## License
 
 This project is licensed under the MIT License.

arm64.Dockerfile

@@ -22,7 +22,7 @@ ENV STATIC_PATH="templates/static"
 RUN touch .env
 
 # Build the Go application
-RUN go build -tags badger -o main .
+RUN go build -o main .
 
 # Expose the port that the application will run on
 EXPOSE 3334

cmd/import-jsonl/main.go

@@ -0,0 +1,73 @@
+package main
+
+import (
+	"bufio"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+
+	"fiatjaf.com/nostr"
+	"fiatjaf.com/nostr/eventstore/lmdb"
+)
+
+func main() {
+	path := flag.String("db", "", "path to the LMDB directory (created if missing)")
+	skipErrors := flag.Bool("skip-errors", false, "skip events that fail to parse or save instead of aborting")
+	flag.Parse()
+	if *path == "" {
+		fmt.Fprintln(os.Stderr, "usage: import-jsonl -db <path> [-skip-errors] < events.jsonl")
+		os.Exit(2)
+	}
+
+	db := &lmdb.LMDBBackend{Path: *path}
+	if err := db.Init(); err != nil {
+		log.Fatalf("init: %v", err)
+	}
+	defer db.Close()
+
+	scanner := bufio.NewScanner(os.Stdin)
+	// events can exceed the default 64KB line limit
+	scanner.Buffer(make([]byte, 0, 1024*1024), 16*1024*1024)
+
+	var saved, failed uint64
+	var line uint64
+	for scanner.Scan() {
+		line++
+		b := scanner.Bytes()
+		if len(b) == 0 {
+			continue
+		}
+
+		var evt nostr.Event
+		if err := json.Unmarshal(b, &evt); err != nil {
+			failed++
+			msg := fmt.Sprintf("line %d: unmarshal: %v", line, err)
+			if *skipErrors {
+				log.Println(msg)
+				continue
+			}
+			log.Fatal(msg)
+		}
+
+		if err := db.SaveEvent(evt); err != nil {
+			failed++
+			msg := fmt.Sprintf("line %d (id=%s): save: %v", line, evt.ID, err)
+			if *skipErrors {
+				log.Println(msg)
+				continue
+			}
+			log.Fatal(msg)
+		}
+		saved++
+
+		if saved%10000 == 0 {
+			log.Printf("imported %d events...", saved)
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		log.Fatalf("scan: %v", err)
+	}
+	log.Printf("done: imported %d events, %d failures, %d lines read", saved, failed, line)
+}

go.mod

@@ -1,55 +1,53 @@
 module github.com/bitvora/wot-relay
 
-go 1.23.1
+go 1.25
 
 require (
-	github.com/fiatjaf/eventstore v0.14.4
-	github.com/fiatjaf/khatru v0.14.0
+	fiatjaf.com/nostr v0.0.0-20260422002040-5b28d08e4720
 	github.com/joho/godotenv v1.5.1
-	github.com/nbd-wtf/go-nostr v0.44.2
 )
 
 require (
-	fiatjaf.com/lib v0.2.0 // indirect
+	fiatjaf.com/lib v0.3.6 // indirect
+	github.com/FastFilter/xorfilter v0.2.1 // indirect
+	github.com/ImVexed/fasturl v0.0.0-20230304231329-4e41488060f3 // indirect
+	github.com/PowerDNS/lmdb-go v1.9.3 // indirect
 	github.com/andybalholm/brotli v1.1.1 // indirect
 	github.com/bep/debounce v1.2.1 // indirect
+	github.com/btcsuite/btcd v0.24.2 // indirect
 	github.com/btcsuite/btcd/btcec/v2 v2.3.4 // indirect
+	github.com/btcsuite/btcd/btcutil v1.1.5 // indirect
 	github.com/btcsuite/btcd/chaincfg/chainhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/coder/websocket v1.8.13 // indirect
 	github.com/decred/dcrd/crypto/blake256 v1.1.0 // indirect
-	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
-	github.com/dgraph-io/badger/v4 v4.5.0 // indirect
-	github.com/dgraph-io/ristretto v1.0.0 // indirect
-	github.com/dgraph-io/ristretto/v2 v2.0.0 // indirect
+	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect
+	github.com/dgraph-io/ristretto/v2 v2.3.0 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
-	github.com/fasthttp/websocket v1.5.11 // indirect
-	github.com/gobwas/httphead v0.1.0 // indirect
-	github.com/gobwas/pool v0.2.1 // indirect
-	github.com/gobwas/ws v1.4.0 // indirect
-	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/glog v1.1.2 // indirect
-	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/flatbuffers v24.3.25+incompatible // indirect
+	github.com/elnosh/gonuts v0.4.2 // indirect
+	github.com/fasthttp/websocket v1.5.12 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/mailru/easyjson v0.9.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/pkg/errors v0.9.1 // indirect
-	github.com/puzpuzpuz/xsync/v3 v3.4.0 // indirect
+	github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect
 	github.com/rs/cors v1.11.1 // indirect
 	github.com/savsgio/gotils v0.0.0-20240704082632-aef3928b8a38 // indirect
+	github.com/templexxx/cpu v0.0.1 // indirect
+	github.com/templexxx/xhex v0.0.0-20200614015412-aed53437177b // indirect
 	github.com/tidwall/gjson v1.18.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.1 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
-	github.com/valyala/fasthttp v1.58.0 // indirect
-	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/exp v0.0.0-20241217172543-b2144cdd0a67 // indirect
-	golang.org/x/net v0.33.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
-	google.golang.org/protobuf v1.35.2 // indirect
+	github.com/valyala/fasthttp v1.59.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	golang.org/x/crypto v0.39.0 // indirect
+	golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/sync v0.15.0 // indirect
+	golang.org/x/sys v0.35.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )