fix(seidb): refuse a corrupted changelog in digest replay instead of repairing it

sei-db/state_db/sc/memiavl/db.go

@@ -213,6 +213,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) {
 	// Even in read-only mode we may need WAL replay to reconstruct non-snapshot versions.
 	streamHandler, err := wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{
 		WriteBufferSize: opts.AsyncCommitBuffer,
+		NoRepairOnOpen:  opts.NoChangelogRepair,
 	})
 	if err != nil {
 		return nil, fmt.Errorf("failed to open changelog WAL: %w", err)

sei-db/state_db/sc/memiavl/opts.go

@@ -18,6 +18,11 @@ type Options struct {
 	InitialVersion uint32
 	// ReadOnly opens the database in read-only mode
 	ReadOnly bool
+	// NoChangelogRepair fails the open with wal.ErrCorrupt instead of repairing a
+	// torn changelog tail. ReadOnly alone does not make an open non-mutating: the
+	// changelog opener repairs the tail whether or not writes through the DB API
+	// are allowed, so a tool that must leave a live directory alone sets both.
+	NoChangelogRepair bool
 	// InitialStores are the initial store names when initializing an empty instance
 	InitialStores []string
 	// ZeroCopy if true, get and iterator methods return slices pointing to mmaped blob files

sei-db/tools/cmd/seidb/operations/evm_logical_digest.go

@@ -20,6 +20,7 @@ import (
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl"
 	"github.com/sei-protocol/sei-chain/sei-db/state_db/sc/migration"
+	"github.com/sei-protocol/sei-chain/sei-db/wal"
 	"github.com/spf13/cobra"
 )
 
@@ -86,7 +87,10 @@ const (
 //     magnitude slower than snapshot (changelog replay + per-leaf tree walk
 //     instead of a sequential file read). Use it only when no snapshot exists
 //     at the target height — e.g. nodes whose snapshot rewrite lags the tip, so
-//     an arbitrary comparison height has no snapshot-<height> on disk.
+//     an arbitrary comparison height has no snapshot-<height> on disk. On a live
+//     node this mode can read a changelog record the node is midway through
+//     writing; it then reports that and asks for a rerun rather than repairing
+//     the changelog under its writer.
 //
 // The flatkv side is always a pebble WAL-replay-to-height and is fast
 // regardless. So when comparing across nodes, pick a height that is an existing
@@ -1104,13 +1108,24 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization
 	}
 }
 
+// openMemiAVLReplayReadOnly opens dbDir for changelog replay without repairing
+// its changelog. The repair is a truncation, and a record a running node is
+// midway through writing is indistinguishable from a corrupt one, so repairing
+// here can discard a block that node has committed.
 func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) {
 	db, err := memiavl.OpenDB(height, memiavl.Options{
-		Dir:      dbDir,
-		ReadOnly: true,
-		ZeroCopy: true,
+		Dir:               dbDir,
+		ReadOnly:          true,
+		ZeroCopy:          true,
+		NoChangelogRepair: true,
 	})
 	if err != nil {
+		if errors.Is(err, wal.ErrCorrupt) {
+			return nil, fmt.Errorf("the changelog under %s ends mid-record, which is what a node "+
+				"writing a block looks like; %s was left as it was found, so rerun this command, and if "+
+				"it keeps failing the changelog is corrupt and the node needs attention: %w",
+				dbDir, dbDir, err)
+		}
 		return nil, fmt.Errorf("open memiavl read-only replay: %w", err)
 	}
 	return db, nil

sei-db/tools/cmd/seidb/operations/memiavl_open_test.go

@@ -0,0 +1,95 @@
+package operations
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/sei-protocol/sei-chain/sei-db/common/keys"
+	"github.com/sei-protocol/sei-chain/sei-db/common/utils"
+	"github.com/sei-protocol/sei-chain/sei-db/proto"
+	"github.com/sei-protocol/sei-chain/sei-db/wal"
+)
+
+// TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutTruncatingIt pins the
+// reason replay does not repair: a record that ends mid-write looks the same
+// whether the node crashed or is committing right now, and truncating it in the
+// second case discards a committed block. The refusal is only worth anything if
+// the tail survives it, so the segment's bytes are compared too.
+func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutTruncatingIt(t *testing.T) {
+	homeDir := t.TempDir()
+	writeMemiavlNonces(t, homeDir, 3)
+
+	dbDir := utils.GetCosmosSCStorePath(homeDir)
+	segment := lastChangelogSegment(t, dbDir)
+	torn := tearFileTail(t, segment)
+
+	db, err := openMemiAVLReplayReadOnly(dbDir, 3)
+	require.Nil(t, db)
+	require.ErrorIs(t, err, wal.ErrCorrupt)
+	require.Contains(t, err.Error(), "rerun this command")
+
+	after, err := os.ReadFile(segment) //nolint:gosec // test-controlled path
+	require.NoError(t, err)
+	require.Equal(t, torn, after, "the open must leave the changelog segment as it found it")
+}
+
+// TestOpenMemiAVLReplayReadOnlyReplaysAnIntactChangelog is the positive control:
+// refusing a torn tail is only the intended change if an intact one still
+// replays.
+func TestOpenMemiAVLReplayReadOnlyReplaysAnIntactChangelog(t *testing.T) {
+	homeDir := t.TempDir()
+	writeMemiavlNonces(t, homeDir, 3)
+
+	db, err := openMemiAVLReplayReadOnly(utils.GetCosmosSCStorePath(homeDir), 3)
+	require.NoError(t, err)
+	defer func() { _ = db.Close() }()
+	require.Equal(t, int64(3), db.Version())
+}
+
+// writeMemiavlNonces commits count blocks to a fresh memiavl store under homeDir
+// and closes it, leaving a changelog with one entry per block.
+func writeMemiavlNonces(t *testing.T, homeDir string, count uint64) {
+	t.Helper()
+	store := newTestMemiavlStore(t, homeDir)
+	for nonce := uint64(1); nonce <= count; nonce++ {
+		require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{
+			Name:      keys.EVMStoreKey,
+			Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}},
+		}}))
+		_, err := store.Commit(store.Version() + 1)
+		require.NoError(t, err)
+	}
+	require.NoError(t, store.Close())
+}
+
+// lastChangelogSegment returns the path of the segment the changelog appends to.
+// Segment names are zero-padded indices, so the highest name sorts last.
+func lastChangelogSegment(t *testing.T, dbDir string) string {
+	t.Helper()
+	changelogDir := utils.GetChangelogPath(dbDir)
+	entries, err := os.ReadDir(changelogDir)
+	require.NoError(t, err)
+	var name string
+	for _, entry := range entries {
+		if !entry.IsDir() && len(entry.Name()) >= 20 {
+			name = entry.Name()
+		}
+	}
+	require.NotEmpty(t, name, "no changelog segment under %s", changelogDir)
+	return filepath.Join(changelogDir, name)
+}
+
+// tearFileTail drops the last byte of path, which is what a reader sees partway
+// through the writer's append, and returns the resulting contents.
+func tearFileTail(t *testing.T, path string) []byte {
+	t.Helper()
+	data, err := os.ReadFile(path) //nolint:gosec // test-controlled path
+	require.NoError(t, err)
+	require.NotEmpty(t, data)
+	torn := data[:len(data)-1]
+	require.NoError(t, os.WriteFile(path, torn, 0o600))
+	return torn
+}

sei-db/wal/utils.go

@@ -24,7 +24,7 @@ func GetLastIndex(dir string) (index uint64, err error) {
 	rlog, err := open(dir, &wal.Options{
 		NoSync: true,
 		NoCopy: true,
-	})
+	}, false)
 	if err != nil {
 		return 0, err
 	}

sei-db/wal/wal.go

@@ -21,6 +21,10 @@ const defaultBufferSize = 1024
 // The size of write batches if the provided write batch size is less than 1.
 const defaultWriteBatchSize = 64
 
+// ErrCorrupt reports that the log ends mid-record. An open returns it only under
+// Config.NoRepairOnOpen; otherwise the tail is truncated and the open succeeds.
+var ErrCorrupt = wal.ErrCorrupt
+
 // WAL is a generic write-ahead log implementation.
 type WAL[T any] struct {
 	ctx    context.Context
@@ -94,6 +98,12 @@ type Config struct {
 	// AllowEmpty permits removing all entries via TruncateAll.
 	// When false (default), at least one entry must remain after truncation.
 	AllowEmpty bool
+
+	// NoRepairOnOpen returns wal.ErrCorrupt from the open instead of truncating a
+	// corrupted tail to recover the log. A reader that must leave the log as it
+	// found it sets it. It does not cover the segment cleanup the open performs for
+	// an interrupted TruncateFront, which reports no error to gate on.
+	NoRepairOnOpen bool
 }
 
 // NewWAL creates a new generic write-ahead log that persists entries.
@@ -120,7 +130,7 @@ func NewWAL[T any](
 		NoSync:     !config.FsyncEnabled,
 		NoCopy:     !config.DeepCopyEnabled,
 		AllowEmpty: config.AllowEmpty,
-	})
+	}, config.NoRepairOnOpen)
 	if err != nil {
 		return nil, err
 	}
@@ -542,13 +552,18 @@ func (walLog *WAL[T]) Close() error {
 	return nil
 }
 
-// open opens the replay log, try to truncate the corrupted tail if there's any
-func open(dir string, opts *wal.Options) (*wal.Log, error) {
+// open opens the replay log, truncating a corrupted tail to recover the log.
+// When noRepair is set it returns wal.ErrCorrupt instead, leaving the tail in
+// place.
+func open(dir string, opts *wal.Options, noRepair bool) (*wal.Log, error) {
 	if opts == nil {
 		opts = wal.DefaultOptions
 	}
 	rlog, err := wal.Open(dir, opts)
 	if errors.Is(err, wal.ErrCorrupt) {
+		if noRepair {
+			return nil, err
+		}
 		// try to truncate corrupted tail
 		var fis []os.DirEntry
 		fis, err = os.ReadDir(dir)

sei-db/wal/wal_test.go

@@ -57,7 +57,7 @@ func TestOpenAndCorruptedTail(t *testing.T) {
 			_, err = wal.Open(dir, opts)
 			require.Equal(t, wal.ErrCorrupt, err)
 
-			log, err := open(dir, opts)
+			log, err := open(dir, opts, false)
 			require.NoError(t, err)
 
 			lastIndex, err := log.LastIndex()
@@ -171,7 +171,7 @@ func TestOpenWithNilOptions(t *testing.T) {
 	dir := t.TempDir()
 
 	// Test that open function handles nil options correctly
-	log, err := open(dir, nil)
+	log, err := open(dir, nil, false)
 	require.NoError(t, err)
 	require.NotNil(t, log)