diff --git a/sei-db/state_db/sc/memiavl/db.go b/sei-db/state_db/sc/memiavl/db.go index b342cbfd76..0d1c8b88d1 100644 --- a/sei-db/state_db/sc/memiavl/db.go +++ b/sei-db/state_db/sc/memiavl/db.go @@ -213,6 +213,7 @@ func OpenDB(targetVersion int64, opts Options) (database *DB, _err error) { // Even in read-only mode we may need WAL replay to reconstruct non-snapshot versions. streamHandler, err := wal.NewChangelogWAL(utils.GetChangelogPath(opts.Dir), wal.Config{ WriteBufferSize: opts.AsyncCommitBuffer, + NoRepairOnOpen: opts.NoChangelogRepair, }) if err != nil { return nil, fmt.Errorf("failed to open changelog WAL: %w", err) diff --git a/sei-db/state_db/sc/memiavl/opts.go b/sei-db/state_db/sc/memiavl/opts.go index 9c4a6f5d31..1495b7f59b 100644 --- a/sei-db/state_db/sc/memiavl/opts.go +++ b/sei-db/state_db/sc/memiavl/opts.go @@ -18,6 +18,11 @@ type Options struct { InitialVersion uint32 // ReadOnly opens the database in read-only mode ReadOnly bool + // NoChangelogRepair fails the open with wal.ErrCorrupt instead of repairing a + // torn changelog tail. ReadOnly alone does not make an open non-mutating: the + // changelog opener repairs the tail whether or not writes through the DB API + // are allowed, so a tool that must leave a live directory alone sets both. + NoChangelogRepair bool // InitialStores are the initial store names when initializing an empty instance InitialStores []string // ZeroCopy if true, get and iterator methods return slices pointing to mmaped blob files diff --git a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go index 79ecd140c6..25fe2e8743 100644 --- a/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go +++ b/sei-db/tools/cmd/seidb/operations/evm_logical_digest.go @@ -20,6 +20,7 @@ import ( "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/vtype" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/memiavl" "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/migration" + "github.com/sei-protocol/sei-chain/sei-db/wal" "github.com/spf13/cobra" ) @@ -86,7 +87,10 @@ const ( // magnitude slower than snapshot (changelog replay + per-leaf tree walk // instead of a sequential file read). Use it only when no snapshot exists // at the target height — e.g. nodes whose snapshot rewrite lags the tip, so -// an arbitrary comparison height has no snapshot- on disk. +// an arbitrary comparison height has no snapshot- on disk. On a live +// node this mode can read a changelog record the node is midway through +// writing; it then reports that and asks for a rerun rather than repairing +// the changelog under its writer. // // The flatkv side is always a pebble WAL-replay-to-height and is fast // regardless. So when comparing across nodes, pick a height that is an existing @@ -1104,13 +1108,24 @@ func digestMemIAVL(dbDir string, height int64, findTarget []byte, normalization } } +// openMemiAVLReplayReadOnly opens dbDir for changelog replay without repairing +// its changelog. The repair is a truncation, and a record a running node is +// midway through writing is indistinguishable from a corrupt one, so repairing +// here can discard a block that node has committed. func openMemiAVLReplayReadOnly(dbDir string, height int64) (*memiavl.DB, error) { db, err := memiavl.OpenDB(height, memiavl.Options{ - Dir: dbDir, - ReadOnly: true, - ZeroCopy: true, + Dir: dbDir, + ReadOnly: true, + ZeroCopy: true, + NoChangelogRepair: true, }) if err != nil { + if errors.Is(err, wal.ErrCorrupt) { + return nil, fmt.Errorf("the changelog under %s ends mid-record, which is what a node "+ + "writing a block looks like; %s was left as it was found, so rerun this command, and if "+ + "it keeps failing the changelog is corrupt and the node needs attention: %w", + dbDir, dbDir, err) + } return nil, fmt.Errorf("open memiavl read-only replay: %w", err) } return db, nil diff --git a/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go new file mode 100644 index 0000000000..2068ed1832 --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/memiavl_open_test.go @@ -0,0 +1,95 @@ +package operations + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/sei-protocol/sei-chain/sei-db/common/keys" + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/proto" + "github.com/sei-protocol/sei-chain/sei-db/wal" +) + +// TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutTruncatingIt pins the +// reason replay does not repair: a record that ends mid-write looks the same +// whether the node crashed or is committing right now, and truncating it in the +// second case discards a committed block. The refusal is only worth anything if +// the tail survives it, so the segment's bytes are compared too. +func TestOpenMemiAVLReplayReadOnlyRefusesATornChangelogWithoutTruncatingIt(t *testing.T) { + homeDir := t.TempDir() + writeMemiavlNonces(t, homeDir, 3) + + dbDir := utils.GetCosmosSCStorePath(homeDir) + segment := lastChangelogSegment(t, dbDir) + torn := tearFileTail(t, segment) + + db, err := openMemiAVLReplayReadOnly(dbDir, 3) + require.Nil(t, db) + require.ErrorIs(t, err, wal.ErrCorrupt) + require.Contains(t, err.Error(), "rerun this command") + + after, err := os.ReadFile(segment) //nolint:gosec // test-controlled path + require.NoError(t, err) + require.Equal(t, torn, after, "the open must leave the changelog segment as it found it") +} + +// TestOpenMemiAVLReplayReadOnlyReplaysAnIntactChangelog is the positive control: +// refusing a torn tail is only the intended change if an intact one still +// replays. +func TestOpenMemiAVLReplayReadOnlyReplaysAnIntactChangelog(t *testing.T) { + homeDir := t.TempDir() + writeMemiavlNonces(t, homeDir, 3) + + db, err := openMemiAVLReplayReadOnly(utils.GetCosmosSCStorePath(homeDir), 3) + require.NoError(t, err) + defer func() { _ = db.Close() }() + require.Equal(t, int64(3), db.Version()) +} + +// writeMemiavlNonces commits count blocks to a fresh memiavl store under homeDir +// and closes it, leaving a changelog with one entry per block. +func writeMemiavlNonces(t *testing.T, homeDir string, count uint64) { + t.Helper() + store := newTestMemiavlStore(t, homeDir) + for nonce := uint64(1); nonce <= count; nonce++ { + require.NoError(t, store.ApplyChangeSets([]*proto.NamedChangeSet{{ + Name: keys.EVMStoreKey, + Changeset: proto.ChangeSet{Pairs: []*proto.KVPair{noncePair(addrN(0xA1), nonce)}}, + }})) + _, err := store.Commit(store.Version() + 1) + require.NoError(t, err) + } + require.NoError(t, store.Close()) +} + +// lastChangelogSegment returns the path of the segment the changelog appends to. +// Segment names are zero-padded indices, so the highest name sorts last. +func lastChangelogSegment(t *testing.T, dbDir string) string { + t.Helper() + changelogDir := utils.GetChangelogPath(dbDir) + entries, err := os.ReadDir(changelogDir) + require.NoError(t, err) + var name string + for _, entry := range entries { + if !entry.IsDir() && len(entry.Name()) >= 20 { + name = entry.Name() + } + } + require.NotEmpty(t, name, "no changelog segment under %s", changelogDir) + return filepath.Join(changelogDir, name) +} + +// tearFileTail drops the last byte of path, which is what a reader sees partway +// through the writer's append, and returns the resulting contents. +func tearFileTail(t *testing.T, path string) []byte { + t.Helper() + data, err := os.ReadFile(path) //nolint:gosec // test-controlled path + require.NoError(t, err) + require.NotEmpty(t, data) + torn := data[:len(data)-1] + require.NoError(t, os.WriteFile(path, torn, 0o600)) + return torn +} diff --git a/sei-db/wal/utils.go b/sei-db/wal/utils.go index a33cf5f33d..e8a81b4d0c 100644 --- a/sei-db/wal/utils.go +++ b/sei-db/wal/utils.go @@ -24,7 +24,7 @@ func GetLastIndex(dir string) (index uint64, err error) { rlog, err := open(dir, &wal.Options{ NoSync: true, NoCopy: true, - }) + }, false) if err != nil { return 0, err } diff --git a/sei-db/wal/wal.go b/sei-db/wal/wal.go index b13fd12af1..7f342c9262 100644 --- a/sei-db/wal/wal.go +++ b/sei-db/wal/wal.go @@ -21,6 +21,10 @@ const defaultBufferSize = 1024 // The size of write batches if the provided write batch size is less than 1. const defaultWriteBatchSize = 64 +// ErrCorrupt reports that the log ends mid-record. An open returns it only under +// Config.NoRepairOnOpen; otherwise the tail is truncated and the open succeeds. +var ErrCorrupt = wal.ErrCorrupt + // WAL is a generic write-ahead log implementation. type WAL[T any] struct { ctx context.Context @@ -94,6 +98,12 @@ type Config struct { // AllowEmpty permits removing all entries via TruncateAll. // When false (default), at least one entry must remain after truncation. AllowEmpty bool + + // NoRepairOnOpen returns wal.ErrCorrupt from the open instead of truncating a + // corrupted tail to recover the log. A reader that must leave the log as it + // found it sets it. It does not cover the segment cleanup the open performs for + // an interrupted TruncateFront, which reports no error to gate on. + NoRepairOnOpen bool } // NewWAL creates a new generic write-ahead log that persists entries. @@ -120,7 +130,7 @@ func NewWAL[T any]( NoSync: !config.FsyncEnabled, NoCopy: !config.DeepCopyEnabled, AllowEmpty: config.AllowEmpty, - }) + }, config.NoRepairOnOpen) if err != nil { return nil, err } @@ -542,13 +552,18 @@ func (walLog *WAL[T]) Close() error { return nil } -// open opens the replay log, try to truncate the corrupted tail if there's any -func open(dir string, opts *wal.Options) (*wal.Log, error) { +// open opens the replay log, truncating a corrupted tail to recover the log. +// When noRepair is set it returns wal.ErrCorrupt instead, leaving the tail in +// place. +func open(dir string, opts *wal.Options, noRepair bool) (*wal.Log, error) { if opts == nil { opts = wal.DefaultOptions } rlog, err := wal.Open(dir, opts) if errors.Is(err, wal.ErrCorrupt) { + if noRepair { + return nil, err + } // try to truncate corrupted tail var fis []os.DirEntry fis, err = os.ReadDir(dir) diff --git a/sei-db/wal/wal_test.go b/sei-db/wal/wal_test.go index a83d5fded3..881abef9bb 100644 --- a/sei-db/wal/wal_test.go +++ b/sei-db/wal/wal_test.go @@ -57,7 +57,7 @@ func TestOpenAndCorruptedTail(t *testing.T) { _, err = wal.Open(dir, opts) require.Equal(t, wal.ErrCorrupt, err) - log, err := open(dir, opts) + log, err := open(dir, opts, false) require.NoError(t, err) lastIndex, err := log.LastIndex() @@ -171,7 +171,7 @@ func TestOpenWithNilOptions(t *testing.T) { dir := t.TempDir() // Test that open function handles nil options correctly - log, err := open(dir, nil) + log, err := open(dir, nil, false) require.NoError(t, err) require.NotNil(t, log)