wip levelcache

carlog/carlog.go

@@ -485,7 +485,7 @@ func (j *CarLog) fixLevelIndex(h Head, w WritableIndex) error {
 
 	err := j.iterate(h.RetiredAt, func(off int64, length uint64, c cid.Cid, data []byte) error {
 		mhsBuf = append(mhsBuf, c.Hash())
-		offsBuf = append(offsBuf, makeOffsetLen(off, int(length)))
+		offsBuf = append(offsBuf, MakeOffsetLen(off, int(length)))
 
 		done++
 
@@ -625,7 +625,7 @@ func (j *CarLog) Put(c []mh.Multihash, b []blocks.Block) error {
 		// todo use a buffer with fixed cid prefix to avoid allocs
 		bcid := cid.NewCidV1(cid.Raw, c[i]).Bytes()
 
-		offsets[i] = makeOffsetLen(j.dataLen, len(bcid)+len(blk.RawData()))
+		offsets[i] = MakeOffsetLen(j.dataLen, len(bcid)+len(blk.RawData()))
 
 		n, err := j.ldWrite(bcid, blk.RawData())
 		if err != nil {
@@ -773,7 +773,7 @@ func (j *CarLog) View(c []mh.Multihash, cb func(cidx int, found bool, data []byt
 			continue
 		}
 
-		off, entLen := fromOffsetLen(locs[i])
+		off, entLen := FromOffsetLen(locs[i])
 
 		if entLen > len(entBuf) {
 			// expand buffer to next power of two if needed
@@ -855,7 +855,7 @@ func (j *CarLog) viewExternal(c []mh.Multihash, cb func(cidx int, found bool, da
 			continue
 		}
 
-		off, entLen := fromOffsetLen(locs[i])
+		off, entLen := FromOffsetLen(locs[i])
 
 		if entLen > len(entBuf) {
 			// expand buffer to next power of two if needed
@@ -1478,7 +1478,7 @@ func (c *carIdxSource) List(f func(c mh.Multihash, offs []int64) error) error {
 			return xerrors.Errorf("decode block cid: %w", err)
 		}
 
-		err = f(c.Hash(), []int64{makeOffsetLen(at, len(d))})
+		err = f(c.Hash(), []int64{MakeOffsetLen(at, len(d))})
 		if err != nil {
 			return err
 		}
@@ -1736,11 +1736,11 @@ func (rs *readSeekerFromReaderAt) Seek(offset int64, whence int) (int64, error)
 
 const MaxEntryLen = 1 << (64 - 40)
 
-func makeOffsetLen(off int64, length int) int64 {
+func MakeOffsetLen(off int64, length int) int64 {
 	return (int64(length) << 40) | (off & 0xFFFF_FFFF_FF)
 }
 
-func fromOffsetLen(offlen int64) (int64, int) {
+func FromOffsetLen(offlen int64) (int64, int) {
 	return offlen & 0xFFFF_FFFF_FF, int(offlen >> 40)
 }
 

carlog/idx_level.go

@@ -136,7 +136,7 @@ func (l *LevelDBIndex) ToTruncate(atOrAbove int64) ([]multihash.Multihash, error
 		if len(it.Value()) != 8 {
 			return nil, xerrors.Errorf("invalid value length")
 		}
-		offs, _ := fromOffsetLen(int64(binary.LittleEndian.Uint64(it.Value())))
+		offs, _ := FromOffsetLen(int64(binary.LittleEndian.Uint64(it.Value())))
 		if offs >= atOrAbove {
 			keyCopy := make([]byte, len(it.Key()))
 			copy(keyCopy, it.Key())

go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/aws/aws-sdk-go v1.44.269
+	github.com/cheggaaa/pb v1.0.29
 	github.com/cockroachdb/pebble v0.0.0-20230503034834-93b977533929
 	github.com/fatih/color v1.13.0
 	github.com/filecoin-project/go-address v1.1.0
@@ -45,6 +46,7 @@ require (
 	github.com/raulk/go-watchdog v1.3.0
 	github.com/stretchr/testify v1.8.4
 	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
+	github.com/urfave/cli/v2 v2.25.5
 	github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc
 	github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa
 	go.uber.org/multierr v1.11.0
@@ -171,6 +173,7 @@ require (
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-runewidth v0.0.10 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/miekg/dns v1.1.54 // indirect
 	github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
@@ -203,12 +206,12 @@ require (
 	github.com/quic-go/quic-go v0.33.0 // indirect
 	github.com/quic-go/webtransport-go v0.5.3 // indirect
 	github.com/raulk/clock v1.1.0 // indirect
+	github.com/rivo/uniseg v0.1.0 // indirect
 	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/rs/cors v1.7.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/shirou/gopsutil v2.18.12+incompatible // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/urfave/cli/v2 v2.25.5 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasttemplate v1.0.1 // indirect
 	github.com/whyrusleeping/bencher v0.0.0-20190829221104-bb6607aa8bba // indirect

go.sum

@@ -111,6 +111,8 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo=
+github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -203,6 +205,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7
 github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw=
 github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
 github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w=
 github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
@@ -1043,6 +1046,7 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
@@ -1052,13 +1056,17 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
 github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
+github.com/mattn/go-runewidth v0.0.10 h1:CoZ3S2P7pvtP45xOtBw+/mDL2z0RKI576gSkzRRpdGg=
+github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
 github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
@@ -1315,6 +1323,8 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB
 github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rivo/uniseg v0.1.0 h1:+2KBaVoUmb9XzDsrx/Ct0W/EYOSFf/nWTauy++DprtY=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=

ributil/levelcache/levelcache.go

@@ -0,0 +1,62 @@
+package levelcache
+
+import (
+	"github.com/lotus-web3/ribs/ributil/logcache"
+	"github.com/lotus-web3/ribs/ributil/mlru"
+	"github.com/multiformats/go-multihash"
+	"sync"
+)
+
+type mhStr string
+
+type LevelCache struct {
+	// root dir
+	// Contents:
+	// /l0.lru
+	// /l0.car
+	// /l0.offs
+	// ...
+	root string
+
+	// settings
+	l0size         int64
+	levelExpansion int
+
+	// mlru group
+	lrugroup *mlru.LRUGroup
+
+	// persistent top level lru
+	// (the top layer is special because it's the only actively mutated layer)
+	topLru *mlru.MLRU[mhStr, bool]
+	topLog *logcache.LogCache
+
+	// levels, with, at least for now, one logcache+mlru per level
+	levels []*cacheLevel
+
+	// compaction stuff
+	compactLk sync.RWMutex
+
+	// todo mem tier
+	// temp memory cache used during compaction
+	compactCache map[mhStr][]byte
+}
+
+type cacheLevel struct {
+	lru *mlru.MLRU[mhStr, bool]
+	log *logcache.LogCache
+}
+
+func (c *LevelCache) compactTop() error {
+	c.compactLk.Lock()
+	defer c.compactLk.Unlock()
+
+	c.levels[0].lru.EvictStats()
+}
+
+func (c *LevelCache) Put(k multihash.Multihash, v []byte) error {
+
+}
+
+func (c *LevelCache) Get(k multihash.Multihash, cb func([]byte) error) error {
+
+}