Skip to content

feat: use pebble db and make storage config unified #7

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fearlessfe merged 1 commit into from
Dec 6, 2024
Merged

feat: use pebble db and make storage config unified #7

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions beacon/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,10 @@ type beaconStorageCache struct {

var _ storage2.ContentStorage = &Storage{}

func NewBeaconStorage(config storage2.PortalStorageConfig) (storage2.ContentStorage, error) {
func NewBeaconStorage(config storage2.PortalStorageConfig, db *sql.DB) (storage2.ContentStorage, error) {
bs := &Storage{
storageCapacityInBytes: config.StorageCapacityMB * BytesInMB,
db: config.DB,
db: db,
log: log.New("beacon_storage"),
spec: config.Spec,
cache: &beaconStorageCache{},
Expand All @@ -47,7 +47,7 @@ func NewBeaconStorage(config storage2.PortalStorageConfig) (storage2.ContentStor
}

var err error
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, config.DB)
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, db)
if err != nil {
return nil, err
}
Expand Down
3 changes: 1 addition & 2 deletions beacon/storage_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,10 @@ func genStorage(testDir string) (storage.ContentStorage, error) {
}
config := &storage.PortalStorageConfig{
StorageCapacityMB: 1000,
DB: db,
NodeId: enode.ID(zeroNodeId),
Spec: configs.Mainnet,
}
return NewBeaconStorage(*config)
return NewBeaconStorage(*config, db)
}

type entry struct {
Expand Down
19 changes: 8 additions & 11 deletions cmd/shisui/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import (
"github.com/optimism-java/shisui2/portalwire"
"github.com/optimism-java/shisui2/state"
"github.com/optimism-java/shisui2/storage"
"github.com/optimism-java/shisui2/storage/sqlite"
"github.com/optimism-java/shisui2/storage/pebble"
"github.com/optimism-java/shisui2/web3"
"github.com/protolambda/zrnt/eth2/configs"
"github.com/urfave/cli/v2"
Expand Down Expand Up @@ -379,16 +379,15 @@ func doPortMapping(natm nat.Interface, ln *enode.LocalNode, addr *net.UDPAddr) {

func initHistory(config Config, server *rpc.Server, conn discover.UDPConn, localNode *enode.LocalNode, discV5 *discover.UDPv5, utp *portalwire.PortalUtp) (*history.Network, error) {
networkName := portalwire.History.Name()
db, err := sqlite.NewDB(config.DataDir, networkName)
db, err := pebble.NewDB(config.DataDir, 16, 400, networkName)
if err != nil {
return nil, err
}
contentStorage, err := sqlite.NewHistoryStorage(storage.PortalStorageConfig{
contentStorage, err := pebble.NewStorage(storage.PortalStorageConfig{
StorageCapacityMB: config.DataCapacity,
DB: db,
NodeId: localNode.ID(),
NetworkName: networkName,
})
}, db)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -435,11 +434,10 @@ func initBeacon(config Config, server *rpc.Server, conn discover.UDPConn, localN

contentStorage, err := beacon.NewBeaconStorage(storage.PortalStorageConfig{
StorageCapacityMB: config.DataCapacity,
DB: sqlDb,
NodeId: localNode.ID(),
Spec: configs.Mainnet,
NetworkName: portalwire.Beacon.Name(),
})
}, sqlDb)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -473,16 +471,15 @@ func initBeacon(config Config, server *rpc.Server, conn discover.UDPConn, localN

func initState(config Config, server *rpc.Server, conn discover.UDPConn, localNode *enode.LocalNode, discV5 *discover.UDPv5, utp *portalwire.PortalUtp) (*state.Network, error) {
networkName := portalwire.State.Name()
db, err := sqlite.NewDB(config.DataDir, networkName)
db, err := pebble.NewDB(config.DataDir, 16, 400, networkName)
if err != nil {
return nil, err
}
contentStorage, err := sqlite.NewHistoryStorage(storage.PortalStorageConfig{
contentStorage, err := pebble.NewStorage(storage.PortalStorageConfig{
StorageCapacityMB: config.DataCapacity,
DB: db,
NodeId: localNode.ID(),
NetworkName: networkName,
})
}, db)
if err != nil {
return nil, err
}
Expand Down
16 changes: 8 additions & 8 deletions state/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package state
import (
"bytes"
"crypto/sha256"
"database/sql"
"errors"

"github.com/cockroachdb/pebble"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
Expand All @@ -24,24 +24,24 @@ var _ storage.ContentStorage = &Storage{}

type Storage struct {
store storage.ContentStorage
db *sql.DB
db *pebble.DB
log log.Logger
}

var portalStorageMetrics *portalwire.PortalStorageMetrics

func NewStateStorage(store storage.ContentStorage, db *sql.DB) *Storage {
func NewStateStorage(store storage.ContentStorage, db *pebble.DB) *Storage {
stateStorage := &Storage{
store: store,
db: db,
log: log.New("storage", "state"),
}

var err error
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics("state", db)
if err != nil {
return nil
}
// var err error
// portalStorageMetrics, err = portalwire.NewPortalStorageMetrics("state", db)
// if err != nil {
// return nil
// }

return stateStorage
}
Expand Down
3 changes: 0 additions & 3 deletions storage/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,12 @@
package storage

import (
"database/sql"

"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/protolambda/zrnt/eth2/beacon/common"
)

type PortalStorageConfig struct {
StorageCapacityMB uint64
DB *sql.DB
NodeId enode.ID
Spec *common.Spec
NetworkName string
Expand Down
15 changes: 4 additions & 11 deletions storage/pebble/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package pepple
package pebble

import (
"bytes"
Expand Down Expand Up @@ -31,14 +31,7 @@ const (

var _ storage.ContentStorage = &ContentStorage{}

type PeppleStorageConfig struct {
StorageCapacityMB uint64
DB *pebble.DB
NodeId enode.ID
NetworkName string
}

func NewPeppleDB(dataDir string, cache, handles int, namespace string) (*pebble.DB, error) {
func NewDB(dataDir string, cache, handles int, namespace string) (*pebble.DB, error) {
// Ensure we have some minimal caching and file guarantees
if cache < minCache {
cache = minCache
Expand Down Expand Up @@ -128,10 +121,10 @@ type ContentStorage struct {
bytePool sync.Pool
}

func NewPeppleStorage(config PeppleStorageConfig) (storage.ContentStorage, error) {
func NewStorage(config storage.PortalStorageConfig, db *pebble.DB) (storage.ContentStorage, error) {
cs := &ContentStorage{
nodeId: config.NodeId,
db: config.DB,
db: db,
storageCapacityInBytes: config.StorageCapacityMB * 1000_000,
log: log.New("storage", config.NetworkName),
writeOptions: &pebble.WriteOptions{Sync: false},
Expand Down
11 changes: 5 additions & 6 deletions storage/pebble/storage_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package pepple
package pebble

import (
"testing"
Expand All @@ -18,26 +18,25 @@ func genBytes(length int) []byte {
}

func TestNewPeppleDB(t *testing.T) {
db, err := NewPeppleDB(t.TempDir(), 16, 16, "test")
db, err := NewDB(t.TempDir(), 16, 16, "test")
assert.NoError(t, err)
defer db.Close()

assert.NotNil(t, db)
}

func setupTestStorage(t *testing.T) storage.ContentStorage {
db, err := NewPeppleDB(t.TempDir(), 16, 16, "test")
db, err := NewDB(t.TempDir(), 16, 16, "test")
assert.NoError(t, err)
t.Cleanup(func() { db.Close() })

config := PeppleStorageConfig{
config := storage.PortalStorageConfig{
StorageCapacityMB: 1,
DB: db,
NodeId: uint256.NewInt(0).Bytes32(),
NetworkName: "test",
}

storage, err := NewPeppleStorage(config)
storage, err := NewStorage(config, db)
assert.NoError(t, err)
return storage
}
Expand Down
6 changes: 3 additions & 3 deletions storage/sqlite/storage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,10 @@ func NewDB(dataDir string, network string) (*sql.DB, error) {
return sqlDb, err
}

func NewHistoryStorage(config storage.PortalStorageConfig) (storage.ContentStorage, error) {
func NewStorage(config storage.PortalStorageConfig, db *sql.DB) (storage.ContentStorage, error) {
hs := &ContentStorage{
nodeId: config.NodeId,
sqliteDB: config.DB,
sqliteDB: db,
storageCapacityInBytes: config.StorageCapacityMB * 1000000,
log: log.New("storage", config.NetworkName),
}
Expand All @@ -124,7 +124,7 @@ func NewHistoryStorage(config storage.PortalStorageConfig) (storage.ContentStora

// necessary to test NetworkName==history because state also initialize HistoryStorage
if strings.ToLower(config.NetworkName) == "history" {
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, config.DB)
portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, db)
if err != nil {
return nil, err
}
Expand Down
5 changes: 2 additions & 3 deletions storage/sqlite/storage_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,10 @@ func newContentStorage(storageCapacityInMB uint64, nodeId enode.ID) (*ContentSto
if err != nil {
return nil, err
}
hs, err := NewHistoryStorage(contentStorage.PortalStorageConfig{
DB: db,
hs, err := NewStorage(contentStorage.PortalStorageConfig{
StorageCapacityMB: storageCapacityInMB,
NodeId: nodeId,
})
}, db)
if err != nil {
return nil, err
}
Expand Down
12 changes: 5 additions & 7 deletions storage/tests/storage_bench_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,13 +48,12 @@ func BenchmarkStorageComparison(b *testing.B) {

b.Run("Pebble_"+tc.name, func(b *testing.B) {
dir, _ := os.MkdirTemp("", "pebble-bench-*")
db, _ := ethpepple.NewPeppleDB(dir, 16, 16, "bench")
storage, _ := ethpepple.NewPeppleStorage(ethpepple.PeppleStorageConfig{
db, _ := ethpepple.NewDB(dir, 16, 16, "bench")
storage, _ := ethpepple.NewStorage(storage.PortalStorageConfig{
StorageCapacityMB: 1000,
DB: db,
NodeId: enode.ID{},
NetworkName: "bench",
})
}, db)
defer func() {
db.Close()
os.RemoveAll(dir)
Expand All @@ -77,12 +76,11 @@ func BenchmarkStorageComparison(b *testing.B) {
b.Run("SQLite_"+tc.name, func(b *testing.B) {
dir, _ := os.MkdirTemp("", "sqlite-bench-*")
db, _ := sqlite.NewDB(dir, "bench")
storage, _ := sqlite.NewHistoryStorage(storage.PortalStorageConfig{
storage, _ := sqlite.NewStorage(storage.PortalStorageConfig{
StorageCapacityMB: 1000,
DB: db,
NodeId: enode.ID{},
NetworkName: "bench",
})
}, db)
defer func() {
storage.Close()
os.RemoveAll(dir)
Expand Down
Loading