diff --git a/lib/block/block.go b/lib/block/block.go index 3b99310ba..261968773 100644 --- a/lib/block/block.go +++ b/lib/block/block.go @@ -78,7 +78,7 @@ func (b Block) NewBlockKeyConfirmed() string { return fmt.Sprintf( "%s%s-%s%s", common.BlockPrefixConfirmed, b.ProposedTime, - common.EncodeUint64ToByteSlice(b.Height), + common.EncodeUint64ToString(b.Height), common.GetUniqueIDFromUUID(), ) } diff --git a/lib/block/genesis.go b/lib/block/genesis.go index 761c9d84d..a941e57fb 100644 --- a/lib/block/genesis.go +++ b/lib/block/genesis.go @@ -117,7 +117,7 @@ func MakeGenesisBlock(st *storage.LevelDBBackend, genesisAccount BlockAccount, c return } - bt := NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx) + bt := NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx, 1) // first tx (not proposer tx) if err = bt.Save(st); err != nil { return } diff --git a/lib/block/operation.go b/lib/block/operation.go index 9df36475a..5e533ddab 100644 --- a/lib/block/operation.go +++ b/lib/block/operation.go @@ -3,6 +3,7 @@ package block import ( "encoding/json" "fmt" + "strconv" "boscoin.io/sebak/lib/common" "boscoin.io/sebak/lib/errors" @@ -24,25 +25,27 @@ type BlockOperation struct { OpHash string `json:"op_hash"` TxHash string `json:"tx_hash"` - Type operation.OperationType `json:"type"` - Source string `json:"source"` - Target string `json:"target"` - Body []byte `json:"body"` - Height uint64 `json:"block_height"` + Type operation.OperationType `json:"type"` + Source string `json:"source"` + Target string `json:"target"` + Body []byte `json:"body"` + Height uint64 `json:"block_height"` + Index uint64 `json:"index"` + TxIndex uint64 `json:"tx_index"` // bellows will be used only for `Save` time. transaction transaction.Transaction operation operation.Operation linked string isSaved bool - opIndex int + order *BlockOrder } -func NewBlockOperationKey(opHash, txHash string) string { - return fmt.Sprintf("%s-%s", opHash, txHash) +func NewBlockOperationKey(opHash, txHash string, index uint64) string { + return common.MustMakeObjectHashString([]string{opHash, txHash, strconv.FormatUint(index, 10)}) } -func NewBlockOperationFromOperation(op operation.Operation, tx transaction.Transaction, blockHeight uint64, opIndex int) (BlockOperation, error) { +func NewBlockOperationFromOperation(op operation.Operation, tx transaction.Transaction, blockHeight uint64, txIndex uint64, opIndex int) (BlockOperation, error) { body, err := json.Marshal(op.B) if err != nil { return BlockOperation{}, err @@ -62,23 +65,26 @@ func NewBlockOperationFromOperation(op operation.Operation, tx transaction.Trans linked = createAccount.Linked } } + order := NewBlockOpOrder(blockHeight, txIndex, uint64(opIndex)) return BlockOperation{ - Hash: NewBlockOperationKey(opHash, txHash), + Hash: NewBlockOperationKey(opHash, txHash, uint64(opIndex)), OpHash: opHash, TxHash: txHash, - Type: op.H.Type, - Source: tx.B.Source, - Target: target, - Body: body, - Height: blockHeight, + Type: op.H.Type, + Source: tx.B.Source, + Target: target, + Body: body, + Height: blockHeight, + Index: uint64(opIndex), + TxIndex: uint64(txIndex), transaction: tx, operation: op, linked: linked, - opIndex: opIndex, + order: order, }, nil } @@ -126,7 +132,7 @@ func (bo *BlockOperation) Save(st *storage.LevelDBBackend) (err error) { if err = st.New(bo.NewBlockOperationPeersKey(bo.Source), bo.Hash); err != nil { return } - if err = st.New(bo.NewBlockOperationPeersAndTypeKey(bo.Source), bo.Hash); err != nil { + if err = st.New(bo.NewBlockOperationPeersAndTypeKey(bo.Source, bo.Type), bo.Hash); err != nil { return } if err = st.New(bo.NewBlockOperationBlockHeightKey(), bo.Hash); err != nil { @@ -137,13 +143,13 @@ func (bo *BlockOperation) Save(st *storage.LevelDBBackend) (err error) { if err = st.New(bo.NewBlockOperationTargetKey(bo.Target), bo.Hash); err != nil { return } - if err = st.New(bo.NewBlockOperationTargetAndTypeKey(bo.Target), bo.Hash); err != nil { + if err = st.New(bo.NewBlockOperationTargetAndTypeKey(bo.Target, bo.Type), bo.Hash); err != nil { return } if err = st.New(bo.NewBlockOperationPeersKey(bo.Target), bo.Hash); err != nil { return } - if err = st.New(bo.NewBlockOperationPeersAndTypeKey(bo.Target), bo.Hash); err != nil { + if err = st.New(bo.NewBlockOperationPeersAndTypeKey(bo.Target, bo.Type), bo.Hash); err != nil { return } } @@ -166,15 +172,6 @@ func key(hash string) string { return fmt.Sprintf("%s%s", common.BlockOperationPrefixHash, hash) } -func GetBlockOperationCreateFrozenKey(hash string, height uint64) string { - return fmt.Sprintf( - "%s%s%s", - common.BlockOperationPrefixCreateFrozen, - common.EncodeUint64ToByteSlice(height), - hash, - ) -} - func keyPrefixFrozenLinked(hash string) string { return fmt.Sprintf( "%s%s", @@ -196,7 +193,7 @@ func keyPrefixSourceAndType(source string, ty operation.OperationType) string { } func keyPrefixBlockHeight(height uint64) string { - return fmt.Sprintf("%s%s-", common.BlockOperationPrefixBlockHeight, common.EncodeUint64ToByteSlice(height)) + return fmt.Sprintf("%s%s-", common.BlockOperationPrefixBlockHeight, common.EncodeUint64ToString(height)) } func keyPrefixTarget(target string) string { @@ -215,89 +212,141 @@ func keyPrefixPeersAndType(addr string, ty operation.OperationType) string { return fmt.Sprintf("%s%s%s-", common.BlockOperationPrefixTypePeers, string(ty), addr) } +func GetBlockOperationKey(hash string) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixHash, hash) + return idx.String() +} + +func GetBlockOperationCreateFrozenKey(hash string, height uint64) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixCreateFrozen) + idx.WritePrefix(common.EncodeUint64ToString(height)) + idx.WritePrefix(hash) + return idx.String() +} + +func GetBlockOperationKeyPrefixFrozenLinked(hash string) string { + idx := storage.NewIndex() + return idx.WritePrefix(common.BlockOperationPrefixFrozenLinked, hash).String() +} + +func GetBlockOperationKeyPrefixTxHash(txHash string) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixTxHash, txHash) + return idx.String() +} + +func GetBlockOperationKeyPrefixSource(source string) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixSource, source) + return idx.String() +} + +func GetBlockOperationKeyPrefixSourceAndType(source string, ty operation.OperationType) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixTypeSource, string(ty), source) + return idx.String() +} + +func GetBlockOperationKeyPrefixBlockHeight(height uint64) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixBlockHeight) + idx.WritePrefix(common.EncodeUint64ToString(height)) + return idx.String() +} + +func GetBlockOperationKeyPrefixTarget(target string) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixTarget, target) + return idx.String() +} + +func GetBlockOperationKeyPrefixTargetAndType(target string, ty operation.OperationType) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixTypeTarget) + idx.WritePrefix(string(ty), target) + return idx.String() +} + +func GetBlockOperationKeyPrefixPeers(addr string) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixPeers, addr) + return idx.String() +} + +func GetBlockOperationKeyPrefixPeersAndType(addr string, ty operation.OperationType) string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockOperationPrefixTypePeers) + idx.WritePrefix(string(ty), addr) + return idx.String() +} + func (bo BlockOperation) NewBlockOperationTxHashKey() string { - return fmt.Sprintf( - "%s%s%s%s", - keyPrefixTxHash(bo.TxHash), - common.EncodeUint64ToByteSlice(bo.Height), - common.EncodeUint64ToByteSlice(bo.transaction.B.SequenceID), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockOperationKeyPrefixTxHash(bo.TxHash)) + bo.order.Index(idx) + return idx.String() } func (bo BlockOperation) NewBlockOperationSourceKey() string { - return fmt.Sprintf( - "%s%s%s%s", - keyPrefixSource(bo.Source), - common.EncodeUint64ToByteSlice(bo.Height), - common.EncodeUint64ToByteSlice(bo.transaction.B.SequenceID), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockOperationKeyPrefixSource(bo.Source)) + bo.order.Index(idx) + return idx.String() } func (bo BlockOperation) NewBlockOperationFrozenLinkedKey(hash string) string { - return fmt.Sprintf( - "%s%s", - keyPrefixFrozenLinked(hash), - common.EncodeUint64ToByteSlice(bo.Height), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockOperationKeyPrefixFrozenLinked(hash)) + bo.order.Index(idx) + return idx.String() } func (bo BlockOperation) NewBlockOperationSourceAndTypeKey() string { - return fmt.Sprintf( - "%s%s%s%s", - keyPrefixSourceAndType(bo.Source, bo.Type), - common.EncodeUint64ToByteSlice(bo.Height), - common.EncodeUint64ToByteSlice(bo.transaction.B.SequenceID), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockOperationKeyPrefixSourceAndType(bo.Source, bo.Type)) + bo.order.Index(idx) + return idx.String() } + func (bo BlockOperation) NewBlockOperationTargetKey(target string) string { - return fmt.Sprintf( - "%s%s%s%s", - keyPrefixTarget(target), - common.EncodeUint64ToByteSlice(bo.Height), - common.EncodeUint64ToByteSlice(bo.transaction.B.SequenceID), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockOperationKeyPrefixTarget(target)) + bo.order.Index(idx) + return idx.String() } -func (bo BlockOperation) NewBlockOperationTargetAndTypeKey(target string) string { - return fmt.Sprintf( - "%s%s%s%s", - keyPrefixTargetAndType(target, bo.Type), - common.EncodeUint64ToByteSlice(bo.Height), - common.EncodeUint64ToByteSlice(bo.transaction.B.SequenceID), - common.GetUniqueIDFromUUID(), - ) +func (bo BlockOperation) NewBlockOperationTargetAndTypeKey(target string, ty operation.OperationType) string { + idx := storage.NewIndex() + idx.WritePrefix(GetBlockOperationKeyPrefixTargetAndType(target, ty)) + bo.order.Index(idx) + return idx.String() } func (bo BlockOperation) NewBlockOperationPeersKey(addr string) string { - return fmt.Sprintf( - "%s%s%s%s", - keyPrefixPeers(addr), - common.EncodeUint64ToByteSlice(bo.Height), - common.EncodeUint64ToByteSlice(bo.transaction.B.SequenceID), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockOperationKeyPrefixPeers(addr)) + bo.order.Index(idx) + return idx.String() } -func (bo BlockOperation) NewBlockOperationPeersAndTypeKey(addr string) string { - return fmt.Sprintf( - "%s%s%s%s", - keyPrefixPeersAndType(addr, bo.Type), - common.EncodeUint64ToByteSlice(bo.Height), - common.EncodeUint64ToByteSlice(bo.transaction.B.SequenceID), - common.GetUniqueIDFromUUID(), - ) +func (bo BlockOperation) NewBlockOperationPeersAndTypeKey(addr string, ty operation.OperationType) string { + idx := storage.NewIndex() + idx.WritePrefix(GetBlockOperationKeyPrefixPeersAndType(addr, ty)) + bo.order.Index(idx) + return idx.String() } + func (bo BlockOperation) NewBlockOperationBlockHeightKey() string { - return fmt.Sprintf( - "%s%s%s", - keyPrefixBlockHeight(bo.Height), - common.EncodeUint64ToByteSlice(bo.transaction.B.SequenceID), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockOperationKeyPrefixBlockHeight(bo.Height)) + bo.order.Index(idx) + return idx.String() +} + +func (bo BlockOperation) BlockOrder() *BlockOrder { + return bo.order } func ExistsBlockOperation(st *storage.LevelDBBackend, hash string) (bool, error) { @@ -310,6 +359,7 @@ func GetBlockOperation(st *storage.LevelDBBackend, hash string) (bo BlockOperati } bo.isSaved = true + bo.order = NewBlockOpOrder(bo.Height, bo.TxIndex, bo.Index) return } @@ -346,7 +396,7 @@ func LoadBlockOperationsInsideIterator( return (func() (BlockOperation, bool, []byte) { item, hasNext := iterFunc() - if !hasNext { + if !hasNext && (item.Key == nil || item.Value == nil) { return BlockOperation{}, false, item.Key } @@ -368,7 +418,7 @@ func GetBlockOperationsByTx(st *storage.LevelDBBackend, txHash string, options s func() (BlockOperation, bool, []byte), func(), ) { - iterFunc, closeFunc := st.GetIterator(keyPrefixTxHash(txHash), options) + iterFunc, closeFunc := st.GetIterator(GetBlockOperationKeyPrefixTxHash(txHash), options) return LoadBlockOperationsInsideIterator(st, iterFunc, closeFunc) } @@ -377,7 +427,7 @@ func GetBlockOperationsBySource(st *storage.LevelDBBackend, source string, optio func() (BlockOperation, bool, []byte), func(), ) { - iterFunc, closeFunc := st.GetIterator(keyPrefixSource(source), options) + iterFunc, closeFunc := st.GetIterator(GetBlockOperationKeyPrefixSource(source), options) return LoadBlockOperationsInsideIterator(st, iterFunc, closeFunc) } @@ -396,7 +446,7 @@ func GetBlockOperationsByLinked(st *storage.LevelDBBackend, hash string, options func() (BlockOperation, bool, []byte), func(), ) { - iterFunc, closeFunc := st.GetIterator(keyPrefixFrozenLinked(hash), options) + iterFunc, closeFunc := st.GetIterator(GetBlockOperationKeyPrefixFrozenLinked(hash), options) return LoadBlockOperationsInsideIterator(st, iterFunc, closeFunc) } @@ -404,7 +454,7 @@ func GetBlockOperationsBySourceAndType(st *storage.LevelDBBackend, source string func() (BlockOperation, bool, []byte), func(), ) { - iterFunc, closeFunc := st.GetIterator(keyPrefixSourceAndType(source, ty), options) + iterFunc, closeFunc := st.GetIterator(GetBlockOperationKeyPrefixSourceAndType(source, ty), options) return LoadBlockOperationsInsideIterator(st, iterFunc, closeFunc) } @@ -412,7 +462,7 @@ func GetBlockOperationsByTarget(st *storage.LevelDBBackend, target string, optio func() (BlockOperation, bool, []byte), func(), ) { - iterFunc, closeFunc := st.GetIterator(keyPrefixTarget(target), options) + iterFunc, closeFunc := st.GetIterator(GetBlockOperationKeyPrefixTarget(target), options) return LoadBlockOperationsInsideIterator(st, iterFunc, closeFunc) } @@ -421,7 +471,7 @@ func GetBlockOperationsByTargetAndType(st *storage.LevelDBBackend, target string func() (BlockOperation, bool, []byte), func(), ) { - iterFunc, closeFunc := st.GetIterator(keyPrefixTargetAndType(target, ty), options) + iterFunc, closeFunc := st.GetIterator(GetBlockOperationKeyPrefixTargetAndType(target, ty), options) return LoadBlockOperationsInsideIterator(st, iterFunc, closeFunc) } @@ -429,7 +479,7 @@ func GetBlockOperationsByPeers(st *storage.LevelDBBackend, addr string, options func() (BlockOperation, bool, []byte), func(), ) { - iterFunc, closeFunc := st.GetIterator(keyPrefixPeers(addr), options) + iterFunc, closeFunc := st.GetIterator(GetBlockOperationKeyPrefixPeers(addr), options) return LoadBlockOperationsInsideIterator(st, iterFunc, closeFunc) } @@ -438,7 +488,7 @@ func GetBlockOperationsByPeersAndType(st *storage.LevelDBBackend, addr string, t func() (BlockOperation, bool, []byte), func(), ) { - iterFunc, closeFunc := st.GetIterator(keyPrefixPeersAndType(addr, ty), options) + iterFunc, closeFunc := st.GetIterator(GetBlockOperationKeyPrefixPeersAndType(addr, ty), options) return LoadBlockOperationsInsideIterator(st, iterFunc, closeFunc) } @@ -446,6 +496,6 @@ func GetBlockOperationsByBlockHeight(st *storage.LevelDBBackend, height uint64, func() (BlockOperation, bool, []byte), func(), ) { - iterFunc, closeFunc := st.GetIterator(keyPrefixBlockHeight(height), options) + iterFunc, closeFunc := st.GetIterator(GetBlockOperationKeyPrefixBlockHeight(height), options) return LoadBlockOperationsInsideIterator(st, iterFunc, closeFunc) } diff --git a/lib/block/operation_test.go b/lib/block/operation_test.go index 43f818f6d..f9d237741 100644 --- a/lib/block/operation_test.go +++ b/lib/block/operation_test.go @@ -15,8 +15,11 @@ func TestNewBlockOperationFromOperation(t *testing.T) { conf := common.NewTestConfig() _, tx := transaction.TestMakeTransaction(conf.NetworkID, 1) + var index int = 0 + var txIndex uint64 = 1 + op := tx.B.Operations[0] - bo, err := NewBlockOperationFromOperation(op, tx, 0, 0) + bo, err := NewBlockOperationFromOperation(op, tx, 0, txIndex, index) require.NoError(t, err) require.Equal(t, bo.Type, op.H.Type) @@ -24,13 +27,15 @@ func TestNewBlockOperationFromOperation(t *testing.T) { require.Equal(t, bo.Source, tx.B.Source) encoded := common.MustMarshalJSON(op.B) require.Equal(t, bo.Body, encoded) + require.Equal(t, bo.Index, uint64(index)) + require.Equal(t, bo.TxIndex, txIndex) } func TestBlockOperationSaveAndGet(t *testing.T) { conf := common.NewTestConfig() st := storage.NewTestStorage() - bos := TestMakeNewBlockOperation(conf.NetworkID, 1) + bos := TestMakeNewBlockOperation(conf.NetworkID, 1, 0) bo := bos[0] bos[0].MustSave(st) @@ -41,13 +46,14 @@ func TestBlockOperationSaveAndGet(t *testing.T) { require.Equal(t, bo.Hash, fetched.Hash) require.Equal(t, bo.Source, fetched.Source) require.Equal(t, bo.Body, fetched.Body) + require.Equal(t, bo.Index, uint64(0)) } func TestBlockOperationSaveExisting(t *testing.T) { conf := common.NewTestConfig() st := storage.NewTestStorage() - bos := TestMakeNewBlockOperation(conf.NetworkID, 1) + bos := TestMakeNewBlockOperation(conf.NetworkID, 1, 0) bo := bos[0] bo.MustSave(st) @@ -67,8 +73,8 @@ func TestGetSortedBlockOperationsByTxHash(t *testing.T) { // create 30 `BlockOperation` var txHashes []string createdOrder := map[string][]string{} - for _ = range [3]int{0, 0, 0} { - bos := TestMakeNewBlockOperation(conf.NetworkID, 10) + for _, i := range [3]uint64{0, 1, 2} { + bos := TestMakeNewBlockOperation(conf.NetworkID, 10, i) txHashes = append(txHashes, bos[0].TxHash) for _, bo := range bos { @@ -93,6 +99,7 @@ func TestGetSortedBlockOperationsByTxHash(t *testing.T) { for i, bo := range saved { require.Equal(t, bo.Hash, createdOrder[bo.TxHash][i]) + require.Equal(t, bo.Index, uint64(i)) } } } @@ -103,7 +110,7 @@ func TestBlockOperationSaveByTransaction(t *testing.T) { _, tx := transaction.TestMakeTransaction(conf.NetworkID, 10) block := TestMakeNewBlockWithPrevBlock(GetLatestBlock(st), []string{tx.GetHash()}) - bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx) + bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx, 0) err := bt.Save(st) require.NoError(t, err) @@ -136,7 +143,7 @@ func TestBlockOperationsByBlockHeight(t *testing.T) { heights := []uint64{1, 2, 3} created := map[uint64][]string{} for _, height := range heights { - bos := TestMakeNewBlockOperation(conf.NetworkID, 10) + bos := TestMakeNewBlockOperation(conf.NetworkID, 10, 0) for _, bo := range bos { bo.Height = height diff --git a/lib/block/order.go b/lib/block/order.go new file mode 100644 index 000000000..35e293e07 --- /dev/null +++ b/lib/block/order.go @@ -0,0 +1,56 @@ +package block + +import ( + "strconv" + "strings" + + "boscoin.io/sebak/lib/common" + "boscoin.io/sebak/lib/storage" +) + +type BlockOrder struct { + parts []uint64 // height,txindex,opindex +} + +func NewBlockOrder(height uint64) *BlockOrder { + b := &BlockOrder{ + parts: []uint64{height}, + } + return b +} + +func NewBlockTxOrder(height, index uint64) *BlockOrder { + b := &BlockOrder{ + parts: []uint64{height, index}, + } + return b +} + +func NewBlockOpOrder(height, txindex, index uint64) *BlockOrder { + b := &BlockOrder{ + parts: []uint64{height, txindex, index}, + } + return b +} + +func (o *BlockOrder) Index(idx *storage.Index) string { + for _, x := range o.parts { + idx.WriteOrder(common.EncodeUint64ToString(x)) + } + return idx.String() +} + +func (o *BlockOrder) formatText(xs []uint64) string { + var ss []string + for _, p := range xs { + ss = append(ss, strconv.FormatUint(p, 10)) + } + return strings.Join(ss, "-") +} + +func (o *BlockOrder) String() string { + if o == nil || o.parts == nil { + return "" + } + return o.formatText(o.parts) +} diff --git a/lib/block/test.go b/lib/block/test.go index 8725d92a6..3e0100892 100644 --- a/lib/block/test.go +++ b/lib/block/test.go @@ -129,11 +129,11 @@ func TestMakeNewBlockWithPrevBlock(prevBlock Block, txs []string) Block { ) } -func TestMakeNewBlockOperation(networkID []byte, n int) (bos []BlockOperation) { +func TestMakeNewBlockOperation(networkID []byte, n int, txindex uint64) (bos []BlockOperation) { _, tx := transaction.TestMakeTransaction(networkID, n) for i, op := range tx.B.Operations { - bo, err := NewBlockOperationFromOperation(op, tx, 0, i) + bo, err := NewBlockOperationFromOperation(op, tx, 0, txindex, i) if err != nil { panic(err) } diff --git a/lib/block/transaction.go b/lib/block/transaction.go index 3473c1002..ea0616152 100644 --- a/lib/block/transaction.go +++ b/lib/block/transaction.go @@ -2,7 +2,6 @@ package block import ( "encoding/json" - "fmt" "boscoin.io/sebak/lib/common" "boscoin.io/sebak/lib/errors" @@ -37,70 +36,83 @@ type BlockTransaction struct { Created string `json:"created"` Message []byte `json:"message"` + Index uint64 `json:"index"` + BlockHeight uint64 `json:"block_height"` + transaction transaction.Transaction isSaved bool - blockHeight uint64 + order *BlockOrder } -func NewBlockTransactionFromTransaction(blockHash string, blockHeight uint64, confirmed string, tx transaction.Transaction) BlockTransaction { +func NewBlockTransactionFromTransaction(blockHash string, blockHeight uint64, confirmed string, tx transaction.Transaction, index uint64) BlockTransaction { var opHashes []string - for _, op := range tx.B.Operations { - opHashes = append(opHashes, NewBlockOperationKey(op.MakeHashString(), tx.GetHash())) + for i, op := range tx.B.Operations { + opHashes = append(opHashes, NewBlockOperationKey(op.MakeHashString(), tx.GetHash(), uint64(i))) } + order := NewBlockTxOrder(blockHeight, index) + return BlockTransaction{ - Hash: tx.H.Hash, - Block: blockHash, - SequenceID: tx.B.SequenceID, - Signature: tx.H.Signature, - Source: tx.B.Source, - Fee: tx.B.Fee, - Operations: opHashes, - Amount: tx.TotalAmount(true), - Confirmed: confirmed, - Created: tx.H.Created, + Hash: tx.H.Hash, + Block: blockHash, + SequenceID: tx.B.SequenceID, + Signature: tx.H.Signature, + Source: tx.B.Source, + Fee: tx.B.Fee, + Operations: opHashes, + Amount: tx.TotalAmount(true), + Confirmed: confirmed, + Created: tx.H.Created, + Index: index, + BlockHeight: blockHeight, transaction: tx, - blockHeight: blockHeight, + order: order, } } func (bt BlockTransaction) NewBlockTransactionKeySource() string { - return fmt.Sprintf( - "%s%s%s%s", - GetBlockTransactionKeyPrefixSource(bt.Source), - common.EncodeUint64ToByteSlice(bt.blockHeight), - common.EncodeUint64ToByteSlice(bt.SequenceID), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockTransactionKeyPrefixSource(bt.Source)) + bt.order.Index(idx) + return idx.String() } func (bt BlockTransaction) NewBlockTransactionKeyConfirmed() string { - return fmt.Sprintf( - "%s%s", - GetBlockTransactionKeyPrefixConfirmed(bt.Confirmed), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockTransactionKeyPrefixConfirmed(bt.Confirmed)) + bt.order.Index(idx) + return idx.String() +} + +func (bt BlockTransaction) NewBlockTransactionKeyAll() string { + idx := storage.NewIndex() + idx.WritePrefix(common.BlockTransactionPrefixAll) + bt.order.Index(idx) + return idx.String() } func (bt BlockTransaction) NewBlockTransactionKeyByAccount(accountAddress string) string { - return fmt.Sprintf( - "%s%s%s%s", - GetBlockTransactionKeyPrefixAccount(accountAddress), - common.EncodeUint64ToByteSlice(bt.blockHeight), - common.EncodeUint64ToByteSlice(bt.SequenceID), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockTransactionKeyPrefixAccount(accountAddress)) + bt.order.Index(idx) + idx.WriteOrder(common.GetUniqueIDFromUUID()) + return idx.String() +} + +func (bt BlockTransaction) NewBlockTransactionKeyByTarget(accountAddress string) string { + idx := storage.NewIndex() + idx.WritePrefix(GetBlockTransactionKeyPrefixTarget(accountAddress)) + bt.order.Index(idx) + idx.WriteOrder(common.GetUniqueIDFromUUID()) + return idx.String() } func (bt BlockTransaction) NewBlockTransactionKeyByBlock(hash string) string { - return fmt.Sprintf( - "%s%s%s%s", - GetBlockTransactionKeyPrefixBlock(hash), - common.EncodeUint64ToByteSlice(bt.blockHeight), - common.EncodeUint64ToByteSlice(bt.SequenceID), - common.GetUniqueIDFromUUID(), - ) + idx := storage.NewIndex() + idx.WritePrefix(GetBlockTransactionKeyPrefixBlock(hash)) + bt.order.Index(idx) + return idx.String() } func (bt *BlockTransaction) Save(st *storage.LevelDBBackend) (err error) { @@ -121,6 +133,9 @@ func (bt *BlockTransaction) Save(st *storage.LevelDBBackend) (err error) { if err = st.New(GetBlockTransactionKey(bt.Hash), bt); err != nil { return } + if err = st.New(bt.NewBlockTransactionKeyAll(), bt.Hash); err != nil { + return + } if err = st.New(bt.NewBlockTransactionKeySource(), bt.Hash); err != nil { return } @@ -169,12 +184,12 @@ func (bt *BlockTransaction) SaveBlockOperations(st *storage.LevelDBBackend) (err return errors.FailedToSaveBlockOperaton } - if bt.blockHeight < 1 { + if bt.BlockHeight < 1 { var blk Block if blk, err = GetBlock(st, bt.Block); err != nil { return } else { - bt.blockHeight = blk.Height + bt.BlockHeight = blk.Height } } @@ -188,17 +203,17 @@ func (bt *BlockTransaction) SaveBlockOperations(st *storage.LevelDBBackend) (err } func (bt *BlockTransaction) SaveBlockOperation(st *storage.LevelDBBackend, op operation.Operation, opIndex int) (err error) { - if bt.blockHeight < 1 { + if bt.BlockHeight < 1 { var blk Block if blk, err = GetBlock(st, bt.Block); err != nil { return } else { - bt.blockHeight = blk.Height + bt.BlockHeight = blk.Height } } var bo BlockOperation - bo, err = NewBlockOperationFromOperation(op, bt.Transaction(), bt.blockHeight, opIndex) + bo, err = NewBlockOperationFromOperation(op, bt.Transaction(), bt.BlockHeight, bt.Index, opIndex) if err != nil { return } @@ -210,6 +225,10 @@ func (bt *BlockTransaction) SaveBlockOperation(st *storage.LevelDBBackend, op op if err != nil { return } + err = st.New(bt.NewBlockTransactionKeyByTarget(pop.TargetAddress()), bt.Hash) + if err != nil { + return + } } return nil @@ -229,24 +248,38 @@ func (bt *BlockTransaction) GetOperationIndex(opHash string) (opIndex int, err e return } +func (bt BlockTransaction) BlockOrder() *BlockOrder { + return bt.order +} + func GetBlockTransactionKeyPrefixSource(source string) string { - return fmt.Sprintf("%s%s-", common.BlockTransactionPrefixSource, source) + idx := storage.NewIndex() + return idx.WritePrefix(common.BlockTransactionPrefixSource, source).String() } func GetBlockTransactionKeyPrefixConfirmed(confirmed string) string { - return fmt.Sprintf("%s%s-", common.BlockTransactionPrefixConfirmed, confirmed) + idx := storage.NewIndex() + return idx.WritePrefix(common.BlockTransactionPrefixConfirmed, confirmed).String() } func GetBlockTransactionKeyPrefixAccount(accountAddress string) string { - return fmt.Sprintf("%s%s-", common.BlockTransactionPrefixAccount, accountAddress) + idx := storage.NewIndex() + return idx.WritePrefix(common.BlockTransactionPrefixAccount, accountAddress).String() +} + +func GetBlockTransactionKeyPrefixTarget(accountAddress string) string { + idx := storage.NewIndex() + return idx.WritePrefix(common.BlockTransactionPrefixTarget, accountAddress).String() } func GetBlockTransactionKeyPrefixBlock(hash string) string { - return fmt.Sprintf("%s%s-", common.BlockTransactionPrefixBlock, hash) + idx := storage.NewIndex() + return idx.WritePrefix(common.BlockTransactionPrefixBlock, hash).String() } func GetBlockTransactionKey(hash string) string { - return fmt.Sprintf("%s%s", common.BlockTransactionPrefixHash, hash) + idx := storage.NewIndex() + return idx.WritePrefix(common.BlockTransactionPrefixHash, hash).String() } func GetBlockTransaction(st *storage.LevelDBBackend, hash string) (bt BlockTransaction, err error) { @@ -255,6 +288,7 @@ func GetBlockTransaction(st *storage.LevelDBBackend, hash string) (bt BlockTrans } bt.isSaved = true + bt.order = NewBlockTxOrder(bt.BlockHeight, bt.Index) return } @@ -273,7 +307,7 @@ func LoadBlockTransactionsInsideIterator( return (func() (BlockTransaction, bool, []byte) { item, hasNext := iterFunc() - if !hasNext { + if !hasNext && (item.Key == nil || item.Value == nil) { return BlockTransaction{}, false, item.Key } @@ -325,4 +359,10 @@ func GetBlockTransactionsByBlock(st *storage.LevelDBBackend, hash string, option return LoadBlockTransactionsInsideIterator(st, iterFunc, closeFunc) } -var GetBlockTransactions = GetBlockTransactionsByConfirmed +func GetBlockTransactions(st *storage.LevelDBBackend, options storage.ListOptions) ( + func() (BlockTransaction, bool, []byte), + func(), +) { + iterFunc, closeFunc := st.GetIterator(common.BlockTransactionPrefixAll, options) + return LoadBlockTransactionsInsideIterator(st, iterFunc, closeFunc) +} diff --git a/lib/block/transaction_test.go b/lib/block/transaction_test.go index 2c45693f6..273eaef9b 100644 --- a/lib/block/transaction_test.go +++ b/lib/block/transaction_test.go @@ -16,7 +16,7 @@ func TestNewBlockTransaction(t *testing.T) { conf := common.NewTestConfig() _, tx := transaction.TestMakeTransaction(conf.NetworkID, 1) block := TestMakeNewBlock([]string{tx.GetHash()}) - bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx) + bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx, 1) require.Equal(t, bt.Hash, tx.H.Hash) require.Equal(t, bt.SequenceID, tx.B.SequenceID) @@ -26,20 +26,25 @@ func TestNewBlockTransaction(t *testing.T) { require.Equal(t, bt.Created, tx.H.Created) var opHashes []string - for _, op := range tx.B.Operations { - opHashes = append(opHashes, NewBlockOperationKey(op.MakeHashString(), tx.GetHash())) + for i, op := range tx.B.Operations { + opHashes = append(opHashes, NewBlockOperationKey(op.MakeHashString(), tx.GetHash(), uint64(i))) } for i, opHash := range bt.Operations { require.Equal(t, opHash, opHashes[i]) } require.Equal(t, bt.Amount, tx.TotalAmount(true)) + + for i, txHash := range block.Transactions { + require.Equal(t, txHash, bt.Hash) + require.Equal(t, uint64(i+1), bt.Index) + } } func TestBlockTransactionSaveAndGet(t *testing.T) { conf := common.NewTestConfig() st := storage.NewTestStorage() - bt := makeNewBlockTransaction(conf.NetworkID, 1) + bt := makeNewBlockTransaction(conf.NetworkID, 1, 0) err := bt.Save(st) require.NoError(t, err) @@ -60,7 +65,7 @@ func TestBlockTransactionSaveExisting(t *testing.T) { conf := common.NewTestConfig() st := storage.NewTestStorage() - bt := makeNewBlockTransaction(conf.NetworkID, 1) + bt := makeNewBlockTransaction(conf.NetworkID, 1, 0) err := bt.Save(st) require.NoError(t, err) @@ -93,8 +98,8 @@ func TestMultipleBlockTransactionSource(t *testing.T) { } block := TestMakeNewBlock(txHashes) - for _, tx := range txs { - bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx) + for i, tx := range txs { + bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx, uint64(i)) err := bt.Save(st) require.NoError(t, err) } @@ -108,9 +113,9 @@ func TestMultipleBlockTransactionSource(t *testing.T) { txHashes = append(txHashes, tx.GetHash()) } - block = TestMakeNewBlock(txHashes) - for _, tx := range txs { - bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx) + block = TestMakeNewBlockWithPrevBlock(block, txHashes) + for i, tx := range txs { + bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx, uint64(i)) err := bt.Save(st) require.NoError(t, err) } @@ -175,8 +180,8 @@ func TestMultipleBlockTransactionConfirmed(t *testing.T) { } block := TestMakeNewBlock(txHashes) - for _, tx := range txs { - bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx) + for i, tx := range txs { + bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx, uint64(i)) err := bt.Save(st) require.NoError(t, err) } @@ -225,7 +230,7 @@ func TestBlockTransactionMultipleSave(t *testing.T) { conf := common.NewTestConfig() st := storage.NewTestStorage() - bt := makeNewBlockTransaction(conf.NetworkID, 1) + bt := makeNewBlockTransaction(conf.NetworkID, 1, 0) err := bt.Save(st) require.NoError(t, err) @@ -258,8 +263,8 @@ func TestMultipleBlockTransactionGetByAccount(t *testing.T) { } blk = TestMakeNewBlock(txHashes) - for _, tx := range txs { - bt := NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx) + for i, tx := range txs { + bt := NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx, uint64(i)) bt.MustSave(st) err := bt.SaveBlockOperations(st) require.NoError(t, err) @@ -277,9 +282,9 @@ func TestMultipleBlockTransactionGetByAccount(t *testing.T) { txHashes = append(txHashes, tx.GetHash()) } - blk = TestMakeNewBlock(txHashes) - for _, tx := range txs { - bt := NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx) + blk = TestMakeNewBlockWithPrevBlock(blk, txHashes) + for i, tx := range txs { + bt := NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx, uint64(i)) bt.MustSave(st) err := bt.SaveBlockOperations(st) require.NoError(t, err) @@ -296,9 +301,9 @@ func TestMultipleBlockTransactionGetByAccount(t *testing.T) { txHashes = append(txHashes, tx.GetHash()) } - blk = TestMakeNewBlock(txHashes) - for _, tx := range txs { - bt := NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx) + blk = TestMakeNewBlockWithPrevBlock(blk, txHashes) + for i, tx := range txs { + bt := NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx, uint64(i)) bt.MustSave(st) err := bt.SaveBlockOperations(st) require.NoError(t, err) @@ -343,8 +348,8 @@ func TestMultipleBlockTransactionGetByBlock(t *testing.T) { } block0 := TestMakeNewBlock(txHashes0) - for _, tx := range txs0 { - bt := NewBlockTransactionFromTransaction(block0.Hash, block0.Height, block0.ProposedTime, tx) + for i, tx := range txs0 { + bt := NewBlockTransactionFromTransaction(block0.Hash, block0.Height, block0.ProposedTime, tx, uint64(i)) bt.MustSave(st) } @@ -358,9 +363,9 @@ func TestMultipleBlockTransactionGetByBlock(t *testing.T) { txHashes1 = append(txHashes1, tx.GetHash()) } - block1 := TestMakeNewBlock(txHashes1) - for _, tx := range txs1 { - bt := NewBlockTransactionFromTransaction(block1.Hash, block1.Height, block1.ProposedTime, tx) + block1 := TestMakeNewBlockWithPrevBlock(block0, txHashes1) + for i, tx := range txs1 { + bt := NewBlockTransactionFromTransaction(block1.Hash, block1.Height, block1.ProposedTime, tx, uint64(i)) bt.MustSave(st) } @@ -380,6 +385,7 @@ func TestMultipleBlockTransactionGetByBlock(t *testing.T) { require.Equal(t, len(saved), len(createdOrder0), "fetched records insufficient") for i, bt := range saved { require.Equal(t, bt.Hash, createdOrder0[i], "order mismatch") + require.Equal(t, bt.Index, uint64(i)) } } @@ -427,8 +433,8 @@ func TestMultipleBlockTransactionsOrderByBlockHeightAndCursor(t *testing.T) { block := TestMakeNewBlock(txHashes) block.Height++ - for _, tx := range txs { - bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx) + for i, tx := range txs { + bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx, uint64(i)) bt.MustSave(st) } transactionOrder = append(transactionOrder, createdOrder...) @@ -448,8 +454,8 @@ func TestMultipleBlockTransactionsOrderByBlockHeightAndCursor(t *testing.T) { } block := TestMakeNewBlock(txHashes) - for _, tx := range txs { - bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx) + for i, tx := range txs { + bt := NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx, uint64(i)) bt.MustSave(st) } @@ -506,9 +512,9 @@ func TestMultipleBlockTransactionsOrderByBlockHeightAndCursor(t *testing.T) { } } -func makeNewBlockTransaction(networkID []byte, n int) BlockTransaction { +func makeNewBlockTransaction(networkID []byte, n int, index uint64) BlockTransaction { _, tx := transaction.TestMakeTransaction(networkID, n) block := TestMakeNewBlock([]string{tx.GetHash()}) - return NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx) + return NewBlockTransactionFromTransaction(block.Hash, block.Height, block.ProposedTime, tx, index) } diff --git a/lib/common/constant.go b/lib/common/constant.go index 49c386e65..ad3af83e9 100644 --- a/lib/common/constant.go +++ b/lib/common/constant.go @@ -67,6 +67,9 @@ const ( // DiscoveryMessageCreatedAllowDuration limit the `DiscoveryMessage.Created` // is allowed or not. DiscoveryMessageCreatedAllowDuration time.Duration = time.Second * 10 + + // ProposerTransactionIndex + ProposerTransactionIndex uint64 = 0 ) var ( diff --git a/lib/common/prefix.go b/lib/common/prefix.go index 8457c96d1..6d57ef7b0 100644 --- a/lib/common/prefix.go +++ b/lib/common/prefix.go @@ -10,6 +10,8 @@ const ( BlockTransactionPrefixConfirmed = string(0x12) BlockTransactionPrefixAccount = string(0x13) BlockTransactionPrefixBlock = string(0x14) + BlockTransactionPrefixAll = string(0x15) + BlockTransactionPrefixTarget = string(0x16) BlockOperationPrefixHash = string(0x20) BlockOperationPrefixTxHash = string(0x21) BlockOperationPrefixSource = string(0x22) diff --git a/lib/common/util.go b/lib/common/util.go index 8174c182a..903b37120 100644 --- a/lib/common/util.go +++ b/lib/common/util.go @@ -2,14 +2,13 @@ package common import ( "bytes" - "encoding/binary" "encoding/json" + "fmt" + "github.com/satori/go.uuid" "io" "net/url" "os" "sort" - - uuid "github.com/satori/go.uuid" ) const MaxUintEncodeByte = 8 @@ -154,10 +153,8 @@ func IsStringMapEqualWithHash(a, b map[string]bool) bool { return bytes.Equal(aHash, bHash) } -func EncodeUint64ToByteSlice(i uint64) [MaxUintEncodeByte]byte { - var b [MaxUintEncodeByte]byte - binary.BigEndian.PutUint64(b[:], i) - return b +func EncodeUint64ToString(i uint64) string { + return fmt.Sprintf("%020d", i) } type KV struct { diff --git a/lib/node/runner/api/api.go b/lib/node/runner/api/api.go index 391acf177..4e9a963ea 100644 --- a/lib/node/runner/api/api.go +++ b/lib/node/runner/api/api.go @@ -85,10 +85,6 @@ func TriggerEvent(st *storage.LevelDBBackend, transactions []*transaction.Transa t(event(cond(obs.Tx, obs.TxHash, txHash)), &bt) for _, op := range tx.B.Operations { - if err != nil { - return - } - if pop, ok := op.B.(operation.Targetable); ok { target := pop.TargetAddress() accountMap[target] = struct{}{} diff --git a/lib/node/runner/api/base_test.go b/lib/node/runner/api/base_test.go index c0d124315..07b30892c 100644 --- a/lib/node/runner/api/base_test.go +++ b/lib/node/runner/api/base_test.go @@ -80,9 +80,9 @@ func prepareOpsWithoutSave(count int, st *storage.LevelDBBackend) (*keypair.Full } theBlock := block.TestMakeNewBlockWithPrevBlock(block.GetLatestBlock(st), txHashes) - for _, tx := range txs { - for i, op := range tx.B.Operations { - bo, err := block.NewBlockOperationFromOperation(op, tx, theBlock.Height, i) + for i, tx := range txs { + for j, op := range tx.B.Operations { + bo, err := block.NewBlockOperationFromOperation(op, tx, theBlock.Height, uint64(i), j) if err != nil { panic(err) } @@ -99,10 +99,10 @@ func prepareBlkTxOpWithoutSave(st *storage.LevelDBBackend) (*keypair.Full, block tx := transaction.TestMakeTransactionWithKeypair(networkID, 1, kp) txHashes = append(txHashes, tx.GetHash()) theBlock := block.TestMakeNewBlockWithPrevBlock(block.GetLatestBlock(st), txHashes) - bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx) + bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx, 0) op := tx.B.Operations[0] - bo, err := block.NewBlockOperationFromOperation(op, tx, theBlock.Height, 0) + bo, err := block.NewBlockOperationFromOperation(op, tx, theBlock.Height, 0, 0) if err != nil { panic(err) } @@ -127,8 +127,8 @@ func prepareTxsWithKeyPair(storage *storage.LevelDBBackend, source, target *keyp theBlock := block.TestMakeNewBlockWithPrevBlock(block.GetLatestBlock(storage), txHashes) theBlock.MustSave(storage) - for _, tx := range txs { - bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx) + for i, tx := range txs { + bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx, uint64(i)) bt.MustSave(storage) if err := bt.SaveBlockOperations(storage); err != nil { return nil, nil, nil @@ -150,7 +150,7 @@ func prepareTxWithOperations(storage *storage.LevelDBBackend, count int) (*keypa theBlock := block.TestMakeNewBlockWithPrevBlock(block.GetLatestBlock(storage), []string{tx.GetHash()}) theBlock.MustSave(storage) - bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx) + bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx, 1) bt.Save(storage) if err := bt.SaveBlockOperations(storage); err != nil { panic(err) @@ -170,8 +170,8 @@ func prepareTxsWithoutSave(count int, st *storage.LevelDBBackend) (*keypair.Full } theBlock := block.TestMakeNewBlockWithPrevBlock(block.GetLatestBlock(st), txHashes) - for _, tx := range txs { - bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx) + for i, tx := range txs { + bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx, uint64(i)) btList = append(btList, bt) } return kp, btList @@ -182,7 +182,7 @@ func prepareTxWithoutSave(st *storage.LevelDBBackend) (*keypair.Full, *transacti tx := transaction.TestMakeTransactionWithKeypair(networkID, 1, kp) theBlock := block.TestMakeNewBlockWithPrevBlock(block.GetLatestBlock(st), []string{tx.GetHash()}) - bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx) + bt := block.NewBlockTransactionFromTransaction(theBlock.Hash, theBlock.Height, theBlock.ProposedTime, tx, 0) return kp, &tx, &bt } diff --git a/lib/node/runner/api/operation.go b/lib/node/runner/api/operation.go index 718ba7056..86a4f1223 100644 --- a/lib/node/runner/api/operation.go +++ b/lib/node/runner/api/operation.go @@ -59,8 +59,6 @@ func (api NetworkHandlerAPI) GetOperationsByAccountHandler(w http.ResponseWriter return } - options := p.ListOptions() - oTypeStr := r.URL.Query().Get("type") if len(oTypeStr) > 0 && !operation.IsValidOperationType(oTypeStr) { httputils.WriteJSONError(w, errors.InvalidQueryString) @@ -75,13 +73,27 @@ func (api NetworkHandlerAPI) GetOperationsByAccountHandler(w http.ResponseWriter return } - var txs []resource.Resource blockCache := map[ /* block.Height */ uint64]*block.Block{} oType := operation.OperationType(oTypeStr) - var firstCursor []byte - var lastCursor []byte - { + prefix := block.GetBlockOperationKeyPrefixPeers(address) + if len(oType) > 0 { + prefix = block.GetBlockOperationKeyPrefixPeersAndType(address, oType) + } + + options, err := p.PageCursorListOptions(prefix) + if err != nil { + httputils.WriteJSONError(w, err) + return + } + + var ( + pOrder *block.BlockOrder + nOrder *block.BlockOrder + ) + + var txs []resource.Resource + { var iterFunc func() (block.BlockOperation, bool, []byte) var closeFunc func() if len(oType) > 0 { @@ -90,14 +102,14 @@ func (api NetworkHandlerAPI) GetOperationsByAccountHandler(w http.ResponseWriter iterFunc, closeFunc = block.GetBlockOperationsByPeers(api.storage, address, options) } for { - t, hasNext, c := iterFunc() + t, hasNext, _ := iterFunc() if !hasNext { break } - if len(firstCursor) == 0 { - firstCursor = append(firstCursor, c...) + if pOrder == nil { + pOrder = t.BlockOrder() } - lastCursor = append([]byte{}, c...) + nOrder = t.BlockOrder() var blk *block.Block var ok bool @@ -127,6 +139,6 @@ func (api NetworkHandlerAPI) GetOperationsByAccountHandler(w http.ResponseWriter closeFunc() } - list := p.ResourceList(txs, firstCursor, lastCursor) + list := p.ResourceListWithOrder(txs, pOrder, nOrder) httputils.MustWriteJSON(w, 200, list) } diff --git a/lib/node/runner/api/operation_test.go b/lib/node/runner/api/operation_test.go index 7aa25ba4a..f75f51383 100644 --- a/lib/node/runner/api/operation_test.go +++ b/lib/node/runner/api/operation_test.go @@ -1,15 +1,17 @@ package api import ( - "boscoin.io/sebak/lib/block" - "boscoin.io/sebak/lib/common" - "boscoin.io/sebak/lib/transaction/operation" "bufio" "fmt" - "github.com/stretchr/testify/require" "io/ioutil" "strings" "testing" + + "boscoin.io/sebak/lib/block" + "boscoin.io/sebak/lib/common" + "boscoin.io/sebak/lib/transaction/operation" + + "github.com/stretchr/testify/require" ) func TestGetOperationsByAccountHandler(t *testing.T) { diff --git a/lib/node/runner/api/pagecursor.go b/lib/node/runner/api/pagecursor.go new file mode 100644 index 000000000..cb4b7fe9c --- /dev/null +++ b/lib/node/runner/api/pagecursor.go @@ -0,0 +1,48 @@ +package api + +import ( + "strconv" + "strings" + + "boscoin.io/sebak/lib/common" + "boscoin.io/sebak/lib/storage" +) + +type PageCursor struct { + input string + prefix string + index *storage.Index +} + +func NewPageCursor(input []byte, prefix string) *PageCursor { + p := &PageCursor{ + input: strings.TrimSpace(string(input)), + prefix: prefix, + index: storage.NewIndex(), + } + + return p +} + +func (p *PageCursor) IndexKey() ([]byte, error) { + p.index.WritePrefix(p.prefix) + err := p.indexOrder() + return p.index.Bytes(), err +} + +func (p *PageCursor) indexOrder() error { + parts := strings.Split(p.input, "-") + for i, part := range parts { + if part == "" { + continue + } + if i < 3 { + partInt, err := strconv.ParseUint(part, 10, 64) + if err != nil { + return err + } + p.index.WriteOrder(common.EncodeUint64ToString(partInt)) + } + } + return nil +} diff --git a/lib/node/runner/api/pagequery.go b/lib/node/runner/api/pagequery.go index d891fff70..1ee071361 100644 --- a/lib/node/runner/api/pagequery.go +++ b/lib/node/runner/api/pagequery.go @@ -7,6 +7,7 @@ import ( "net/url" "strconv" + "boscoin.io/sebak/lib/block" "boscoin.io/sebak/lib/common" "boscoin.io/sebak/lib/errors" "boscoin.io/sebak/lib/node/runner/api/resource" @@ -88,6 +89,19 @@ func (p *PageQuery) ListOptions() storage.ListOptions { return storage.NewDefaultListOptions(p.Reverse(), p.Cursor(), p.Limit()) } +func (p *PageQuery) PageCursorListOptions(prefix string) (storage.ListOptions, error) { + var indexCursor []byte + var err error + if p.Cursor() != nil { + c := NewPageCursor(p.Cursor(), prefix) + indexCursor, err = c.IndexKey() + if err != nil { + return nil, err + } + } + return storage.NewDefaultListOptions(p.Reverse(), indexCursor, p.Limit()), nil +} + func (p *PageQuery) WalkOption() *storage.WalkOption { return storage.NewWalkOption(string(p.Cursor()), p.Limit(), p.Reverse()) } @@ -100,6 +114,12 @@ func (p *PageQuery) ResourceList(rs []resource.Resource, firstCursor, lastCursor } } +func (p *PageQuery) ResourceListWithOrder(rs []resource.Resource, prevOrder *block.BlockOrder, nextOrder *block.BlockOrder) *resource.ResourceList { + pcursor := []byte(prevOrder.String()) + ncursor := []byte(nextOrder.String()) + return p.ResourceList(rs, pcursor, ncursor) +} + func (p *PageQuery) parseRequest() error { q := p.request.URL.Query() r := q.Get("reverse") diff --git a/lib/node/runner/api/resource/operation.go b/lib/node/runner/api/resource/operation.go index 777df34e1..ffa77fab3 100644 --- a/lib/node/runner/api/resource/operation.go +++ b/lib/node/runner/api/resource/operation.go @@ -38,6 +38,7 @@ func (o Operation) GetMap() hal.Entry { "target": o.bo.Target, "type": o.bo.Type, "tx_hash": o.bo.TxHash, + "index": o.bo.Index, "body": body, "block_height": o.bo.Height, } diff --git a/lib/node/runner/api/resource/resource_test.go b/lib/node/runner/api/resource/resource_test.go index c301f3432..b0d0593d8 100644 --- a/lib/node/runner/api/resource/resource_test.go +++ b/lib/node/runner/api/resource/resource_test.go @@ -47,7 +47,7 @@ func TestResourceAccount(t *testing.T) { // Transaction { _, tx := transaction.TestMakeTransaction([]byte{0x00}, 1) - bt := block.NewBlockTransactionFromTransaction("dummy", 0, common.NowISO8601(), tx) + bt := block.NewBlockTransactionFromTransaction("dummy", 0, common.NowISO8601(), tx, 0) bt.MustSave(storage) rt := NewTransaction(&bt) @@ -73,7 +73,7 @@ func TestResourceAccount(t *testing.T) { // Operation { _, tx := transaction.TestMakeTransaction([]byte{0x00}, 1) - bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, common.NowISO8601(), tx) + bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, common.NowISO8601(), tx, 1) bt.MustSave(storage) err := bt.SaveBlockOperations(storage) @@ -104,7 +104,7 @@ func TestResourceAccount(t *testing.T) { { var err error _, tx := transaction.TestMakeTransaction([]byte{0x00}, 3) - bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, common.NowISO8601(), tx) + bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, common.NowISO8601(), tx, 2) bt.MustSave(storage) err = bt.SaveBlockOperations(storage) require.NoError(t, err) diff --git a/lib/node/runner/api/resource/transaction.go b/lib/node/runner/api/resource/transaction.go index 0fc302edf..bb183edd0 100644 --- a/lib/node/runner/api/resource/transaction.go +++ b/lib/node/runner/api/resource/transaction.go @@ -27,6 +27,8 @@ func (t Transaction) GetMap() hal.Entry { "sequence_id": t.bt.SequenceID, "created": t.bt.Created, "operation_count": len(t.bt.Operations), + "index": t.bt.Index, + "block_height": t.bt.BlockHeight, } } func (t Transaction) Resource() *hal.Resource { diff --git a/lib/node/runner/api/transaction.go b/lib/node/runner/api/transaction.go index 3364bdff6..a7a1f61c6 100644 --- a/lib/node/runner/api/transaction.go +++ b/lib/node/runner/api/transaction.go @@ -8,6 +8,7 @@ import ( "github.com/gorilla/mux" "boscoin.io/sebak/lib/block" + "boscoin.io/sebak/lib/common" o "boscoin.io/sebak/lib/common/observer" "boscoin.io/sebak/lib/errors" "boscoin.io/sebak/lib/network/httputils" @@ -21,21 +22,30 @@ func (api NetworkHandlerAPI) GetTransactionsHandler(w http.ResponseWriter, r *ht return } - var options = p.ListOptions() - var firstCursor []byte - var cursor []byte + options, err := p.PageCursorListOptions(common.BlockTransactionPrefixAll) + if err != nil { + //TODO: more correct err for it + httputils.WriteJSONError(w, err) + return + } + + var ( + prevOrder *block.BlockOrder + nextOrder *block.BlockOrder + ) + readFunc := func() []resource.Resource { var txs []resource.Resource iterFunc, closeFunc := block.GetBlockTransactions(api.storage, options) for { - t, hasNext, c := iterFunc() + t, hasNext, _ := iterFunc() if !hasNext { break } - cursor = append([]byte{}, c...) - if len(firstCursor) == 0 { - firstCursor = append(firstCursor, c...) + if prevOrder == nil { + prevOrder = t.BlockOrder() } + nextOrder = t.BlockOrder() txs = append(txs, resource.NewTransaction(&t)) } closeFunc() @@ -43,8 +53,7 @@ func (api NetworkHandlerAPI) GetTransactionsHandler(w http.ResponseWriter, r *ht } txs := readFunc() - - list := p.ResourceList(txs, firstCursor, cursor) + list := p.ResourceListWithOrder(txs, prevOrder, nextOrder) httputils.MustWriteJSON(w, 200, list) } @@ -86,22 +95,27 @@ func (api NetworkHandlerAPI) GetTransactionsByAccountHandler(w http.ResponseWrit return } - var options = p.ListOptions() - var firstCursor []byte - var cursor []byte + options, err := p.PageCursorListOptions(block.GetBlockTransactionKeyPrefixAccount(address)) + if err != nil { + httputils.WriteJSONError(w, err) + return + } + var ( + pOrder *block.BlockOrder + nOrder *block.BlockOrder + ) readFunc := func() []resource.Resource { var txs []resource.Resource iterFunc, closeFunc := block.GetBlockTransactionsByAccount(api.storage, address, options) for { - t, hasNext, c := iterFunc() + t, hasNext, _ := iterFunc() if !hasNext { break } - cursor = append([]byte{}, c...) - if len(firstCursor) == 0 { - firstCursor = append(firstCursor, c...) + if pOrder == nil { + pOrder = t.BlockOrder() } - + nOrder = t.BlockOrder() txs = append(txs, resource.NewTransaction(&t)) } closeFunc() @@ -109,7 +123,7 @@ func (api NetworkHandlerAPI) GetTransactionsByAccountHandler(w http.ResponseWrit } txs := readFunc() - list := p.ResourceList(txs, firstCursor, cursor) + list := p.ResourceListWithOrder(txs, pOrder, nOrder) httputils.MustWriteJSON(w, 200, list) } diff --git a/lib/node/runner/api/tx_operations.go b/lib/node/runner/api/tx_operations.go index 7dcfd0469..029962e7a 100644 --- a/lib/node/runner/api/tx_operations.go +++ b/lib/node/runner/api/tx_operations.go @@ -27,7 +27,11 @@ func (api NetworkHandlerAPI) GetOperationsByTxHandler(w http.ResponseWriter, r * return } - options := p.ListOptions() + options, err := p.PageCursorListOptions(block.GetBlockOperationKeyPrefixTxHash(hash)) + if err != nil { + httputils.WriteJSONError(w, err) + return + } var blk *block.Block if blk, err = api.getBlockByTx(hash); err != nil { @@ -35,27 +39,28 @@ func (api NetworkHandlerAPI) GetOperationsByTxHandler(w http.ResponseWriter, r * return } - ops, firstCursor, cursor := api.getOperationsByTx(hash, blk, options) + ops, pOrder, nOrder := api.getOperationsByTx(hash, blk, options) + if len(ops) < 1 { httputils.WriteJSONError(w, errors.BlockTransactionDoesNotExists) return } - list := p.ResourceList(ops, firstCursor, cursor) + list := p.ResourceListWithOrder(ops, pOrder, nOrder) httputils.MustWriteJSON(w, 200, list) } -func (api NetworkHandlerAPI) getOperationsByTx(txHash string, blk *block.Block, options storage.ListOptions) (txs []resource.Resource, firstCursor, cursor []byte) { +func (api NetworkHandlerAPI) getOperationsByTx(txHash string, blk *block.Block, options storage.ListOptions) (txs []resource.Resource, pOrder *block.BlockOrder, nOrder *block.BlockOrder) { iterFunc, closeFunc := block.GetBlockOperationsByTx(api.storage, txHash, options) for idx := 0; ; idx++ { - o, hasNext, c := iterFunc() + o, hasNext, _ := iterFunc() if !hasNext { break } - cursor = append([]byte{}, c...) - if len(firstCursor) == 0 { - firstCursor = append(firstCursor, c...) + if pOrder == nil { + pOrder = o.BlockOrder() } + nOrder = o.BlockOrder() rs := resource.NewOperation(&o, idx) rs.Block = blk diff --git a/lib/node/runner/api_block_test.go b/lib/node/runner/api_block_test.go index e2a805a16..6e6f23f97 100644 --- a/lib/node/runner/api_block_test.go +++ b/lib/node/runner/api_block_test.go @@ -57,8 +57,8 @@ func (p *HelperTestGetBlocksHandler) createBlock() block.Block { bk := block.TestMakeNewBlockWithPrevBlock(block.GetLatestBlock(p.st), txHashes) bk.MustSave(p.st) - for _, tx := range txs { - btx := block.NewBlockTransactionFromTransaction(bk.Hash, bk.Height, bk.ProposedTime, tx) + for i, tx := range txs { + btx := block.NewBlockTransactionFromTransaction(bk.Hash, bk.Height, bk.ProposedTime, tx, uint64(i)) btx.MustSave(p.st) btx.SaveBlockOperations(p.st) block.SaveTransactionPool(p.st, tx) diff --git a/lib/node/runner/api_transactions_test.go b/lib/node/runner/api_transactions_test.go index ee9844e4b..da81dc6af 100644 --- a/lib/node/runner/api_transactions_test.go +++ b/lib/node/runner/api_transactions_test.go @@ -109,8 +109,8 @@ func (p *HelperTestGetNodeTransactionsHandler) createBlock() block.Block { bk.Height = uint64(height + 1) bk.MustSave(p.st) - for _, tx := range txs { - btx := block.NewBlockTransactionFromTransaction(bk.Hash, bk.Height, bk.ProposedTime, tx) + for i, tx := range txs { + btx := block.NewBlockTransactionFromTransaction(bk.Hash, bk.Height, bk.ProposedTime, tx, uint64(i)) if err := btx.Save(p.st); err != nil { panic(err) } diff --git a/lib/node/runner/block_operations.go b/lib/node/runner/block_operations.go index fde530305..4d4e5e8b3 100644 --- a/lib/node/runner/block_operations.go +++ b/lib/node/runner/block_operations.go @@ -257,7 +257,7 @@ func (sb *SavingBlockOperations) CheckTransactionByBlock(st *storage.LevelDBBack } for i, op := range bt.Transaction().B.Operations { - opHash := block.NewBlockOperationKey(op.MakeHashString(), hash) + opHash := block.NewBlockOperationKey(op.MakeHashString(), hash, uint64(i)) var exists bool if exists, err = block.ExistsBlockOperation(st, opHash); err != nil { diff --git a/lib/node/runner/block_operations_test.go b/lib/node/runner/block_operations_test.go index a72a27acf..f1bc041ee 100644 --- a/lib/node/runner/block_operations_test.go +++ b/lib/node/runner/block_operations_test.go @@ -54,7 +54,7 @@ func (p *TestSavingBlockOperationHelper) makeBlock(prevBlock block.Block, numTxs opi, _ := ballot.NewInflationFromBallot(*blt, block.CommonKP.Address(), common.BaseReserve) opc, _ := ballot.NewCollectTxFeeFromBallot(*blt, block.CommonKP.Address(), txs...) ptx, _ := ballot.NewProposerTransactionFromBallot(*blt, opc, opi) - bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, ptx.Transaction) + bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, ptx.Transaction, common.ProposerTransactionIndex) bt.MustSave(p.st) block.SaveTransactionPool(p.st, ptx.Transaction) @@ -62,8 +62,8 @@ func (p *TestSavingBlockOperationHelper) makeBlock(prevBlock block.Block, numTxs blk.MustSave(p.st) - for _, tx := range txs { - bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx) + for i, tx := range txs { + bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, tx, uint64(i+1)) bt.MustSave(p.st) } diff --git a/lib/node/runner/finish_ballot.go b/lib/node/runner/finish_ballot.go index 3111dffef..b1cf3b96b 100644 --- a/lib/node/runner/finish_ballot.go +++ b/lib/node/runner/finish_ballot.go @@ -5,6 +5,7 @@ import ( "boscoin.io/sebak/lib/ballot" "boscoin.io/sebak/lib/block" + "boscoin.io/sebak/lib/common" "boscoin.io/sebak/lib/errors" "boscoin.io/sebak/lib/metrics" "boscoin.io/sebak/lib/storage" @@ -126,8 +127,9 @@ func getProposedTransactions(st *storage.LevelDBBackend, pTxHashes []string, tra } func FinishTransactions(blk block.Block, transactions []*transaction.Transaction, st *storage.LevelDBBackend) (err error) { - for _, tx := range transactions { - bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, *tx) + for i, tx := range transactions { + // i = 0 is an index of proposerTransaction + bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, *tx, uint64(i+1)) if err = bt.Save(st); err != nil { return } @@ -290,7 +292,7 @@ func FinishProposerTransaction(st *storage.LevelDBBackend, blk block.Block, ptx } } - bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, ptx.Transaction) + bt := block.NewBlockTransactionFromTransaction(blk.Hash, blk.Height, blk.ProposedTime, ptx.Transaction, common.ProposerTransactionIndex) if err = bt.Save(st); err != nil { return } diff --git a/lib/storage/index.go b/lib/storage/index.go new file mode 100644 index 000000000..8727ae86b --- /dev/null +++ b/lib/storage/index.go @@ -0,0 +1,56 @@ +package storage + +import ( + "strings" +) + +const ( + IndexPrefixOrderDelimiter = "/" + IndexElementDelimiter = "-" +) + +type Index struct { + prefix []string + order []string +} + +func NewIndex() *Index { + idx := &Index{ + prefix: make([]string, 0, 0), + order: make([]string, 0, 0), + } + return idx +} + +func (idx Index) Bytes() []byte { + return []byte(idx.String()) +} + +func (idx Index) String() string { + prefix := strings.Join(idx.prefix, IndexElementDelimiter) + order := strings.Join(idx.order, IndexElementDelimiter) + var strs []string + if len(prefix) != 0 { + strs = append(strs, prefix) + } + if len(order) != 0 { + strs = append(strs, order) + } + + index := strings.Join(strs, IndexPrefixOrderDelimiter) + return index +} + +func (idx *Index) WritePrefix(ss ...string) *Index { + for _, s := range ss { + idx.prefix = append(idx.prefix, s) + } + return idx +} + +func (idx *Index) WriteOrder(ss ...string) *Index { + for _, s := range ss { + idx.order = append(idx.order, s) + } + return idx +}