From 9c2da86a4c7bf0824722fb8cc27592a19423001d Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Sat, 13 Jul 2024 05:48:54 -0700 Subject: [PATCH 01/23] Beginning of DB unit tests. --- db/store/main.go | 12 +++++++++- db/store/message_test.go | 52 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 db/store/message_test.go diff --git a/db/store/main.go b/db/store/main.go index d89c0a3f..a09ed7f0 100644 --- a/db/store/main.go +++ b/db/store/main.go @@ -29,10 +29,20 @@ func CloseAll() { } } +func SetConn(connId string, db *leveldb.DB) { + connsMutex.Lock() + defer connsMutex.Unlock() + conns[connId] = db +} + +func GetConnId(topic string, shard uint) string { + return fmt.Sprintf("%d:%s", shard, topic) +} + func getDb(topic string, shard uint) (*leveldb.DB, error) { connsMutex.Lock() defer connsMutex.Unlock() - connId := fmt.Sprintf("%d:%s", shard, topic) + connId := GetConnId(topic, shard) if conns[connId] == nil { if conns[connId] != nil { return conns[connId], nil diff --git a/db/store/message_test.go b/db/store/message_test.go new file mode 100644 index 00000000..bb122e53 --- /dev/null +++ b/db/store/message_test.go @@ -0,0 +1,52 @@ +package store_test + +import ( + "fmt" + "github.com/memocash/index/db/store" + "github.com/syndtr/goleveldb/leveldb" + "os" + "path/filepath" + "testing" +) + +const TestTopic = "test" +const TestShard = 0 + +func initTestDb() error { + connId := store.GetConnId(TestTopic, TestShard) + testDbPath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtest-%d", os.Getuid())) + if err := os.RemoveAll(testDbPath); err != nil { + return fmt.Errorf("error removing old db; %w", err) + } + db, err := leveldb.OpenFile(testDbPath, nil) + if err != nil { + return fmt.Errorf("error opening level db; %w", err) + } + store.SetConn(connId, db) + return nil +} + +func TestGetMessage(t *testing.T) { + if err := initTestDb(); err != nil { + t.Errorf("error initializing test db; %v", err) + } + if err := store.SaveMessages(TestTopic, TestShard, []*store.Message{{ + Uid: []byte("test-uid"), + Message: []byte("test-message"), + }}); err != nil { + t.Errorf("error saving message; %v", err) + } + message, err := store.GetMessage(TestTopic, TestShard, []byte("test-uid")) + if err != nil { + t.Errorf("error getting message; %v", err) + return + } + if message == nil { + t.Errorf("message not found") + return + } + if string(message.Message) != "test-message" { + t.Errorf("message not correct") + return + } +} From fa9ab9376be63f649007dac7aeeee11957be5bf5 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Sat, 13 Jul 2024 23:43:03 -0700 Subject: [PATCH 02/23] Test get by prefixes --- db/store/message_test.go | 52 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/db/store/message_test.go b/db/store/message_test.go index bb122e53..65a4dd67 100644 --- a/db/store/message_test.go +++ b/db/store/message_test.go @@ -13,16 +13,18 @@ const TestTopic = "test" const TestShard = 0 func initTestDb() error { - connId := store.GetConnId(TestTopic, TestShard) testDbPath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtest-%d", os.Getuid())) if err := os.RemoveAll(testDbPath); err != nil { return fmt.Errorf("error removing old db; %w", err) } + db, err := leveldb.OpenFile(testDbPath, nil) if err != nil { return fmt.Errorf("error opening level db; %w", err) } - store.SetConn(connId, db) + + store.SetConn(store.GetConnId(TestTopic, TestShard), db) + return nil } @@ -30,23 +32,69 @@ func TestGetMessage(t *testing.T) { if err := initTestDb(); err != nil { t.Errorf("error initializing test db; %v", err) } + if err := store.SaveMessages(TestTopic, TestShard, []*store.Message{{ Uid: []byte("test-uid"), Message: []byte("test-message"), }}); err != nil { t.Errorf("error saving message; %v", err) } + message, err := store.GetMessage(TestTopic, TestShard, []byte("test-uid")) if err != nil { t.Errorf("error getting message; %v", err) return } + if message == nil { t.Errorf("message not found") return } + if string(message.Message) != "test-message" { t.Errorf("message not correct") return } } + +func TestGetByPrefixes(t *testing.T) { + if err := initTestDb(); err != nil { + t.Errorf("error initializing test db; %v", err) + } + + const MessageCount = 10 + for i := 0; i < MessageCount; i++ { + if err := store.SaveMessages(TestTopic, TestShard, []*store.Message{{ + Uid: []byte(fmt.Sprintf("test-%d", i)), + Message: []byte(fmt.Sprintf("test-message-%d", i)), + }}); err != nil { + t.Errorf("error saving prefix messages; %v", err) + } + } + + prefix := []byte("test-") + messages, err := store.GetMessages(TestTopic, TestShard, [][]byte{prefix}, nil, 0, false) + if err != nil { + t.Errorf("error getting message; %v", err) + return + } + + if len(messages) != MessageCount { + t.Errorf("unexpected number of messages: %d, expected %d\n", len(messages), MessageCount) + return + } + + for i := range messages { + message := messages[i] + expectedUid := fmt.Sprintf("test-%d", i) + if string(message.Uid) != expectedUid { + t.Errorf("unexpected message uid: %s, expected %s\n", message.Uid, expectedUid) + return + } + expectedMessage := fmt.Sprintf("test-message-%d", i) + if string(message.Message) != expectedMessage { + t.Errorf("unexpected message: %s, expected %s\n", message.Message, expectedMessage) + return + } + } +} From 34883c414ef443b3b9f9793b666f0f3b77971a88 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Sun, 14 Jul 2024 08:38:35 -0700 Subject: [PATCH 03/23] Test with multiple prefixes. --- db/store/message_test.go | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/db/store/message_test.go b/db/store/message_test.go index 65a4dd67..9e2b5037 100644 --- a/db/store/message_test.go +++ b/db/store/message_test.go @@ -62,18 +62,26 @@ func TestGetByPrefixes(t *testing.T) { t.Errorf("error initializing test db; %v", err) } + const prefixTest = "test" + const prefixOther = "other" + const MessageCount = 10 for i := 0; i < MessageCount; i++ { if err := store.SaveMessages(TestTopic, TestShard, []*store.Message{{ - Uid: []byte(fmt.Sprintf("test-%d", i)), - Message: []byte(fmt.Sprintf("test-message-%d", i)), + Uid: []byte(fmt.Sprintf("%s-%d", prefixTest, i)), + Message: []byte(fmt.Sprintf("message-%d", i)), + }, { + Uid: []byte(fmt.Sprintf("%s-%d", prefixOther, i)), + Message: []byte(fmt.Sprintf("message-%d", i)), }}); err != nil { t.Errorf("error saving prefix messages; %v", err) } } - prefix := []byte("test-") - messages, err := store.GetMessages(TestTopic, TestShard, [][]byte{prefix}, nil, 0, false) + messages, err := store.GetMessages(TestTopic, TestShard, [][]byte{ + []byte(prefixTest), + []byte(prefixOther), + }, nil, 5, false) if err != nil { t.Errorf("error getting message; %v", err) return @@ -86,12 +94,18 @@ func TestGetByPrefixes(t *testing.T) { for i := range messages { message := messages[i] - expectedUid := fmt.Sprintf("test-%d", i) + var prefix = prefixTest + var id = i + if i >= MessageCount/2 { + prefix = prefixOther + id = i - MessageCount/2 + } + expectedUid := fmt.Sprintf("%s-%d", prefix, id) if string(message.Uid) != expectedUid { t.Errorf("unexpected message uid: %s, expected %s\n", message.Uid, expectedUid) return } - expectedMessage := fmt.Sprintf("test-message-%d", i) + expectedMessage := fmt.Sprintf("message-%d", id) if string(message.Message) != expectedMessage { t.Errorf("unexpected message: %s, expected %s\n", message.Message, expectedMessage) return From 6723e0ced8917be5de872a98b57d3c831f596b12 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Sun, 14 Jul 2024 16:37:49 -0700 Subject: [PATCH 04/23] Add multiple GetByPrefixes tests. --- db/store/message_test.go | 144 +++++++++++++++++++++++++-------------- 1 file changed, 92 insertions(+), 52 deletions(-) diff --git a/db/store/message_test.go b/db/store/message_test.go index 9e2b5037..35be1978 100644 --- a/db/store/message_test.go +++ b/db/store/message_test.go @@ -1,6 +1,7 @@ package store_test import ( + "bytes" "fmt" "github.com/memocash/index/db/store" "github.com/syndtr/goleveldb/leveldb" @@ -12,6 +13,65 @@ import ( const TestTopic = "test" const TestShard = 0 +const PrefixTest = "test" +const PrefixOther = "other" + +var ( + testMessageTest0 = &store.Message{Uid: []byte("test-0"), Message: []byte("message-0")} + testMessageTest1 = &store.Message{Uid: []byte("test-1"), Message: []byte("message-1")} + testMessageTest2 = &store.Message{Uid: []byte("test-2"), Message: []byte("message-2")} + testMessageTest3 = &store.Message{Uid: []byte("test-3"), Message: []byte("message-3")} + testMessageTest4 = &store.Message{Uid: []byte("test-4"), Message: []byte("message-4")} + testMessageTest5 = &store.Message{Uid: []byte("test-5"), Message: []byte("message-5")} + testMessageTest6 = &store.Message{Uid: []byte("test-6"), Message: []byte("message-6")} + testMessageTest7 = &store.Message{Uid: []byte("test-7"), Message: []byte("message-7")} + testMessageTest8 = &store.Message{Uid: []byte("test-8"), Message: []byte("message-8")} + testMessageTest9 = &store.Message{Uid: []byte("test-9"), Message: []byte("message-9")} + + testMessageOther0 = &store.Message{Uid: []byte("other-0"), Message: []byte("message-0")} + testMessageOther1 = &store.Message{Uid: []byte("other-1"), Message: []byte("message-1")} + testMessageOther2 = &store.Message{Uid: []byte("other-2"), Message: []byte("message-2")} + testMessageOther3 = &store.Message{Uid: []byte("other-3"), Message: []byte("message-3")} + testMessageOther4 = &store.Message{Uid: []byte("other-4"), Message: []byte("message-4")} + testMessageOther5 = &store.Message{Uid: []byte("other-5"), Message: []byte("message-5")} + testMessageOther6 = &store.Message{Uid: []byte("other-6"), Message: []byte("message-6")} + testMessageOther7 = &store.Message{Uid: []byte("other-7"), Message: []byte("message-7")} + testMessageOther8 = &store.Message{Uid: []byte("other-8"), Message: []byte("message-8")} + testMessageOther9 = &store.Message{Uid: []byte("other-9"), Message: []byte("message-9")} +) + +type GetMessagesTest struct { + Prefixes [][]byte + Start []byte + Max int + Newest bool + Expected []*store.Message +} + +var tests = []GetMessagesTest{{ + Prefixes: [][]byte{[]byte(PrefixTest), []byte(PrefixOther)}, + Max: 5, + Expected: []*store.Message{ + testMessageTest0, testMessageTest1, testMessageTest2, testMessageTest3, testMessageTest4, + testMessageOther0, testMessageOther1, testMessageOther2, testMessageOther3, testMessageOther4, + }, +}, { + Prefixes: [][]byte{[]byte(PrefixTest), []byte(PrefixOther)}, + Start: []byte(fmt.Sprintf("%s-%d", PrefixOther, 1)), + Max: 5, + Expected: []*store.Message{ + testMessageTest0, testMessageTest1, testMessageTest2, testMessageTest3, testMessageTest4, + testMessageOther1, testMessageOther2, testMessageOther3, testMessageOther4, testMessageOther5, + }, +}, { + Prefixes: [][]byte{[]byte(PrefixTest), []byte(PrefixOther)}, + Start: []byte(fmt.Sprintf("%s-%d", PrefixTest, 1)), + Max: 5, + Expected: []*store.Message{ + testMessageTest1, testMessageTest2, testMessageTest3, testMessageTest4, testMessageTest5, + }, +}} + func initTestDb() error { testDbPath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtest-%d", os.Getuid())) if err := os.RemoveAll(testDbPath); err != nil { @@ -25,6 +85,15 @@ func initTestDb() error { store.SetConn(store.GetConnId(TestTopic, TestShard), db) + if err := store.SaveMessages(TestTopic, TestShard, []*store.Message{ + testMessageTest0, testMessageTest1, testMessageTest2, testMessageTest3, testMessageTest4, + testMessageTest5, testMessageTest6, testMessageTest7, testMessageTest8, testMessageTest9, + testMessageOther0, testMessageOther1, testMessageOther2, testMessageOther3, testMessageOther4, + testMessageOther5, testMessageOther6, testMessageOther7, testMessageOther8, testMessageOther9, + }); err != nil { + return fmt.Errorf("error saving prefix messages; %w", err) + } + return nil } @@ -33,14 +102,7 @@ func TestGetMessage(t *testing.T) { t.Errorf("error initializing test db; %v", err) } - if err := store.SaveMessages(TestTopic, TestShard, []*store.Message{{ - Uid: []byte("test-uid"), - Message: []byte("test-message"), - }}); err != nil { - t.Errorf("error saving message; %v", err) - } - - message, err := store.GetMessage(TestTopic, TestShard, []byte("test-uid")) + message, err := store.GetMessage(TestTopic, TestShard, testMessageTest1.Uid) if err != nil { t.Errorf("error getting message; %v", err) return @@ -51,7 +113,7 @@ func TestGetMessage(t *testing.T) { return } - if string(message.Message) != "test-message" { + if !bytes.Equal(message.Message, testMessageTest1.Message) { t.Errorf("message not correct") return } @@ -62,53 +124,31 @@ func TestGetByPrefixes(t *testing.T) { t.Errorf("error initializing test db; %v", err) } - const prefixTest = "test" - const prefixOther = "other" - - const MessageCount = 10 - for i := 0; i < MessageCount; i++ { - if err := store.SaveMessages(TestTopic, TestShard, []*store.Message{{ - Uid: []byte(fmt.Sprintf("%s-%d", prefixTest, i)), - Message: []byte(fmt.Sprintf("message-%d", i)), - }, { - Uid: []byte(fmt.Sprintf("%s-%d", prefixOther, i)), - Message: []byte(fmt.Sprintf("message-%d", i)), - }}); err != nil { - t.Errorf("error saving prefix messages; %v", err) + for _, test := range tests { + messages, err := store.GetMessages(TestTopic, TestShard, test.Prefixes, test.Start, test.Max, test.Newest) + if err != nil { + t.Errorf("error getting message; %v", err) + return } - } - - messages, err := store.GetMessages(TestTopic, TestShard, [][]byte{ - []byte(prefixTest), - []byte(prefixOther), - }, nil, 5, false) - if err != nil { - t.Errorf("error getting message; %v", err) - return - } - - if len(messages) != MessageCount { - t.Errorf("unexpected number of messages: %d, expected %d\n", len(messages), MessageCount) - return - } - for i := range messages { - message := messages[i] - var prefix = prefixTest - var id = i - if i >= MessageCount/2 { - prefix = prefixOther - id = i - MessageCount/2 - } - expectedUid := fmt.Sprintf("%s-%d", prefix, id) - if string(message.Uid) != expectedUid { - t.Errorf("unexpected message uid: %s, expected %s\n", message.Uid, expectedUid) + if len(messages) != len(test.Expected) { + t.Errorf("unexpected number of messages: %d, expected %d\n", len(messages), len(test.Expected)) return } - expectedMessage := fmt.Sprintf("message-%d", id) - if string(message.Message) != expectedMessage { - t.Errorf("unexpected message: %s, expected %s\n", message.Message, expectedMessage) - return + + for i := range messages { + message := messages[i] + expected := test.Expected[i] + + if !bytes.Equal(message.Uid, expected.Uid) { + t.Errorf("unexpected message uid: %s, expected %s\n", message.Uid, expected.Uid) + return + } + + if !bytes.Equal(message.Message, expected.Message) { + t.Errorf("unexpected message: %s, expected %s\n", message.Message, expected.Message) + return + } } } } From 854adbd51765dbbb0718d230c21df509a2e93c07 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Sun, 14 Jul 2024 21:34:52 -0700 Subject: [PATCH 05/23] Newest prefix messages test. --- db/store/message_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/db/store/message_test.go b/db/store/message_test.go index 35be1978..8c8a20f1 100644 --- a/db/store/message_test.go +++ b/db/store/message_test.go @@ -70,6 +70,14 @@ var tests = []GetMessagesTest{{ Expected: []*store.Message{ testMessageTest1, testMessageTest2, testMessageTest3, testMessageTest4, testMessageTest5, }, +}, { + Prefixes: [][]byte{[]byte(PrefixTest), []byte(PrefixOther)}, + Max: 5, + Newest: true, + Expected: []*store.Message{ + testMessageTest9, testMessageTest8, testMessageTest7, testMessageTest6, testMessageTest5, + testMessageOther9, testMessageOther8, testMessageOther7, testMessageOther6, testMessageOther5, + }, }} func initTestDb() error { From 1af629e791e8e42e73ac915ad5f194f1142e5fc3 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Tue, 16 Jul 2024 12:35:57 -0700 Subject: [PATCH 06/23] New store get by prefixes interface. --- db/server/server.go | 15 ++++- db/store/message.go | 121 ++++++++++++++++++++++++++------------- db/store/message_test.go | 48 ++++++++++------ 3 files changed, 125 insertions(+), 59 deletions(-) diff --git a/db/server/server.go b/db/server/server.go index 42e0b0d7..0f685f41 100644 --- a/db/server/server.go +++ b/db/server/server.go @@ -129,8 +129,19 @@ func (s *Server) GetMessages(ctx context.Context, request *queue_pb.Request) (*q } } else { for i := 0; i < 2; i++ { - messages, err = store.GetMessages(request.Topic, s.Shard, request.Prefixes, request.Start, int(request.Max), - request.Newest) + var requestByPrefixes = store.RequestByPrefixes{ + Topic: request.Topic, + Shard: s.Shard, + Max: int(request.Max), + Newest: request.Newest, + } + for _, prefix := range request.Prefixes { + requestByPrefixes.Prefixes = append(requestByPrefixes.Prefixes, store.Prefix{ + Prefix: prefix, + Start: request.Start, + }) + } + messages, err = store.GetByPrefixes(requestByPrefixes) if err != nil { return nil, fmt.Errorf("error getting messages for topic: %s (shard %d); %w", request.Topic, s.Shard, err) } diff --git a/db/store/message.go b/db/store/message.go index d5e5dc1d..b5452955 100644 --- a/db/store/message.go +++ b/db/store/message.go @@ -81,65 +81,106 @@ func GetMessagesByUids(topic string, shard uint, uids [][]byte) ([]*Message, err return messages, nil } -// GetMessages returns messages. Options: -// - Topic: required -// - Shard: required -// - Prefixes: optional, limits results -// - Start: optional, where to start -// - Max: optional, number of results -// - Newest: optional, reverse order (fff first, instead of 000) -func GetMessages(topic string, shard uint, prefixes [][]byte, start []byte, max int, newest bool) ([]*Message, error) { - db, err := getDb(topic, shard) +type RequestByPrefixes struct { + Topic string // required + Shard uint // required + Prefixes []Prefix + Max int + Newest bool +} + +type Prefix struct { + Prefix []byte + Start []byte + Max int +} + +// GetByPrefixes returns messages. +func GetByPrefixes(request RequestByPrefixes) ([]*Message, error) { + db, err := getDb(request.Topic, request.Shard) if err != nil { - return nil, fmt.Errorf("error getting db shard %d; %w", shard, err) + return nil, fmt.Errorf("error getting db shard %d; %w", request.Shard, err) } - if max == 0 { - max = client.HugeLimit + + var maxResults = request.Max + if maxResults == 0 { + maxResults = client.HugeLimit } + + var prefixes = request.Prefixes if len(prefixes) == 0 { - prefixes = append(prefixes, []byte{}) + prefixes = append(prefixes, Prefix{}) } + var messages []*Message defer func() { metric.AddTopicRead(metric.TopicRead{ - Topic: topic, + Topic: request.Topic, Quantity: len(messages), }) }() + for _, prefix := range prefixes { - var prefixMessages []*Message - iterRange := util.BytesPrefix(prefix) - if len(start) > 0 { - if newest && (len(prefix) == 0 || bytes.Compare(start, prefix) != 1) { - iterRange.Limit = start - } else if !newest && (len(prefix) == 0 || bytes.Compare(start, prefix) != -1) { - iterRange.Start = start - } - } - iter := db.NewIterator(iterRange, nil) - storePrefixMessage := func() bool { - prefixMessages = append(prefixMessages, &Message{ - Uid: GetPtrSlice(iter.Key()), - Message: GetPtrSlice(iter.Value()), - }) - return len(prefixMessages) < max - } - if newest { - for ok := iter.Last(); ok && storePrefixMessage(); ok = iter.Prev() { - } - } else { - for iter.Next() && storePrefixMessage() { - } + prefixMessages, err := getPrefixMessages(db, prefix, request.Newest, maxResults-len(messages)) + if err != nil { + return nil, fmt.Errorf("error getting prefix messages; %w", err) } + messages = append(messages, prefixMessages...) - iter.Release() - if err = iter.Error(); err != nil { - return nil, fmt.Errorf("error with releasing iterator; %w", err) + + if len(messages) >= maxResults { + break } } + return messages, nil } +func getPrefixMessages(db *leveldb.DB, prefix Prefix, newest bool, totalMaxLeft int) ([]*Message, error) { + var maxPrefixResults = prefix.Max + if maxPrefixResults == 0 { + maxPrefixResults = client.HugeLimit + } + if maxPrefixResults > totalMaxLeft { + maxPrefixResults = totalMaxLeft + } + + iterRange := util.BytesPrefix(prefix.Prefix) + if len(prefix.Start) > 0 { + if newest && (len(prefix.Prefix) == 0 || bytes.Compare(prefix.Start, prefix.Prefix) != 1) { + iterRange.Limit = prefix.Start + } else if !newest && (len(prefix.Prefix) == 0 || bytes.Compare(prefix.Start, prefix.Prefix) != -1) { + iterRange.Start = prefix.Start + } + } + + iter := db.NewIterator(iterRange, nil) + var prefixMessages []*Message + + storePrefixMessage := func() bool { + prefixMessages = append(prefixMessages, &Message{ + Uid: GetPtrSlice(iter.Key()), + Message: GetPtrSlice(iter.Value()), + }) + return len(prefixMessages) < maxPrefixResults + } + + if newest { + for ok := iter.Last(); ok && storePrefixMessage(); ok = iter.Prev() { + } + } else { + for iter.Next() && storePrefixMessage() { + } + } + + iter.Release() + if err := iter.Error(); err != nil { + return nil, fmt.Errorf("error with releasing iterator; %w", err) + } + + return prefixMessages, nil +} + func DeleteMessages(topic string, shard uint, uids [][]byte) error { db, err := getDb(topic, shard) if err != nil { diff --git a/db/store/message_test.go b/db/store/message_test.go index 8c8a20f1..220f9446 100644 --- a/db/store/message_test.go +++ b/db/store/message_test.go @@ -41,38 +41,43 @@ var ( ) type GetMessagesTest struct { - Prefixes [][]byte - Start []byte + Name string + Prefixes []store.Prefix Max int Newest bool Expected []*store.Message } var tests = []GetMessagesTest{{ - Prefixes: [][]byte{[]byte(PrefixTest), []byte(PrefixOther)}, - Max: 5, + Name: "Basic", + Prefixes: []store.Prefix{{Prefix: []byte(PrefixTest), Max: 5}, {Prefix: []byte(PrefixOther), Max: 5}}, Expected: []*store.Message{ testMessageTest0, testMessageTest1, testMessageTest2, testMessageTest3, testMessageTest4, testMessageOther0, testMessageOther1, testMessageOther2, testMessageOther3, testMessageOther4, }, }, { - Prefixes: [][]byte{[]byte(PrefixTest), []byte(PrefixOther)}, - Start: []byte(fmt.Sprintf("%s-%d", PrefixOther, 1)), - Max: 5, + Name: "Start 1", + Prefixes: []store.Prefix{ + {Prefix: []byte(PrefixTest), Start: []byte(fmt.Sprintf("%s-%d", PrefixOther, 1)), Max: 5}, + {Prefix: []byte(PrefixOther), Start: []byte(fmt.Sprintf("%s-%d", PrefixOther, 1)), Max: 5}, + }, Expected: []*store.Message{ testMessageTest0, testMessageTest1, testMessageTest2, testMessageTest3, testMessageTest4, testMessageOther1, testMessageOther2, testMessageOther3, testMessageOther4, testMessageOther5, }, }, { - Prefixes: [][]byte{[]byte(PrefixTest), []byte(PrefixOther)}, - Start: []byte(fmt.Sprintf("%s-%d", PrefixTest, 1)), - Max: 5, + Name: "Start 2", + Prefixes: []store.Prefix{ + {Prefix: []byte(PrefixTest), Start: []byte(fmt.Sprintf("%s-%d", PrefixTest, 1))}, + {Prefix: []byte(PrefixOther), Start: []byte(fmt.Sprintf("%s-%d", PrefixTest, 1))}, + }, + Max: 5, Expected: []*store.Message{ testMessageTest1, testMessageTest2, testMessageTest3, testMessageTest4, testMessageTest5, }, }, { - Prefixes: [][]byte{[]byte(PrefixTest), []byte(PrefixOther)}, - Max: 5, + Name: "Newest", + Prefixes: []store.Prefix{{Prefix: []byte(PrefixTest), Max: 5}, {Prefix: []byte(PrefixOther), Max: 5}}, Newest: true, Expected: []*store.Message{ testMessageTest9, testMessageTest8, testMessageTest7, testMessageTest6, testMessageTest5, @@ -133,14 +138,21 @@ func TestGetByPrefixes(t *testing.T) { } for _, test := range tests { - messages, err := store.GetMessages(TestTopic, TestShard, test.Prefixes, test.Start, test.Max, test.Newest) + messages, err := store.GetByPrefixes(store.RequestByPrefixes{ + Topic: TestTopic, + Shard: TestShard, + Prefixes: test.Prefixes, + Max: test.Max, + Newest: test.Newest, + }) if err != nil { - t.Errorf("error getting message; %v", err) + t.Errorf("%s test error getting message; %v", test.Name, err) return } if len(messages) != len(test.Expected) { - t.Errorf("unexpected number of messages: %d, expected %d\n", len(messages), len(test.Expected)) + t.Errorf("%s test error unexpected number of messages: %d, expected %d\n", + test.Name, len(messages), len(test.Expected)) return } @@ -149,12 +161,14 @@ func TestGetByPrefixes(t *testing.T) { expected := test.Expected[i] if !bytes.Equal(message.Uid, expected.Uid) { - t.Errorf("unexpected message uid: %s, expected %s\n", message.Uid, expected.Uid) + t.Errorf("%s test error unexpected message uid: %s, expected %s\n", + test.Name, message.Uid, expected.Uid) return } if !bytes.Equal(message.Message, expected.Message) { - t.Errorf("unexpected message: %s, expected %s\n", message.Message, expected.Message) + t.Errorf("%s test error unexpected message: %s, expected %s\n", + test.Name, message.Message, expected.Message) return } } From bb10c556570d28350be4d66aef20b211ef9261d8 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Thu, 18 Jul 2024 09:48:46 -0700 Subject: [PATCH 07/23] Close DBs after tests. --- db/store/message_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/db/store/message_test.go b/db/store/message_test.go index 220f9446..c17a0d4a 100644 --- a/db/store/message_test.go +++ b/db/store/message_test.go @@ -114,6 +114,7 @@ func TestGetMessage(t *testing.T) { if err := initTestDb(); err != nil { t.Errorf("error initializing test db; %v", err) } + defer store.CloseAll() message, err := store.GetMessage(TestTopic, TestShard, testMessageTest1.Uid) if err != nil { @@ -136,6 +137,7 @@ func TestGetByPrefixes(t *testing.T) { if err := initTestDb(); err != nil { t.Errorf("error initializing test db; %v", err) } + defer store.CloseAll() for _, test := range tests { messages, err := store.GetByPrefixes(store.RequestByPrefixes{ From 3b360e1c5dfce08e948121667bcbcc9516df4509 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Thu, 18 Jul 2024 10:03:50 -0700 Subject: [PATCH 08/23] Add GetByPrefixes to queue protobuf interface. Implement interface. Update README. --- db/proto/README.md | 12 + db/proto/queue.proto | 15 ++ db/proto/queue_pb/queue.pb.go | 347 ++++++++++++++++++++++------- db/proto/queue_pb/queue_grpc.pb.go | 93 ++++++-- db/server/server.go | 30 ++- go.mod | 15 +- go.sum | 27 ++- 7 files changed, 415 insertions(+), 124 deletions(-) diff --git a/db/proto/README.md b/db/proto/README.md index cacb0272..1fee9333 100644 --- a/db/proto/README.md +++ b/db/proto/README.md @@ -1,7 +1,19 @@ # Directions +## Install the Protocol Buffers Compiler + +```bash +brew install protobuf +export GOPATH=~/go +go install google.golang.org/protobuf/cmd/protoc-gen-go@latest +go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@latest +``` + +## Generate the Go Code + ```bash src="/usr/local/src" +cd $src/github.com/memocash/index/db/proto mkdir -p queue_pb protoc --go_out=$src ./queue.proto protoc --go-grpc_out=$src ./queue.proto diff --git a/db/proto/queue.proto b/db/proto/queue.proto index e654aa2b..b28b4034 100644 --- a/db/proto/queue.proto +++ b/db/proto/queue.proto @@ -13,6 +13,8 @@ service Queue { } rpc GetMessages (Request) returns (Messages) { } + rpc GetByPrefixes (RequestPrefixes) returns (Messages) { + } rpc GetStreamMessages(RequestStream) returns (stream Message) { } rpc GetTopicList (EmptyRequest) returns (TopicListReply) { @@ -81,3 +83,16 @@ message CountRequest { message TopicCount { uint64 count = 1; } + +message RequestPrefixes { + string topic = 1; + repeated RequestPrefix prefixes = 2; + uint32 max = 3; + bool newest = 4; +} + +message RequestPrefix { + bytes prefix = 1; + bytes start = 2; + uint32 max = 3; +} diff --git a/db/proto/queue_pb/queue.pb.go b/db/proto/queue_pb/queue.pb.go index 0a4c533b..f530097f 100644 --- a/db/proto/queue_pb/queue.pb.go +++ b/db/proto/queue_pb/queue.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.26.0 -// protoc v3.17.3 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: queue.proto package queue_pb @@ -55,7 +55,7 @@ func (x *Messages) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use UidChan.ProtoReflect.Descriptor instead. +// Deprecated: Use Messages.ProtoReflect.Descriptor instead. func (*Messages) Descriptor() ([]byte, []int) { return file_queue_proto_rawDescGZIP(), []int{0} } @@ -687,6 +687,140 @@ func (x *TopicCount) GetCount() uint64 { return 0 } +type RequestPrefixes struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Topic string `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` + Prefixes []*RequestPrefix `protobuf:"bytes,2,rep,name=prefixes,proto3" json:"prefixes,omitempty"` + Max uint32 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` + Newest bool `protobuf:"varint,4,opt,name=newest,proto3" json:"newest,omitempty"` +} + +func (x *RequestPrefixes) Reset() { + *x = RequestPrefixes{} + if protoimpl.UnsafeEnabled { + mi := &file_queue_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RequestPrefixes) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestPrefixes) ProtoMessage() {} + +func (x *RequestPrefixes) ProtoReflect() protoreflect.Message { + mi := &file_queue_proto_msgTypes[12] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestPrefixes.ProtoReflect.Descriptor instead. +func (*RequestPrefixes) Descriptor() ([]byte, []int) { + return file_queue_proto_rawDescGZIP(), []int{12} +} + +func (x *RequestPrefixes) GetTopic() string { + if x != nil { + return x.Topic + } + return "" +} + +func (x *RequestPrefixes) GetPrefixes() []*RequestPrefix { + if x != nil { + return x.Prefixes + } + return nil +} + +func (x *RequestPrefixes) GetMax() uint32 { + if x != nil { + return x.Max + } + return 0 +} + +func (x *RequestPrefixes) GetNewest() bool { + if x != nil { + return x.Newest + } + return false +} + +type RequestPrefix struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Prefix []byte `protobuf:"bytes,1,opt,name=prefix,proto3" json:"prefix,omitempty"` + Start []byte `protobuf:"bytes,2,opt,name=start,proto3" json:"start,omitempty"` + Max uint32 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` +} + +func (x *RequestPrefix) Reset() { + *x = RequestPrefix{} + if protoimpl.UnsafeEnabled { + mi := &file_queue_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RequestPrefix) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestPrefix) ProtoMessage() {} + +func (x *RequestPrefix) ProtoReflect() protoreflect.Message { + mi := &file_queue_proto_msgTypes[13] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestPrefix.ProtoReflect.Descriptor instead. +func (*RequestPrefix) Descriptor() ([]byte, []int) { + return file_queue_proto_rawDescGZIP(), []int{13} +} + +func (x *RequestPrefix) GetPrefix() []byte { + if x != nil { + return x.Prefix + } + return nil +} + +func (x *RequestPrefix) GetStart() []byte { + if x != nil { + return x.Start + } + return nil +} + +func (x *RequestPrefix) GetMax() uint32 { + if x != nil { + return x.Max + } + return 0 +} + var File_queue_proto protoreflect.FileDescriptor var file_queue_proto_rawDesc = []byte{ @@ -739,39 +873,57 @@ var file_queue_proto_rawDesc = []byte{ 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x22, 0x0a, 0x0a, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x32, 0xc4, 0x03, 0x0a, - 0x05, 0x51, 0x75, 0x65, 0x75, 0x65, 0x12, 0x3a, 0x0a, 0x0c, 0x53, 0x61, 0x76, 0x65, 0x4d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, - 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, - 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x73, 0x12, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x55, 0x69, 0x64, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, - 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, - 0x79, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x12, 0x17, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, - 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, - 0x36, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x11, - 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x1a, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x73, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x17, 0x2e, 0x71, - 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, - 0x74, 0x72, 0x65, 0x61, 0x6d, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, - 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, - 0x47, 0x65, 0x74, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x71, - 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, - 0x54, 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, - 0x12, 0x41, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x12, 0x16, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x43, - 0x6f, 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x71, 0x75, - 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x43, 0x6f, 0x75, 0x6e, - 0x74, 0x22, 0x00, 0x42, 0x2d, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, - 0x6d, 0x2f, 0x6d, 0x65, 0x6d, 0x6f, 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, - 0x2f, 0x64, 0x62, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, - 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x86, 0x01, 0x0a, + 0x0f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, + 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x33, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, + 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, + 0x78, 0x52, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x6d, + 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x12, 0x16, 0x0a, + 0x06, 0x6e, 0x65, 0x77, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x6e, + 0x65, 0x77, 0x65, 0x73, 0x74, 0x22, 0x4f, 0x0a, 0x0d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x14, + 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, + 0x74, 0x61, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x32, 0x86, 0x04, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x75, 0x65, + 0x12, 0x3a, 0x0a, 0x0c, 0x53, 0x61, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, + 0x12, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x0e, + 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x15, + 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, + 0x65, 0x55, 0x69, 0x64, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, + 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3a, 0x0a, + 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, 0x2e, 0x71, 0x75, + 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, 0x69, + 0x6e, 0x67, 0x6c, 0x65, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0b, 0x47, 0x65, 0x74, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, + 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x71, 0x75, + 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x22, + 0x00, 0x12, 0x40, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x42, 0x79, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, + 0x65, 0x73, 0x12, 0x19, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x1a, 0x12, 0x2e, + 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x73, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x17, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, + 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x54, + 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, + 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x1a, 0x18, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, + 0x63, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0f, + 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, + 0x16, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, + 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x00, 0x42, + 0x2d, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x65, + 0x6d, 0x6f, 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x64, 0x62, 0x2f, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -786,43 +938,48 @@ func file_queue_proto_rawDescGZIP() []byte { return file_queue_proto_rawDescData } -var file_queue_proto_msgTypes = make([]protoimpl.MessageInfo, 12) -var file_queue_proto_goTypes = []interface{}{ - (*Messages)(nil), // 0: queue_pb.UidChan - (*Message)(nil), // 1: queue_pb.Message - (*ErrorReply)(nil), // 2: queue_pb.ErrorReply - (*MessageUids)(nil), // 3: queue_pb.MessageUids - (*RequestSingle)(nil), // 4: queue_pb.RequestSingle - (*Request)(nil), // 5: queue_pb.Request - (*RequestStream)(nil), // 6: queue_pb.RequestStream - (*EmptyRequest)(nil), // 7: queue_pb.EmptyRequest - (*Topic)(nil), // 8: queue_pb.Topic - (*TopicListReply)(nil), // 9: queue_pb.TopicListReply - (*CountRequest)(nil), // 10: queue_pb.CountRequest - (*TopicCount)(nil), // 11: queue_pb.TopicCount +var file_queue_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_queue_proto_goTypes = []any{ + (*Messages)(nil), // 0: queue_pb.Messages + (*Message)(nil), // 1: queue_pb.Message + (*ErrorReply)(nil), // 2: queue_pb.ErrorReply + (*MessageUids)(nil), // 3: queue_pb.MessageUids + (*RequestSingle)(nil), // 4: queue_pb.RequestSingle + (*Request)(nil), // 5: queue_pb.Request + (*RequestStream)(nil), // 6: queue_pb.RequestStream + (*EmptyRequest)(nil), // 7: queue_pb.EmptyRequest + (*Topic)(nil), // 8: queue_pb.Topic + (*TopicListReply)(nil), // 9: queue_pb.TopicListReply + (*CountRequest)(nil), // 10: queue_pb.CountRequest + (*TopicCount)(nil), // 11: queue_pb.TopicCount + (*RequestPrefixes)(nil), // 12: queue_pb.RequestPrefixes + (*RequestPrefix)(nil), // 13: queue_pb.RequestPrefix } var file_queue_proto_depIdxs = []int32{ - 1, // 0: queue_pb.UidChan.messages:type_name -> queue_pb.Message + 1, // 0: queue_pb.Messages.messages:type_name -> queue_pb.Message 8, // 1: queue_pb.TopicListReply.topics:type_name -> queue_pb.Topic - 0, // 2: queue_pb.Queue.SaveMessages:input_type -> queue_pb.UidChan - 3, // 3: queue_pb.Queue.DeleteMessages:input_type -> queue_pb.MessageUids - 4, // 4: queue_pb.Queue.GetMessage:input_type -> queue_pb.RequestSingle - 5, // 5: queue_pb.Queue.GetMessages:input_type -> queue_pb.Request - 6, // 6: queue_pb.Queue.GetStreamMessages:input_type -> queue_pb.RequestStream - 7, // 7: queue_pb.Queue.GetTopicList:input_type -> queue_pb.EmptyRequest - 10, // 8: queue_pb.Queue.GetMessageCount:input_type -> queue_pb.CountRequest - 2, // 9: queue_pb.Queue.SaveMessages:output_type -> queue_pb.ErrorReply - 2, // 10: queue_pb.Queue.DeleteMessages:output_type -> queue_pb.ErrorReply - 1, // 11: queue_pb.Queue.GetMessage:output_type -> queue_pb.Message - 0, // 12: queue_pb.Queue.GetMessages:output_type -> queue_pb.UidChan - 1, // 13: queue_pb.Queue.GetStreamMessages:output_type -> queue_pb.Message - 9, // 14: queue_pb.Queue.GetTopicList:output_type -> queue_pb.TopicListReply - 11, // 15: queue_pb.Queue.GetMessageCount:output_type -> queue_pb.TopicCount - 9, // [9:16] is the sub-list for method output_type - 2, // [2:9] is the sub-list for method input_type - 2, // [2:2] is the sub-list for extension type_name - 2, // [2:2] is the sub-list for extension extendee - 0, // [0:2] is the sub-list for field type_name + 13, // 2: queue_pb.RequestPrefixes.prefixes:type_name -> queue_pb.RequestPrefix + 0, // 3: queue_pb.Queue.SaveMessages:input_type -> queue_pb.Messages + 3, // 4: queue_pb.Queue.DeleteMessages:input_type -> queue_pb.MessageUids + 4, // 5: queue_pb.Queue.GetMessage:input_type -> queue_pb.RequestSingle + 5, // 6: queue_pb.Queue.GetMessages:input_type -> queue_pb.Request + 12, // 7: queue_pb.Queue.GetByPrefixes:input_type -> queue_pb.RequestPrefixes + 6, // 8: queue_pb.Queue.GetStreamMessages:input_type -> queue_pb.RequestStream + 7, // 9: queue_pb.Queue.GetTopicList:input_type -> queue_pb.EmptyRequest + 10, // 10: queue_pb.Queue.GetMessageCount:input_type -> queue_pb.CountRequest + 2, // 11: queue_pb.Queue.SaveMessages:output_type -> queue_pb.ErrorReply + 2, // 12: queue_pb.Queue.DeleteMessages:output_type -> queue_pb.ErrorReply + 1, // 13: queue_pb.Queue.GetMessage:output_type -> queue_pb.Message + 0, // 14: queue_pb.Queue.GetMessages:output_type -> queue_pb.Messages + 0, // 15: queue_pb.Queue.GetByPrefixes:output_type -> queue_pb.Messages + 1, // 16: queue_pb.Queue.GetStreamMessages:output_type -> queue_pb.Message + 9, // 17: queue_pb.Queue.GetTopicList:output_type -> queue_pb.TopicListReply + 11, // 18: queue_pb.Queue.GetMessageCount:output_type -> queue_pb.TopicCount + 11, // [11:19] is the sub-list for method output_type + 3, // [3:11] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name } func init() { file_queue_proto_init() } @@ -831,7 +988,7 @@ func file_queue_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_queue_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Messages); i { case 0: return &v.state @@ -843,7 +1000,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*Message); i { case 0: return &v.state @@ -855,7 +1012,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*ErrorReply); i { case 0: return &v.state @@ -867,7 +1024,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*MessageUids); i { case 0: return &v.state @@ -879,7 +1036,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*RequestSingle); i { case 0: return &v.state @@ -891,7 +1048,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*Request); i { case 0: return &v.state @@ -903,7 +1060,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*RequestStream); i { case 0: return &v.state @@ -915,7 +1072,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*EmptyRequest); i { case 0: return &v.state @@ -927,7 +1084,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[8].Exporter = func(v any, i int) any { switch v := v.(*Topic); i { case 0: return &v.state @@ -939,7 +1096,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[9].Exporter = func(v any, i int) any { switch v := v.(*TopicListReply); i { case 0: return &v.state @@ -951,7 +1108,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[10].Exporter = func(v any, i int) any { switch v := v.(*CountRequest); i { case 0: return &v.state @@ -963,7 +1120,7 @@ func file_queue_proto_init() { return nil } } - file_queue_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + file_queue_proto_msgTypes[11].Exporter = func(v any, i int) any { switch v := v.(*TopicCount); i { case 0: return &v.state @@ -975,6 +1132,30 @@ func file_queue_proto_init() { return nil } } + file_queue_proto_msgTypes[12].Exporter = func(v any, i int) any { + switch v := v.(*RequestPrefixes); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_queue_proto_msgTypes[13].Exporter = func(v any, i int) any { + switch v := v.(*RequestPrefix); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -982,7 +1163,7 @@ func file_queue_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_queue_proto_rawDesc, NumEnums: 0, - NumMessages: 12, + NumMessages: 14, NumExtensions: 0, NumServices: 1, }, diff --git a/db/proto/queue_pb/queue_grpc.pb.go b/db/proto/queue_pb/queue_grpc.pb.go index 30abb710..1d2ec3e2 100644 --- a/db/proto/queue_pb/queue_grpc.pb.go +++ b/db/proto/queue_pb/queue_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.4.0 +// - protoc v5.27.1 +// source: queue.proto package queue_pb @@ -11,8 +15,19 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + Queue_SaveMessages_FullMethodName = "/queue_pb.Queue/SaveMessages" + Queue_DeleteMessages_FullMethodName = "/queue_pb.Queue/DeleteMessages" + Queue_GetMessage_FullMethodName = "/queue_pb.Queue/GetMessage" + Queue_GetMessages_FullMethodName = "/queue_pb.Queue/GetMessages" + Queue_GetByPrefixes_FullMethodName = "/queue_pb.Queue/GetByPrefixes" + Queue_GetStreamMessages_FullMethodName = "/queue_pb.Queue/GetStreamMessages" + Queue_GetTopicList_FullMethodName = "/queue_pb.Queue/GetTopicList" + Queue_GetMessageCount_FullMethodName = "/queue_pb.Queue/GetMessageCount" +) // QueueClient is the client API for Queue service. // @@ -22,6 +37,7 @@ type QueueClient interface { DeleteMessages(ctx context.Context, in *MessageUids, opts ...grpc.CallOption) (*ErrorReply, error) GetMessage(ctx context.Context, in *RequestSingle, opts ...grpc.CallOption) (*Message, error) GetMessages(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Messages, error) + GetByPrefixes(ctx context.Context, in *RequestPrefixes, opts ...grpc.CallOption) (*Messages, error) GetStreamMessages(ctx context.Context, in *RequestStream, opts ...grpc.CallOption) (Queue_GetStreamMessagesClient, error) GetTopicList(ctx context.Context, in *EmptyRequest, opts ...grpc.CallOption) (*TopicListReply, error) GetMessageCount(ctx context.Context, in *CountRequest, opts ...grpc.CallOption) (*TopicCount, error) @@ -36,8 +52,9 @@ func NewQueueClient(cc grpc.ClientConnInterface) QueueClient { } func (c *queueClient) SaveMessages(ctx context.Context, in *Messages, opts ...grpc.CallOption) (*ErrorReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ErrorReply) - err := c.cc.Invoke(ctx, "/queue_pb.Queue/SaveMessages", in, out, opts...) + err := c.cc.Invoke(ctx, Queue_SaveMessages_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -45,8 +62,9 @@ func (c *queueClient) SaveMessages(ctx context.Context, in *Messages, opts ...gr } func (c *queueClient) DeleteMessages(ctx context.Context, in *MessageUids, opts ...grpc.CallOption) (*ErrorReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ErrorReply) - err := c.cc.Invoke(ctx, "/queue_pb.Queue/DeleteMessages", in, out, opts...) + err := c.cc.Invoke(ctx, Queue_DeleteMessages_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -54,8 +72,9 @@ func (c *queueClient) DeleteMessages(ctx context.Context, in *MessageUids, opts } func (c *queueClient) GetMessage(ctx context.Context, in *RequestSingle, opts ...grpc.CallOption) (*Message, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Message) - err := c.cc.Invoke(ctx, "/queue_pb.Queue/GetMessage", in, out, opts...) + err := c.cc.Invoke(ctx, Queue_GetMessage_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -63,8 +82,19 @@ func (c *queueClient) GetMessage(ctx context.Context, in *RequestSingle, opts .. } func (c *queueClient) GetMessages(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Messages, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Messages) + err := c.cc.Invoke(ctx, Queue_GetMessages_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *queueClient) GetByPrefixes(ctx context.Context, in *RequestPrefixes, opts ...grpc.CallOption) (*Messages, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(Messages) - err := c.cc.Invoke(ctx, "/queue_pb.Queue/GetMessages", in, out, opts...) + err := c.cc.Invoke(ctx, Queue_GetByPrefixes_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -72,11 +102,12 @@ func (c *queueClient) GetMessages(ctx context.Context, in *Request, opts ...grpc } func (c *queueClient) GetStreamMessages(ctx context.Context, in *RequestStream, opts ...grpc.CallOption) (Queue_GetStreamMessagesClient, error) { - stream, err := c.cc.NewStream(ctx, &Queue_ServiceDesc.Streams[0], "/queue_pb.Queue/GetStreamMessages", opts...) + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Queue_ServiceDesc.Streams[0], Queue_GetStreamMessages_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &queueGetStreamMessagesClient{stream} + x := &queueGetStreamMessagesClient{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -104,8 +135,9 @@ func (x *queueGetStreamMessagesClient) Recv() (*Message, error) { } func (c *queueClient) GetTopicList(ctx context.Context, in *EmptyRequest, opts ...grpc.CallOption) (*TopicListReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TopicListReply) - err := c.cc.Invoke(ctx, "/queue_pb.Queue/GetTopicList", in, out, opts...) + err := c.cc.Invoke(ctx, Queue_GetTopicList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -113,8 +145,9 @@ func (c *queueClient) GetTopicList(ctx context.Context, in *EmptyRequest, opts . } func (c *queueClient) GetMessageCount(ctx context.Context, in *CountRequest, opts ...grpc.CallOption) (*TopicCount, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TopicCount) - err := c.cc.Invoke(ctx, "/queue_pb.Queue/GetMessageCount", in, out, opts...) + err := c.cc.Invoke(ctx, Queue_GetMessageCount_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -129,6 +162,7 @@ type QueueServer interface { DeleteMessages(context.Context, *MessageUids) (*ErrorReply, error) GetMessage(context.Context, *RequestSingle) (*Message, error) GetMessages(context.Context, *Request) (*Messages, error) + GetByPrefixes(context.Context, *RequestPrefixes) (*Messages, error) GetStreamMessages(*RequestStream, Queue_GetStreamMessagesServer) error GetTopicList(context.Context, *EmptyRequest) (*TopicListReply, error) GetMessageCount(context.Context, *CountRequest) (*TopicCount, error) @@ -151,6 +185,9 @@ func (UnimplementedQueueServer) GetMessage(context.Context, *RequestSingle) (*Me func (UnimplementedQueueServer) GetMessages(context.Context, *Request) (*Messages, error) { return nil, status.Errorf(codes.Unimplemented, "method GetMessages not implemented") } +func (UnimplementedQueueServer) GetByPrefixes(context.Context, *RequestPrefixes) (*Messages, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetByPrefixes not implemented") +} func (UnimplementedQueueServer) GetStreamMessages(*RequestStream, Queue_GetStreamMessagesServer) error { return status.Errorf(codes.Unimplemented, "method GetStreamMessages not implemented") } @@ -183,7 +220,7 @@ func _Queue_SaveMessages_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/queue_pb.Queue/SaveMessages", + FullMethod: Queue_SaveMessages_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(QueueServer).SaveMessages(ctx, req.(*Messages)) @@ -201,7 +238,7 @@ func _Queue_DeleteMessages_Handler(srv interface{}, ctx context.Context, dec fun } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/queue_pb.Queue/DeleteMessages", + FullMethod: Queue_DeleteMessages_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(QueueServer).DeleteMessages(ctx, req.(*MessageUids)) @@ -219,7 +256,7 @@ func _Queue_GetMessage_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/queue_pb.Queue/GetMessage", + FullMethod: Queue_GetMessage_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(QueueServer).GetMessage(ctx, req.(*RequestSingle)) @@ -237,7 +274,7 @@ func _Queue_GetMessages_Handler(srv interface{}, ctx context.Context, dec func(i } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/queue_pb.Queue/GetMessages", + FullMethod: Queue_GetMessages_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(QueueServer).GetMessages(ctx, req.(*Request)) @@ -245,12 +282,30 @@ func _Queue_GetMessages_Handler(srv interface{}, ctx context.Context, dec func(i return interceptor(ctx, in, info, handler) } +func _Queue_GetByPrefixes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RequestPrefixes) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(QueueServer).GetByPrefixes(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Queue_GetByPrefixes_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(QueueServer).GetByPrefixes(ctx, req.(*RequestPrefixes)) + } + return interceptor(ctx, in, info, handler) +} + func _Queue_GetStreamMessages_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(RequestStream) if err := stream.RecvMsg(m); err != nil { return err } - return srv.(QueueServer).GetStreamMessages(m, &queueGetStreamMessagesServer{stream}) + return srv.(QueueServer).GetStreamMessages(m, &queueGetStreamMessagesServer{ServerStream: stream}) } type Queue_GetStreamMessagesServer interface { @@ -276,7 +331,7 @@ func _Queue_GetTopicList_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/queue_pb.Queue/GetTopicList", + FullMethod: Queue_GetTopicList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(QueueServer).GetTopicList(ctx, req.(*EmptyRequest)) @@ -294,7 +349,7 @@ func _Queue_GetMessageCount_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/queue_pb.Queue/GetMessageCount", + FullMethod: Queue_GetMessageCount_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(QueueServer).GetMessageCount(ctx, req.(*CountRequest)) @@ -325,6 +380,10 @@ var Queue_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetMessages", Handler: _Queue_GetMessages_Handler, }, + { + MethodName: "GetByPrefixes", + Handler: _Queue_GetByPrefixes_Handler, + }, { MethodName: "GetTopicList", Handler: _Queue_GetTopicList_Handler, diff --git a/db/server/server.go b/db/server/server.go index 0f685f41..579b96e1 100644 --- a/db/server/server.go +++ b/db/server/server.go @@ -155,17 +155,43 @@ func (s *Server) GetMessages(ctx context.Context, request *queue_pb.Request) (*q } } } + + return getQueueMessagesFromStoreMessages(request.Topic, messages), nil +} + +func getQueueMessagesFromStoreMessages(topic string, messages []*store.Message) *queue_pb.Messages { var queueMessages = make([]*queue_pb.Message, len(messages)) for i := range messages { queueMessages[i] = &queue_pb.Message{ - Topic: request.Topic, + Topic: topic, Uid: messages[i].Uid, Message: messages[i].Message, } } return &queue_pb.Messages{ Messages: queueMessages, - }, nil + } +} + +func (s *Server) GetByPrefixes(ctx context.Context, req *queue_pb.RequestPrefixes) (*queue_pb.Messages, error) { + var requestByPrefixes = store.RequestByPrefixes{ + Topic: req.Topic, + Shard: s.Shard, + Max: int(req.Max), + Newest: req.Newest, + } + for _, prefix := range req.Prefixes { + requestByPrefixes.Prefixes = append(requestByPrefixes.Prefixes, store.Prefix{ + Prefix: prefix.Prefix, + Start: prefix.Start, + }) + } + messages, err := store.GetByPrefixes(requestByPrefixes) + if err != nil { + return nil, fmt.Errorf("error getting messages by prefixes: %s (shard %d); %w", req.Topic, s.Shard, err) + } + + return getQueueMessagesFromStoreMessages(req.Topic, messages), nil } func (s *Server) GetStreamMessages(request *queue_pb.RequestStream, server queue_pb.Queue_GetStreamMessagesServer) error { diff --git a/go.mod b/go.mod index 1b54c6ad..ca9cb4ea 100644 --- a/go.mod +++ b/go.mod @@ -12,14 +12,15 @@ require ( github.com/jchavannes/btcutil v1.1.4 github.com/jchavannes/go-mnemonic v0.0.0-20191017214729-76f026914b65 github.com/jchavannes/jgo v0.0.0-20240515195449-361d07b9e227 + github.com/mitchellh/mapstructure v1.4.1 github.com/pkg/profile v1.6.0 github.com/spf13/cobra v1.2.1 github.com/spf13/viper v1.8.1 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tyler-smith/go-bip32 v1.0.0 github.com/vektah/gqlparser/v2 v2.5.16 - google.golang.org/grpc v1.56.3 - google.golang.org/protobuf v1.33.0 + google.golang.org/grpc v1.65.0 + google.golang.org/protobuf v1.34.1 ) require ( @@ -30,7 +31,6 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/fsnotify/fsnotify v1.5.1 // indirect - github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect @@ -40,7 +40,6 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/magiconair/properties v1.8.5 // indirect github.com/mattn/go-sqlite3 v1.14.16 // indirect - github.com/mitchellh/mapstructure v1.4.1 // indirect github.com/pelletier/go-toml v1.9.3 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/spf13/afero v1.6.0 // indirect @@ -48,10 +47,10 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.2.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.23.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/crypto v0.23.0 // indirect + golang.org/x/net v0.25.0 // indirect + golang.org/x/sys v0.20.0 // indirect + golang.org/x/text v0.15.0 // indirect google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/ini.v1 v1.62.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 6ddaa82b..8aa079d9 100644 --- a/go.sum +++ b/go.sum @@ -155,8 +155,6 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= @@ -173,7 +171,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -424,8 +422,8 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -511,8 +509,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -601,8 +599,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -622,8 +620,9 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -780,8 +779,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= +google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -795,8 +794,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= From e7da508204809df1a8cd4f006cffc0472c983768 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Mon, 22 Jul 2024 11:20:48 -0700 Subject: [PATCH 09/23] New client GetByPrefixes, convert some uses. --- cmd/maint/queue_profile.go | 6 ++-- db/client/client.go | 62 ++++++++++++++++++++++++++++++++++++++ db/item/addr/seen_tx.go | 12 +++----- db/item/chain/block.go | 1 - db/item/chain/block_tx.go | 12 +++----- 5 files changed, 75 insertions(+), 18 deletions(-) diff --git a/cmd/maint/queue_profile.go b/cmd/maint/queue_profile.go index ec0c2d14..020057f4 100644 --- a/cmd/maint/queue_profile.go +++ b/cmd/maint/queue_profile.go @@ -37,14 +37,14 @@ var queueProfileCmd = &cobra.Command{ if err != nil { log.Fatalf("fatal error getting block txs; %v", err) } - var uids [][]byte + var prefixes [][]byte for _, blockTx := range blockTxs { if db.GetShardIdFromByte(blockTx.TxHash[:]) == Shard { - uids = append(uids, jutil.ByteReverse(blockTx.TxHash[:])) + prefixes = append(prefixes, jutil.ByteReverse(blockTx.TxHash[:])) } } dbClient := client.NewClient(config.GetShardConfig(Shard, config.GetQueueShards()).GetHost()) - if err := dbClient.GetByPrefixes(db.TopicChainTxOutput, uids); err != nil { + if err := dbClient.GetByPrefixes(db.TopicChainTxOutput, prefixes); err != nil { log.Fatalf("fatal error getting db message tx outputs; %v", err) } log.Printf("%d outputs retrieved for shard 0 (height: %d, txs: %d)\n", diff --git a/db/client/client.go b/db/client/client.go index d005efbb..47db2a9d 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -125,6 +125,68 @@ func (s *Client) GetByPrefixes(topic string, prefixes [][]byte) error { return nil } +type Prefix struct { + Prefix []byte + Start []byte + Limit uint32 +} + +type Option interface { + Apply(*queue_pb.RequestPrefixes) +} + +type OptionMax struct { + Max int +} + +func (o *OptionMax) Apply(r *queue_pb.RequestPrefixes) { + r.Max = uint32(o.Max) +} + +type OptionNewest struct { + Newest bool +} + +func (o *OptionNewest) Apply(r *queue_pb.RequestPrefixes) { + r.Newest = o.Newest +} + +func (s *Client) GetByPrefixesNew(ctx context.Context, topic string, prefixes []Prefix, opts ...Option) error { + c := queue_pb.NewQueueClient(s.conn) + ctxNew, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + s.Messages = nil + var reqPrefixes = make([]*queue_pb.RequestPrefix, len(prefixes)) + for i := range prefixes { + reqPrefixes[i] = &queue_pb.RequestPrefix{ + Prefix: prefixes[i].Prefix, + Start: prefixes[i].Start, + Max: prefixes[i].Limit, + } + } + req := &queue_pb.RequestPrefixes{ + Topic: topic, + Prefixes: reqPrefixes, + } + for _, opt := range opts { + opt.Apply(req) + } + message, err := c.GetByPrefixes(ctxNew, req, grpc.MaxCallRecvMsgSize(MaxMessageSize)) + if err != nil { + return fmt.Errorf("error getting messages rpc; %w", err) + } + var messages = make([]Message, len(message.Messages)) + for i := range message.Messages { + messages[i] = Message{ + Topic: message.Messages[i].Topic, + Uid: message.Messages[i].Uid, + Message: message.Messages[i].Message, + } + } + s.Messages = append(s.Messages, messages...) + return nil +} + func (s *Client) GetByPrefix(topic string, prefix []byte) error { if err := s.GetWOpts(Opts{ Topic: topic, diff --git a/db/item/addr/seen_tx.go b/db/item/addr/seen_tx.go index e59d6e01..fb21ec03 100644 --- a/db/item/addr/seen_tx.go +++ b/db/item/addr/seen_tx.go @@ -50,13 +50,11 @@ func (i *SeenTx) Deserialize([]byte) {} func GetSeenTxs(ctx context.Context, addr [25]byte, start []byte) ([]*SeenTx, error) { shardConfig := config.GetShardConfig(client.GenShardSource32(addr[:]), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: db.TopicAddrSeenTx, - Start: start, - Prefixes: [][]byte{addr[:]}, - Max: client.ExLargeLimit, - }); err != nil { + if err := dbClient.GetByPrefixesNew(ctx, db.TopicAddrSeenTx, []client.Prefix{{ + Prefix: addr[:], + Start: start, + Limit: client.ExLargeLimit, + }}); err != nil { return nil, fmt.Errorf("error getting db addr seen txs by prefix; %w", err) } var seenTxs = make([]*SeenTx, len(dbClient.Messages)) diff --git a/db/item/chain/block.go b/db/item/chain/block.go index 357cbbab..d7fc1b81 100644 --- a/db/item/chain/block.go +++ b/db/item/chain/block.go @@ -36,7 +36,6 @@ func (b *Block) Serialize() []byte { func (b *Block) Deserialize(data []byte) { b.Raw = data - } func GetBlock(blockHash [32]byte) (*Block, error) { diff --git a/db/item/chain/block_tx.go b/db/item/chain/block_tx.go index ceb14672..1a61988d 100644 --- a/db/item/chain/block_tx.go +++ b/db/item/chain/block_tx.go @@ -91,13 +91,11 @@ func GetBlockTxs(request BlockTxsRequest) ([]*BlockTx, error) { if request.StartIndex > 0 { startUid = GetBlockTxUid(request.BlockHash, request.StartIndex) } - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicChainBlockTx, - Prefixes: [][]byte{jutil.ByteReverse(request.BlockHash[:])}, - Start: startUid, - Max: limit, - Context: request.Context, - }); err != nil { + if err := dbClient.GetByPrefixesNew(request.Context, db.TopicChainBlockTx, []client.Prefix{{ + Prefix: jutil.ByteReverse(request.BlockHash[:]), + Start: startUid, + Limit: limit, + }}); err != nil { return nil, fmt.Errorf("error getting client message; %w", err) } var blocks = make([]*BlockTx, len(dbClient.Messages)) From bd452dcb18bc73c12d5bcc3441dff0d4f4edfc04 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Tue, 23 Jul 2024 06:26:01 -0700 Subject: [PATCH 10/23] Convert GetByPrefixes uses to GetByPrefixesNew. --- cmd/maint/queue_profile.go | 8 +++++--- db/item/chain/output_input.go | 8 ++++---- db/item/memo/like_tip.go | 10 ++++++---- graph/attach/memo_like.go | 2 +- 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/cmd/maint/queue_profile.go b/cmd/maint/queue_profile.go index 020057f4..7e0bd245 100644 --- a/cmd/maint/queue_profile.go +++ b/cmd/maint/queue_profile.go @@ -1,6 +1,7 @@ package maint import ( + "context" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/chain" @@ -26,6 +27,7 @@ var queueProfileCmd = &cobra.Command{ var outputs int start := time.Now() log.Println("starting queue profile...") + ctx := context.Background() for _, heightBlock := range heightBlocks { if db.GetShardIdFromByte(heightBlock.BlockHash[:]) != Shard { continue @@ -37,14 +39,14 @@ var queueProfileCmd = &cobra.Command{ if err != nil { log.Fatalf("fatal error getting block txs; %v", err) } - var prefixes [][]byte + var prefixes []client.Prefix for _, blockTx := range blockTxs { if db.GetShardIdFromByte(blockTx.TxHash[:]) == Shard { - prefixes = append(prefixes, jutil.ByteReverse(blockTx.TxHash[:])) + prefixes = append(prefixes, client.Prefix{Prefix: jutil.ByteReverse(blockTx.TxHash[:])}) } } dbClient := client.NewClient(config.GetShardConfig(Shard, config.GetQueueShards()).GetHost()) - if err := dbClient.GetByPrefixes(db.TopicChainTxOutput, prefixes); err != nil { + if err := dbClient.GetByPrefixesNew(ctx, db.TopicChainTxOutput, prefixes); err != nil { log.Fatalf("fatal error getting db message tx outputs; %v", err) } log.Printf("%d outputs retrieved for shard 0 (height: %d, txs: %d)\n", diff --git a/db/item/chain/output_input.go b/db/item/chain/output_input.go index 6b847b6d..3709b05c 100644 --- a/db/item/chain/output_input.go +++ b/db/item/chain/output_input.go @@ -88,7 +88,7 @@ func GetOutputInputs(ctx context.Context, outs []memo.Out) ([]*OutputInput, erro return outputInputs, nil } -func GetOutputInputsForTxHashes(txHashes [][]byte) ([]*OutputInput, error) { +func GetOutputInputsForTxHashes(ctx context.Context, txHashes [][]byte) ([]*OutputInput, error) { var shardOutGroups = make(map[uint32][][]byte) for _, txHash := range txHashes { shard := db.GetShardIdFromByte32(txHash) @@ -98,11 +98,11 @@ func GetOutputInputsForTxHashes(txHashes [][]byte) ([]*OutputInput, error) { for shard, outGroup := range shardOutGroups { shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - var prefixes = make([][]byte, len(outGroup)) + var prefixes = make([]client.Prefix, len(outGroup)) for i := range outGroup { - prefixes[i] = jutil.ByteReverse(outGroup[i]) + prefixes[i] = client.Prefix{Prefix: jutil.ByteReverse(outGroup[i])} } - if err := dbClient.GetByPrefixes(db.TopicChainOutputInput, prefixes); err != nil { + if err := dbClient.GetByPrefixesNew(ctx, db.TopicChainOutputInput, prefixes); err != nil { return nil, fmt.Errorf("error getting by prefixes for chain output inputs by tx hashes; %w", err) } for i := range dbClient.Messages { diff --git a/db/item/memo/like_tip.go b/db/item/memo/like_tip.go index 0029c729..aa2a58c6 100644 --- a/db/item/memo/like_tip.go +++ b/db/item/memo/like_tip.go @@ -1,6 +1,7 @@ package memo import ( + "context" "fmt" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" @@ -46,17 +47,18 @@ func (t *LikeTip) Deserialize(data []byte) { t.Tip = jutil.GetInt64(data) } -func GetLikeTips(likeTxHashes [][32]byte) ([]*LikeTip, error) { - var shardPrefixes = make(map[uint32][][]byte) +func GetLikeTips(ctx context.Context, likeTxHashes [][32]byte) ([]*LikeTip, error) { + var shardPrefixes = make(map[uint32][]client.Prefix) for i := range likeTxHashes { shard := db.GetShardIdFromByte32(likeTxHashes[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.ByteReverse(likeTxHashes[i][:])) + prefix := jutil.ByteReverse(likeTxHashes[i][:]) + shardPrefixes[shard] = append(shardPrefixes[shard], client.Prefix{Prefix: prefix}) } var likeTips []*LikeTip for shard, prefixes := range shardPrefixes { shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetByPrefixes(db.TopicMemoLikeTip, prefixes); err != nil { + if err := dbClient.GetByPrefixesNew(ctx, db.TopicMemoLikeTip, prefixes); err != nil { return nil, fmt.Errorf("error getting client message memo like tips; %w", err) } for _, msg := range dbClient.Messages { diff --git a/graph/attach/memo_like.go b/graph/attach/memo_like.go index 1e48759d..5e427a77 100644 --- a/graph/attach/memo_like.go +++ b/graph/attach/memo_like.go @@ -72,7 +72,7 @@ func (a *MemoLike) AttachTips() { if !a.HasField([]string{"tip"}) { return } - memoLikeTips, err := memo.GetLikeTips(a.getTxHashes(false, true)) + memoLikeTips, err := memo.GetLikeTips(a.Ctx, a.getTxHashes(false, true)) if err != nil { a.AddError(fmt.Errorf("error getting memo like tips for post resolver; %w", err)) return From 0577549722c543c8011f919bf8afa822fbbc3525 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Tue, 23 Jul 2024 11:31:14 -0700 Subject: [PATCH 11/23] Update GetByPrefix to use new prefixes. Rename GetShardPrefixesNew and replace GetShardPrefixes. Helper prefix functions to simplify usage. --- cmd/maint/queue_profile.go | 4 +-- db/client/client.go | 27 ++++++---------- db/item/addr/seen_tx.go | 4 +-- db/item/chain/block_height.go | 12 +++---- db/item/chain/block_tx.go | 4 +-- db/item/chain/height_block.go | 10 +++--- db/item/chain/output_input.go | 52 +++++++----------------------- db/item/chain/tx_block.go | 12 +++---- db/item/chain/tx_input.go | 7 +--- db/item/chain/tx_output.go | 7 +--- db/item/chain/tx_seen.go | 7 +--- db/item/db/item.go | 44 +++++++++++++++++++------ db/item/memo/like_tip.go | 25 ++++---------- node/obj/get/tx.go | 2 +- node/obj/saver/block.go | 9 ++++-- ref/bitcoin/tx/hs/hash.go | 9 ++++++ ref/cluster/lead/node.go | 2 +- ref/cluster/lead/processor.go | 2 +- ref/network/network_server/main.go | 12 +++---- 19 files changed, 110 insertions(+), 141 deletions(-) diff --git a/cmd/maint/queue_profile.go b/cmd/maint/queue_profile.go index 7e0bd245..a1fc71be 100644 --- a/cmd/maint/queue_profile.go +++ b/cmd/maint/queue_profile.go @@ -42,11 +42,11 @@ var queueProfileCmd = &cobra.Command{ var prefixes []client.Prefix for _, blockTx := range blockTxs { if db.GetShardIdFromByte(blockTx.TxHash[:]) == Shard { - prefixes = append(prefixes, client.Prefix{Prefix: jutil.ByteReverse(blockTx.TxHash[:])}) + prefixes = append(prefixes, client.NewPrefix(jutil.ByteReverse(blockTx.TxHash[:]))) } } dbClient := client.NewClient(config.GetShardConfig(Shard, config.GetQueueShards()).GetHost()) - if err := dbClient.GetByPrefixesNew(ctx, db.TopicChainTxOutput, prefixes); err != nil { + if err := dbClient.GetByPrefixes(ctx, db.TopicChainTxOutput, prefixes); err != nil { log.Fatalf("fatal error getting db message tx outputs; %v", err) } log.Printf("%d outputs retrieved for shard 0 (height: %d, txs: %d)\n", diff --git a/db/client/client.go b/db/client/client.go index 47db2a9d..0291de38 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -115,22 +115,18 @@ func (s *Client) GetSpecific(topic string, uids [][]byte) error { return nil } -func (s *Client) GetByPrefixes(topic string, prefixes [][]byte) error { - if err := s.GetWOpts(Opts{ - Topic: topic, - Prefixes: prefixes, - }); err != nil { - return fmt.Errorf("error getting with opts prefixes; %w", err) - } - return nil -} - type Prefix struct { Prefix []byte Start []byte Limit uint32 } +func NewPrefix(prefix []byte) Prefix { + return Prefix{ + Prefix: prefix, + } +} + type Option interface { Apply(*queue_pb.RequestPrefixes) } @@ -151,7 +147,7 @@ func (o *OptionNewest) Apply(r *queue_pb.RequestPrefixes) { r.Newest = o.Newest } -func (s *Client) GetByPrefixesNew(ctx context.Context, topic string, prefixes []Prefix, opts ...Option) error { +func (s *Client) GetByPrefixes(ctx context.Context, topic string, prefixes []Prefix, opts ...Option) error { c := queue_pb.NewQueueClient(s.conn) ctxNew, cancel := context.WithTimeout(ctx, 10*time.Second) defer cancel() @@ -187,12 +183,9 @@ func (s *Client) GetByPrefixesNew(ctx context.Context, topic string, prefixes [] return nil } -func (s *Client) GetByPrefix(topic string, prefix []byte) error { - if err := s.GetWOpts(Opts{ - Topic: topic, - Prefixes: [][]byte{prefix}, - }); err != nil { - return fmt.Errorf("error getting with opts prefix; %w", err) +func (s *Client) GetByPrefix(ctx context.Context, topic string, prefix Prefix) error { + if err := s.GetByPrefixes(ctx, topic, []Prefix{prefix}); err != nil { + return fmt.Errorf("error getting with single prefix; %w", err) } return nil } diff --git a/db/item/addr/seen_tx.go b/db/item/addr/seen_tx.go index fb21ec03..b0861563 100644 --- a/db/item/addr/seen_tx.go +++ b/db/item/addr/seen_tx.go @@ -50,11 +50,11 @@ func (i *SeenTx) Deserialize([]byte) {} func GetSeenTxs(ctx context.Context, addr [25]byte, start []byte) ([]*SeenTx, error) { shardConfig := config.GetShardConfig(client.GenShardSource32(addr[:]), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetByPrefixesNew(ctx, db.TopicAddrSeenTx, []client.Prefix{{ + if err := dbClient.GetByPrefix(ctx, db.TopicAddrSeenTx, client.Prefix{ Prefix: addr[:], Start: start, Limit: client.ExLargeLimit, - }}); err != nil { + }); err != nil { return nil, fmt.Errorf("error getting db addr seen txs by prefix; %w", err) } var seenTxs = make([]*SeenTx, len(dbClient.Messages)) diff --git a/db/item/chain/block_height.go b/db/item/chain/block_height.go index c5e64a4b..633f2466 100644 --- a/db/item/chain/block_height.go +++ b/db/item/chain/block_height.go @@ -41,10 +41,11 @@ func (b *BlockHeight) Serialize() []byte { func (b *BlockHeight) Deserialize([]byte) {} -func GetBlockHeight(blockHash [32]byte) (*BlockHeight, error) { +func GetBlockHeight(ctx context.Context, blockHash [32]byte) (*BlockHeight, error) { shardConfig := config.GetShardConfig(client.GenShardSource32(blockHash[:]), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetByPrefix(db.TopicChainBlockHeight, jutil.ByteReverse(blockHash[:])); err != nil { + prefix := client.NewPrefix(jutil.ByteReverse(blockHash[:])) + if err := dbClient.GetByPrefix(ctx, db.TopicChainBlockHeight, prefix); err != nil { return nil, fmt.Errorf("error getting client message for block height; %w", err) } if len(dbClient.Messages) == 0 { @@ -63,12 +64,7 @@ func GetBlockHeight(blockHash [32]byte) (*BlockHeight, error) { } func GetBlockHeights(ctx context.Context, blockHashes [][32]byte) ([]*BlockHeight, error) { - var shardPrefixes = make(map[uint32][][]byte) - for _, blockHash := range blockHashes { - shard := db.GetShardIdFromByte32(blockHash[:]) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.ByteReverse(blockHash[:])) - } - messages, err := db.GetByPrefixes(ctx, db.TopicChainBlockHeight, shardPrefixes) + messages, err := db.GetByPrefixes(ctx, db.TopicChainBlockHeight, db.ShardPrefixesTxHashes(blockHashes)) if err != nil { return nil, fmt.Errorf("error getting client message chain block heights; %w", err) } diff --git a/db/item/chain/block_tx.go b/db/item/chain/block_tx.go index 1a61988d..ddba87ab 100644 --- a/db/item/chain/block_tx.go +++ b/db/item/chain/block_tx.go @@ -91,11 +91,11 @@ func GetBlockTxs(request BlockTxsRequest) ([]*BlockTx, error) { if request.StartIndex > 0 { startUid = GetBlockTxUid(request.BlockHash, request.StartIndex) } - if err := dbClient.GetByPrefixesNew(request.Context, db.TopicChainBlockTx, []client.Prefix{{ + if err := dbClient.GetByPrefix(request.Context, db.TopicChainBlockTx, client.Prefix{ Prefix: jutil.ByteReverse(request.BlockHash[:]), Start: startUid, Limit: limit, - }}); err != nil { + }); err != nil { return nil, fmt.Errorf("error getting client message; %w", err) } var blocks = make([]*BlockTx, len(dbClient.Messages)) diff --git a/db/item/chain/height_block.go b/db/item/chain/height_block.go index 38fb7325..c2390e55 100644 --- a/db/item/chain/height_block.go +++ b/db/item/chain/height_block.go @@ -1,6 +1,7 @@ package chain import ( + "context" "fmt" "github.com/jchavannes/btcd/chaincfg/chainhash" "github.com/jchavannes/jgo/jutil" @@ -103,8 +104,8 @@ func GetOldestHeightBlock() (*HeightBlock, error) { return oldestHeightBlock, nil } -func GetHeightBlockSingle(height int64) (*HeightBlock, error) { - heightBlocks, err := GetHeightBlock(height) +func GetHeightBlockSingle(ctx context.Context, height int64) (*HeightBlock, error) { + heightBlocks, err := GetHeightBlock(ctx, height) if err != nil { return nil, fmt.Errorf("error getting height block; %w", err) } @@ -121,10 +122,11 @@ func GetHeightBlockSingle(height int64) (*HeightBlock, error) { return heightBlocks[0], nil } -func GetHeightBlock(height int64) ([]*HeightBlock, error) { +func GetHeightBlock(ctx context.Context, height int64) ([]*HeightBlock, error) { shardConfig := config.GetShardConfig(uint32(height), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetByPrefix(db.TopicChainHeightBlock, jutil.GetInt64DataBig(height)); err != nil { + prefix := client.NewPrefix(jutil.GetInt64DataBig(height)) + if err := dbClient.GetByPrefix(ctx, db.TopicChainHeightBlock, prefix); err != nil { return nil, fmt.Errorf("error getting height blocks for height from queue client; %w", err) } var heightBlocks = make([]*HeightBlock, len(dbClient.Messages)) diff --git a/db/item/chain/output_input.go b/db/item/chain/output_input.go index 3709b05c..c257e473 100644 --- a/db/item/chain/output_input.go +++ b/db/item/chain/output_input.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" ) type OutputInput struct { @@ -50,30 +49,14 @@ func (t *OutputInput) Serialize() []byte { func (t *OutputInput) Deserialize([]byte) {} -func GetOutputInput(out memo.Out) ([]*OutputInput, error) { - shard := db.GetShardIdFromByte32(out.TxHash) - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - prefix := jutil.CombineBytes(jutil.ByteReverse(out.TxHash), jutil.GetUint32Data(out.Index)) - if err := dbClient.GetByPrefix(db.TopicChainOutputInput, prefix); err != nil { - return nil, fmt.Errorf("error getting by prefix for chain output input; %w", err) - } - var outputInputs = make([]*OutputInput, len(dbClient.Messages)) - for i := range dbClient.Messages { - outputInputs[i] = new(OutputInput) - db.Set(outputInputs[i], dbClient.Messages[i]) - } - return outputInputs, nil -} - func GetOutputInputs(ctx context.Context, outs []memo.Out) ([]*OutputInput, error) { - var shardPrefixes = make(map[uint32][][]byte) + var shardPrefixes = make(map[uint32][]client.Prefix) for _, out := range outs { shard := db.GetShardIdFromByte32(out.TxHash) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.CombineBytes( + shardPrefixes[shard] = append(shardPrefixes[shard], client.NewPrefix(jutil.CombineBytes( jutil.ByteReverse(out.TxHash), jutil.GetUint32DataBig(out.Index), - )) + ))) } messages, err := db.GetByPrefixes(ctx, db.TopicChainOutputInput, shardPrefixes) if err != nil { @@ -88,28 +71,15 @@ func GetOutputInputs(ctx context.Context, outs []memo.Out) ([]*OutputInput, erro return outputInputs, nil } -func GetOutputInputsForTxHashes(ctx context.Context, txHashes [][]byte) ([]*OutputInput, error) { - var shardOutGroups = make(map[uint32][][]byte) - for _, txHash := range txHashes { - shard := db.GetShardIdFromByte32(txHash) - shardOutGroups[shard] = append(shardOutGroups[shard], txHash) +func GetOutputInputsForTxHashes(ctx context.Context, txHashes [][32]byte) ([]*OutputInput, error) { + messages, err := db.GetByPrefixes(ctx, db.TopicChainOutputInput, db.ShardPrefixesTxHashes(txHashes)) + if err != nil { + return nil, fmt.Errorf("error getting client messages memo output inputs; %w", err) } - var outputInputs []*OutputInput - for shard, outGroup := range shardOutGroups { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - var prefixes = make([]client.Prefix, len(outGroup)) - for i := range outGroup { - prefixes[i] = client.Prefix{Prefix: jutil.ByteReverse(outGroup[i])} - } - if err := dbClient.GetByPrefixesNew(ctx, db.TopicChainOutputInput, prefixes); err != nil { - return nil, fmt.Errorf("error getting by prefixes for chain output inputs by tx hashes; %w", err) - } - for i := range dbClient.Messages { - var outputInput = new(OutputInput) - db.Set(outputInput, dbClient.Messages[i]) - outputInputs = append(outputInputs, outputInput) - } + var outputInputs = make([]*OutputInput, len(messages)) + for i := range messages { + outputInputs[i] = new(OutputInput) + db.Set(outputInputs[i], messages[i]) } return outputInputs, nil } diff --git a/db/item/chain/tx_block.go b/db/item/chain/tx_block.go index 37990809..263b2616 100644 --- a/db/item/chain/tx_block.go +++ b/db/item/chain/tx_block.go @@ -59,10 +59,11 @@ func GetSingleTxBlock(txHash, blockHash [32]byte) (*TxBlock, error) { return txBlock, nil } -func GetSingleTxBlocks(txHash [32]byte) ([]*TxBlock, error) { +func GetSingleTxBlocks(ctx context.Context, txHash [32]byte) ([]*TxBlock, error) { shardConfig := config.GetShardConfig(client.GenShardSource32(txHash[:]), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetByPrefix(db.TopicChainTxBlock, jutil.ByteReverse(txHash[:])); err != nil { + prefix := client.NewPrefix(jutil.ByteReverse(txHash[:])) + if err := dbClient.GetByPrefix(ctx, db.TopicChainTxBlock, prefix); err != nil { return nil, fmt.Errorf("error getting client message chain tx block by prefix; %w", err) } var txBlocks []*TxBlock @@ -75,12 +76,7 @@ func GetSingleTxBlocks(txHash [32]byte) ([]*TxBlock, error) { } func GetTxBlocks(ctx context.Context, txHashes [][32]byte) ([]*TxBlock, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range txHashes { - shard := uint32(db.GetShardIdFromByte(txHashes[i][:])) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.ByteReverse(txHashes[i][:])) - } - messages, err := db.GetByPrefixes(ctx, db.TopicChainTxBlock, shardPrefixes) + messages, err := db.GetByPrefixes(ctx, db.TopicChainTxBlock, db.ShardPrefixesTxHashes(txHashes)) if err != nil { return nil, fmt.Errorf("error getting client message chain tx blocks; %w", err) } diff --git a/db/item/chain/tx_input.go b/db/item/chain/tx_input.go index 5381058f..cc51c4c7 100644 --- a/db/item/chain/tx_input.go +++ b/db/item/chain/tx_input.go @@ -81,12 +81,7 @@ func GetTxInputUid(txHash [32]byte, index uint32) []byte { } func GetTxInputsByHashes(ctx context.Context, txHashes [][32]byte) ([]*TxInput, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range txHashes { - shard := uint32(db.GetShardIdFromByte(txHashes[i][:])) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.ByteReverse(txHashes[i][:])) - } - messages, err := db.GetByPrefixes(ctx, db.TopicChainTxInput, shardPrefixes) + messages, err := db.GetByPrefixes(ctx, db.TopicChainTxInput, db.ShardPrefixesTxHashes(txHashes)) if err != nil { return nil, fmt.Errorf("error getting client message chain tx input; %w", err) } diff --git a/db/item/chain/tx_output.go b/db/item/chain/tx_output.go index 561e22bd..697d4379 100644 --- a/db/item/chain/tx_output.go +++ b/db/item/chain/tx_output.go @@ -71,12 +71,7 @@ func GetAllTxOutputs(shard uint32, startUid []byte) ([]*TxOutput, error) { } func GetTxOutputsByHashes(ctx context.Context, txHashes [][32]byte) ([]*TxOutput, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range txHashes { - shard := uint32(db.GetShardIdFromByte(txHashes[i][:])) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.ByteReverse(txHashes[i][:])) - } - messages, err := db.GetByPrefixes(ctx, db.TopicChainTxOutput, shardPrefixes) + messages, err := db.GetByPrefixes(ctx, db.TopicChainTxOutput, db.ShardPrefixesTxHashes(txHashes)) if err != nil { return nil, fmt.Errorf("error getting client message chain tx output; %w", err) } diff --git a/db/item/chain/tx_seen.go b/db/item/chain/tx_seen.go index 920d57c9..257c6bda 100644 --- a/db/item/chain/tx_seen.go +++ b/db/item/chain/tx_seen.go @@ -44,12 +44,7 @@ func (s *TxSeen) SetUid(uid []byte) { func (s *TxSeen) Deserialize([]byte) {} func GetTxSeens(ctx context.Context, txHashes [][32]byte) ([]*TxSeen, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range txHashes { - shard := db.GetShardIdFromByte32(txHashes[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.ByteReverse(txHashes[i][:])) - } - messages, err := db.GetByPrefixes(ctx, db.TopicChainTxSeen, shardPrefixes) + messages, err := db.GetByPrefixes(ctx, db.TopicChainTxSeen, db.ShardPrefixesTxHashes(txHashes)) if err != nil { return nil, fmt.Errorf("error getting client message chain tx seen; %w", err) } diff --git a/db/item/db/item.go b/db/item/db/item.go index 4dd02577..504be942 100644 --- a/db/item/db/item.go +++ b/db/item/db/item.go @@ -6,6 +6,7 @@ import ( "github.com/jchavannes/jgo/jerr" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" + "github.com/memocash/index/ref/bitcoin/tx/hs" "github.com/memocash/index/ref/config" "sync" ) @@ -60,27 +61,52 @@ func GetSpecific(ctx context.Context, topic string, shardUids map[uint32][][]byt return messages, nil } -func GetByPrefixes(ctx context.Context, topic string, shardPrefixes map[uint32][][]byte) ([]client.Message, error) { +func removeDupeAndEmptyPrefixes(prefixes []client.Prefix) []client.Prefix { + var seen = make(map[string]bool) + var newPrefixes []client.Prefix + for _, prefix := range prefixes { + if len(prefix.Prefix) == 0 { + continue + } + if _, ok := seen[string(prefix.Prefix)]; ok { + continue + } + seen[string(prefix.Prefix)] = true + newPrefixes = append(newPrefixes, prefix) + } + return newPrefixes +} + +func ShardPrefixesTxHashes(txHashes [][32]byte) map[uint32][]client.Prefix { + return ShardPrefixes(hs.HashesToSlices(txHashes)) +} + +func ShardPrefixes(bytePrefixes [][]byte) map[uint32][]client.Prefix { + var shardPrefixes = make(map[uint32][]client.Prefix) + for _, bytePrefix := range bytePrefixes { + shard := GetShardIdFromByte32(bytePrefix) + shardPrefixes[shard] = append(shardPrefixes[shard], client.NewPrefix(bytePrefix)) + } + return shardPrefixes +} + +func GetByPrefixes(ctx context.Context, topic string, shardPrefixes map[uint32][]client.Prefix) ([]client.Message, error) { wait := NewWait(len(shardPrefixes)) var messages []client.Message for shardT, prefixesT := range shardPrefixes { - go func(shard uint32, prefixes [][]byte) { + go func(shard uint32, prefixes []client.Prefix) { defer wait.Group.Done() - prefixes = jutil.RemoveDupesAndEmpties(prefixes) + prefixes = removeDupeAndEmptyPrefixes(prefixes) shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) for len(prefixes) > 0 { - var prefixesToUse [][]byte + var prefixesToUse []client.Prefix if len(prefixes) > client.HugeLimit { prefixesToUse, prefixes = prefixes[:client.HugeLimit], prefixes[client.HugeLimit:] } else { prefixesToUse, prefixes = prefixes, nil } - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: topic, - Prefixes: prefixesToUse, - }); err != nil { + if err := dbClient.GetByPrefixes(ctx, topic, prefixesToUse); err != nil { wait.AddError(fmt.Errorf("error getting client message get by prefixes; %w", err)) return } diff --git a/db/item/memo/like_tip.go b/db/item/memo/like_tip.go index aa2a58c6..87b448cc 100644 --- a/db/item/memo/like_tip.go +++ b/db/item/memo/like_tip.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" ) type LikeTip struct { @@ -48,24 +47,14 @@ func (t *LikeTip) Deserialize(data []byte) { } func GetLikeTips(ctx context.Context, likeTxHashes [][32]byte) ([]*LikeTip, error) { - var shardPrefixes = make(map[uint32][]client.Prefix) - for i := range likeTxHashes { - shard := db.GetShardIdFromByte32(likeTxHashes[i][:]) - prefix := jutil.ByteReverse(likeTxHashes[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], client.Prefix{Prefix: prefix}) + messages, err := db.GetByPrefixes(ctx, db.TopicMemoLikeTip, db.ShardPrefixesTxHashes(likeTxHashes)) + if err != nil { + return nil, fmt.Errorf("error getting client messages memo like tips; %w", err) } - var likeTips []*LikeTip - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetByPrefixesNew(ctx, db.TopicMemoLikeTip, prefixes); err != nil { - return nil, fmt.Errorf("error getting client message memo like tips; %w", err) - } - for _, msg := range dbClient.Messages { - var likeTip = new(LikeTip) - db.Set(likeTip, msg) - likeTips = append(likeTips, likeTip) - } + var likeTips = make([]*LikeTip, len(messages)) + for i := range messages { + likeTips[i] = new(LikeTip) + db.Set(likeTips[i], messages[i]) } return likeTips, nil } diff --git a/node/obj/get/tx.go b/node/obj/get/tx.go index fe201036..3e3bf8ae 100644 --- a/node/obj/get/tx.go +++ b/node/obj/get/tx.go @@ -14,7 +14,7 @@ type Tx struct { } func (t *Tx) Get(ctx context.Context) error { - txBlocks, err := chain.GetSingleTxBlocks(t.TxHash) + txBlocks, err := chain.GetSingleTxBlocks(ctx, t.TxHash) if err != nil { return fmt.Errorf("error getting tx blocks; %w", err) } diff --git a/node/obj/saver/block.go b/node/obj/saver/block.go index f8379619..f55991c8 100644 --- a/node/obj/saver/block.go +++ b/node/obj/saver/block.go @@ -1,6 +1,7 @@ package saver import ( + "context" "fmt" "github.com/jchavannes/btcd/chaincfg/chainhash" "github.com/memocash/index/db/client" @@ -13,6 +14,7 @@ import ( ) type Block struct { + Context context.Context Verbose bool BlockHash chainhash.Hash PrevBlockHash chainhash.Hash @@ -44,7 +46,7 @@ func (b *Block) saveBlockObjects(info dbi.BlockInfo) error { parentHeight = b.PrevBlockHeight hasParent = true } else { - parentBlockHeight, err := chain.GetBlockHeight(info.Header.PrevBlock) + parentBlockHeight, err := chain.GetBlockHeight(b.Context, info.Header.PrevBlock) if err != nil && !client.IsEntryNotFoundError(err) { return fmt.Errorf("error getting parent block height for potential orphan; %w", err) } @@ -116,7 +118,7 @@ func (b *Block) GetBlock(heightBack int64) (*chainhash.Hash, error) { } if heightBack > 0 { height := heightBlock.Height - heightBack - heightBlock, err = chain.GetHeightBlockSingle(height) + heightBlock, err = chain.GetHeightBlockSingle(b.Context, height) if err != nil { return nil, fmt.Errorf("error getting height back height block (height: %d, back: %d); %w", height, heightBack, err) @@ -128,8 +130,9 @@ func (b *Block) GetBlock(heightBack int64) (*chainhash.Hash, error) { return &blockHash, nil } -func NewBlock(verbose bool) *Block { +func NewBlock(ctx context.Context, verbose bool) *Block { return &Block{ + Context: ctx, Verbose: verbose, } } diff --git a/ref/bitcoin/tx/hs/hash.go b/ref/bitcoin/tx/hs/hash.go index a1736e32..a9dcc252 100644 --- a/ref/bitcoin/tx/hs/hash.go +++ b/ref/bitcoin/tx/hs/hash.go @@ -77,3 +77,12 @@ func GetHashIndexString(txHash []byte, index uint32) string { func GetHashIndexWithString(txHash string, index uint32) string { return fmt.Sprintf("%s:%d", txHash, index) } + +// HashesToSlices converts a slice of 32 byte hashes to a slice of byte slices +func HashesToSlices(txHashes [][32]byte) [][]byte { + var slices = make([][]byte, len(txHashes)) + for i, txHash := range txHashes { + slices[i] = txHash[:] + } + return slices +} diff --git a/ref/cluster/lead/node.go b/ref/cluster/lead/node.go index e523bfd8..d0af5c76 100644 --- a/ref/cluster/lead/node.go +++ b/ref/cluster/lead/node.go @@ -38,7 +38,7 @@ func (n *Node) GetBlock(heightBack int64) (*chainhash.Hash, error) { if n.Off { return nil, nil } - hash, err := saver.NewBlock(n.Verbose).GetBlock(heightBack + 1) + hash, err := saver.NewBlock(context.TODO(), n.Verbose).GetBlock(heightBack + 1) if err != nil { return nil, fmt.Errorf("error getting block for lead node; %w", err) } diff --git a/ref/cluster/lead/processor.go b/ref/cluster/lead/processor.go index 4a8eeb7d..6a6f7772 100644 --- a/ref/cluster/lead/processor.go +++ b/ref/cluster/lead/processor.go @@ -127,7 +127,7 @@ func (p *Processor) ProcessBlock(block *dbi.Block, loc string) bool { } var height int64 if dbi.BlockHeaderSet(block.Header) { - blockSaver := saver.NewBlock(p.Verbose) + blockSaver := saver.NewBlock(context.TODO(), p.Verbose) if err := blockSaver.SaveBlock(blockInfo); err != nil { log.Printf("error saving block for lead node; %v", err) return false diff --git a/ref/network/network_server/main.go b/ref/network/network_server/main.go index 36231326..d5b17234 100644 --- a/ref/network/network_server/main.go +++ b/ref/network/network_server/main.go @@ -49,7 +49,7 @@ func (s *Server) SaveTxs(ctx context.Context, txs *network_pb.Txs) (*network_pb. } blockTxs[blockHashStr] = append(blockTxs[blockHashStr], txMsg) } - blockSaver := saver.NewBlock(false) + blockSaver := saver.NewBlock(ctx, false) combinedSaver := saver.NewCombinedTx(false) for blockHashStr, msgTxs := range blockTxs { var blockHeader *wire.BlockHeader @@ -179,7 +179,7 @@ func (s *Server) SaveTxBlock(ctx context.Context, txBlock *network_pb.TxBlock) ( if err != nil { return nil, fmt.Errorf("error parsing block header; %w", err) } - blockSaver := saver.NewBlock(true) + blockSaver := saver.NewBlock(ctx, true) if err := blockSaver.SaveBlock(dbi.BlockInfo{Header: *blockHeader}); err != nil { return nil, fmt.Errorf("error saving block; %w", err) } @@ -298,12 +298,12 @@ func GetBlockTxCount(blockHash [32]byte) (int64, error) { return int64(blockInfo.TxCount), nil } -func (s *Server) GetBlockByHash(_ context.Context, req *network_pb.BlockHashRequest) (*network_pb.BlockInfo, error) { +func (s *Server) GetBlockByHash(ctx context.Context, req *network_pb.BlockHashRequest) (*network_pb.BlockInfo, error) { blockHash, err := chainhash.NewHash(req.GetHash()) if err != nil { return nil, fmt.Errorf("error getting block hash for network server block by hash; %w", err) } - blockHeight, err := chain.GetBlockHeight(*blockHash) + blockHeight, err := chain.GetBlockHeight(ctx, *blockHash) if err != nil && !client.IsEntryNotFoundError(err) { return nil, fmt.Errorf("error getting block height by hash; %w", err) } @@ -327,8 +327,8 @@ func (s *Server) GetBlockByHash(_ context.Context, req *network_pb.BlockHashRequ }, nil } -func (s *Server) GetBlockByHeight(_ context.Context, req *network_pb.BlockRequest) (*network_pb.BlockInfo, error) { - heightBlock, err := chain.GetHeightBlockSingle(req.GetHeight()) +func (s *Server) GetBlockByHeight(ctx context.Context, req *network_pb.BlockRequest) (*network_pb.BlockInfo, error) { + heightBlock, err := chain.GetHeightBlockSingle(ctx, req.GetHeight()) if err != nil { return nil, fmt.Errorf("error getting height block by height; %w", err) } From c40773422e1f43f7fdc28225239cea7203a0c1a5 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Tue, 23 Jul 2024 16:27:44 -0700 Subject: [PATCH 12/23] Remove some old client functions. Move prefix and options into new file. More context handling. --- admin/server/node/peers.go | 2 +- admin/server/topic/view.go | 2 +- cmd/maint/queue_profile.go | 6 +- cmd/peer/list.go | 4 +- cmd/serve/all.go | 4 +- db/client/client.go | 98 +++--------------------------- db/client/peer/list.go | 10 +-- db/client/request.go | 53 ++++++++++++++++ db/item/chain/height_block.go | 35 ++--------- db/item/chain/height_duplicate.go | 20 ------ db/item/peer.go | 12 ++-- db/item/peer_found.go | 8 ++- node/group.go | 9 ++- node/obj/run/server.go | 6 +- ref/network/network_server/main.go | 4 +- 15 files changed, 105 insertions(+), 168 deletions(-) create mode 100644 db/client/request.go diff --git a/admin/server/node/peers.go b/admin/server/node/peers.go index 8e0b66f2..2720aa0a 100644 --- a/admin/server/node/peers.go +++ b/admin/server/node/peers.go @@ -17,7 +17,7 @@ var peersRoute = admin.Route{ r.Error(fmt.Errorf("error unmarshalling peers request; %w", err)) return } - peerList := peer.NewList() + peerList := peer.NewList(r.Request.Context()) if err := peerList.GetPeers(request.Filter); err != nil { r.Error(fmt.Errorf("error getting list of peers with filter; %w", err)) return diff --git a/admin/server/topic/view.go b/admin/server/topic/view.go index 93e8da93..8e7e680e 100644 --- a/admin/server/topic/view.go +++ b/admin/server/topic/view.go @@ -32,7 +32,7 @@ var viewRoute = admin.Route{ continue } db := client.NewClient(shardConfig.GetHost()) - if err := db.Get(topicViewRequest.Topic, start, false); err != nil { + if err := db.GetByPrefix(r.Request.Context(), topicViewRequest.Topic, client.NewStart(start)); err != nil { log.Printf("error getting topic items for admin view; %v", err) return } diff --git a/cmd/maint/queue_profile.go b/cmd/maint/queue_profile.go index a1fc71be..d5342d33 100644 --- a/cmd/maint/queue_profile.go +++ b/cmd/maint/queue_profile.go @@ -1,7 +1,6 @@ package maint import ( - "context" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/chain" @@ -20,14 +19,13 @@ var queueProfileCmd = &cobra.Command{ startHeight = jutil.GetInt64FromString(args[0]) } const Shard = 0 - heightBlocks, err := chain.GetHeightBlocks(Shard, startHeight, false) + heightBlocks, err := chain.GetHeightBlocks(cmd.Context(), Shard, startHeight, false) if err != nil { log.Fatalf("fatal error getting height blocks; %v", err) } var outputs int start := time.Now() log.Println("starting queue profile...") - ctx := context.Background() for _, heightBlock := range heightBlocks { if db.GetShardIdFromByte(heightBlock.BlockHash[:]) != Shard { continue @@ -46,7 +44,7 @@ var queueProfileCmd = &cobra.Command{ } } dbClient := client.NewClient(config.GetShardConfig(Shard, config.GetQueueShards()).GetHost()) - if err := dbClient.GetByPrefixes(ctx, db.TopicChainTxOutput, prefixes); err != nil { + if err := dbClient.GetByPrefixes(cmd.Context(), db.TopicChainTxOutput, prefixes); err != nil { log.Fatalf("fatal error getting db message tx outputs; %v", err) } log.Printf("%d outputs retrieved for shard 0 (height: %d, txs: %d)\n", diff --git a/cmd/peer/list.go b/cmd/peer/list.go index bb3a0295..c4b1101e 100644 --- a/cmd/peer/list.go +++ b/cmd/peer/list.go @@ -16,7 +16,7 @@ var listCmd = &cobra.Command{ if len(args) > 0 { shard = jutil.GetUInt32FromString(args[0]) } - peers, err := item.GetPeers(shard, nil) + peers, err := item.GetPeers(cmd.Context(), shard, nil) if err != nil { log.Fatalf("fatal error getting peers; %v", err) } @@ -53,7 +53,7 @@ var listPeerFoundsCmd = &cobra.Command{ if len(args) > 0 { shard = jutil.GetUInt32FromString(args[0]) } - foundPeers, err := item.GetPeerFounds(shard, nil) + foundPeers, err := item.GetPeerFounds(cmd.Context(), shard, nil) if err != nil { log.Fatalf("fatal error getting peer founds; %v", err) } diff --git a/cmd/serve/all.go b/cmd/serve/all.go index 92b35fcc..68d0c35a 100644 --- a/cmd/serve/all.go +++ b/cmd/serve/all.go @@ -10,7 +10,7 @@ var allCmd = &cobra.Command{ Use: "all", Run: func(c *cobra.Command, args []string) { verbose, _ := c.Flags().GetBool(FlagVerbose) - server := run.NewServer(true, verbose) + server := run.NewServer(c.Context(), true, verbose) log.Fatalf("fatal memo server error encountered (dev); %v", server.Run()) }, } @@ -19,7 +19,7 @@ var liveCmd = &cobra.Command{ Use: "live", Run: func(c *cobra.Command, args []string) { verbose, _ := c.Flags().GetBool(FlagVerbose) - server := run.NewServer(false, verbose) + server := run.NewServer(c.Context(), false, verbose) log.Fatalf("fatal memo server error encountered (live); %v", server.Run()) }, } diff --git a/db/client/client.go b/db/client/client.go index 0291de38..52a8fdb3 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -2,7 +2,6 @@ package client import ( "context" - "encoding/hex" "fmt" "github.com/jchavannes/jgo/jerr" "github.com/jchavannes/jgo/jutil" @@ -21,13 +20,6 @@ type Client struct { Topics []Topic } -func (s *Client) GetLast() string { - if len(s.Messages) == 0 { - return "" - } - return hex.EncodeToString(s.Messages[len(s.Messages)-1].Uid) -} - func (s *Client) SetConn() error { if connHandler == nil { connHandler = new(ConnHandler) @@ -48,13 +40,6 @@ func (s *Client) SetConn() error { return nil } -func (s *Client) SaveSingle(message *Message, timestamp time.Time) error { - if err := s.Save([]*Message{message}, timestamp); err != nil { - return fmt.Errorf("error saving single client message; %w", err) - } - return nil -} - func (s *Client) Save(messages []*Message, timestamp time.Time) error { if err := s.SetConn(); err != nil { return fmt.Errorf("error setting connection; %w", err) @@ -94,59 +79,6 @@ func (s *Client) Save(messages []*Message, timestamp time.Time) error { return nil } -func (s *Client) Get(topic string, start []byte, wait bool) error { - if err := s.GetWOpts(Opts{ - Topic: topic, - Start: start, - Wait: wait, - }); err != nil { - return fmt.Errorf("error getting with opts; %w", err) - } - return nil -} - -func (s *Client) GetSpecific(topic string, uids [][]byte) error { - if err := s.GetWOpts(Opts{ - Topic: topic, - Uids: uids, - }); err != nil { - return fmt.Errorf("error getting with opts specific; %w", err) - } - return nil -} - -type Prefix struct { - Prefix []byte - Start []byte - Limit uint32 -} - -func NewPrefix(prefix []byte) Prefix { - return Prefix{ - Prefix: prefix, - } -} - -type Option interface { - Apply(*queue_pb.RequestPrefixes) -} - -type OptionMax struct { - Max int -} - -func (o *OptionMax) Apply(r *queue_pb.RequestPrefixes) { - r.Max = uint32(o.Max) -} - -type OptionNewest struct { - Newest bool -} - -func (o *OptionNewest) Apply(r *queue_pb.RequestPrefixes) { - r.Newest = o.Newest -} - func (s *Client) GetByPrefixes(ctx context.Context, topic string, prefixes []Prefix, opts ...Option) error { c := queue_pb.NewQueueClient(s.conn) ctxNew, cancel := context.WithTimeout(ctx, 10*time.Second) @@ -183,8 +115,8 @@ func (s *Client) GetByPrefixes(ctx context.Context, topic string, prefixes []Pre return nil } -func (s *Client) GetByPrefix(ctx context.Context, topic string, prefix Prefix) error { - if err := s.GetByPrefixes(ctx, topic, []Prefix{prefix}); err != nil { +func (s *Client) GetByPrefix(ctx context.Context, topic string, prefix Prefix, opts ...Option) error { + if err := s.GetByPrefixes(ctx, topic, []Prefix{prefix}, opts...); err != nil { return fmt.Errorf("error getting with single prefix; %w", err) } return nil @@ -223,29 +155,13 @@ func (s *Client) GetSingleContext(ctx context.Context, topic string, uid []byte) return nil } -func (s *Client) GetLarge(topic string, start []byte, wait bool, newest bool) error { - if err := s.GetWOpts(Opts{ - Topic: topic, - Start: start, - Wait: wait, - Max: LargeLimit, - Newest: newest, - }); err != nil { - return fmt.Errorf("error getting with opts; %w", err) - } - return nil -} - -func (s *Client) GetNext(topic string, start []byte, wait bool, newest bool) error { +func (s *Client) GetNext(ctx context.Context, topic string, start []byte) error { startPlusOne := jutil.CombineBytes(start, []byte{0x0}) - if err := s.GetWOpts(Opts{ - Topic: topic, - Start: startPlusOne, - Wait: wait, - Max: 1, - Newest: newest, + if err := s.GetByPrefix(ctx, topic, Prefix{ + Start: startPlusOne, + Limit: 1, }); err != nil { - return fmt.Errorf("error getting next with opts; %w", err) + return fmt.Errorf("error getting next with prefix; %w", err) } return nil } diff --git a/db/client/peer/list.go b/db/client/peer/list.go index 3e7349d4..c9201595 100644 --- a/db/client/peer/list.go +++ b/db/client/peer/list.go @@ -2,6 +2,7 @@ package peer import ( "bytes" + "context" "fmt" "github.com/memocash/index/db/client" "github.com/memocash/index/db/item" @@ -24,14 +25,15 @@ type Peer struct { } type List struct { - Peers []*Peer + Context context.Context + Peers []*Peer } func (l *List) GetPeers(filter string) error { var startId []byte var shard uint32 for { - peers, err := item.GetPeers(shard, startId) + peers, err := item.GetPeers(l.Context, shard, startId) if err != nil { return fmt.Errorf("error getting next set of peers; %w", err) } @@ -89,6 +91,6 @@ func (l *List) GetPeers(filter string) error { return nil } -func NewList() *List { - return &List{} +func NewList(ctx context.Context) *List { + return &List{Context: ctx} } diff --git a/db/client/request.go b/db/client/request.go new file mode 100644 index 00000000..6535b40b --- /dev/null +++ b/db/client/request.go @@ -0,0 +1,53 @@ +package client + +import "github.com/memocash/index/db/proto/queue_pb" + +type Prefix struct { + Prefix []byte + Start []byte + Limit uint32 +} + +func NewPrefix(prefix []byte) Prefix { + return Prefix{ + Prefix: prefix, + } +} + +func NewStart(start []byte) Prefix { + return Prefix{ + Start: start, + } +} + +type Option interface { + Apply(*queue_pb.RequestPrefixes) +} + +type OptionMax struct { + Max int +} + +func (o *OptionMax) Apply(r *queue_pb.RequestPrefixes) { + r.Max = uint32(o.Max) +} + +func NewOptionMax(max int) *OptionMax { + return &OptionMax{ + Max: max, + } +} + +type OptionNewest struct { + Newest bool +} + +func (o *OptionNewest) Apply(r *queue_pb.RequestPrefixes) { + r.Newest = o.Newest +} + +func NewOptionNewest(newest bool) *OptionNewest { + return &OptionNewest{ + Newest: newest, + } +} diff --git a/db/item/chain/height_block.go b/db/item/chain/height_block.go index c2390e55..816f6d3d 100644 --- a/db/item/chain/height_block.go +++ b/db/item/chain/height_block.go @@ -76,34 +76,6 @@ func GetRecentHeightBlock() (*HeightBlock, error) { return newestHeightBlock, nil } -func GetOldestHeightBlock() (*HeightBlock, error) { - var heightBlocks []*HeightBlock - for i, shardConfig := range config.GetQueueShards() { - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetNext(db.TopicChainHeightBlock, nil, false, false); err != nil { - return nil, fmt.Errorf("error getting oldest height block for shard: %d; %w", i, err) - } - for i := range dbClient.Messages { - var heightBlock = new(HeightBlock) - db.Set(heightBlock, dbClient.Messages[i]) - heightBlocks = append(heightBlocks, heightBlock) - } - } - if len(heightBlocks) == 0 { - return nil, nil - } - var oldestHeightBlock *HeightBlock - for _, heightBlock := range heightBlocks { - if oldestHeightBlock == nil || oldestHeightBlock.Height > heightBlock.Height { - oldestHeightBlock = heightBlock - } - } - if oldestHeightBlock == nil { - return nil, nil - } - return oldestHeightBlock, nil -} - func GetHeightBlockSingle(ctx context.Context, height int64) (*HeightBlock, error) { heightBlocks, err := GetHeightBlock(ctx, height) if err != nil { @@ -137,14 +109,17 @@ func GetHeightBlock(ctx context.Context, height int64) ([]*HeightBlock, error) { return heightBlocks, nil } -func GetHeightBlocks(shard uint32, startHeight int64, newest bool) ([]*HeightBlock, error) { +func GetHeightBlocks(ctx context.Context, shard uint32, startHeight int64, newest bool) ([]*HeightBlock, error) { shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) var startHeightBytes []byte if startHeight > 0 || !newest { startHeightBytes = jutil.GetInt64DataBig(startHeight) } - if err := dbClient.GetLarge(db.TopicChainHeightBlock, startHeightBytes, false, newest); err != nil { + if err := dbClient.GetByPrefix(ctx, db.TopicChainHeightBlock, client.Prefix{ + Start: startHeightBytes, + Limit: client.LargeLimit, + }, client.NewOptionNewest(newest)); err != nil { return nil, fmt.Errorf("error getting height blocks from queue client; %w", err) } var heightBlocks = make([]*HeightBlock, len(dbClient.Messages)) diff --git a/db/item/chain/height_duplicate.go b/db/item/chain/height_duplicate.go index 2c7f468a..f78c9529 100644 --- a/db/item/chain/height_duplicate.go +++ b/db/item/chain/height_duplicate.go @@ -1,11 +1,8 @@ package chain import ( - "fmt" "github.com/jchavannes/jgo/jutil" - "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" - "github.com/memocash/index/ref/config" ) type HeightDuplicate struct { @@ -38,20 +35,3 @@ func (d *HeightDuplicate) Serialize() []byte { } func (d *HeightDuplicate) Deserialize([]byte) {} - -func GetHeightDuplicatesAll(startHeight int64) ([]*HeightDuplicate, error) { - var heightDuplicates []*HeightDuplicate - for _, shardConfig := range config.GetQueueShards() { - dbClient := client.NewClient(shardConfig.GetHost()) - err := dbClient.GetLarge(db.TopicChainHeightDuplicate, jutil.GetInt64DataBig(startHeight), false, false) - if err != nil { - return nil, fmt.Errorf("error getting height duplicates from queue client; %w", err) - } - for i := range dbClient.Messages { - var heightDuplicate = new(HeightDuplicate) - db.Set(heightDuplicate, dbClient.Messages[i]) - heightDuplicates = append(heightDuplicates, heightDuplicate) - } - } - return heightDuplicates, nil -} diff --git a/db/item/peer.go b/db/item/peer.go index 0d818877..a0c3e359 100644 --- a/db/item/peer.go +++ b/db/item/peer.go @@ -1,6 +1,7 @@ package item import ( + "context" "fmt" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" @@ -42,14 +43,17 @@ func (p *Peer) Deserialize(data []byte) { p.Services = jutil.GetUint64(data) } -func GetPeers(shard uint32, startId []byte) ([]*Peer, error) { +func GetPeers(ctx context.Context, shard uint32, startId []byte) ([]*Peer, error) { shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) var startIdBytes []byte if len(startId) > 0 { startIdBytes = startId } - if err := dbClient.GetLarge(db.TopicPeer, startIdBytes, false, false); err != nil { + if err := dbClient.GetByPrefix(ctx, db.TopicPeer, client.Prefix{ + Start: startIdBytes, + Limit: client.LargeLimit, + }); err != nil { return nil, fmt.Errorf("error getting peers from queue client; %w", err) } var peers = make([]*Peer, len(dbClient.Messages)) @@ -60,10 +64,10 @@ func GetPeers(shard uint32, startId []byte) ([]*Peer, error) { return peers, nil } -func GetNextPeer(shard uint32, startId []byte) (*Peer, error) { +func GetNextPeer(ctx context.Context, shard uint32, startId []byte) (*Peer, error) { shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetNext(db.TopicPeer, startId, false, false); err != nil { + if err := dbClient.GetNext(ctx, db.TopicPeer, startId); err != nil { return nil, fmt.Errorf("error getting peers from queue client; %w", err) } else if len(dbClient.Messages) == 0 { return nil, fmt.Errorf("error next peer not found; %w", client.EntryNotFoundError) diff --git a/db/item/peer_found.go b/db/item/peer_found.go index 2910a8c1..fdc9200a 100644 --- a/db/item/peer_found.go +++ b/db/item/peer_found.go @@ -1,6 +1,7 @@ package item import ( + "context" "fmt" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" @@ -52,14 +53,17 @@ func (p *PeerFound) SetUid(uid []byte) { func (p *PeerFound) Deserialize([]byte) {} -func GetPeerFounds(shard uint32, startId []byte) ([]*PeerFound, error) { +func GetPeerFounds(ctx context.Context, shard uint32, startId []byte) ([]*PeerFound, error) { shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) var startIdBytes []byte if len(startId) > 0 { startIdBytes = startId } - if err := dbClient.GetLarge(db.TopicPeerFound, startIdBytes, false, false); err != nil { + if err := dbClient.GetByPrefix(ctx, db.TopicPeerFound, client.Prefix{ + Start: startIdBytes, + Limit: client.LargeLimit, + }); err != nil { return nil, fmt.Errorf("error getting peer founds from queue client; %w", err) } var peerFounds = make([]*PeerFound, len(dbClient.Messages)) diff --git a/node/group.go b/node/group.go index 34151a93..ebb02973 100644 --- a/node/group.go +++ b/node/group.go @@ -1,6 +1,7 @@ package node import ( + "context" "fmt" "github.com/memocash/index/db/client" "github.com/memocash/index/db/item" @@ -10,6 +11,7 @@ import ( ) type Group struct { + Context context.Context Nodes map[string]*Server Looping bool LastPeerId []byte @@ -32,7 +34,7 @@ func (g *Group) AddDefaultNode() { func (g *Group) AddNextNode() error { var peerToUse *item.Peer for attempt := 1; ; attempt++ { - newPeer, err := item.GetNextPeer(0, g.LastPeerId) + newPeer, err := item.GetNextPeer(g.Context, 0, g.LastPeerId) if err != nil && !client.IsEntryNotFoundError(err) { return fmt.Errorf("error getting next peer; %w", err) } @@ -89,9 +91,10 @@ func (g *Group) AddNode(ip []byte, port uint16) { }() } -func NewGroup() *Group { +func NewGroup(ctx context.Context) *Group { return &Group{ - Nodes: make(map[string]*Server), + Context: ctx, + Nodes: make(map[string]*Server), } } diff --git a/node/obj/run/server.go b/node/obj/run/server.go index 150eb39c..7cd8aac5 100644 --- a/node/obj/run/server.go +++ b/node/obj/run/server.go @@ -16,6 +16,7 @@ import ( ) type Server struct { + Context context.Context Dev bool Verbose bool } @@ -23,7 +24,7 @@ type Server struct { func (s *Server) Run() error { var errorHandler = make(chan error) // Admin server - adminServer := admin.NewServer(node.NewGroup()) + adminServer := admin.NewServer(node.NewGroup(s.Context)) if err := adminServer.Start(); err != nil { return fmt.Errorf("fatal error starting admin server; %w", err) } @@ -85,8 +86,9 @@ func (s *Server) Run() error { return <-errorHandler } -func NewServer(dev, verbose bool) *Server { +func NewServer(ctx context.Context, dev, verbose bool) *Server { return &Server{ + Context: ctx, Dev: dev, Verbose: verbose, } diff --git a/ref/network/network_server/main.go b/ref/network/network_server/main.go index d5b17234..cd69d427 100644 --- a/ref/network/network_server/main.go +++ b/ref/network/network_server/main.go @@ -223,10 +223,10 @@ BlockTxsLoop: return &network_pb.ErrorReply{}, nil } -func (s *Server) GetBlockInfos(_ context.Context, req *network_pb.BlockRequest) (*network_pb.BlockInfoReply, error) { +func (s *Server) GetBlockInfos(ctx context.Context, req *network_pb.BlockRequest) (*network_pb.BlockInfoReply, error) { var heightBlocks []*chain.HeightBlock for _, shardConfig := range config.GetQueueShards() { - shardHeightBlocks, err := chain.GetHeightBlocks(shardConfig.Shard, req.GetHeight(), req.Newest) + shardHeightBlocks, err := chain.GetHeightBlocks(ctx, shardConfig.Shard, req.GetHeight(), req.Newest) if err != nil { return nil, fmt.Errorf("error getting height block raws; %w", err) } From be76f25ba4b5d42707ae35f926c3d9e0dde1030a Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Thu, 25 Jul 2024 05:51:27 -0700 Subject: [PATCH 13/23] Always pass context with GetSingle. --- admin/server/topic/item.go | 2 +- cmd/maint/populate_addr.go | 4 ++-- cmd/serve/lead.go | 2 +- db/client/client.go | 9 +-------- db/item/chain/block.go | 4 ++-- db/item/chain/block_info.go | 4 ++-- db/item/chain/block_tx.go | 4 ++-- db/item/chain/tx_block.go | 4 ++-- db/item/db/item.go | 4 ++-- db/item/message.go | 5 +++-- db/item/process_status.go | 5 +++-- db/item/sync_status.go | 5 +++-- node/act/maint/populate_addr.go | 11 +++++++---- node/act/maint/populate_p2sh.go | 2 +- node/act/maint/populate_p2sh_direct.go | 2 +- node/obj/get/block_tx.go | 7 ++++--- node/obj/run/server.go | 2 +- ref/cluster/lead/processor.go | 10 ++++++---- ref/network/network_server/main.go | 12 ++++++------ test/tasks/save_message.go | 3 ++- 20 files changed, 52 insertions(+), 49 deletions(-) diff --git a/admin/server/topic/item.go b/admin/server/topic/item.go index 9a47aa93..bc112089 100644 --- a/admin/server/topic/item.go +++ b/admin/server/topic/item.go @@ -36,7 +36,7 @@ var itemRoute = admin.Route{ var topicItemResponse = new(admin.TopicItemResponse) shardConfig := config.GetShardConfig(uint32(topicItemRequest.Shard), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetSingle(topicItemRequest.Topic, uid); err != nil { + if err := dbClient.GetSingle(r.Request.Context(), topicItemRequest.Topic, uid); err != nil { log.Printf("error getting topic item for admin view; %v", err) return } diff --git a/cmd/maint/populate_addr.go b/cmd/maint/populate_addr.go index 761dc97d..f3ca522d 100644 --- a/cmd/maint/populate_addr.go +++ b/cmd/maint/populate_addr.go @@ -11,7 +11,7 @@ var populateAddrOutputsCmd = &cobra.Command{ Short: "populate-addr-outputs", Run: func(c *cobra.Command, args []string) { restart, _ := c.Flags().GetBool(FlagRestart) - populateAddr := maint.NewPopulateAddr(false) + populateAddr := maint.NewPopulateAddr(c.Context(), false) log.Printf("Starting populate addr outputs...\n") if err := populateAddr.Populate(restart); err != nil { log.Fatalf("error populate addr outputs; %v", err) @@ -25,7 +25,7 @@ var populateAddrInputsCmd = &cobra.Command{ Short: "populate-addr-inputs", Run: func(c *cobra.Command, args []string) { restart, _ := c.Flags().GetBool(FlagRestart) - populateAddr := maint.NewPopulateAddr(true) + populateAddr := maint.NewPopulateAddr(c.Context(), true) log.Printf("Starting populate addr inputs...\n") if err := populateAddr.Populate(restart); err != nil { log.Fatalf("error populate addr inputs; %v", err) diff --git a/cmd/serve/lead.go b/cmd/serve/lead.go index 53f64d4f..fcf42b76 100644 --- a/cmd/serve/lead.go +++ b/cmd/serve/lead.go @@ -11,7 +11,7 @@ var leadCmd = &cobra.Command{ Short: "lead", Run: func(c *cobra.Command, args []string) { verbose, _ := c.Flags().GetBool(FlagVerbose) - p := lead.NewProcessor(verbose) + p := lead.NewProcessor(c.Context(), verbose) log.Fatalf("fatal error running lead processor; %v", p.Run()) }, } diff --git a/db/client/client.go b/db/client/client.go index 52a8fdb3..e2de0686 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -122,14 +122,7 @@ func (s *Client) GetByPrefix(ctx context.Context, topic string, prefix Prefix, o return nil } -func (s *Client) GetSingle(topic string, uid []byte) error { - if err := s.GetSingleContext(context.Background(), topic, uid); err != nil { - return fmt.Errorf("error getting single for topic / uid: %s, %x; %w", topic, uid, err) - } - return nil -} - -func (s *Client) GetSingleContext(ctx context.Context, topic string, uid []byte) error { +func (s *Client) GetSingle(ctx context.Context, topic string, uid []byte) error { if err := s.SetConn(); err != nil { return fmt.Errorf("error setting connection; %w", err) } diff --git a/db/item/chain/block.go b/db/item/chain/block.go index d7fc1b81..2aaeeeb9 100644 --- a/db/item/chain/block.go +++ b/db/item/chain/block.go @@ -38,10 +38,10 @@ func (b *Block) Deserialize(data []byte) { b.Raw = data } -func GetBlock(blockHash [32]byte) (*Block, error) { +func GetBlock(ctx context.Context, blockHash [32]byte) (*Block, error) { shardConfig := config.GetShardConfig(client.GenShardSource32(blockHash[:]), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetSingle(db.TopicChainBlock, jutil.ByteReverse(blockHash[:])); err != nil { + if err := dbClient.GetSingle(ctx, db.TopicChainBlock, jutil.ByteReverse(blockHash[:])); err != nil { return nil, fmt.Errorf("error getting client message block; %w", err) } if len(dbClient.Messages) != 1 { diff --git a/db/item/chain/block_info.go b/db/item/chain/block_info.go index cc2e95c9..981e6f81 100644 --- a/db/item/chain/block_info.go +++ b/db/item/chain/block_info.go @@ -49,10 +49,10 @@ func (b *BlockInfo) Deserialize(data []byte) { b.TxCount = jutil.GetInt(data[8:12]) } -func GetBlockInfo(blockHash [32]byte) (*BlockInfo, error) { +func GetBlockInfo(ctx context.Context, blockHash [32]byte) (*BlockInfo, error) { shardConfig := config.GetShardConfig(client.GenShardSource32(blockHash[:]), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetSingle(db.TopicChainBlockInfo, jutil.ByteReverse(blockHash[:])); err != nil { + if err := dbClient.GetSingle(ctx, db.TopicChainBlockInfo, jutil.ByteReverse(blockHash[:])); err != nil { return nil, fmt.Errorf("error getting client message block info; %w", err) } if len(dbClient.Messages) != 1 { diff --git a/db/item/chain/block_tx.go b/db/item/chain/block_tx.go index ddba87ab..01b94271 100644 --- a/db/item/chain/block_tx.go +++ b/db/item/chain/block_tx.go @@ -55,11 +55,11 @@ func GetBlockTxUid(blockHash [32]byte, index uint32) []byte { ) } -func GetBlockTx(blockHash [32]byte, index uint32) (*BlockTx, error) { +func GetBlockTx(ctx context.Context, blockHash [32]byte, index uint32) (*BlockTx, error) { shard := client.GenShardSource32(blockHash[:]) shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetSingle(db.TopicChainBlockTx, GetBlockTxUid(blockHash, index)); err != nil { + if err := dbClient.GetSingle(ctx, db.TopicChainBlockTx, GetBlockTxUid(blockHash, index)); err != nil { return nil, fmt.Errorf("error getting client message chain block tx single; %w", err) } if len(dbClient.Messages) != 1 { diff --git a/db/item/chain/tx_block.go b/db/item/chain/tx_block.go index 263b2616..e72e7493 100644 --- a/db/item/chain/tx_block.go +++ b/db/item/chain/tx_block.go @@ -45,10 +45,10 @@ func GetTxBlockUid(txHash, blockHash [32]byte) []byte { return jutil.CombineBytes(jutil.ByteReverse(txHash[:]), jutil.ByteReverse(blockHash[:])) } -func GetSingleTxBlock(txHash, blockHash [32]byte) (*TxBlock, error) { +func GetSingleTxBlock(ctx context.Context, txHash, blockHash [32]byte) (*TxBlock, error) { shardConfig := config.GetShardConfig(client.GenShardSource32(txHash[:]), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetSingle(db.TopicChainTxBlock, GetTxBlockUid(txHash, blockHash)); err != nil { + if err := dbClient.GetSingle(ctx, db.TopicChainTxBlock, GetTxBlockUid(txHash, blockHash)); err != nil { return nil, fmt.Errorf("error getting client message single tx block; %w", err) } if len(dbClient.Messages) != 1 { diff --git a/db/item/db/item.go b/db/item/db/item.go index 504be942..3b160c9f 100644 --- a/db/item/db/item.go +++ b/db/item/db/item.go @@ -11,10 +11,10 @@ import ( "sync" ) -func GetItem(obj Object) error { +func GetItem(ctx context.Context, obj Object) error { shardConfig := config.GetShardConfig(uint32(obj.GetShardSource()), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetSingle(obj.GetTopic(), obj.GetUid()); err != nil && !client.IsMessageNotSetError(err) { + if err := dbClient.GetSingle(ctx, obj.GetTopic(), obj.GetUid()); err != nil && !client.IsMessageNotSetError(err) { return fmt.Errorf("error getting db item single; %w", err) } if len(dbClient.Messages) != 1 { diff --git a/db/item/message.go b/db/item/message.go index 4a7f4fd0..737426b1 100644 --- a/db/item/message.go +++ b/db/item/message.go @@ -1,6 +1,7 @@ package item import ( + "context" "fmt" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" @@ -43,10 +44,10 @@ func (t *Message) Deserialize(data []byte) { t.Message = string(data[8:]) } -func GetMessage(id uint) (*Message, error) { +func GetMessage(ctx context.Context, id uint) (*Message, error) { shardConfig := config.GetShardConfig(client.GenShardSource32(jutil.GetUintData(id)), config.GetQueueShards()) queueClient := client.NewClient(shardConfig.GetHost()) - if err := queueClient.GetSingle(db.TopicMessage, jutil.GetUintData(id)); err != nil { + if err := queueClient.GetSingle(ctx, db.TopicMessage, jutil.GetUintData(id)); err != nil { return nil, fmt.Errorf("error getting single client message; %w", err) } if len(queueClient.Messages) != 1 { diff --git a/db/item/process_status.go b/db/item/process_status.go index 9f82afa2..30144932 100644 --- a/db/item/process_status.go +++ b/db/item/process_status.go @@ -1,6 +1,7 @@ package item import ( + "context" "fmt" "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" @@ -57,10 +58,10 @@ func NewProcessStatus(shard uint, name string) *ProcessStatus { } } -func GetProcessStatus(shard uint, name string) (*ProcessStatus, error) { +func GetProcessStatus(ctx context.Context, shard uint, name string) (*ProcessStatus, error) { shardConfig := config.GetShardConfig(uint32(shard), config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetSingle(db.TopicProcessStatus, []byte(name)); err != nil { + if err := dbClient.GetSingle(ctx, db.TopicProcessStatus, []byte(name)); err != nil { return nil, fmt.Errorf("error getting db message process status; %w", err) } if len(dbClient.Messages) == 0 || len(dbClient.Messages[0].Uid) == 0 { diff --git a/db/item/sync_status.go b/db/item/sync_status.go index ff1cace6..6f5dd793 100644 --- a/db/item/sync_status.go +++ b/db/item/sync_status.go @@ -1,6 +1,7 @@ package item import ( + "context" "fmt" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" @@ -43,9 +44,9 @@ func (s *SyncStatus) Deserialize(data []byte) { s.Height = jutil.GetInt64Big(data) } -func GetSyncStatus(name string) (*SyncStatus, error) { +func GetSyncStatus(ctx context.Context, name string) (*SyncStatus, error) { var syncStatus = &SyncStatus{Name: name} - if err := db.GetItem(syncStatus); err != nil { + if err := db.GetItem(ctx, syncStatus); err != nil { return nil, fmt.Errorf("error getting item sync status; %w", err) } return syncStatus, nil diff --git a/node/act/maint/populate_addr.go b/node/act/maint/populate_addr.go index b2865252..112d33dd 100644 --- a/node/act/maint/populate_addr.go +++ b/node/act/maint/populate_addr.go @@ -1,6 +1,7 @@ package maint import ( + "context" "fmt" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" @@ -16,6 +17,7 @@ import ( ) type PopulateAddr struct { + Context context.Context status map[uint]*item.ProcessStatus hasError bool mu sync.Mutex @@ -24,10 +26,11 @@ type PopulateAddr struct { Inputs bool } -func NewPopulateAddr(inputs bool) *PopulateAddr { +func NewPopulateAddr(ctx context.Context, inputs bool) *PopulateAddr { return &PopulateAddr{ - status: make(map[uint]*item.ProcessStatus), - Inputs: inputs, + Context: ctx, + status: make(map[uint]*item.ProcessStatus), + Inputs: inputs, } } @@ -72,7 +75,7 @@ func (p *PopulateAddr) Populate(newRun bool) error { shardConfigs := config.GetQueueShards() if !newRun { for _, shardConfig := range shardConfigs { - syncStatus, err := item.GetProcessStatus(uint(shardConfig.Shard), p.GetStatusName()) + syncStatus, err := item.GetProcessStatus(p.Context, uint(shardConfig.Shard), p.GetStatusName()) if err != nil && !client.IsMessageNotSetError(err) { return fmt.Errorf("error getting sync status; %w", err) } else if syncStatus != nil { diff --git a/node/act/maint/populate_p2sh.go b/node/act/maint/populate_p2sh.go index ad5b018b..3ba99af3 100644 --- a/node/act/maint/populate_p2sh.go +++ b/node/act/maint/populate_p2sh.go @@ -48,7 +48,7 @@ func (p *PopulateP2sh) Populate(startHeight int64) error { if err != nil { return fmt.Errorf("error getting block txs for populate p2sh; %w", err) } - block, err := chain.GetBlock(heightBlock.BlockHash) + block, err := chain.GetBlock(p.Ctx, heightBlock.BlockHash) if err != nil { return fmt.Errorf("error getting block info for populate p2sh; %w", err) } diff --git a/node/act/maint/populate_p2sh_direct.go b/node/act/maint/populate_p2sh_direct.go index 3a275aa8..d5710840 100644 --- a/node/act/maint/populate_p2sh_direct.go +++ b/node/act/maint/populate_p2sh_direct.go @@ -66,7 +66,7 @@ func (p *PopulateP2shDirect) Populate(newRun bool) error { shardConfigs := config.GetQueueShards() if !newRun { for _, shardConfig := range shardConfigs { - syncStatus, err := item.GetProcessStatus(uint(shardConfig.Shard), item.ProcessStatusPopulateP2sh) + syncStatus, err := item.GetProcessStatus(p.Ctx, uint(shardConfig.Shard), item.ProcessStatusPopulateP2sh) if err != nil && !client.IsMessageNotSetError(err) { return fmt.Errorf("error getting sync status; %w", err) } else if syncStatus != nil { diff --git a/node/obj/get/block_tx.go b/node/obj/get/block_tx.go index 681331b7..eca0e61c 100644 --- a/node/obj/get/block_tx.go +++ b/node/obj/get/block_tx.go @@ -1,6 +1,7 @@ package get import ( + "context" "fmt" "github.com/memocash/index/db/item/chain" ) @@ -12,13 +13,13 @@ type BlockTx struct { TxBlock *chain.TxBlock } -func (b *BlockTx) Get() error { +func (b *BlockTx) Get(ctx context.Context) error { var err error - b.TxBlock, err = chain.GetSingleTxBlock(b.TxHash, b.BlockHash) + b.TxBlock, err = chain.GetSingleTxBlock(ctx, b.TxHash, b.BlockHash) if err != nil { return fmt.Errorf("error getting tx block from queue; %w", err) } - b.BlockTx, err = chain.GetBlockTx(b.BlockHash, b.TxBlock.Index) + b.BlockTx, err = chain.GetBlockTx(ctx, b.BlockHash, b.TxBlock.Index) if err != nil { return fmt.Errorf("error getting block tx from queue; %w", err) } diff --git a/node/obj/run/server.go b/node/obj/run/server.go index 7cd8aac5..a276bde9 100644 --- a/node/obj/run/server.go +++ b/node/obj/run/server.go @@ -62,7 +62,7 @@ func (s *Server) Run() error { errorHandler <- fmt.Errorf("error running network server; %w", networkServer.Serve()) }() if !s.Dev { - processor := lead.NewProcessor(s.Verbose) + processor := lead.NewProcessor(s.Context, s.Verbose) log.Printf("Cluster lead processor starting...\n") go func() { errorHandler <- fmt.Errorf("error running cluster lead processor; %w", processor.Run()) diff --git a/ref/cluster/lead/processor.go b/ref/cluster/lead/processor.go index 6a6f7772..c12cc0ea 100644 --- a/ref/cluster/lead/processor.go +++ b/ref/cluster/lead/processor.go @@ -21,6 +21,7 @@ import ( ) type Processor struct { + Context context.Context Clients map[int]*Client ErrorChan chan error BlockNode *Node @@ -44,7 +45,7 @@ func (p *Processor) Run() error { var syncStatusComplete *item.SyncStatus if err := ExecWithRetry(func() error { var err error - syncStatusComplete, err = item.GetSyncStatus(item.SyncStatusComplete) + syncStatusComplete, err = item.GetSyncStatus(p.Context, item.SyncStatusComplete) if err != nil && !client.IsEntryNotFoundError(err) { return fmt.Errorf("error getting sync status complete; %w", err) } @@ -127,7 +128,7 @@ func (p *Processor) ProcessBlock(block *dbi.Block, loc string) bool { } var height int64 if dbi.BlockHeaderSet(block.Header) { - blockSaver := saver.NewBlock(context.TODO(), p.Verbose) + blockSaver := saver.NewBlock(p.Context, p.Verbose) if err := blockSaver.SaveBlock(blockInfo); err != nil { log.Printf("error saving block for lead node; %v", err) return false @@ -160,7 +161,7 @@ func (p *Processor) SaveBlockShards(height int64, seen time.Time, shardBlocks ma return } if err := ExecWithRetry(func() error { - if _, err := c.Client.SaveTxs(context.Background(), &cluster_pb.SaveReq{ + if _, err := c.Client.SaveTxs(p.Context, &cluster_pb.SaveReq{ Block: shardBlocks[c.Config.Shard], IsInitial: !p.Synced, Height: height, @@ -178,8 +179,9 @@ func (p *Processor) SaveBlockShards(height int64, seen time.Time, shardBlocks ma return !hadError } -func NewProcessor(verbose bool) *Processor { +func NewProcessor(ctx context.Context, verbose bool) *Processor { return &Processor{ + Context: ctx, ErrorChan: make(chan error), Verbose: verbose, } diff --git a/ref/network/network_server/main.go b/ref/network/network_server/main.go index cd69d427..9684ce07 100644 --- a/ref/network/network_server/main.go +++ b/ref/network/network_server/main.go @@ -58,7 +58,7 @@ func (s *Server) SaveTxs(ctx context.Context, txs *network_pb.Txs) (*network_pb. if err != nil { return nil, fmt.Errorf("error decoding block hash for tx: %s; %w", msgTxs[0].TxHash(), err) } - block, err := chain.GetBlock(*blockHash) + block, err := chain.GetBlock(ctx, *blockHash) if err != nil { return nil, fmt.Errorf("error getting block; %w", err) } @@ -290,8 +290,8 @@ func (s *Server) GetHeightBlocks(_ context.Context, req *network_pb.BlockHeightR return response, nil } -func GetBlockTxCount(blockHash [32]byte) (int64, error) { - blockInfo, err := chain.GetBlockInfo(blockHash) +func GetBlockTxCount(ctx context.Context, blockHash [32]byte) (int64, error) { + blockInfo, err := chain.GetBlockInfo(ctx, blockHash) if err != nil { return 0, fmt.Errorf("error getting block txes; %w", err) } @@ -311,11 +311,11 @@ func (s *Server) GetBlockByHash(ctx context.Context, req *network_pb.BlockHashRe if blockHeight != nil { height = blockHeight.Height } - txCount, err := GetBlockTxCount(*blockHash) + txCount, err := GetBlockTxCount(ctx, *blockHash) if err != nil { return nil, fmt.Errorf("error getting block tx count; %w", err) } - block, err := chain.GetBlock(*blockHash) + block, err := chain.GetBlock(ctx, *blockHash) if err != nil { return nil, fmt.Errorf("error getting block; %w", err) } @@ -332,7 +332,7 @@ func (s *Server) GetBlockByHeight(ctx context.Context, req *network_pb.BlockRequ if err != nil { return nil, fmt.Errorf("error getting height block by height; %w", err) } - block, err := chain.GetBlock(heightBlock.BlockHash) + block, err := chain.GetBlock(ctx, heightBlock.BlockHash) if err != nil { return nil, fmt.Errorf("error getting block; %w", err) } diff --git a/test/tasks/save_message.go b/test/tasks/save_message.go index 91c5b84e..c3b4171a 100644 --- a/test/tasks/save_message.go +++ b/test/tasks/save_message.go @@ -1,6 +1,7 @@ package tasks import ( + "context" "fmt" "github.com/memocash/index/db/item" "github.com/memocash/index/db/item/db" @@ -21,7 +22,7 @@ var SaveMessage = suite.Test{ if err := db.Save(messages); err != nil { return fmt.Errorf("error saving message to client; %w", err) } - message, err := item.GetMessage(0) + message, err := item.GetMessage(context.TODO(), 0) if err != nil { return fmt.Errorf("error getting message from client; %w", err) } From 92f092adf58fe7720693e2176227b172e12f1f90 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Thu, 25 Jul 2024 15:07:07 -0700 Subject: [PATCH 14/23] Rename max to limit and newest to order descending. --- db/client/client.go | 2 +- db/client/request.go | 32 +++-- db/item/chain/height_block.go | 6 +- db/proto/queue.proto | 11 +- db/proto/queue_pb/queue.pb.go | 241 +++++++++++++++++++++------------- db/server/server.go | 16 +-- db/store/message.go | 8 +- db/store/message_test.go | 14 +- 8 files changed, 196 insertions(+), 134 deletions(-) diff --git a/db/client/client.go b/db/client/client.go index e2de0686..03bc4656 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -89,7 +89,7 @@ func (s *Client) GetByPrefixes(ctx context.Context, topic string, prefixes []Pre reqPrefixes[i] = &queue_pb.RequestPrefix{ Prefix: prefixes[i].Prefix, Start: prefixes[i].Start, - Max: prefixes[i].Limit, + Limit: prefixes[i].Limit, } } req := &queue_pb.RequestPrefixes{ diff --git a/db/client/request.go b/db/client/request.go index 6535b40b..f36c6232 100644 --- a/db/client/request.go +++ b/db/client/request.go @@ -24,30 +24,34 @@ type Option interface { Apply(*queue_pb.RequestPrefixes) } -type OptionMax struct { - Max int +type OptionLimit struct { + Limit int } -func (o *OptionMax) Apply(r *queue_pb.RequestPrefixes) { - r.Max = uint32(o.Max) +func (o *OptionLimit) Apply(r *queue_pb.RequestPrefixes) { + r.Limit = uint32(o.Limit) } -func NewOptionMax(max int) *OptionMax { - return &OptionMax{ - Max: max, +func NewOptionLimit(limit int) *OptionLimit { + return &OptionLimit{ + Limit: limit, } } -type OptionNewest struct { - Newest bool +type OptionOrder struct { + Desc bool } -func (o *OptionNewest) Apply(r *queue_pb.RequestPrefixes) { - r.Newest = o.Newest +func (o *OptionOrder) Apply(r *queue_pb.RequestPrefixes) { + if o.Desc { + r.Order = queue_pb.Order_DESC + } else { + r.Order = queue_pb.Order_ASC + } } -func NewOptionNewest(newest bool) *OptionNewest { - return &OptionNewest{ - Newest: newest, +func NewOptionOrder(desc bool) *OptionOrder { + return &OptionOrder{ + Desc: desc, } } diff --git a/db/item/chain/height_block.go b/db/item/chain/height_block.go index 816f6d3d..067ca57e 100644 --- a/db/item/chain/height_block.go +++ b/db/item/chain/height_block.go @@ -109,17 +109,17 @@ func GetHeightBlock(ctx context.Context, height int64) ([]*HeightBlock, error) { return heightBlocks, nil } -func GetHeightBlocks(ctx context.Context, shard uint32, startHeight int64, newest bool) ([]*HeightBlock, error) { +func GetHeightBlocks(ctx context.Context, shard uint32, startHeight int64, desc bool) ([]*HeightBlock, error) { shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) var startHeightBytes []byte - if startHeight > 0 || !newest { + if startHeight > 0 || !desc { startHeightBytes = jutil.GetInt64DataBig(startHeight) } if err := dbClient.GetByPrefix(ctx, db.TopicChainHeightBlock, client.Prefix{ Start: startHeightBytes, Limit: client.LargeLimit, - }, client.NewOptionNewest(newest)); err != nil { + }, client.NewOptionOrder(desc)); err != nil { return nil, fmt.Errorf("error getting height blocks from queue client; %w", err) } var heightBlocks = make([]*HeightBlock, len(dbClient.Messages)) diff --git a/db/proto/queue.proto b/db/proto/queue.proto index b28b4034..a2af1ead 100644 --- a/db/proto/queue.proto +++ b/db/proto/queue.proto @@ -87,12 +87,17 @@ message TopicCount { message RequestPrefixes { string topic = 1; repeated RequestPrefix prefixes = 2; - uint32 max = 3; - bool newest = 4; + uint32 limit = 3; + Order order = 4; +} + +enum Order { + ASC = 0; + DESC = 1; } message RequestPrefix { bytes prefix = 1; bytes start = 2; - uint32 max = 3; + uint32 limit = 3; } diff --git a/db/proto/queue_pb/queue.pb.go b/db/proto/queue_pb/queue.pb.go index f530097f..938d74b6 100644 --- a/db/proto/queue_pb/queue.pb.go +++ b/db/proto/queue_pb/queue.pb.go @@ -20,6 +20,52 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type Order int32 + +const ( + Order_ASC Order = 0 + Order_DESC Order = 1 +) + +// Enum value maps for Order. +var ( + Order_name = map[int32]string{ + 0: "ASC", + 1: "DESC", + } + Order_value = map[string]int32{ + "ASC": 0, + "DESC": 1, + } +) + +func (x Order) Enum() *Order { + p := new(Order) + *p = x + return p +} + +func (x Order) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Order) Descriptor() protoreflect.EnumDescriptor { + return file_queue_proto_enumTypes[0].Descriptor() +} + +func (Order) Type() protoreflect.EnumType { + return &file_queue_proto_enumTypes[0] +} + +func (x Order) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Order.Descriptor instead. +func (Order) EnumDescriptor() ([]byte, []int) { + return file_queue_proto_rawDescGZIP(), []int{0} +} + type Messages struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -694,8 +740,8 @@ type RequestPrefixes struct { Topic string `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"` Prefixes []*RequestPrefix `protobuf:"bytes,2,rep,name=prefixes,proto3" json:"prefixes,omitempty"` - Max uint32 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` - Newest bool `protobuf:"varint,4,opt,name=newest,proto3" json:"newest,omitempty"` + Limit uint32 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` + Order Order `protobuf:"varint,4,opt,name=order,proto3,enum=queue_pb.Order" json:"order,omitempty"` } func (x *RequestPrefixes) Reset() { @@ -744,18 +790,18 @@ func (x *RequestPrefixes) GetPrefixes() []*RequestPrefix { return nil } -func (x *RequestPrefixes) GetMax() uint32 { +func (x *RequestPrefixes) GetLimit() uint32 { if x != nil { - return x.Max + return x.Limit } return 0 } -func (x *RequestPrefixes) GetNewest() bool { +func (x *RequestPrefixes) GetOrder() Order { if x != nil { - return x.Newest + return x.Order } - return false + return Order_ASC } type RequestPrefix struct { @@ -765,7 +811,7 @@ type RequestPrefix struct { Prefix []byte `protobuf:"bytes,1,opt,name=prefix,proto3" json:"prefix,omitempty"` Start []byte `protobuf:"bytes,2,opt,name=start,proto3" json:"start,omitempty"` - Max uint32 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` + Limit uint32 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` } func (x *RequestPrefix) Reset() { @@ -814,9 +860,9 @@ func (x *RequestPrefix) GetStart() []byte { return nil } -func (x *RequestPrefix) GetMax() uint32 { +func (x *RequestPrefix) GetLimit() uint32 { if x != nil { - return x.Max + return x.Limit } return 0 } @@ -873,57 +919,60 @@ var file_queue_proto_rawDesc = []byte{ 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x22, 0x0a, 0x0a, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x86, 0x01, 0x0a, + 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x99, 0x01, 0x0a, 0x0f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x33, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, - 0x78, 0x52, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x6d, - 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x12, 0x16, 0x0a, - 0x06, 0x6e, 0x65, 0x77, 0x65, 0x73, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x6e, - 0x65, 0x77, 0x65, 0x73, 0x74, 0x22, 0x4f, 0x0a, 0x0d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x14, - 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, - 0x74, 0x61, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x32, 0x86, 0x04, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x75, 0x65, - 0x12, 0x3a, 0x0a, 0x0c, 0x53, 0x61, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, - 0x12, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, - 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x0e, - 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x15, + 0x78, 0x52, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, + 0x69, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, + 0x74, 0x12, 0x25, 0x0a, 0x05, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, + 0x32, 0x0f, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4f, 0x72, 0x64, 0x65, + 0x72, 0x52, 0x05, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x22, 0x53, 0x0a, 0x0d, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, + 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, + 0x78, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, + 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x2a, 0x1a, 0x0a, + 0x05, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x53, 0x43, 0x10, 0x00, 0x12, + 0x08, 0x0a, 0x04, 0x44, 0x45, 0x53, 0x43, 0x10, 0x01, 0x32, 0x86, 0x04, 0x0a, 0x05, 0x51, 0x75, + 0x65, 0x75, 0x65, 0x12, 0x3a, 0x0a, 0x0c, 0x53, 0x61, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x73, 0x12, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, + 0x70, 0x62, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, + 0x3f, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x73, 0x12, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, + 0x73, 0x61, 0x67, 0x65, 0x55, 0x69, 0x64, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, + 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, + 0x12, 0x3a, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, + 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, + 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0b, + 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x11, 0x2e, 0x71, 0x75, + 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x55, 0x69, 0x64, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, - 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3a, 0x0a, - 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, 0x2e, 0x71, 0x75, - 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, 0x69, - 0x6e, 0x67, 0x6c, 0x65, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0b, 0x47, 0x65, 0x74, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, - 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x71, 0x75, - 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x22, - 0x00, 0x12, 0x40, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x42, 0x79, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x65, 0x73, 0x12, 0x19, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x1a, 0x12, 0x2e, - 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x73, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x17, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, - 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x54, - 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, - 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x18, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, - 0x63, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0f, - 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, - 0x16, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, - 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x00, 0x42, - 0x2d, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x65, - 0x6d, 0x6f, 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x64, 0x62, 0x2f, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x65, 0x73, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x42, 0x79, 0x50, 0x72, 0x65, + 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, + 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, + 0x1a, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x73, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x17, 0x2e, 0x71, 0x75, + 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, 0x74, + 0x72, 0x65, 0x61, 0x6d, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, + 0x65, 0x74, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x71, 0x75, + 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, + 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, + 0x41, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x12, 0x16, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, + 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x22, 0x00, 0x42, 0x2d, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, + 0x2f, 0x6d, 0x65, 0x6d, 0x6f, 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, + 0x64, 0x62, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, + 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -938,48 +987,51 @@ func file_queue_proto_rawDescGZIP() []byte { return file_queue_proto_rawDescData } +var file_queue_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_queue_proto_msgTypes = make([]protoimpl.MessageInfo, 14) var file_queue_proto_goTypes = []any{ - (*Messages)(nil), // 0: queue_pb.Messages - (*Message)(nil), // 1: queue_pb.Message - (*ErrorReply)(nil), // 2: queue_pb.ErrorReply - (*MessageUids)(nil), // 3: queue_pb.MessageUids - (*RequestSingle)(nil), // 4: queue_pb.RequestSingle - (*Request)(nil), // 5: queue_pb.Request - (*RequestStream)(nil), // 6: queue_pb.RequestStream - (*EmptyRequest)(nil), // 7: queue_pb.EmptyRequest - (*Topic)(nil), // 8: queue_pb.Topic - (*TopicListReply)(nil), // 9: queue_pb.TopicListReply - (*CountRequest)(nil), // 10: queue_pb.CountRequest - (*TopicCount)(nil), // 11: queue_pb.TopicCount - (*RequestPrefixes)(nil), // 12: queue_pb.RequestPrefixes - (*RequestPrefix)(nil), // 13: queue_pb.RequestPrefix + (Order)(0), // 0: queue_pb.Order + (*Messages)(nil), // 1: queue_pb.Messages + (*Message)(nil), // 2: queue_pb.Message + (*ErrorReply)(nil), // 3: queue_pb.ErrorReply + (*MessageUids)(nil), // 4: queue_pb.MessageUids + (*RequestSingle)(nil), // 5: queue_pb.RequestSingle + (*Request)(nil), // 6: queue_pb.Request + (*RequestStream)(nil), // 7: queue_pb.RequestStream + (*EmptyRequest)(nil), // 8: queue_pb.EmptyRequest + (*Topic)(nil), // 9: queue_pb.Topic + (*TopicListReply)(nil), // 10: queue_pb.TopicListReply + (*CountRequest)(nil), // 11: queue_pb.CountRequest + (*TopicCount)(nil), // 12: queue_pb.TopicCount + (*RequestPrefixes)(nil), // 13: queue_pb.RequestPrefixes + (*RequestPrefix)(nil), // 14: queue_pb.RequestPrefix } var file_queue_proto_depIdxs = []int32{ - 1, // 0: queue_pb.Messages.messages:type_name -> queue_pb.Message - 8, // 1: queue_pb.TopicListReply.topics:type_name -> queue_pb.Topic - 13, // 2: queue_pb.RequestPrefixes.prefixes:type_name -> queue_pb.RequestPrefix - 0, // 3: queue_pb.Queue.SaveMessages:input_type -> queue_pb.Messages - 3, // 4: queue_pb.Queue.DeleteMessages:input_type -> queue_pb.MessageUids - 4, // 5: queue_pb.Queue.GetMessage:input_type -> queue_pb.RequestSingle - 5, // 6: queue_pb.Queue.GetMessages:input_type -> queue_pb.Request - 12, // 7: queue_pb.Queue.GetByPrefixes:input_type -> queue_pb.RequestPrefixes - 6, // 8: queue_pb.Queue.GetStreamMessages:input_type -> queue_pb.RequestStream - 7, // 9: queue_pb.Queue.GetTopicList:input_type -> queue_pb.EmptyRequest - 10, // 10: queue_pb.Queue.GetMessageCount:input_type -> queue_pb.CountRequest - 2, // 11: queue_pb.Queue.SaveMessages:output_type -> queue_pb.ErrorReply - 2, // 12: queue_pb.Queue.DeleteMessages:output_type -> queue_pb.ErrorReply - 1, // 13: queue_pb.Queue.GetMessage:output_type -> queue_pb.Message - 0, // 14: queue_pb.Queue.GetMessages:output_type -> queue_pb.Messages - 0, // 15: queue_pb.Queue.GetByPrefixes:output_type -> queue_pb.Messages - 1, // 16: queue_pb.Queue.GetStreamMessages:output_type -> queue_pb.Message - 9, // 17: queue_pb.Queue.GetTopicList:output_type -> queue_pb.TopicListReply - 11, // 18: queue_pb.Queue.GetMessageCount:output_type -> queue_pb.TopicCount - 11, // [11:19] is the sub-list for method output_type - 3, // [3:11] is the sub-list for method input_type - 3, // [3:3] is the sub-list for extension type_name - 3, // [3:3] is the sub-list for extension extendee - 0, // [0:3] is the sub-list for field type_name + 2, // 0: queue_pb.Messages.messages:type_name -> queue_pb.Message + 9, // 1: queue_pb.TopicListReply.topics:type_name -> queue_pb.Topic + 14, // 2: queue_pb.RequestPrefixes.prefixes:type_name -> queue_pb.RequestPrefix + 0, // 3: queue_pb.RequestPrefixes.order:type_name -> queue_pb.Order + 1, // 4: queue_pb.Queue.SaveMessages:input_type -> queue_pb.Messages + 4, // 5: queue_pb.Queue.DeleteMessages:input_type -> queue_pb.MessageUids + 5, // 6: queue_pb.Queue.GetMessage:input_type -> queue_pb.RequestSingle + 6, // 7: queue_pb.Queue.GetMessages:input_type -> queue_pb.Request + 13, // 8: queue_pb.Queue.GetByPrefixes:input_type -> queue_pb.RequestPrefixes + 7, // 9: queue_pb.Queue.GetStreamMessages:input_type -> queue_pb.RequestStream + 8, // 10: queue_pb.Queue.GetTopicList:input_type -> queue_pb.EmptyRequest + 11, // 11: queue_pb.Queue.GetMessageCount:input_type -> queue_pb.CountRequest + 3, // 12: queue_pb.Queue.SaveMessages:output_type -> queue_pb.ErrorReply + 3, // 13: queue_pb.Queue.DeleteMessages:output_type -> queue_pb.ErrorReply + 2, // 14: queue_pb.Queue.GetMessage:output_type -> queue_pb.Message + 1, // 15: queue_pb.Queue.GetMessages:output_type -> queue_pb.Messages + 1, // 16: queue_pb.Queue.GetByPrefixes:output_type -> queue_pb.Messages + 2, // 17: queue_pb.Queue.GetStreamMessages:output_type -> queue_pb.Message + 10, // 18: queue_pb.Queue.GetTopicList:output_type -> queue_pb.TopicListReply + 12, // 19: queue_pb.Queue.GetMessageCount:output_type -> queue_pb.TopicCount + 12, // [12:20] is the sub-list for method output_type + 4, // [4:12] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name } func init() { file_queue_proto_init() } @@ -1162,13 +1214,14 @@ func file_queue_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_queue_proto_rawDesc, - NumEnums: 0, + NumEnums: 1, NumMessages: 14, NumExtensions: 0, NumServices: 1, }, GoTypes: file_queue_proto_goTypes, DependencyIndexes: file_queue_proto_depIdxs, + EnumInfos: file_queue_proto_enumTypes, MessageInfos: file_queue_proto_msgTypes, }.Build() File_queue_proto = out.File diff --git a/db/server/server.go b/db/server/server.go index 579b96e1..97c2bb60 100644 --- a/db/server/server.go +++ b/db/server/server.go @@ -130,10 +130,10 @@ func (s *Server) GetMessages(ctx context.Context, request *queue_pb.Request) (*q } else { for i := 0; i < 2; i++ { var requestByPrefixes = store.RequestByPrefixes{ - Topic: request.Topic, - Shard: s.Shard, - Max: int(request.Max), - Newest: request.Newest, + Topic: request.Topic, + Shard: s.Shard, + Limit: int(request.Max), + Desc: request.Newest, } for _, prefix := range request.Prefixes { requestByPrefixes.Prefixes = append(requestByPrefixes.Prefixes, store.Prefix{ @@ -175,10 +175,10 @@ func getQueueMessagesFromStoreMessages(topic string, messages []*store.Message) func (s *Server) GetByPrefixes(ctx context.Context, req *queue_pb.RequestPrefixes) (*queue_pb.Messages, error) { var requestByPrefixes = store.RequestByPrefixes{ - Topic: req.Topic, - Shard: s.Shard, - Max: int(req.Max), - Newest: req.Newest, + Topic: req.Topic, + Shard: s.Shard, + Limit: int(req.Limit), + Desc: req.Order == queue_pb.Order_DESC, } for _, prefix := range req.Prefixes { requestByPrefixes.Prefixes = append(requestByPrefixes.Prefixes, store.Prefix{ diff --git a/db/store/message.go b/db/store/message.go index b5452955..28933d78 100644 --- a/db/store/message.go +++ b/db/store/message.go @@ -85,8 +85,8 @@ type RequestByPrefixes struct { Topic string // required Shard uint // required Prefixes []Prefix - Max int - Newest bool + Limit int + Desc bool } type Prefix struct { @@ -102,7 +102,7 @@ func GetByPrefixes(request RequestByPrefixes) ([]*Message, error) { return nil, fmt.Errorf("error getting db shard %d; %w", request.Shard, err) } - var maxResults = request.Max + var maxResults = request.Limit if maxResults == 0 { maxResults = client.HugeLimit } @@ -121,7 +121,7 @@ func GetByPrefixes(request RequestByPrefixes) ([]*Message, error) { }() for _, prefix := range prefixes { - prefixMessages, err := getPrefixMessages(db, prefix, request.Newest, maxResults-len(messages)) + prefixMessages, err := getPrefixMessages(db, prefix, request.Desc, maxResults-len(messages)) if err != nil { return nil, fmt.Errorf("error getting prefix messages; %w", err) } diff --git a/db/store/message_test.go b/db/store/message_test.go index c17a0d4a..afc541ac 100644 --- a/db/store/message_test.go +++ b/db/store/message_test.go @@ -43,8 +43,8 @@ var ( type GetMessagesTest struct { Name string Prefixes []store.Prefix - Max int - Newest bool + Limit int + Desc bool Expected []*store.Message } @@ -71,14 +71,14 @@ var tests = []GetMessagesTest{{ {Prefix: []byte(PrefixTest), Start: []byte(fmt.Sprintf("%s-%d", PrefixTest, 1))}, {Prefix: []byte(PrefixOther), Start: []byte(fmt.Sprintf("%s-%d", PrefixTest, 1))}, }, - Max: 5, + Limit: 5, Expected: []*store.Message{ testMessageTest1, testMessageTest2, testMessageTest3, testMessageTest4, testMessageTest5, }, }, { - Name: "Newest", + Name: "Descending", Prefixes: []store.Prefix{{Prefix: []byte(PrefixTest), Max: 5}, {Prefix: []byte(PrefixOther), Max: 5}}, - Newest: true, + Desc: true, Expected: []*store.Message{ testMessageTest9, testMessageTest8, testMessageTest7, testMessageTest6, testMessageTest5, testMessageOther9, testMessageOther8, testMessageOther7, testMessageOther6, testMessageOther5, @@ -144,8 +144,8 @@ func TestGetByPrefixes(t *testing.T) { Topic: TestTopic, Shard: TestShard, Prefixes: test.Prefixes, - Max: test.Max, - Newest: test.Newest, + Limit: test.Limit, + Desc: test.Desc, }) if err != nil { t.Errorf("%s test error getting message; %v", test.Name, err) From 2d9ead2587bae19fe3a9d8c84d8ef6e4e7f71be9 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Fri, 26 Jul 2024 04:30:37 -0700 Subject: [PATCH 15/23] Remove wait option from queue get messages. --- db/client/client.go | 7 +- db/client/main.go | 1 - db/item/chain/height_block.go | 20 +-- db/item/chain/tx_processed.go | 22 --- db/proto/queue.proto | 3 +- db/proto/queue_pb/queue.pb.go | 161 +++++++++--------- db/server/server.go | 40 ++--- graph/resolver/query.resolvers.go | 2 +- node/act/maint/populate_p2sh.go | 2 +- ref/network/block.proto | 1 - ref/network/gen/network_pb/block.pb.go | 104 +++++------ ref/network/gen/network_pb/block_tx.pb.go | 12 +- ref/network/gen/network_pb/double_spend.pb.go | 12 +- ref/network/gen/network_pb/mempool.pb.go | 12 +- ref/network/gen/network_pb/metric.pb.go | 18 +- ref/network/gen/network_pb/network.pb.go | 83 +++++---- ref/network/gen/network_pb/network_grpc.pb.go | 149 ++++++++-------- ref/network/gen/network_pb/output.pb.go | 20 +-- ref/network/gen/network_pb/tx.pb.go | 127 ++++---------- ref/network/network.proto | 1 - ref/network/network_client/block_height.go | 1 - ref/network/network_client/tx_processed.go | 33 ---- ref/network/network_server/main.go | 12 +- ref/network/tx.proto | 4 - test/run/queue/get.go | 23 --- test/tasks/queue.go | 46 ----- 26 files changed, 332 insertions(+), 584 deletions(-) delete mode 100644 ref/network/network_client/tx_processed.go diff --git a/db/client/client.go b/db/client/client.go index 03bc4656..e6b056f6 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -185,7 +185,6 @@ func (s *Client) GetWOpts(opts Opts) error { Prefixes: opts.Prefixes[i:end], Max: opts.Max, Uids: opts.Uids, - Wait: opts.Wait, Newest: opts.Newest, }) } @@ -201,7 +200,6 @@ func (s *Client) GetWOpts(opts Opts) error { Start: opts.Start, Max: opts.Max, Uids: opts.Uids[i:end], - Wait: opts.Wait, Newest: opts.Newest, }) } @@ -214,10 +212,8 @@ func (s *Client) GetWOpts(opts Opts) error { var timeout time.Duration if opts.Timeout > 0 { timeout = opts.Timeout - } else if !opts.Wait { - timeout = DefaultGetTimeout } else { - timeout = DefaultWaitTimeout + timeout = DefaultGetTimeout } c := queue_pb.NewQueueClient(s.conn) var bgCtx = opts.Context @@ -234,7 +230,6 @@ func (s *Client) GetWOpts(opts Opts) error { Start: optGroup.Start, Max: optGroup.Max, Uids: optGroup.Uids, - Wait: optGroup.Wait, Newest: optGroup.Newest, }, grpc.MaxCallRecvMsgSize(MaxMessageSize)) if err != nil { diff --git a/db/client/main.go b/db/client/main.go index 8f5e49d2..3176ce3d 100644 --- a/db/client/main.go +++ b/db/client/main.go @@ -12,7 +12,6 @@ import ( const ( DefaultGetTimeout = 60 * time.Second DefaultSetTimeout = 10 * time.Minute - DefaultWaitTimeout = 5 * time.Minute DefaultStreamTimeout = 7 * 24 * time.Hour ) diff --git a/db/item/chain/height_block.go b/db/item/chain/height_block.go index 067ca57e..d27473d3 100644 --- a/db/item/chain/height_block.go +++ b/db/item/chain/height_block.go @@ -10,7 +10,6 @@ import ( "github.com/memocash/index/ref/config" "sort" "strings" - "time" ) type HeightBlock struct { @@ -130,35 +129,28 @@ func GetHeightBlocks(ctx context.Context, shard uint32, startHeight int64, desc return heightBlocks, nil } -func GetHeightBlocksAll(startHeight int64, waitSingle bool) ([]*HeightBlock, error) { - heightBlocks, err := GetHeightBlocksAllLimit(startHeight, waitSingle, client.LargeLimit, false) +func GetHeightBlocksAll(startHeight int64) ([]*HeightBlock, error) { + heightBlocks, err := GetHeightBlocksAllLimit(startHeight, client.LargeLimit, false) if err != nil { return nil, fmt.Errorf("error getting height blocks all large limit; %w", err) } return heightBlocks, nil } -func GetHeightBlocksAllDefault(startHeight int64, waitSingle bool, newest bool) ([]*HeightBlock, error) { - heightBlocks, err := GetHeightBlocksAllLimit(startHeight, waitSingle, client.DefaultLimit, newest) +func GetHeightBlocksAllDefault(startHeight int64, newest bool) ([]*HeightBlock, error) { + heightBlocks, err := GetHeightBlocksAllLimit(startHeight, client.DefaultLimit, newest) if err != nil { return nil, fmt.Errorf("error getting height blocks all default limit; %w", err) } return heightBlocks, nil } -func GetHeightBlocksAllLimit(startHeight int64, waitSingle bool, limit uint32, newest bool) ([]*HeightBlock, error) { +func GetHeightBlocksAllLimit(startHeight int64, limit uint32, newest bool) ([]*HeightBlock, error) { var heightBlocks []*HeightBlock shardConfigs := config.GetQueueShards() shardLimit := limit / uint32(len(shardConfigs)) for _, shardConfig := range shardConfigs { - if waitSingle && db.GetShardId32(uint(startHeight)) != shardConfig.Shard { - continue - } dbClient := client.NewClient(shardConfig.GetHost()) - var timeout time.Duration - if waitSingle { - timeout = time.Hour - } var start []byte if startHeight != 0 { start = jutil.GetInt64DataBig(startHeight) @@ -166,10 +158,8 @@ func GetHeightBlocksAllLimit(startHeight int64, waitSingle bool, limit uint32, n if err := dbClient.GetWOpts(client.Opts{ Topic: db.TopicChainHeightBlock, Start: start, - Wait: waitSingle, Max: shardLimit, Newest: newest, - Timeout: timeout, }); err != nil { return nil, fmt.Errorf("error getting height blocks from queue client all; %w", err) } diff --git a/db/item/chain/tx_processed.go b/db/item/chain/tx_processed.go index 1aaa7bfe..a8edb6e9 100644 --- a/db/item/chain/tx_processed.go +++ b/db/item/chain/tx_processed.go @@ -1,12 +1,9 @@ package chain import ( - "context" - "fmt" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" - "github.com/memocash/index/ref/config" "time" ) @@ -44,22 +41,3 @@ func (s *TxProcessed) Deserialize([]byte) {} func GetTxProcessedUid(txHash []byte, timestamp time.Time) []byte { return jutil.CombineBytes(jutil.ByteReverse(txHash), jutil.GetTimeByteNanoBig(timestamp)) } - -func WaitForTxProcessed(ctx context.Context, txHash []byte) (*TxProcessed, error) { - shardConfig := config.GetShardConfig(db.GetShardIdFromByte32(txHash), config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: db.TopicChainTxProcessed, - Prefixes: [][]byte{jutil.ByteReverse(txHash)}, - Wait: true, - }); err != nil { - return nil, fmt.Errorf("error getting tx processed with wait db message; %w", err) - } - if len(dbClient.Messages) == 0 { - return nil, fmt.Errorf("error with tx processed wait, empty message; %w", client.EntryNotFoundError) - } - var txProcessed = new(TxProcessed) - db.Set(txProcessed, dbClient.Messages[0]) - return txProcessed, nil -} diff --git a/db/proto/queue.proto b/db/proto/queue.proto index a2af1ead..251607cf 100644 --- a/db/proto/queue.proto +++ b/db/proto/queue.proto @@ -54,8 +54,7 @@ message Request { uint32 max = 3; repeated bytes prefixes = 4; repeated bytes uids = 5; - bool wait = 6; - bool newest = 7; + bool newest = 6; } message RequestStream { diff --git a/db/proto/queue_pb/queue.pb.go b/db/proto/queue_pb/queue.pb.go index 938d74b6..d419b341 100644 --- a/db/proto/queue_pb/queue.pb.go +++ b/db/proto/queue_pb/queue.pb.go @@ -351,8 +351,7 @@ type Request struct { Max uint32 `protobuf:"varint,3,opt,name=max,proto3" json:"max,omitempty"` Prefixes [][]byte `protobuf:"bytes,4,rep,name=prefixes,proto3" json:"prefixes,omitempty"` Uids [][]byte `protobuf:"bytes,5,rep,name=uids,proto3" json:"uids,omitempty"` - Wait bool `protobuf:"varint,6,opt,name=wait,proto3" json:"wait,omitempty"` - Newest bool `protobuf:"varint,7,opt,name=newest,proto3" json:"newest,omitempty"` + Newest bool `protobuf:"varint,6,opt,name=newest,proto3" json:"newest,omitempty"` } func (x *Request) Reset() { @@ -422,13 +421,6 @@ func (x *Request) GetUids() [][]byte { return nil } -func (x *Request) GetWait() bool { - if x != nil { - return x.Wait - } - return false -} - func (x *Request) GetNewest() bool { if x != nil { return x.Newest @@ -891,88 +883,87 @@ var file_queue_proto_rawDesc = []byte{ 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, - 0x75, 0x69, 0x64, 0x22, 0xa3, 0x01, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, + 0x75, 0x69, 0x64, 0x22, 0x8f, 0x01, 0x0a, 0x07, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6d, 0x61, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6d, 0x61, 0x78, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x75, 0x69, 0x64, - 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x75, 0x69, 0x64, 0x73, 0x12, 0x12, 0x0a, - 0x04, 0x77, 0x61, 0x69, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x77, 0x61, 0x69, - 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x65, 0x77, 0x65, 0x73, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x06, 0x6e, 0x65, 0x77, 0x65, 0x73, 0x74, 0x22, 0x41, 0x0a, 0x0d, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, - 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, - 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, - 0x28, 0x0c, 0x52, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x22, 0x0e, 0x0a, 0x0c, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x05, - 0x54, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, - 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, - 0x39, 0x0a, 0x0e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x70, 0x6c, - 0x79, 0x12, 0x27, 0x0a, 0x06, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x0f, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, - 0x69, 0x63, 0x52, 0x06, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x22, 0x3c, 0x0a, 0x0c, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, - 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, - 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x22, 0x22, 0x0a, 0x0a, 0x54, 0x6f, 0x70, 0x69, - 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x99, 0x01, 0x0a, - 0x0f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, - 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x33, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, - 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, - 0x78, 0x52, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, - 0x69, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, - 0x74, 0x12, 0x25, 0x0a, 0x05, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, - 0x32, 0x0f, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4f, 0x72, 0x64, 0x65, - 0x72, 0x52, 0x05, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x22, 0x53, 0x0a, 0x0d, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, - 0x66, 0x69, 0x78, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, - 0x78, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x2a, 0x1a, 0x0a, - 0x05, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x53, 0x43, 0x10, 0x00, 0x12, - 0x08, 0x0a, 0x04, 0x44, 0x45, 0x53, 0x43, 0x10, 0x01, 0x32, 0x86, 0x04, 0x0a, 0x05, 0x51, 0x75, - 0x65, 0x75, 0x65, 0x12, 0x3a, 0x0a, 0x0c, 0x53, 0x61, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x73, 0x12, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, - 0x70, 0x62, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, - 0x3f, 0x0a, 0x0e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x73, 0x12, 0x15, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x55, 0x69, 0x64, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, - 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, - 0x12, 0x3a, 0x0a, 0x0a, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, - 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, - 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0b, - 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x11, 0x2e, 0x71, 0x75, - 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, - 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, - 0x65, 0x73, 0x22, 0x00, 0x12, 0x40, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x42, 0x79, 0x50, 0x72, 0x65, - 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x19, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, - 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, - 0x1a, 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, - 0x61, 0x67, 0x65, 0x73, 0x22, 0x00, 0x12, 0x43, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x17, 0x2e, 0x71, 0x75, - 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, 0x74, - 0x72, 0x65, 0x61, 0x6d, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, - 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, - 0x65, 0x74, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x71, 0x75, - 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x18, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, - 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, - 0x41, 0x0a, 0x0f, 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, - 0x6e, 0x74, 0x12, 0x16, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, - 0x75, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, - 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, - 0x22, 0x00, 0x42, 0x2d, 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x6d, 0x65, 0x6d, 0x6f, 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, - 0x64, 0x62, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, - 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x04, 0x75, 0x69, 0x64, 0x73, 0x12, 0x16, 0x0a, + 0x06, 0x6e, 0x65, 0x77, 0x65, 0x73, 0x74, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x6e, + 0x65, 0x77, 0x65, 0x73, 0x74, 0x22, 0x41, 0x0a, 0x0d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x1a, 0x0a, 0x08, + 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x08, + 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x22, 0x0e, 0x0a, 0x0c, 0x45, 0x6d, 0x70, 0x74, + 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, 0x31, 0x0a, 0x05, 0x54, 0x6f, 0x70, 0x69, + 0x63, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x39, 0x0a, 0x0e, 0x54, + 0x6f, 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x27, 0x0a, + 0x06, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, + 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x52, 0x06, + 0x74, 0x6f, 0x70, 0x69, 0x63, 0x73, 0x22, 0x3c, 0x0a, 0x0c, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, 0x69, 0x63, 0x12, 0x16, 0x0a, 0x06, + 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x72, + 0x65, 0x66, 0x69, 0x78, 0x22, 0x22, 0x0a, 0x0a, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x43, 0x6f, 0x75, + 0x6e, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x99, 0x01, 0x0a, 0x0f, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, + 0x74, 0x6f, 0x70, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x74, 0x6f, 0x70, + 0x69, 0x63, 0x12, 0x33, 0x0a, 0x08, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x18, 0x02, + 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x52, 0x08, 0x70, + 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x25, 0x0a, + 0x05, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x0f, 0x2e, 0x71, + 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4f, 0x72, 0x64, 0x65, 0x72, 0x52, 0x05, 0x6f, + 0x72, 0x64, 0x65, 0x72, 0x22, 0x53, 0x0a, 0x0d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x50, + 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x14, 0x0a, + 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x73, 0x74, + 0x61, 0x72, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x05, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x2a, 0x1a, 0x0a, 0x05, 0x4f, 0x72, 0x64, + 0x65, 0x72, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x53, 0x43, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x44, + 0x45, 0x53, 0x43, 0x10, 0x01, 0x32, 0x86, 0x04, 0x0a, 0x05, 0x51, 0x75, 0x65, 0x75, 0x65, 0x12, + 0x3a, 0x0a, 0x0c, 0x53, 0x61, 0x76, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, + 0x12, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, + 0x67, 0x65, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x45, + 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3f, 0x0a, 0x0e, 0x44, + 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x15, 0x2e, + 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x55, 0x69, 0x64, 0x73, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, + 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x3a, 0x0a, 0x0a, + 0x47, 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x17, 0x2e, 0x71, 0x75, 0x65, + 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, 0x69, 0x6e, + 0x67, 0x6c, 0x65, 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x00, 0x12, 0x36, 0x0a, 0x0b, 0x47, 0x65, 0x74, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, + 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x71, 0x75, 0x65, + 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x22, 0x00, + 0x12, 0x40, 0x0a, 0x0d, 0x47, 0x65, 0x74, 0x42, 0x79, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, + 0x73, 0x12, 0x19, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, + 0x75, 0x65, 0x73, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73, 0x1a, 0x12, 0x2e, 0x71, + 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, + 0x22, 0x00, 0x12, 0x43, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4d, + 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x12, 0x17, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, + 0x70, 0x62, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, + 0x1a, 0x11, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x4d, 0x65, 0x73, 0x73, + 0x61, 0x67, 0x65, 0x22, 0x00, 0x30, 0x01, 0x12, 0x42, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x54, 0x6f, + 0x70, 0x69, 0x63, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x16, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, + 0x70, 0x62, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, + 0x18, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, + 0x4c, 0x69, 0x73, 0x74, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x22, 0x00, 0x12, 0x41, 0x0a, 0x0f, 0x47, + 0x65, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x16, + 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x52, + 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, + 0x62, 0x2e, 0x54, 0x6f, 0x70, 0x69, 0x63, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x00, 0x42, 0x2d, + 0x5a, 0x2b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x65, 0x6d, + 0x6f, 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x64, 0x62, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x71, 0x75, 0x65, 0x75, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/db/server/server.go b/db/server/server.go index 97c2bb60..b5736869 100644 --- a/db/server/server.go +++ b/db/server/server.go @@ -128,31 +128,21 @@ func (s *Server) GetMessages(ctx context.Context, request *queue_pb.Request) (*q return nil, fmt.Errorf("error getting messages by uids; %w", err) } } else { - for i := 0; i < 2; i++ { - var requestByPrefixes = store.RequestByPrefixes{ - Topic: request.Topic, - Shard: s.Shard, - Limit: int(request.Max), - Desc: request.Newest, - } - for _, prefix := range request.Prefixes { - requestByPrefixes.Prefixes = append(requestByPrefixes.Prefixes, store.Prefix{ - Prefix: prefix, - Start: request.Start, - }) - } - messages, err = store.GetByPrefixes(requestByPrefixes) - if err != nil { - return nil, fmt.Errorf("error getting messages for topic: %s (shard %d); %w", request.Topic, s.Shard, err) - } - if len(messages) == 0 && request.Wait && i == 0 { - metric.AddTopicListen(metric.TopicListen{Topic: request.Topic}) - if err := ListenSingle(ctx, s.Shard, request.Topic, request.Start, request.Prefixes); err != nil { - return nil, fmt.Errorf("error listening for new topic item; %w", err) - } - } else { - break - } + var requestByPrefixes = store.RequestByPrefixes{ + Topic: request.Topic, + Shard: s.Shard, + Limit: int(request.Max), + Desc: request.Newest, + } + for _, prefix := range request.Prefixes { + requestByPrefixes.Prefixes = append(requestByPrefixes.Prefixes, store.Prefix{ + Prefix: prefix, + Start: request.Start, + }) + } + messages, err = store.GetByPrefixes(requestByPrefixes) + if err != nil { + return nil, fmt.Errorf("error getting messages for topic: %s (shard %d); %w", request.Topic, s.Shard, err) } } diff --git a/graph/resolver/query.resolvers.go b/graph/resolver/query.resolvers.go index 531d25d2..1d1633ce 100644 --- a/graph/resolver/query.resolvers.go +++ b/graph/resolver/query.resolvers.go @@ -106,7 +106,7 @@ func (r *queryResolver) Blocks(ctx context.Context, newest *bool, start *uint32) if newest != nil { newestBool = *newest } - heightBlocks, err := chain.GetHeightBlocksAllDefault(startInt, false, newestBool) + heightBlocks, err := chain.GetHeightBlocksAllDefault(startInt, newestBool) if err != nil { return nil, InternalError{fmt.Errorf("error getting height blocks for query; %w", err)} } diff --git a/node/act/maint/populate_p2sh.go b/node/act/maint/populate_p2sh.go index 3ba99af3..edd8992a 100644 --- a/node/act/maint/populate_p2sh.go +++ b/node/act/maint/populate_p2sh.go @@ -32,7 +32,7 @@ func (p *PopulateP2sh) Populate(startHeight int64) error { addressSaver.SkipP2pkh = true var maxHeight = startHeight for { - heightBlocks, err := chain.GetHeightBlocksAllLimit(maxHeight, false, client.HugeLimit, false) + heightBlocks, err := chain.GetHeightBlocksAllLimit(maxHeight, client.HugeLimit, false) if err != nil { return fmt.Errorf("fatal error getting height blocks all for populate p2sh; %w", err) } diff --git a/ref/network/block.proto b/ref/network/block.proto index 57dd1ef5..3a1a92f9 100644 --- a/ref/network/block.proto +++ b/ref/network/block.proto @@ -8,7 +8,6 @@ option go_package = "github.com/memocash/index/ref/network/gen/network_pb"; message BlockHeightRequest { int64 start = 1; - bool wait = 2; } message BlockHeightResponse { diff --git a/ref/network/gen/network_pb/block.pb.go b/ref/network/gen/network_pb/block.pb.go index 2ff25550..fde7b89c 100644 --- a/ref/network/gen/network_pb/block.pb.go +++ b/ref/network/gen/network_pb/block.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.6.1 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: block.proto package network_pb @@ -26,7 +26,6 @@ type BlockHeightRequest struct { unknownFields protoimpl.UnknownFields Start int64 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"` - Wait bool `protobuf:"varint,2,opt,name=wait,proto3" json:"wait,omitempty"` } func (x *BlockHeightRequest) Reset() { @@ -68,13 +67,6 @@ func (x *BlockHeightRequest) GetStart() int64 { return 0 } -func (x *BlockHeightRequest) GetWait() bool { - if x != nil { - return x.Wait - } - return false -} - type BlockHeightResponse struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -457,45 +449,43 @@ var File_block_proto protoreflect.FileDescriptor var file_block_proto_rawDesc = []byte{ 0x0a, 0x0b, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x0a, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x1a, 0x08, 0x74, 0x78, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x22, 0x3e, 0x0a, 0x12, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, + 0x6f, 0x74, 0x6f, 0x22, 0x2a, 0x0a, 0x12, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, - 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, - 0x12, 0x0a, 0x04, 0x77, 0x61, 0x69, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x04, 0x77, - 0x61, 0x69, 0x74, 0x22, 0x46, 0x0a, 0x13, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, - 0x68, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x06, 0x62, 0x6c, - 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, - 0x67, 0x68, 0x74, 0x52, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x22, 0x39, 0x0a, 0x0b, 0x42, - 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, - 0x69, 0x67, 0x68, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, - 0x68, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, - 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x22, 0x3e, 0x0a, 0x0c, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x16, - 0x0a, 0x06, 0x6e, 0x65, 0x77, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, - 0x6e, 0x65, 0x77, 0x65, 0x73, 0x74, 0x22, 0x61, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, - 0x10, 0x0a, 0x03, 0x74, 0x78, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x74, 0x78, - 0x73, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x3f, 0x0a, 0x0e, 0x42, 0x6c, 0x6f, - 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x2d, 0x0a, 0x06, 0x62, - 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6e, 0x65, - 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, - 0x66, 0x6f, 0x52, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x22, 0x1f, 0x0a, 0x05, 0x42, 0x6c, - 0x6f, 0x63, 0x6b, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0c, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x54, 0x0a, 0x07, 0x54, - 0x78, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x27, 0x0a, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, - 0x70, 0x62, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x12, - 0x20, 0x0a, 0x03, 0x74, 0x78, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x78, 0x52, 0x03, 0x74, 0x78, - 0x73, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x6d, 0x65, 0x6d, 0x6f, 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x72, - 0x65, 0x66, 0x2f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, + 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x22, + 0x46, 0x0a, 0x13, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2f, 0x0a, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, + 0x5f, 0x70, 0x62, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x52, + 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x22, 0x39, 0x0a, 0x0b, 0x42, 0x6c, 0x6f, 0x63, 0x6b, + 0x48, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x12, + 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, + 0x73, 0x68, 0x22, 0x3e, 0x0a, 0x0c, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, + 0x73, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x65, + 0x77, 0x65, 0x73, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x6e, 0x65, 0x77, 0x65, + 0x73, 0x74, 0x22, 0x61, 0x0a, 0x09, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x12, + 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, + 0x61, 0x73, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x06, 0x68, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x74, + 0x78, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x03, 0x74, 0x78, 0x73, 0x12, 0x16, 0x0a, + 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x06, 0x68, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x3f, 0x0a, 0x0e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, + 0x66, 0x6f, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x2d, 0x0a, 0x06, 0x62, 0x6c, 0x6f, 0x63, 0x6b, + 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, + 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x06, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x73, 0x22, 0x1f, 0x0a, 0x05, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x12, + 0x16, 0x0a, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, + 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x22, 0x54, 0x0a, 0x07, 0x54, 0x78, 0x42, 0x6c, 0x6f, + 0x63, 0x6b, 0x12, 0x27, 0x0a, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x11, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x42, + 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x12, 0x20, 0x0a, 0x03, 0x74, + 0x78, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0e, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x78, 0x52, 0x03, 0x74, 0x78, 0x73, 0x42, 0x36, 0x5a, + 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x65, 0x6d, 0x6f, + 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x72, 0x65, 0x66, 0x2f, 0x6e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6e, 0x65, 0x74, 0x77, 0x6f, + 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -511,7 +501,7 @@ func file_block_proto_rawDescGZIP() []byte { } var file_block_proto_msgTypes = make([]protoimpl.MessageInfo, 8) -var file_block_proto_goTypes = []interface{}{ +var file_block_proto_goTypes = []any{ (*BlockHeightRequest)(nil), // 0: network_pb.BlockHeightRequest (*BlockHeightResponse)(nil), // 1: network_pb.BlockHeightResponse (*BlockHeight)(nil), // 2: network_pb.BlockHeight @@ -541,7 +531,7 @@ func file_block_proto_init() { } file_tx_proto_init() if !protoimpl.UnsafeEnabled { - file_block_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_block_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*BlockHeightRequest); i { case 0: return &v.state @@ -553,7 +543,7 @@ func file_block_proto_init() { return nil } } - file_block_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_block_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*BlockHeightResponse); i { case 0: return &v.state @@ -565,7 +555,7 @@ func file_block_proto_init() { return nil } } - file_block_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_block_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*BlockHeight); i { case 0: return &v.state @@ -577,7 +567,7 @@ func file_block_proto_init() { return nil } } - file_block_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_block_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*BlockRequest); i { case 0: return &v.state @@ -589,7 +579,7 @@ func file_block_proto_init() { return nil } } - file_block_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_block_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*BlockInfo); i { case 0: return &v.state @@ -601,7 +591,7 @@ func file_block_proto_init() { return nil } } - file_block_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_block_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*BlockInfoReply); i { case 0: return &v.state @@ -613,7 +603,7 @@ func file_block_proto_init() { return nil } } - file_block_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_block_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*Block); i { case 0: return &v.state @@ -625,7 +615,7 @@ func file_block_proto_init() { return nil } } - file_block_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_block_proto_msgTypes[7].Exporter = func(v any, i int) any { switch v := v.(*TxBlock); i { case 0: return &v.state diff --git a/ref/network/gen/network_pb/block_tx.pb.go b/ref/network/gen/network_pb/block_tx.pb.go index e7dac71a..501be4ae 100644 --- a/ref/network/gen/network_pb/block_tx.pb.go +++ b/ref/network/gen/network_pb/block_tx.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.6.1 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: block_tx.proto package network_pb @@ -221,7 +221,7 @@ func file_block_tx_proto_rawDescGZIP() []byte { } var file_block_tx_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_block_tx_proto_goTypes = []interface{}{ +var file_block_tx_proto_goTypes = []any{ (*BlockTxRequest)(nil), // 0: network_pb.BlockTxRequest (*BlockTxResponse)(nil), // 1: network_pb.BlockTxResponse (*BlockTx)(nil), // 2: network_pb.BlockTx @@ -241,7 +241,7 @@ func file_block_tx_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_block_tx_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_block_tx_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*BlockTxRequest); i { case 0: return &v.state @@ -253,7 +253,7 @@ func file_block_tx_proto_init() { return nil } } - file_block_tx_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_block_tx_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*BlockTxResponse); i { case 0: return &v.state @@ -265,7 +265,7 @@ func file_block_tx_proto_init() { return nil } } - file_block_tx_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_block_tx_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*BlockTx); i { case 0: return &v.state diff --git a/ref/network/gen/network_pb/double_spend.pb.go b/ref/network/gen/network_pb/double_spend.pb.go index 4069df3a..e0de58f5 100644 --- a/ref/network/gen/network_pb/double_spend.pb.go +++ b/ref/network/gen/network_pb/double_spend.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.6.1 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: double_spend.proto package network_pb @@ -194,7 +194,7 @@ func file_double_spend_proto_rawDescGZIP() []byte { } var file_double_spend_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_double_spend_proto_goTypes = []interface{}{ +var file_double_spend_proto_goTypes = []any{ (*DoubleSpendRequest)(nil), // 0: network_pb.DoubleSpendRequest (*DoubleSpendResponse)(nil), // 1: network_pb.DoubleSpendResponse (*DoubleSpend)(nil), // 2: network_pb.DoubleSpend @@ -214,7 +214,7 @@ func file_double_spend_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_double_spend_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_double_spend_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*DoubleSpendRequest); i { case 0: return &v.state @@ -226,7 +226,7 @@ func file_double_spend_proto_init() { return nil } } - file_double_spend_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_double_spend_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*DoubleSpendResponse); i { case 0: return &v.state @@ -238,7 +238,7 @@ func file_double_spend_proto_init() { return nil } } - file_double_spend_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_double_spend_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*DoubleSpend); i { case 0: return &v.state diff --git a/ref/network/gen/network_pb/mempool.pb.go b/ref/network/gen/network_pb/mempool.pb.go index 28c3a421..c2f8c5b6 100644 --- a/ref/network/gen/network_pb/mempool.pb.go +++ b/ref/network/gen/network_pb/mempool.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.6.1 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: mempool.proto package network_pb @@ -193,7 +193,7 @@ func file_mempool_proto_rawDescGZIP() []byte { } var file_mempool_proto_msgTypes = make([]protoimpl.MessageInfo, 3) -var file_mempool_proto_goTypes = []interface{}{ +var file_mempool_proto_goTypes = []any{ (*MempoolTxRequest)(nil), // 0: network_pb.MempoolTxRequest (*MempoolTxResponse)(nil), // 1: network_pb.MempoolTxResponse (*MempoolTx)(nil), // 2: network_pb.MempoolTx @@ -213,7 +213,7 @@ func file_mempool_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_mempool_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_mempool_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*MempoolTxRequest); i { case 0: return &v.state @@ -225,7 +225,7 @@ func file_mempool_proto_init() { return nil } } - file_mempool_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_mempool_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*MempoolTxResponse); i { case 0: return &v.state @@ -237,7 +237,7 @@ func file_mempool_proto_init() { return nil } } - file_mempool_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_mempool_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*MempoolTx); i { case 0: return &v.state diff --git a/ref/network/gen/network_pb/metric.pb.go b/ref/network/gen/network_pb/metric.pb.go index 6a9ed8f6..82507971 100644 --- a/ref/network/gen/network_pb/metric.pb.go +++ b/ref/network/gen/network_pb/metric.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.6.1 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: metric.proto package network_pb @@ -419,7 +419,7 @@ func file_metric_proto_rawDescGZIP() []byte { } var file_metric_proto_msgTypes = make([]protoimpl.MessageInfo, 6) -var file_metric_proto_goTypes = []interface{}{ +var file_metric_proto_goTypes = []any{ (*MetricRequest)(nil), // 0: network_pb.MetricRequest (*MetricResponse)(nil), // 1: network_pb.MetricResponse (*MetricInfo)(nil), // 2: network_pb.MetricInfo @@ -443,7 +443,7 @@ func file_metric_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_metric_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_metric_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*MetricRequest); i { case 0: return &v.state @@ -455,7 +455,7 @@ func file_metric_proto_init() { return nil } } - file_metric_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_metric_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*MetricResponse); i { case 0: return &v.state @@ -467,7 +467,7 @@ func file_metric_proto_init() { return nil } } - file_metric_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_metric_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*MetricInfo); i { case 0: return &v.state @@ -479,7 +479,7 @@ func file_metric_proto_init() { return nil } } - file_metric_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_metric_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*MetricTimeRequest); i { case 0: return &v.state @@ -491,7 +491,7 @@ func file_metric_proto_init() { return nil } } - file_metric_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_metric_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*MetricTimeResponse); i { case 0: return &v.state @@ -503,7 +503,7 @@ func file_metric_proto_init() { return nil } } - file_metric_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_metric_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*MetricTime); i { case 0: return &v.state diff --git a/ref/network/gen/network_pb/network.pb.go b/ref/network/gen/network_pb/network.pb.go index 6182e5a2..9094c3f1 100644 --- a/ref/network/gen/network_pb/network.pb.go +++ b/ref/network/gen/network_pb/network.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.6.1 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: network.proto package network_pb @@ -335,7 +335,7 @@ var file_network_proto_rawDesc = []byte{ 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, 0x26, 0x0a, 0x10, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x48, 0x61, 0x73, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x68, 0x61, 0x73, 0x68, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x32, 0xf0, 0x09, 0x0a, 0x07, 0x4e, + 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x68, 0x61, 0x73, 0x68, 0x32, 0xb2, 0x09, 0x0a, 0x07, 0x4e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x12, 0x34, 0x0a, 0x07, 0x53, 0x61, 0x76, 0x65, 0x54, 0x78, 0x73, 0x12, 0x0f, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x78, 0x73, 0x1a, 0x18, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, @@ -406,19 +406,15 @@ var file_network_proto_rawDesc = []byte{ 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x20, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x49, 0x6e, 0x70, - 0x75, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x08, 0x4c, - 0x69, 0x73, 0x74, 0x65, 0x6e, 0x54, 0x78, 0x12, 0x15, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, - 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x54, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x19, - 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x4c, 0x69, 0x73, 0x74, - 0x65, 0x6e, 0x54, 0x78, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x42, 0x0a, 0x0d, 0x4f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x19, 0x2e, 0x6e, 0x65, 0x74, - 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x4d, 0x65, - 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, - 0x70, 0x62, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x42, 0x36, 0x5a, - 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x65, 0x6d, 0x6f, - 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x72, 0x65, 0x66, 0x2f, 0x6e, - 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x75, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x42, 0x0a, 0x0d, 0x4f, + 0x75, 0x74, 0x70, 0x75, 0x74, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x19, 0x2e, 0x6e, + 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, + 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x1a, 0x16, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, + 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x42, + 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x65, + 0x6d, 0x6f, 0x63, 0x61, 0x73, 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x72, 0x65, 0x66, + 0x2f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6e, 0x65, 0x74, + 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -434,7 +430,7 @@ func file_network_proto_rawDescGZIP() []byte { } var file_network_proto_msgTypes = make([]protoimpl.MessageInfo, 5) -var file_network_proto_goTypes = []interface{}{ +var file_network_proto_goTypes = []any{ (*Address)(nil), // 0: network_pb.Address (*BalanceReply)(nil), // 1: network_pb.BalanceReply (*StringMessage)(nil), // 2: network_pb.StringMessage @@ -466,7 +462,6 @@ var file_network_proto_goTypes = []interface{}{ (*DoubleSpendResponse)(nil), // 28: network_pb.DoubleSpendResponse (*UtxosResponse)(nil), // 29: network_pb.UtxosResponse (*OutputInputsResponse)(nil), // 30: network_pb.OutputInputsResponse - (*ListenTxReply)(nil), // 31: network_pb.ListenTxReply } var file_network_proto_depIdxs = []int32{ 5, // 0: network_pb.Network.SaveTxs:input_type -> network_pb.Txs @@ -485,28 +480,26 @@ var file_network_proto_depIdxs = []int32{ 15, // 13: network_pb.Network.GetDoubleSpends:input_type -> network_pb.DoubleSpendRequest 16, // 14: network_pb.Network.GetUtxos:input_type -> network_pb.UtxosRequest 17, // 15: network_pb.Network.GetOutputInputs:input_type -> network_pb.OutputInputsRequest - 6, // 16: network_pb.Network.ListenTx:input_type -> network_pb.TxRequest - 2, // 17: network_pb.Network.OutputMessage:input_type -> network_pb.StringMessage - 18, // 18: network_pb.Network.SaveTxs:output_type -> network_pb.SaveTxsReply - 19, // 19: network_pb.Network.GetTx:output_type -> network_pb.TxReply - 3, // 20: network_pb.Network.SaveTxBlock:output_type -> network_pb.ErrorReply - 20, // 21: network_pb.Network.GetTxBlock:output_type -> network_pb.TxBlockReply - 1, // 22: network_pb.Network.GetBalance:output_type -> network_pb.BalanceReply - 21, // 23: network_pb.Network.GetBlockInfos:output_type -> network_pb.BlockInfoReply - 22, // 24: network_pb.Network.GetBlockByHash:output_type -> network_pb.BlockInfo - 22, // 25: network_pb.Network.GetBlockByHeight:output_type -> network_pb.BlockInfo - 23, // 26: network_pb.Network.GetMetrics:output_type -> network_pb.MetricResponse - 24, // 27: network_pb.Network.GetMetricList:output_type -> network_pb.MetricTimeResponse - 25, // 28: network_pb.Network.GetHeightBlocks:output_type -> network_pb.BlockHeightResponse - 26, // 29: network_pb.Network.GetBlockTxs:output_type -> network_pb.BlockTxResponse - 27, // 30: network_pb.Network.GetMempoolTxs:output_type -> network_pb.MempoolTxResponse - 28, // 31: network_pb.Network.GetDoubleSpends:output_type -> network_pb.DoubleSpendResponse - 29, // 32: network_pb.Network.GetUtxos:output_type -> network_pb.UtxosResponse - 30, // 33: network_pb.Network.GetOutputInputs:output_type -> network_pb.OutputInputsResponse - 31, // 34: network_pb.Network.ListenTx:output_type -> network_pb.ListenTxReply - 3, // 35: network_pb.Network.OutputMessage:output_type -> network_pb.ErrorReply - 18, // [18:36] is the sub-list for method output_type - 0, // [0:18] is the sub-list for method input_type + 2, // 16: network_pb.Network.OutputMessage:input_type -> network_pb.StringMessage + 18, // 17: network_pb.Network.SaveTxs:output_type -> network_pb.SaveTxsReply + 19, // 18: network_pb.Network.GetTx:output_type -> network_pb.TxReply + 3, // 19: network_pb.Network.SaveTxBlock:output_type -> network_pb.ErrorReply + 20, // 20: network_pb.Network.GetTxBlock:output_type -> network_pb.TxBlockReply + 1, // 21: network_pb.Network.GetBalance:output_type -> network_pb.BalanceReply + 21, // 22: network_pb.Network.GetBlockInfos:output_type -> network_pb.BlockInfoReply + 22, // 23: network_pb.Network.GetBlockByHash:output_type -> network_pb.BlockInfo + 22, // 24: network_pb.Network.GetBlockByHeight:output_type -> network_pb.BlockInfo + 23, // 25: network_pb.Network.GetMetrics:output_type -> network_pb.MetricResponse + 24, // 26: network_pb.Network.GetMetricList:output_type -> network_pb.MetricTimeResponse + 25, // 27: network_pb.Network.GetHeightBlocks:output_type -> network_pb.BlockHeightResponse + 26, // 28: network_pb.Network.GetBlockTxs:output_type -> network_pb.BlockTxResponse + 27, // 29: network_pb.Network.GetMempoolTxs:output_type -> network_pb.MempoolTxResponse + 28, // 30: network_pb.Network.GetDoubleSpends:output_type -> network_pb.DoubleSpendResponse + 29, // 31: network_pb.Network.GetUtxos:output_type -> network_pb.UtxosResponse + 30, // 32: network_pb.Network.GetOutputInputs:output_type -> network_pb.OutputInputsResponse + 3, // 33: network_pb.Network.OutputMessage:output_type -> network_pb.ErrorReply + 17, // [17:34] is the sub-list for method output_type + 0, // [0:17] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name @@ -525,7 +518,7 @@ func file_network_proto_init() { file_double_spend_proto_init() file_tx_proto_init() if !protoimpl.UnsafeEnabled { - file_network_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_network_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*Address); i { case 0: return &v.state @@ -537,7 +530,7 @@ func file_network_proto_init() { return nil } } - file_network_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_network_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*BalanceReply); i { case 0: return &v.state @@ -549,7 +542,7 @@ func file_network_proto_init() { return nil } } - file_network_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_network_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*StringMessage); i { case 0: return &v.state @@ -561,7 +554,7 @@ func file_network_proto_init() { return nil } } - file_network_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_network_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*ErrorReply); i { case 0: return &v.state @@ -573,7 +566,7 @@ func file_network_proto_init() { return nil } } - file_network_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_network_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*BlockHashRequest); i { case 0: return &v.state diff --git a/ref/network/gen/network_pb/network_grpc.pb.go b/ref/network/gen/network_pb/network_grpc.pb.go index d774b8e0..d7cf72f6 100644 --- a/ref/network/gen/network_pb/network_grpc.pb.go +++ b/ref/network/gen/network_pb/network_grpc.pb.go @@ -1,4 +1,8 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.4.0 +// - protoc v5.27.1 +// source: network.proto package network_pb @@ -11,8 +15,28 @@ import ( // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.62.0 or later. +const _ = grpc.SupportPackageIsVersion8 + +const ( + Network_SaveTxs_FullMethodName = "/network_pb.Network/SaveTxs" + Network_GetTx_FullMethodName = "/network_pb.Network/GetTx" + Network_SaveTxBlock_FullMethodName = "/network_pb.Network/SaveTxBlock" + Network_GetTxBlock_FullMethodName = "/network_pb.Network/GetTxBlock" + Network_GetBalance_FullMethodName = "/network_pb.Network/GetBalance" + Network_GetBlockInfos_FullMethodName = "/network_pb.Network/GetBlockInfos" + Network_GetBlockByHash_FullMethodName = "/network_pb.Network/GetBlockByHash" + Network_GetBlockByHeight_FullMethodName = "/network_pb.Network/GetBlockByHeight" + Network_GetMetrics_FullMethodName = "/network_pb.Network/GetMetrics" + Network_GetMetricList_FullMethodName = "/network_pb.Network/GetMetricList" + Network_GetHeightBlocks_FullMethodName = "/network_pb.Network/GetHeightBlocks" + Network_GetBlockTxs_FullMethodName = "/network_pb.Network/GetBlockTxs" + Network_GetMempoolTxs_FullMethodName = "/network_pb.Network/GetMempoolTxs" + Network_GetDoubleSpends_FullMethodName = "/network_pb.Network/GetDoubleSpends" + Network_GetUtxos_FullMethodName = "/network_pb.Network/GetUtxos" + Network_GetOutputInputs_FullMethodName = "/network_pb.Network/GetOutputInputs" + Network_OutputMessage_FullMethodName = "/network_pb.Network/OutputMessage" +) // NetworkClient is the client API for Network service. // @@ -34,7 +58,6 @@ type NetworkClient interface { GetDoubleSpends(ctx context.Context, in *DoubleSpendRequest, opts ...grpc.CallOption) (*DoubleSpendResponse, error) GetUtxos(ctx context.Context, in *UtxosRequest, opts ...grpc.CallOption) (*UtxosResponse, error) GetOutputInputs(ctx context.Context, in *OutputInputsRequest, opts ...grpc.CallOption) (*OutputInputsResponse, error) - ListenTx(ctx context.Context, in *TxRequest, opts ...grpc.CallOption) (*ListenTxReply, error) OutputMessage(ctx context.Context, in *StringMessage, opts ...grpc.CallOption) (*ErrorReply, error) } @@ -47,8 +70,9 @@ func NewNetworkClient(cc grpc.ClientConnInterface) NetworkClient { } func (c *networkClient) SaveTxs(ctx context.Context, in *Txs, opts ...grpc.CallOption) (*SaveTxsReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(SaveTxsReply) - err := c.cc.Invoke(ctx, "/network_pb.Network/SaveTxs", in, out, opts...) + err := c.cc.Invoke(ctx, Network_SaveTxs_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -56,8 +80,9 @@ func (c *networkClient) SaveTxs(ctx context.Context, in *Txs, opts ...grpc.CallO } func (c *networkClient) GetTx(ctx context.Context, in *TxRequest, opts ...grpc.CallOption) (*TxReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TxReply) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetTx", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetTx_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -65,8 +90,9 @@ func (c *networkClient) GetTx(ctx context.Context, in *TxRequest, opts ...grpc.C } func (c *networkClient) SaveTxBlock(ctx context.Context, in *TxBlock, opts ...grpc.CallOption) (*ErrorReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ErrorReply) - err := c.cc.Invoke(ctx, "/network_pb.Network/SaveTxBlock", in, out, opts...) + err := c.cc.Invoke(ctx, Network_SaveTxBlock_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -74,8 +100,9 @@ func (c *networkClient) SaveTxBlock(ctx context.Context, in *TxBlock, opts ...gr } func (c *networkClient) GetTxBlock(ctx context.Context, in *TxBlockRequest, opts ...grpc.CallOption) (*TxBlockReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TxBlockReply) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetTxBlock", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetTxBlock_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -83,8 +110,9 @@ func (c *networkClient) GetTxBlock(ctx context.Context, in *TxBlockRequest, opts } func (c *networkClient) GetBalance(ctx context.Context, in *Address, opts ...grpc.CallOption) (*BalanceReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(BalanceReply) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetBalance", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetBalance_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -92,8 +120,9 @@ func (c *networkClient) GetBalance(ctx context.Context, in *Address, opts ...grp } func (c *networkClient) GetBlockInfos(ctx context.Context, in *BlockRequest, opts ...grpc.CallOption) (*BlockInfoReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(BlockInfoReply) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetBlockInfos", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetBlockInfos_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -101,8 +130,9 @@ func (c *networkClient) GetBlockInfos(ctx context.Context, in *BlockRequest, opt } func (c *networkClient) GetBlockByHash(ctx context.Context, in *BlockHashRequest, opts ...grpc.CallOption) (*BlockInfo, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(BlockInfo) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetBlockByHash", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetBlockByHash_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -110,8 +140,9 @@ func (c *networkClient) GetBlockByHash(ctx context.Context, in *BlockHashRequest } func (c *networkClient) GetBlockByHeight(ctx context.Context, in *BlockRequest, opts ...grpc.CallOption) (*BlockInfo, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(BlockInfo) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetBlockByHeight", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetBlockByHeight_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -119,8 +150,9 @@ func (c *networkClient) GetBlockByHeight(ctx context.Context, in *BlockRequest, } func (c *networkClient) GetMetrics(ctx context.Context, in *MetricRequest, opts ...grpc.CallOption) (*MetricResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MetricResponse) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetMetrics", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetMetrics_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -128,8 +160,9 @@ func (c *networkClient) GetMetrics(ctx context.Context, in *MetricRequest, opts } func (c *networkClient) GetMetricList(ctx context.Context, in *MetricTimeRequest, opts ...grpc.CallOption) (*MetricTimeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MetricTimeResponse) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetMetricList", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetMetricList_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -137,8 +170,9 @@ func (c *networkClient) GetMetricList(ctx context.Context, in *MetricTimeRequest } func (c *networkClient) GetHeightBlocks(ctx context.Context, in *BlockHeightRequest, opts ...grpc.CallOption) (*BlockHeightResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(BlockHeightResponse) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetHeightBlocks", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetHeightBlocks_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -146,8 +180,9 @@ func (c *networkClient) GetHeightBlocks(ctx context.Context, in *BlockHeightRequ } func (c *networkClient) GetBlockTxs(ctx context.Context, in *BlockTxRequest, opts ...grpc.CallOption) (*BlockTxResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(BlockTxResponse) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetBlockTxs", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetBlockTxs_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -155,8 +190,9 @@ func (c *networkClient) GetBlockTxs(ctx context.Context, in *BlockTxRequest, opt } func (c *networkClient) GetMempoolTxs(ctx context.Context, in *MempoolTxRequest, opts ...grpc.CallOption) (*MempoolTxResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(MempoolTxResponse) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetMempoolTxs", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetMempoolTxs_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -164,8 +200,9 @@ func (c *networkClient) GetMempoolTxs(ctx context.Context, in *MempoolTxRequest, } func (c *networkClient) GetDoubleSpends(ctx context.Context, in *DoubleSpendRequest, opts ...grpc.CallOption) (*DoubleSpendResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DoubleSpendResponse) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetDoubleSpends", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetDoubleSpends_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -173,8 +210,9 @@ func (c *networkClient) GetDoubleSpends(ctx context.Context, in *DoubleSpendRequ } func (c *networkClient) GetUtxos(ctx context.Context, in *UtxosRequest, opts ...grpc.CallOption) (*UtxosResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(UtxosResponse) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetUtxos", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetUtxos_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -182,17 +220,9 @@ func (c *networkClient) GetUtxos(ctx context.Context, in *UtxosRequest, opts ... } func (c *networkClient) GetOutputInputs(ctx context.Context, in *OutputInputsRequest, opts ...grpc.CallOption) (*OutputInputsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(OutputInputsResponse) - err := c.cc.Invoke(ctx, "/network_pb.Network/GetOutputInputs", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *networkClient) ListenTx(ctx context.Context, in *TxRequest, opts ...grpc.CallOption) (*ListenTxReply, error) { - out := new(ListenTxReply) - err := c.cc.Invoke(ctx, "/network_pb.Network/ListenTx", in, out, opts...) + err := c.cc.Invoke(ctx, Network_GetOutputInputs_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -200,8 +230,9 @@ func (c *networkClient) ListenTx(ctx context.Context, in *TxRequest, opts ...grp } func (c *networkClient) OutputMessage(ctx context.Context, in *StringMessage, opts ...grpc.CallOption) (*ErrorReply, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ErrorReply) - err := c.cc.Invoke(ctx, "/network_pb.Network/OutputMessage", in, out, opts...) + err := c.cc.Invoke(ctx, Network_OutputMessage_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -228,7 +259,6 @@ type NetworkServer interface { GetDoubleSpends(context.Context, *DoubleSpendRequest) (*DoubleSpendResponse, error) GetUtxos(context.Context, *UtxosRequest) (*UtxosResponse, error) GetOutputInputs(context.Context, *OutputInputsRequest) (*OutputInputsResponse, error) - ListenTx(context.Context, *TxRequest) (*ListenTxReply, error) OutputMessage(context.Context, *StringMessage) (*ErrorReply, error) mustEmbedUnimplementedNetworkServer() } @@ -285,9 +315,6 @@ func (UnimplementedNetworkServer) GetUtxos(context.Context, *UtxosRequest) (*Utx func (UnimplementedNetworkServer) GetOutputInputs(context.Context, *OutputInputsRequest) (*OutputInputsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetOutputInputs not implemented") } -func (UnimplementedNetworkServer) ListenTx(context.Context, *TxRequest) (*ListenTxReply, error) { - return nil, status.Errorf(codes.Unimplemented, "method ListenTx not implemented") -} func (UnimplementedNetworkServer) OutputMessage(context.Context, *StringMessage) (*ErrorReply, error) { return nil, status.Errorf(codes.Unimplemented, "method OutputMessage not implemented") } @@ -314,7 +341,7 @@ func _Network_SaveTxs_Handler(srv interface{}, ctx context.Context, dec func(int } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/SaveTxs", + FullMethod: Network_SaveTxs_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).SaveTxs(ctx, req.(*Txs)) @@ -332,7 +359,7 @@ func _Network_GetTx_Handler(srv interface{}, ctx context.Context, dec func(inter } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetTx", + FullMethod: Network_GetTx_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetTx(ctx, req.(*TxRequest)) @@ -350,7 +377,7 @@ func _Network_SaveTxBlock_Handler(srv interface{}, ctx context.Context, dec func } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/SaveTxBlock", + FullMethod: Network_SaveTxBlock_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).SaveTxBlock(ctx, req.(*TxBlock)) @@ -368,7 +395,7 @@ func _Network_GetTxBlock_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetTxBlock", + FullMethod: Network_GetTxBlock_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetTxBlock(ctx, req.(*TxBlockRequest)) @@ -386,7 +413,7 @@ func _Network_GetBalance_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetBalance", + FullMethod: Network_GetBalance_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetBalance(ctx, req.(*Address)) @@ -404,7 +431,7 @@ func _Network_GetBlockInfos_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetBlockInfos", + FullMethod: Network_GetBlockInfos_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetBlockInfos(ctx, req.(*BlockRequest)) @@ -422,7 +449,7 @@ func _Network_GetBlockByHash_Handler(srv interface{}, ctx context.Context, dec f } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetBlockByHash", + FullMethod: Network_GetBlockByHash_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetBlockByHash(ctx, req.(*BlockHashRequest)) @@ -440,7 +467,7 @@ func _Network_GetBlockByHeight_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetBlockByHeight", + FullMethod: Network_GetBlockByHeight_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetBlockByHeight(ctx, req.(*BlockRequest)) @@ -458,7 +485,7 @@ func _Network_GetMetrics_Handler(srv interface{}, ctx context.Context, dec func( } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetMetrics", + FullMethod: Network_GetMetrics_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetMetrics(ctx, req.(*MetricRequest)) @@ -476,7 +503,7 @@ func _Network_GetMetricList_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetMetricList", + FullMethod: Network_GetMetricList_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetMetricList(ctx, req.(*MetricTimeRequest)) @@ -494,7 +521,7 @@ func _Network_GetHeightBlocks_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetHeightBlocks", + FullMethod: Network_GetHeightBlocks_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetHeightBlocks(ctx, req.(*BlockHeightRequest)) @@ -512,7 +539,7 @@ func _Network_GetBlockTxs_Handler(srv interface{}, ctx context.Context, dec func } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetBlockTxs", + FullMethod: Network_GetBlockTxs_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetBlockTxs(ctx, req.(*BlockTxRequest)) @@ -530,7 +557,7 @@ func _Network_GetMempoolTxs_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetMempoolTxs", + FullMethod: Network_GetMempoolTxs_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetMempoolTxs(ctx, req.(*MempoolTxRequest)) @@ -548,7 +575,7 @@ func _Network_GetDoubleSpends_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetDoubleSpends", + FullMethod: Network_GetDoubleSpends_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetDoubleSpends(ctx, req.(*DoubleSpendRequest)) @@ -566,7 +593,7 @@ func _Network_GetUtxos_Handler(srv interface{}, ctx context.Context, dec func(in } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetUtxos", + FullMethod: Network_GetUtxos_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetUtxos(ctx, req.(*UtxosRequest)) @@ -584,7 +611,7 @@ func _Network_GetOutputInputs_Handler(srv interface{}, ctx context.Context, dec } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/GetOutputInputs", + FullMethod: Network_GetOutputInputs_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).GetOutputInputs(ctx, req.(*OutputInputsRequest)) @@ -592,24 +619,6 @@ func _Network_GetOutputInputs_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _Network_ListenTx_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(TxRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(NetworkServer).ListenTx(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/network_pb.Network/ListenTx", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(NetworkServer).ListenTx(ctx, req.(*TxRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _Network_OutputMessage_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(StringMessage) if err := dec(in); err != nil { @@ -620,7 +629,7 @@ func _Network_OutputMessage_Handler(srv interface{}, ctx context.Context, dec fu } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/network_pb.Network/OutputMessage", + FullMethod: Network_OutputMessage_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { return srv.(NetworkServer).OutputMessage(ctx, req.(*StringMessage)) @@ -699,10 +708,6 @@ var Network_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetOutputInputs", Handler: _Network_GetOutputInputs_Handler, }, - { - MethodName: "ListenTx", - Handler: _Network_ListenTx_Handler, - }, { MethodName: "OutputMessage", Handler: _Network_OutputMessage_Handler, diff --git a/ref/network/gen/network_pb/output.pb.go b/ref/network/gen/network_pb/output.pb.go index 4125acfb..abb9e0c1 100644 --- a/ref/network/gen/network_pb/output.pb.go +++ b/ref/network/gen/network_pb/output.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.6.1 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: output.proto package network_pb @@ -471,7 +471,7 @@ func file_output_proto_rawDescGZIP() []byte { } var file_output_proto_msgTypes = make([]protoimpl.MessageInfo, 7) -var file_output_proto_goTypes = []interface{}{ +var file_output_proto_goTypes = []any{ (*UtxosRequest)(nil), // 0: network_pb.UtxosRequest (*UtxosResponse)(nil), // 1: network_pb.UtxosResponse (*Output)(nil), // 2: network_pb.Output @@ -497,7 +497,7 @@ func file_output_proto_init() { return } if !protoimpl.UnsafeEnabled { - file_output_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_output_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*UtxosRequest); i { case 0: return &v.state @@ -509,7 +509,7 @@ func file_output_proto_init() { return nil } } - file_output_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_output_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*UtxosResponse); i { case 0: return &v.state @@ -521,7 +521,7 @@ func file_output_proto_init() { return nil } } - file_output_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_output_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*Output); i { case 0: return &v.state @@ -533,7 +533,7 @@ func file_output_proto_init() { return nil } } - file_output_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_output_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*Input); i { case 0: return &v.state @@ -545,7 +545,7 @@ func file_output_proto_init() { return nil } } - file_output_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_output_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*TxHashIndex); i { case 0: return &v.state @@ -557,7 +557,7 @@ func file_output_proto_init() { return nil } } - file_output_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + file_output_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*OutputInputsRequest); i { case 0: return &v.state @@ -569,7 +569,7 @@ func file_output_proto_init() { return nil } } - file_output_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_output_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*OutputInputsResponse); i { case 0: return &v.state diff --git a/ref/network/gen/network_pb/tx.pb.go b/ref/network/gen/network_pb/tx.pb.go index 690d8afc..af2d82b0 100644 --- a/ref/network/gen/network_pb/tx.pb.go +++ b/ref/network/gen/network_pb/tx.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.27.1 -// protoc v3.6.1 +// protoc-gen-go v1.34.2 +// protoc v5.27.1 // source: tx.proto package network_pb @@ -263,53 +263,6 @@ func (x *SaveTxsReply) GetError() string { return "" } -type ListenTxReply struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Timestamp int64 `protobuf:"varint,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"` -} - -func (x *ListenTxReply) Reset() { - *x = ListenTxReply{} - if protoimpl.UnsafeEnabled { - mi := &file_tx_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *ListenTxReply) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*ListenTxReply) ProtoMessage() {} - -func (x *ListenTxReply) ProtoReflect() protoreflect.Message { - mi := &file_tx_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) -} - -// Deprecated: Use ListenTxReply.ProtoReflect.Descriptor instead. -func (*ListenTxReply) Descriptor() ([]byte, []int) { - return file_tx_proto_rawDescGZIP(), []int{5} -} - -func (x *ListenTxReply) GetTimestamp() int64 { - if x != nil { - return x.Timestamp - } - return 0 -} - type TxBlockRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -321,7 +274,7 @@ type TxBlockRequest struct { func (x *TxBlockRequest) Reset() { *x = TxBlockRequest{} if protoimpl.UnsafeEnabled { - mi := &file_tx_proto_msgTypes[6] + mi := &file_tx_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -334,7 +287,7 @@ func (x *TxBlockRequest) String() string { func (*TxBlockRequest) ProtoMessage() {} func (x *TxBlockRequest) ProtoReflect() protoreflect.Message { - mi := &file_tx_proto_msgTypes[6] + mi := &file_tx_proto_msgTypes[5] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -347,7 +300,7 @@ func (x *TxBlockRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TxBlockRequest.ProtoReflect.Descriptor instead. func (*TxBlockRequest) Descriptor() ([]byte, []int) { - return file_tx_proto_rawDescGZIP(), []int{6} + return file_tx_proto_rawDescGZIP(), []int{5} } func (x *TxBlockRequest) GetTxs() [][]byte { @@ -368,7 +321,7 @@ type TxBlockReply struct { func (x *TxBlockReply) Reset() { *x = TxBlockReply{} if protoimpl.UnsafeEnabled { - mi := &file_tx_proto_msgTypes[7] + mi := &file_tx_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -381,7 +334,7 @@ func (x *TxBlockReply) String() string { func (*TxBlockReply) ProtoMessage() {} func (x *TxBlockReply) ProtoReflect() protoreflect.Message { - mi := &file_tx_proto_msgTypes[7] + mi := &file_tx_proto_msgTypes[6] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -394,7 +347,7 @@ func (x *TxBlockReply) ProtoReflect() protoreflect.Message { // Deprecated: Use TxBlockReply.ProtoReflect.Descriptor instead. func (*TxBlockReply) Descriptor() ([]byte, []int) { - return file_tx_proto_rawDescGZIP(), []int{7} + return file_tx_proto_rawDescGZIP(), []int{6} } func (x *TxBlockReply) GetTxs() []*BlockTx { @@ -422,19 +375,16 @@ var file_tx_proto_rawDesc = []byte{ 0x28, 0x0c, 0x52, 0x05, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x22, 0x24, 0x0a, 0x0c, 0x53, 0x61, 0x76, 0x65, 0x54, 0x78, 0x73, 0x52, 0x65, 0x70, 0x6c, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x22, - 0x2d, 0x0a, 0x0d, 0x4c, 0x69, 0x73, 0x74, 0x65, 0x6e, 0x54, 0x78, 0x52, 0x65, 0x70, 0x6c, 0x79, - 0x12, 0x1c, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x03, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0x22, - 0x0a, 0x0e, 0x54, 0x78, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x10, 0x0a, 0x03, 0x74, 0x78, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x03, 0x74, - 0x78, 0x73, 0x22, 0x35, 0x0a, 0x0c, 0x54, 0x78, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x70, - 0x6c, 0x79, 0x12, 0x25, 0x0a, 0x03, 0x74, 0x78, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, - 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x42, 0x6c, 0x6f, - 0x63, 0x6b, 0x54, 0x78, 0x52, 0x03, 0x74, 0x78, 0x73, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, - 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x65, 0x6d, 0x6f, 0x63, 0x61, 0x73, 0x68, - 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x72, 0x65, 0x66, 0x2f, 0x6e, 0x65, 0x74, 0x77, 0x6f, - 0x72, 0x6b, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, - 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x22, 0x0a, 0x0e, 0x54, 0x78, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, + 0x74, 0x12, 0x10, 0x0a, 0x03, 0x74, 0x78, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0c, 0x52, 0x03, + 0x74, 0x78, 0x73, 0x22, 0x35, 0x0a, 0x0c, 0x54, 0x78, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x52, 0x65, + 0x70, 0x6c, 0x79, 0x12, 0x25, 0x0a, 0x03, 0x74, 0x78, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x13, 0x2e, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, 0x70, 0x62, 0x2e, 0x42, 0x6c, + 0x6f, 0x63, 0x6b, 0x54, 0x78, 0x52, 0x03, 0x74, 0x78, 0x73, 0x42, 0x36, 0x5a, 0x34, 0x67, 0x69, + 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x65, 0x6d, 0x6f, 0x63, 0x61, 0x73, + 0x68, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2f, 0x72, 0x65, 0x66, 0x2f, 0x6e, 0x65, 0x74, 0x77, + 0x6f, 0x72, 0x6b, 0x2f, 0x67, 0x65, 0x6e, 0x2f, 0x6e, 0x65, 0x74, 0x77, 0x6f, 0x72, 0x6b, 0x5f, + 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -449,22 +399,21 @@ func file_tx_proto_rawDescGZIP() []byte { return file_tx_proto_rawDescData } -var file_tx_proto_msgTypes = make([]protoimpl.MessageInfo, 8) -var file_tx_proto_goTypes = []interface{}{ +var file_tx_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_tx_proto_goTypes = []any{ (*TxRequest)(nil), // 0: network_pb.TxRequest (*TxReply)(nil), // 1: network_pb.TxReply (*Txs)(nil), // 2: network_pb.Txs (*Tx)(nil), // 3: network_pb.Tx (*SaveTxsReply)(nil), // 4: network_pb.SaveTxsReply - (*ListenTxReply)(nil), // 5: network_pb.ListenTxReply - (*TxBlockRequest)(nil), // 6: network_pb.TxBlockRequest - (*TxBlockReply)(nil), // 7: network_pb.TxBlockReply - (*BlockTx)(nil), // 8: network_pb.BlockTx + (*TxBlockRequest)(nil), // 5: network_pb.TxBlockRequest + (*TxBlockReply)(nil), // 6: network_pb.TxBlockReply + (*BlockTx)(nil), // 7: network_pb.BlockTx } var file_tx_proto_depIdxs = []int32{ 3, // 0: network_pb.TxReply.tx:type_name -> network_pb.Tx 3, // 1: network_pb.Txs.txs:type_name -> network_pb.Tx - 8, // 2: network_pb.TxBlockReply.txs:type_name -> network_pb.BlockTx + 7, // 2: network_pb.TxBlockReply.txs:type_name -> network_pb.BlockTx 3, // [3:3] is the sub-list for method output_type 3, // [3:3] is the sub-list for method input_type 3, // [3:3] is the sub-list for extension type_name @@ -479,7 +428,7 @@ func file_tx_proto_init() { } file_block_tx_proto_init() if !protoimpl.UnsafeEnabled { - file_tx_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + file_tx_proto_msgTypes[0].Exporter = func(v any, i int) any { switch v := v.(*TxRequest); i { case 0: return &v.state @@ -491,7 +440,7 @@ func file_tx_proto_init() { return nil } } - file_tx_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + file_tx_proto_msgTypes[1].Exporter = func(v any, i int) any { switch v := v.(*TxReply); i { case 0: return &v.state @@ -503,7 +452,7 @@ func file_tx_proto_init() { return nil } } - file_tx_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + file_tx_proto_msgTypes[2].Exporter = func(v any, i int) any { switch v := v.(*Txs); i { case 0: return &v.state @@ -515,7 +464,7 @@ func file_tx_proto_init() { return nil } } - file_tx_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + file_tx_proto_msgTypes[3].Exporter = func(v any, i int) any { switch v := v.(*Tx); i { case 0: return &v.state @@ -527,7 +476,7 @@ func file_tx_proto_init() { return nil } } - file_tx_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + file_tx_proto_msgTypes[4].Exporter = func(v any, i int) any { switch v := v.(*SaveTxsReply); i { case 0: return &v.state @@ -539,19 +488,7 @@ func file_tx_proto_init() { return nil } } - file_tx_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ListenTxReply); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_tx_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + file_tx_proto_msgTypes[5].Exporter = func(v any, i int) any { switch v := v.(*TxBlockRequest); i { case 0: return &v.state @@ -563,7 +500,7 @@ func file_tx_proto_init() { return nil } } - file_tx_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + file_tx_proto_msgTypes[6].Exporter = func(v any, i int) any { switch v := v.(*TxBlockReply); i { case 0: return &v.state @@ -582,7 +519,7 @@ func file_tx_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_tx_proto_rawDesc, NumEnums: 0, - NumMessages: 8, + NumMessages: 7, NumExtensions: 0, NumServices: 0, }, diff --git a/ref/network/network.proto b/ref/network/network.proto index 0cfa78ed..b3c17b56 100644 --- a/ref/network/network.proto +++ b/ref/network/network.proto @@ -29,7 +29,6 @@ service Network { rpc GetDoubleSpends (DoubleSpendRequest) returns (DoubleSpendResponse); rpc GetUtxos (UtxosRequest) returns (UtxosResponse); rpc GetOutputInputs (OutputInputsRequest) returns (OutputInputsResponse); - rpc ListenTx (TxRequest) returns (ListenTxReply); rpc OutputMessage (StringMessage) returns (ErrorReply); } diff --git a/ref/network/network_client/block_height.go b/ref/network/network_client/block_height.go index 179cb407..ddc85676 100644 --- a/ref/network/network_client/block_height.go +++ b/ref/network/network_client/block_height.go @@ -22,7 +22,6 @@ func (h *BlockHeightGetter) Get(startHeight int64) error { defer connection.Close() if reply, err := connection.Client.GetHeightBlocks(connection.GetDefaultContext(), &network_pb.BlockHeightRequest{ Start: startHeight, - Wait: false, }); err != nil { return fmt.Errorf("could not greet network; %w", err) } else { diff --git a/ref/network/network_client/tx_processed.go b/ref/network/network_client/tx_processed.go deleted file mode 100644 index 12ad52af..00000000 --- a/ref/network/network_client/tx_processed.go +++ /dev/null @@ -1,33 +0,0 @@ -package network_client - -import ( - "fmt" - "github.com/memocash/index/ref/network/gen/network_pb" - "time" -) - -type TxProcessed struct { - TxHash []byte - Timestamp time.Time -} - -func (t *TxProcessed) Get(txHash []byte) error { - conn, err := NewConnection() - if err != nil { - return fmt.Errorf("error connecting to network; %w", err) - } - defer conn.Close() - response, err := conn.Client.ListenTx(conn.GetDefaultContext(), &network_pb.TxRequest{ - Hash: txHash, - }) - if err != nil { - return fmt.Errorf("error getting rpc network listen tx; %w", err) - } - t.Timestamp = time.Unix(response.Timestamp, 0) - return nil -} - -func NewTxProcessed() *TxProcessed { - return &TxProcessed{ - } -} diff --git a/ref/network/network_server/main.go b/ref/network/network_server/main.go index 9684ce07..3aca4d8a 100644 --- a/ref/network/network_server/main.go +++ b/ref/network/network_server/main.go @@ -110,16 +110,6 @@ func (s *Server) GetTxBlock(ctx context.Context, req *network_pb.TxBlockRequest) return &network_pb.TxBlockReply{Txs: txs}, nil } -func (s *Server) ListenTx(ctx context.Context, req *network_pb.TxRequest) (*network_pb.ListenTxReply, error) { - txProcessed, err := chain.WaitForTxProcessed(ctx, req.GetHash()) - if err != nil { - return nil, fmt.Errorf("error waiting for tx processed; %w", err) - } - return &network_pb.ListenTxReply{ - Timestamp: txProcessed.Timestamp.Unix(), - }, nil -} - func (s *Server) GetOutputInputs(ctx context.Context, req *network_pb.OutputInputsRequest) (*network_pb.OutputInputsResponse, error) { var outs = make([]memo.Out, len(req.Outputs)) for i := range req.Outputs { @@ -276,7 +266,7 @@ func (s *Server) GetBlockInfos(ctx context.Context, req *network_pb.BlockRequest func (s *Server) GetHeightBlocks(_ context.Context, req *network_pb.BlockHeightRequest) (*network_pb.BlockHeightResponse, error) { var response = new(network_pb.BlockHeightResponse) - if heightBlocks, err := chain.GetHeightBlocksAll(req.Start, req.Wait); err != nil { + if heightBlocks, err := chain.GetHeightBlocksAll(req.Start); err != nil { return nil, fmt.Errorf("error getting height blocks all; %w", err) } else { response.Blocks = make([]*network_pb.BlockHeight, len(heightBlocks)) diff --git a/ref/network/tx.proto b/ref/network/tx.proto index f3a64545..91f33e0d 100644 --- a/ref/network/tx.proto +++ b/ref/network/tx.proto @@ -27,10 +27,6 @@ message SaveTxsReply { string error = 1; } -message ListenTxReply { - int64 timestamp = 1; -} - message TxBlockRequest { repeated bytes txs = 1; } diff --git a/test/run/queue/get.go b/test/run/queue/get.go index 6d0cb894..32460cfc 100644 --- a/test/run/queue/get.go +++ b/test/run/queue/get.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/memocash/index/db/client" "github.com/memocash/index/ref/config" - "time" ) type Get struct { @@ -32,28 +31,6 @@ func (r *Get) GetByPrefixes(topic string, prefixes [][]byte) error { return nil } -func (r *Get) GetAndWait(topic string, start []byte) error { - shardConfig := config.GetShardConfig(r.Shard, config.GetQueueShards()) - db := client.NewClient(fmt.Sprintf("127.0.0.1:%d", shardConfig.Port)) - if err := db.GetWOpts(client.Opts{ - Topic: topic, - Start: start, - Wait: true, - Timeout: time.Second, - }); err != nil { - return fmt.Errorf("error getting and waiting with start using queue client; %w", err) - } - r.Items = make([]Item, len(db.Messages)) - for i := range db.Messages { - r.Items[i] = Item{ - Topic: db.Messages[i].Topic, - Uid: db.Messages[i].Uid, - Data: db.Messages[i].Message, - } - } - return nil -} - func NewGet(shard uint32) *Get { return &Get{ Shard: shard, diff --git a/test/tasks/queue.go b/test/tasks/queue.go index 94d0cf6a..b15308f9 100644 --- a/test/tasks/queue.go +++ b/test/tasks/queue.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "github.com/jchavannes/jgo/jutil" - "github.com/memocash/index/db/client" "github.com/memocash/index/test/run/queue" "github.com/memocash/index/test/suite" ) @@ -86,48 +85,3 @@ var queueTest = suite.Test{ return nil }, } - -var waitTest = suite.Test{ - Name: TestQueueWait, - Test: func(r *suite.TestRequest) error { - add := queue.NewAdd(shard) - var itemList1 = []queue.Item{ - itemAA, - itemAB, - } - if err := add.Add(itemList1); err != nil { - return fmt.Errorf("error adding items 1 to queue; %w", err) - } - get := queue.NewGet(shard) - if err := get.GetAndWait(topic, nil); err != nil { - return fmt.Errorf("error getting and waiting 1; %w", err) - } - if err := checkExpectedItems(get.Items, itemList1); err != nil { - return fmt.Errorf("error checking expected items 1; %w", err) - } - var response = make(chan error) - go func() { - err := get.GetAndWait(topic, client.IncrementBytes(bytesAB)) - if err != nil { - response <- fmt.Errorf("error getting and waiting 2; %w", err) - } else { - response <- nil - } - }() - var itemList2 = []queue.Item{ - itemBA, - itemCA, - itemCB, - } - if err := add.Add(itemList2); err != nil { - return fmt.Errorf("error adding items 2 to queue; %w", err) - } - if err := <-response; err != nil { - return fmt.Errorf("error with wait response; %w", err) - } - if err := checkExpectedItems(get.Items, itemList2); err != nil { - return fmt.Errorf("error checking expected items 2; %w", err) - } - return nil - }, -} From aaccdc0f677d9321ca8acb1a723b7e58468c0572 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Fri, 26 Jul 2024 04:34:05 -0700 Subject: [PATCH 16/23] Remove timeout option. --- db/client/client.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/db/client/client.go b/db/client/client.go index e6b056f6..efc89856 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -165,10 +165,8 @@ type Opts struct { Prefixes [][]byte Max uint32 Uids [][]byte - Wait bool Newest bool Context context.Context - Timeout time.Duration } func (s *Client) GetWOpts(opts Opts) error { @@ -209,18 +207,12 @@ func (s *Client) GetWOpts(opts Opts) error { if err := s.SetConn(); err != nil { return fmt.Errorf("error setting connection; %w", err) } - var timeout time.Duration - if opts.Timeout > 0 { - timeout = opts.Timeout - } else { - timeout = DefaultGetTimeout - } c := queue_pb.NewQueueClient(s.conn) var bgCtx = opts.Context if jutil.IsNil(bgCtx) { bgCtx = context.Background() } - ctx, cancel := context.WithTimeout(bgCtx, timeout) + ctx, cancel := context.WithTimeout(bgCtx, DefaultGetTimeout) defer cancel() s.Messages = nil for _, optGroup := range optGroups { From 45ff52de3cf549ac60dfc13fa996934023207849 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Fri, 26 Jul 2024 05:00:00 -0700 Subject: [PATCH 17/23] Start converting calls to GetWOpts to new functions. --- db/client/client.go | 21 +++++++++++++++++++++ db/item/chain/height_block.go | 30 ++++++++++++------------------ db/item/chain/tx.go | 25 +++++++------------------ db/item/chain/tx_input.go | 13 ++++--------- db/item/db/client.go | 10 ++++++++++ graph/resolver/query.resolvers.go | 4 ++-- node/act/maint/populate_addr.go | 2 +- node/act/maint/populate_p2sh.go | 2 +- node/obj/saver/block.go | 2 +- ref/cluster/lead/processor.go | 2 +- ref/network/network_server/main.go | 4 ++-- 11 files changed, 62 insertions(+), 53 deletions(-) create mode 100644 db/item/db/client.go diff --git a/db/client/client.go b/db/client/client.go index efc89856..fbcc5a73 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -122,6 +122,27 @@ func (s *Client) GetByPrefix(ctx context.Context, topic string, prefix Prefix, o return nil } +func (s *Client) GetFirst(ctx context.Context, topic string) error { + if err := s.GetByPrefix(ctx, topic, Prefix{Limit: 1}); err != nil { + return fmt.Errorf("error getting first; %w", err) + } + return nil +} + +func (s *Client) GetLast(ctx context.Context, topic string) error { + if err := s.GetByPrefix(ctx, topic, Prefix{Limit: 1}, NewOptionOrder(true)); err != nil { + return fmt.Errorf("error getting last; %w", err) + } + return nil +} + +func (s *Client) GetAll(ctx context.Context, topic string, start []byte, opts ...Option) error { + if err := s.GetByPrefix(ctx, topic, Prefix{Start: start}, opts...); err != nil { + return fmt.Errorf("error getting all; %w", err) + } + return nil +} + func (s *Client) GetSingle(ctx context.Context, topic string, uid []byte) error { if err := s.SetConn(); err != nil { return fmt.Errorf("error setting connection; %w", err) diff --git a/db/item/chain/height_block.go b/db/item/chain/height_block.go index d27473d3..2f040918 100644 --- a/db/item/chain/height_block.go +++ b/db/item/chain/height_block.go @@ -43,15 +43,11 @@ func (b *HeightBlock) Serialize() []byte { func (b *HeightBlock) Deserialize([]byte) {} -func GetRecentHeightBlock() (*HeightBlock, error) { +func GetRecentHeightBlock(ctx context.Context) (*HeightBlock, error) { var heightBlocks []*HeightBlock for i, shardConfig := range config.GetQueueShards() { dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicChainHeightBlock, - Max: 1, - Newest: true, - }); err != nil { + if err := dbClient.GetLast(ctx, db.TopicChainHeightBlock); err != nil { return nil, fmt.Errorf("error getting recent height block for shard: %d; %w", i, err) } for i := range dbClient.Messages { @@ -129,38 +125,36 @@ func GetHeightBlocks(ctx context.Context, shard uint32, startHeight int64, desc return heightBlocks, nil } -func GetHeightBlocksAll(startHeight int64) ([]*HeightBlock, error) { - heightBlocks, err := GetHeightBlocksAllLimit(startHeight, client.LargeLimit, false) +func GetHeightBlocksAll(ctx context.Context, startHeight int64) ([]*HeightBlock, error) { + heightBlocks, err := GetHeightBlocksAllLimit(ctx, startHeight, client.LargeLimit, false) if err != nil { return nil, fmt.Errorf("error getting height blocks all large limit; %w", err) } return heightBlocks, nil } -func GetHeightBlocksAllDefault(startHeight int64, newest bool) ([]*HeightBlock, error) { - heightBlocks, err := GetHeightBlocksAllLimit(startHeight, client.DefaultLimit, newest) +func GetHeightBlocksAllDefault(ctx context.Context, startHeight int64, newest bool) ([]*HeightBlock, error) { + heightBlocks, err := GetHeightBlocksAllLimit(ctx, startHeight, client.DefaultLimit, newest) if err != nil { return nil, fmt.Errorf("error getting height blocks all default limit; %w", err) } return heightBlocks, nil } -func GetHeightBlocksAllLimit(startHeight int64, limit uint32, newest bool) ([]*HeightBlock, error) { +func GetHeightBlocksAllLimit(ctx context.Context, startHeight int64, limit int, newest bool) ([]*HeightBlock, error) { var heightBlocks []*HeightBlock shardConfigs := config.GetQueueShards() - shardLimit := limit / uint32(len(shardConfigs)) + shardLimit := limit / len(shardConfigs) for _, shardConfig := range shardConfigs { dbClient := client.NewClient(shardConfig.GetHost()) var start []byte if startHeight != 0 { start = jutil.GetInt64DataBig(startHeight) } - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicChainHeightBlock, - Start: start, - Max: shardLimit, - Newest: newest, - }); err != nil { + if err := dbClient.GetAll(ctx, db.TopicChainHeightBlock, start, + client.NewOptionOrder(newest), + client.NewOptionLimit(shardLimit), + ); err != nil { return nil, fmt.Errorf("error getting height blocks from queue client all; %w", err) } for i := range dbClient.Messages { diff --git a/db/item/chain/tx.go b/db/item/chain/tx.go index b92026e1..b76b901c 100644 --- a/db/item/chain/tx.go +++ b/db/item/chain/tx.go @@ -6,7 +6,6 @@ import ( "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" - "github.com/memocash/index/ref/config" ) type Tx struct { @@ -50,24 +49,14 @@ func (t *Tx) Deserialize(data []byte) { } func GetTxsByHashes(ctx context.Context, txHashes [][32]byte) ([]*Tx, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range txHashes { - shard := uint32(db.GetShardIdFromByte(txHashes[i][:])) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.ByteReverse(txHashes[i][:])) + messages, err := db.GetByPrefixes(ctx, db.TopicChainTx, db.ShardPrefixesTxHashes(txHashes)) + if err != nil { + return nil, fmt.Errorf("error getting db message chain txs by hashes; %w", err) } - var txs []*Tx - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - err := dbClient.GetWOpts(client.Opts{Context: ctx, Topic: db.TopicChainTx, Prefixes: prefixes}) - if err != nil { - return nil, fmt.Errorf("error getting db message chain txs by hashes; %w", err) - } - for _, msg := range dbClient.Messages { - var tx = new(Tx) - db.Set(tx, msg) - txs = append(txs, tx) - } + var txs = make([]*Tx, len(messages)) + for i := range messages { + txs[i] = new(Tx) + db.Set(txs[i], messages[i]) } return txs, nil } diff --git a/db/item/chain/tx_input.go b/db/item/chain/tx_input.go index cc51c4c7..2f8f160e 100644 --- a/db/item/chain/tx_input.go +++ b/db/item/chain/tx_input.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" ) type TxInput struct { @@ -58,14 +57,10 @@ func (t *TxInput) Deserialize(data []byte) { t.UnlockScript = data[40:] } -func GetAllTxInputs(shard uint32, startUid []byte) ([]*TxInput, error) { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicChainTxInput, - Start: startUid, - Max: client.HugeLimit, - }); err != nil { +func GetAllTxInputs(ctx context.Context, shard uint32, startUid []byte) ([]*TxInput, error) { + dbClient := db.GetShardClient(shard) + optionLimit := client.NewOptionLimit(client.HugeLimit) + if err := dbClient.GetAll(ctx, db.TopicChainTxInput, startUid, optionLimit); err != nil { return nil, fmt.Errorf("error getting db message chain tx inputs for all; %w", err) } var txInputs = make([]*TxInput, len(dbClient.Messages)) diff --git a/db/item/db/client.go b/db/item/db/client.go new file mode 100644 index 00000000..e602116d --- /dev/null +++ b/db/item/db/client.go @@ -0,0 +1,10 @@ +package db + +import ( + "github.com/memocash/index/db/client" + "github.com/memocash/index/ref/config" +) + +func GetShardClient(shard uint32) *client.Client { + return client.NewClient(config.GetShardConfig(shard, config.GetQueueShards()).GetHost()) +} diff --git a/graph/resolver/query.resolvers.go b/graph/resolver/query.resolvers.go index 1d1633ce..09c98d01 100644 --- a/graph/resolver/query.resolvers.go +++ b/graph/resolver/query.resolvers.go @@ -78,7 +78,7 @@ func (r *queryResolver) Block(ctx context.Context, hash model.Hash) (*model.Bloc // BlockNewest is the resolver for the block_newest field. func (r *queryResolver) BlockNewest(ctx context.Context) (*model.Block, error) { SetEndPoint(ctx, metric.EndPointBlockNewest) - heightBlock, err := chain.GetRecentHeightBlock() + heightBlock, err := chain.GetRecentHeightBlock(ctx) if err != nil { return nil, InternalError{fmt.Errorf("error getting recent height block for query; %w", err)} } @@ -106,7 +106,7 @@ func (r *queryResolver) Blocks(ctx context.Context, newest *bool, start *uint32) if newest != nil { newestBool = *newest } - heightBlocks, err := chain.GetHeightBlocksAllDefault(startInt, newestBool) + heightBlocks, err := chain.GetHeightBlocksAllDefault(ctx, startInt, newestBool) if err != nil { return nil, InternalError{fmt.Errorf("error getting height blocks for query; %w", err)} } diff --git a/node/act/maint/populate_addr.go b/node/act/maint/populate_addr.go index 112d33dd..3cf9b70f 100644 --- a/node/act/maint/populate_addr.go +++ b/node/act/maint/populate_addr.go @@ -133,7 +133,7 @@ func (p *PopulateAddr) populateShardSingle(shard uint32) (bool, error) { var objMap = make(map[[57]byte]*addr.SeenTx) var checked int if p.Inputs { - txInputs, err := chain.GetAllTxInputs(shard, shardStatus.Status) + txInputs, err := chain.GetAllTxInputs(p.Context, shard, shardStatus.Status) if err != nil { return false, fmt.Errorf("error getting tx outputs for populate addr shard: %d; %w", shard, err) } diff --git a/node/act/maint/populate_p2sh.go b/node/act/maint/populate_p2sh.go index edd8992a..a62f2ffe 100644 --- a/node/act/maint/populate_p2sh.go +++ b/node/act/maint/populate_p2sh.go @@ -32,7 +32,7 @@ func (p *PopulateP2sh) Populate(startHeight int64) error { addressSaver.SkipP2pkh = true var maxHeight = startHeight for { - heightBlocks, err := chain.GetHeightBlocksAllLimit(maxHeight, client.HugeLimit, false) + heightBlocks, err := chain.GetHeightBlocksAllLimit(p.Ctx, maxHeight, client.HugeLimit, false) if err != nil { return fmt.Errorf("fatal error getting height blocks all for populate p2sh; %w", err) } diff --git a/node/obj/saver/block.go b/node/obj/saver/block.go index f55991c8..b5c146ee 100644 --- a/node/obj/saver/block.go +++ b/node/obj/saver/block.go @@ -109,7 +109,7 @@ func (b *Block) saveBlockObjects(info dbi.BlockInfo) error { } func (b *Block) GetBlock(heightBack int64) (*chainhash.Hash, error) { - heightBlock, err := chain.GetRecentHeightBlock() + heightBlock, err := chain.GetRecentHeightBlock(b.Context) if err != nil { return nil, fmt.Errorf("error getting recent height block from queue; %w", err) } diff --git a/ref/cluster/lead/processor.go b/ref/cluster/lead/processor.go index c12cc0ea..cbaaff94 100644 --- a/ref/cluster/lead/processor.go +++ b/ref/cluster/lead/processor.go @@ -78,7 +78,7 @@ func (p *Processor) Run() error { case <-p.BlockNode.SyncDone: log.Printf("Node sync done\n") p.Synced = true - recentBlock, err := chain.GetRecentHeightBlock() + recentBlock, err := chain.GetRecentHeightBlock(p.Context) if err != nil { p.ErrorChan <- fmt.Errorf("error getting recent height block; %w", err) break diff --git a/ref/network/network_server/main.go b/ref/network/network_server/main.go index 3aca4d8a..ba2a65d4 100644 --- a/ref/network/network_server/main.go +++ b/ref/network/network_server/main.go @@ -264,9 +264,9 @@ func (s *Server) GetBlockInfos(ctx context.Context, req *network_pb.BlockRequest }, nil } -func (s *Server) GetHeightBlocks(_ context.Context, req *network_pb.BlockHeightRequest) (*network_pb.BlockHeightResponse, error) { +func (s *Server) GetHeightBlocks(ctx context.Context, req *network_pb.BlockHeightRequest) (*network_pb.BlockHeightResponse, error) { var response = new(network_pb.BlockHeightResponse) - if heightBlocks, err := chain.GetHeightBlocksAll(req.Start); err != nil { + if heightBlocks, err := chain.GetHeightBlocksAll(ctx, req.Start); err != nil { return nil, fmt.Errorf("error getting height blocks all; %w", err) } else { response.Blocks = make([]*network_pb.BlockHeight, len(heightBlocks)) From f7e93e0d5ca44b0ed5a5639981a5bf8fd3b1aa2a Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Fri, 26 Jul 2024 13:25:19 -0700 Subject: [PATCH 18/23] Convert some GetWOpts calls to new interface. --- admin/server/node/found_peers.go | 2 +- cmd/peer/list.go | 2 +- db/client/request.go | 8 ++++++++ db/item/chain/tx_input.go | 3 +-- db/item/chain/tx_output.go | 12 +++--------- db/item/found_peer.go | 24 ++++++++---------------- node/act/maint/populate_addr.go | 2 +- node/act/maint/populate_p2sh_direct.go | 2 +- 8 files changed, 24 insertions(+), 31 deletions(-) diff --git a/admin/server/node/found_peers.go b/admin/server/node/found_peers.go index 44523f73..961dfdb0 100644 --- a/admin/server/node/found_peers.go +++ b/admin/server/node/found_peers.go @@ -22,7 +22,7 @@ var foundPeersRoute = admin.Route{ var shard uint32 FoundPeersLoop: for { - foundPeers, err := item.GetFoundPeers(shard, startId, request.Ip, request.Port) + foundPeers, err := item.GetFoundPeers(r.Request.Context(), shard, startId, request.Ip, request.Port) if err != nil { log.Fatalf("fatal error getting found peers; %v", err) } diff --git a/cmd/peer/list.go b/cmd/peer/list.go index c4b1101e..6f85d7fe 100644 --- a/cmd/peer/list.go +++ b/cmd/peer/list.go @@ -34,7 +34,7 @@ var listFoundPeersCmd = &cobra.Command{ if len(args) > 0 { shard = jutil.GetUInt32FromString(args[0]) } - foundPeers, err := item.GetFoundPeers(shard, nil, nil, 0) + foundPeers, err := item.GetFoundPeers(cmd.Context(), shard, nil, nil, 0) if err != nil { log.Fatalf("fatal error getting found peers; %v", err) } diff --git a/db/client/request.go b/db/client/request.go index f36c6232..5ef7d876 100644 --- a/db/client/request.go +++ b/db/client/request.go @@ -38,6 +38,14 @@ func NewOptionLimit(limit int) *OptionLimit { } } +func OptionHugeLimit() *OptionLimit { + return NewOptionLimit(HugeLimit) +} + +func OptionLargeLimit() *OptionLimit { + return NewOptionLimit(LargeLimit) +} + type OptionOrder struct { Desc bool } diff --git a/db/item/chain/tx_input.go b/db/item/chain/tx_input.go index 2f8f160e..6bca8a5c 100644 --- a/db/item/chain/tx_input.go +++ b/db/item/chain/tx_input.go @@ -59,8 +59,7 @@ func (t *TxInput) Deserialize(data []byte) { func GetAllTxInputs(ctx context.Context, shard uint32, startUid []byte) ([]*TxInput, error) { dbClient := db.GetShardClient(shard) - optionLimit := client.NewOptionLimit(client.HugeLimit) - if err := dbClient.GetAll(ctx, db.TopicChainTxInput, startUid, optionLimit); err != nil { + if err := dbClient.GetAll(ctx, db.TopicChainTxInput, startUid, client.OptionHugeLimit()); err != nil { return nil, fmt.Errorf("error getting db message chain tx inputs for all; %w", err) } var txInputs = make([]*TxInput, len(dbClient.Messages)) diff --git a/db/item/chain/tx_output.go b/db/item/chain/tx_output.go index 697d4379..6f369894 100644 --- a/db/item/chain/tx_output.go +++ b/db/item/chain/tx_output.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" ) type TxOutput struct { @@ -52,14 +51,9 @@ func (t *TxOutput) Deserialize(data []byte) { t.LockScript = data[8:] } -func GetAllTxOutputs(shard uint32, startUid []byte) ([]*TxOutput, error) { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicChainTxOutput, - Start: startUid, - Max: client.HugeLimit, - }); err != nil { +func GetAllTxOutputs(ctx context.Context, shard uint32, startUid []byte) ([]*TxOutput, error) { + dbClient := db.GetShardClient(shard) + if err := dbClient.GetAll(ctx, db.TopicChainTxOutput, startUid, client.OptionHugeLimit()); err != nil { return nil, fmt.Errorf("error getting db message chain tx outputs for all; %w", err) } var txOutputs = make([]*TxOutput, len(dbClient.Messages)) diff --git a/db/item/found_peer.go b/db/item/found_peer.go index 69c11cf8..6cf330b7 100644 --- a/db/item/found_peer.go +++ b/db/item/found_peer.go @@ -1,11 +1,11 @@ package item import ( + "context" "fmt" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" - "github.com/memocash/index/ref/config" ) type FoundPeer struct { @@ -48,27 +48,19 @@ func (p *FoundPeer) Serialize() []byte { func (p *FoundPeer) Deserialize([]byte) {} -func GetFoundPeers(shard uint32, startId []byte, ip []byte, port uint16) ([]*FoundPeer, error) { - var prefix []byte +func GetFoundPeers(ctx context.Context, shard uint32, startId []byte, ip []byte, port uint16) ([]*FoundPeer, error) { + var prefix client.Prefix if len(ip) > 0 { - prefix = append(prefix, jutil.BytePadPrefix(ip, IpBytePadSize)...) + prefix.Prefix = jutil.BytePadPrefix(ip, IpBytePadSize) if port > 0 { - prefix = append(prefix, jutil.GetUintData(uint(port))...) + prefix.Prefix = append(prefix.Prefix, jutil.GetUintData(uint(port))...) } } - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - var startIdBytes []byte if len(startId) > 0 { - startIdBytes = startId + prefix.Start = startId } - opts := client.Opts{ - Topic: db.TopicFoundPeer, - Start: startIdBytes, - Max: client.LargeLimit, - Prefixes: [][]byte{prefix}, - } - if err := dbClient.GetWOpts(opts); err != nil { + dbClient := db.GetShardClient(shard) + if err := dbClient.GetByPrefix(ctx, db.TopicFoundPeer, prefix, client.OptionLargeLimit()); err != nil { return nil, fmt.Errorf("error getting found peers from queue client; %w", err) } var foundPeers = make([]*FoundPeer, len(dbClient.Messages)) diff --git a/node/act/maint/populate_addr.go b/node/act/maint/populate_addr.go index 3cf9b70f..b996ad61 100644 --- a/node/act/maint/populate_addr.go +++ b/node/act/maint/populate_addr.go @@ -154,7 +154,7 @@ func (p *PopulateAddr) populateShardSingle(shard uint32) (bool, error) { } checked = len(txInputs) } else { - txOutputs, err := chain.GetAllTxOutputs(shard, shardStatus.Status) + txOutputs, err := chain.GetAllTxOutputs(p.Context, shard, shardStatus.Status) if err != nil { return false, fmt.Errorf("error getting tx outputs for populate addr shard: %d; %w", shard, err) } diff --git a/node/act/maint/populate_p2sh_direct.go b/node/act/maint/populate_p2sh_direct.go index d5710840..c3a74e9c 100644 --- a/node/act/maint/populate_p2sh_direct.go +++ b/node/act/maint/populate_p2sh_direct.go @@ -120,7 +120,7 @@ func (p *PopulateP2shDirect) populateShardSingle(shard uint32) (bool, error) { if shardStatus == nil { shardStatus = item.NewProcessStatus(uint(shard), item.ProcessStatusPopulateP2sh) } - txOutputs, err := chain.GetAllTxOutputs(shard, shardStatus.Status) + txOutputs, err := chain.GetAllTxOutputs(p.Ctx, shard, shardStatus.Status) if err != nil { return false, fmt.Errorf("error getting tx outputs for populate p2sh shard: %d; %w", shard, err) } From 73ad58414bf9a7541a81c8cddb8e07076cef7d5e Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Fri, 26 Jul 2024 22:42:34 -0700 Subject: [PATCH 19/23] GraphQL Playground. --- graph/server/main.go | 5 +++++ graph/server/playground.go | 15 +++++++++++++++ ref/bitcoin/wallet/addr.go | 2 +- test/tasks/main.go | 1 - 4 files changed, 21 insertions(+), 2 deletions(-) create mode 100644 graph/server/playground.go diff --git a/graph/server/main.go b/graph/server/main.go index 03cdaa15..321e3c5c 100644 --- a/graph/server/main.go +++ b/graph/server/main.go @@ -26,10 +26,15 @@ func (s *Server) Run() error { return fmt.Errorf("error serving admin server; %w", s.Serve()) } +var AddHandlers []func(*http.ServeMux) + func (s *Server) Start() error { mux := http.NewServeMux() mux.HandleFunc("/", GetIndexHandler()) mux.HandleFunc("/graphql", GetGraphQLHandler()) + for _, addHandler := range AddHandlers { + addHandler(mux) + } s.server = http.Server{Handler: mux} var err error if s.listener, err = net.Listen("tcp", s.GetHost()); err != nil { diff --git a/graph/server/playground.go b/graph/server/playground.go new file mode 100644 index 00000000..d11328c2 --- /dev/null +++ b/graph/server/playground.go @@ -0,0 +1,15 @@ +//go:build debug + +package server + +import ( + "github.com/99designs/gqlgen/graphql/playground" + "net/http" +) + +func init() { + AddHandlers = append(AddHandlers, func(mux *http.ServeMux) { + mux.Handle("/playground", playground.Handler("GraphQL playground", "/graphql")) + mux.Handle("/playground-live", playground.Handler("GraphQL playground", "https://graph.cash/graphql")) + }) +} diff --git a/ref/bitcoin/wallet/addr.go b/ref/bitcoin/wallet/addr.go index b77ac123..247bace3 100644 --- a/ref/bitcoin/wallet/addr.go +++ b/ref/bitcoin/wallet/addr.go @@ -164,7 +164,7 @@ func GetAddrFromBytes(b []byte) *Addr { return addr } -func AddrsToSlices(addrs []Addr) [][]byte { +func AddrsToSlices(addrs [][25]byte) [][]byte { var addrSlices = make([][]byte, len(addrs)) for i := range addrs { addrSlices[i] = addrs[i][:] diff --git a/test/tasks/main.go b/test/tasks/main.go index fa1626f8..2d47980c 100644 --- a/test/tasks/main.go +++ b/test/tasks/main.go @@ -12,6 +12,5 @@ func GetTests() []suite.Test { return []suite.Test{ SaveMessage, queueTest, - waitTest, } } From b93b6ce9cf0f6f6946ae33759578375dcced4439 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Mon, 29 Jul 2024 04:45:03 -0700 Subject: [PATCH 20/23] Convert addr GetWOpts uses. --- db/client/request.go | 32 +++++++++++++++-- db/item/db/item.go | 12 ++++--- db/item/memo/addr_follow.go | 47 ++++++------------------ db/item/memo/addr_followed.go | 49 +++++++------------------ db/item/memo/addr_name.go | 33 +++++------------ db/item/memo/addr_post.go | 61 ++++++++++---------------------- db/item/memo/addr_profile.go | 33 +++++------------ db/item/memo/addr_profile_pic.go | 33 +++++------------ db/item/memo/addr_room_follow.go | 29 ++++----------- 9 files changed, 115 insertions(+), 214 deletions(-) diff --git a/db/client/request.go b/db/client/request.go index 5ef7d876..6a04121b 100644 --- a/db/client/request.go +++ b/db/client/request.go @@ -38,12 +38,36 @@ func NewOptionLimit(limit int) *OptionLimit { } } +func OptionLargeLimit() *OptionLimit { + return NewOptionLimit(LargeLimit) +} + +func OptionExLargeLimit() *OptionLimit { + return NewOptionLimit(ExLargeLimit) +} + func OptionHugeLimit() *OptionLimit { return NewOptionLimit(HugeLimit) } -func OptionLargeLimit() *OptionLimit { - return NewOptionLimit(LargeLimit) +type OptionPrefixLimit struct { + Limit int +} + +func (o *OptionPrefixLimit) Apply(r *queue_pb.RequestPrefixes) { + for i := range r.Prefixes { + r.Prefixes[i].Limit = uint32(o.Limit) + } +} + +func NewOptionPrefixLimit(limit int) *OptionPrefixLimit { + return &OptionPrefixLimit{ + Limit: limit, + } +} + +func OptionSinglePrefixLimit() *OptionPrefixLimit { + return NewOptionPrefixLimit(1) } type OptionOrder struct { @@ -63,3 +87,7 @@ func NewOptionOrder(desc bool) *OptionOrder { Desc: desc, } } + +func OptionNewest() *OptionOrder { + return NewOptionOrder(true) +} diff --git a/db/item/db/item.go b/db/item/db/item.go index 3b160c9f..8a9cdde4 100644 --- a/db/item/db/item.go +++ b/db/item/db/item.go @@ -7,6 +7,7 @@ import ( "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" "github.com/memocash/index/ref/bitcoin/tx/hs" + "github.com/memocash/index/ref/bitcoin/wallet" "github.com/memocash/index/ref/config" "sync" ) @@ -81,6 +82,10 @@ func ShardPrefixesTxHashes(txHashes [][32]byte) map[uint32][]client.Prefix { return ShardPrefixes(hs.HashesToSlices(txHashes)) } +func ShardPrefixesAddrs(addrs [][25]byte) map[uint32][]client.Prefix { + return ShardPrefixes(wallet.AddrsToSlices(addrs)) +} + func ShardPrefixes(bytePrefixes [][]byte) map[uint32][]client.Prefix { var shardPrefixes = make(map[uint32][]client.Prefix) for _, bytePrefix := range bytePrefixes { @@ -90,15 +95,14 @@ func ShardPrefixes(bytePrefixes [][]byte) map[uint32][]client.Prefix { return shardPrefixes } -func GetByPrefixes(ctx context.Context, topic string, shardPrefixes map[uint32][]client.Prefix) ([]client.Message, error) { +func GetByPrefixes(ctx context.Context, topic string, shardPrefixes map[uint32][]client.Prefix, opts ...client.Option) ([]client.Message, error) { wait := NewWait(len(shardPrefixes)) var messages []client.Message for shardT, prefixesT := range shardPrefixes { go func(shard uint32, prefixes []client.Prefix) { defer wait.Group.Done() prefixes = removeDupeAndEmptyPrefixes(prefixes) - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) + dbClient := GetShardClient(shard) for len(prefixes) > 0 { var prefixesToUse []client.Prefix if len(prefixes) > client.HugeLimit { @@ -106,7 +110,7 @@ func GetByPrefixes(ctx context.Context, topic string, shardPrefixes map[uint32][ } else { prefixesToUse, prefixes = prefixes, nil } - if err := dbClient.GetByPrefixes(ctx, topic, prefixesToUse); err != nil { + if err := dbClient.GetByPrefixes(ctx, topic, prefixesToUse, opts...); err != nil { wait.AddError(fmt.Errorf("error getting client message get by prefixes; %w", err)) return } diff --git a/db/item/memo/addr_follow.go b/db/item/memo/addr_follow.go index 33ba8967..61730d0b 100644 --- a/db/item/memo/addr_follow.go +++ b/db/item/memo/addr_follow.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" "time" ) @@ -64,49 +63,25 @@ func (f *AddrFollow) Deserialize(data []byte) { } func GetAddrFollows(ctx context.Context, addrs [][25]byte) ([]*AddrFollow, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range addrs { - shard := client.GenShardSource32(addrs[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], addrs[i][:]) + messages, err := db.GetByPrefixes(ctx, db.TopicMemoAddrFollow, db.ShardPrefixesAddrs(addrs)) + if err != nil { + return nil, fmt.Errorf("error getting db addr memo follow by prefixes; %w", err) } - shardConfigs := config.GetQueueShards() - var addrFollows []*AddrFollow - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, shardConfigs) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrFollow, - Prefixes: prefixes, - Max: client.ExLargeLimit, - Context: ctx, - }); err != nil { - return nil, fmt.Errorf("error getting db addr memo follow by prefix; %w", err) - } - for _, msg := range dbClient.Messages { - var addrFollow = new(AddrFollow) - db.Set(addrFollow, msg) - addrFollows = append(addrFollows, addrFollow) - } + var addrFollows = make([]*AddrFollow, len(messages)) + for i := range messages { + addrFollows[i] = new(AddrFollow) + db.Set(addrFollows[i], messages[i]) } return addrFollows, nil } func GetAddrFollowsSingle(ctx context.Context, addr [25]byte, start time.Time) ([]*AddrFollow, error) { - shardConfig := config.GetShardConfig(client.GenShardSource32(addr[:]), config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - var startByte []byte + dbClient := db.GetShardClient(client.GenShardSource32(addr[:])) + prefix := client.NewPrefix(addr[:]) if !jutil.IsTimeZero(start) { - startByte = jutil.CombineBytes(addr[:], jutil.GetTimeByteNanoBig(start)) - } else { - startByte = addr[:] + prefix.Start = jutil.CombineBytes(addr[:], jutil.GetTimeByteNanoBig(start)) } - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrFollow, - Prefixes: [][]byte{addr[:]}, - Start: startByte, - Max: client.ExLargeLimit, - Context: ctx, - }); err != nil { + if err := dbClient.GetByPrefix(ctx, db.TopicMemoAddrFollow, prefix, client.OptionExLargeLimit()); err != nil { return nil, fmt.Errorf("error getting db addr memo follow by prefix; %w", err) } var addrFollows = make([]*AddrFollow, len(dbClient.Messages)) diff --git a/db/item/memo/addr_followed.go b/db/item/memo/addr_followed.go index cbf0a7ef..30bb5d69 100644 --- a/db/item/memo/addr_followed.go +++ b/db/item/memo/addr_followed.go @@ -64,50 +64,27 @@ func (f *AddrFollowed) Deserialize(data []byte) { } func GetAddrFolloweds(ctx context.Context, followAddresses [][25]byte) ([]*AddrFollowed, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range followAddresses { - shard := client.GenShardSource32(followAddresses[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], followAddresses[i][:]) + shardPrefixes := db.ShardPrefixesAddrs(followAddresses) + messages, err := db.GetByPrefixes(ctx, db.TopicMemoAddrFollowed, shardPrefixes, client.OptionExLargeLimit()) + if err != nil { + return nil, fmt.Errorf("error getting db addr memo followed by prefix; %w", err) } - shardConfigs := config.GetQueueShards() - var addrFolloweds []*AddrFollowed - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, shardConfigs) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrFollowed, - Prefixes: prefixes, - Max: client.ExLargeLimit, - Context: ctx, - }); err != nil { - return nil, fmt.Errorf("error getting db addr memo followed by prefix; %w", err) - } - for _, msg := range dbClient.Messages { - var addrFollowed = new(AddrFollowed) - db.Set(addrFollowed, msg) - addrFolloweds = append(addrFolloweds, addrFollowed) - } + var addrFolloweds = make([]*AddrFollowed, len(messages)) + for i := range messages { + addrFolloweds[i] = new(AddrFollowed) + db.Set(addrFolloweds[i], messages[i]) } return addrFolloweds, nil } func GetAddrFollowedsSingle(ctx context.Context, followAddr [25]byte, start time.Time) ([]*AddrFollowed, error) { - shardConfig := config.GetShardConfig(client.GenShardSource32(followAddr[:]), config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - var startByte []byte + dbClient := db.GetShardClient(client.GenShardSource32(followAddr[:])) + var prefix = client.NewPrefix(followAddr[:]) if !jutil.IsTimeZero(start) { - startByte = jutil.CombineBytes(followAddr[:], jutil.GetTimeByteNanoBig(start)) - } else { - startByte = followAddr[:] + prefix.Start = jutil.CombineBytes(followAddr[:], jutil.GetTimeByteNanoBig(start)) } - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrFollowed, - Prefixes: [][]byte{followAddr[:]}, - Start: startByte, - Max: client.ExLargeLimit, - Context: ctx, - }); err != nil { - return nil, fmt.Errorf("error getting db addr memo follow by prefix; %w", err) + if err := dbClient.GetByPrefix(ctx, db.TopicMemoAddrFollowed, prefix, client.OptionExLargeLimit()); err != nil { + return nil, fmt.Errorf("error getting db addr memo followed by prefix; %w", err) } var addrFolloweds = make([]*AddrFollowed, len(dbClient.Messages)) for i := range dbClient.Messages { diff --git a/db/item/memo/addr_name.go b/db/item/memo/addr_name.go index c38530a2..9e0bdd4f 100644 --- a/db/item/memo/addr_name.go +++ b/db/item/memo/addr_name.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" "time" ) @@ -52,30 +51,16 @@ func (n *AddrName) Deserialize(data []byte) { } func GetAddrNames(ctx context.Context, addrs [][25]byte) ([]*AddrName, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range addrs { - shard := db.GetShardIdFromByte32(addrs[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], addrs[i][:]) + shardPrefixes := db.ShardPrefixesAddrs(addrs) + var options = []client.Option{client.OptionSinglePrefixLimit(), client.OptionNewest()} + messages, err := db.GetByPrefixes(ctx, db.TopicMemoAddrName, shardPrefixes, options...) + if err != nil { + return nil, fmt.Errorf("error getting db addr memo names by prefix; %w", err) } - shardConfigs := config.GetQueueShards() - var addrNames []*AddrName - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, shardConfigs) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrName, - Prefixes: prefixes, - Max: 1, - Newest: true, - Context: ctx, - }); err != nil { - return nil, fmt.Errorf("error getting db addr memo names by prefix; %w", err) - } - for _, msg := range dbClient.Messages { - var addrName = new(AddrName) - db.Set(addrName, msg) - addrNames = append(addrNames, addrName) - } + var addrNames = make([]*AddrName, len(messages)) + for i := range messages { + addrNames[i] = new(AddrName) + db.Set(addrNames[i], messages[i]) } return addrNames, nil } diff --git a/db/item/memo/addr_post.go b/db/item/memo/addr_post.go index 6721379e..d5e0a039 100644 --- a/db/item/memo/addr_post.go +++ b/db/item/memo/addr_post.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" "time" ) @@ -49,57 +48,35 @@ func (p *AddrPost) Serialize() []byte { func (p *AddrPost) Deserialize([]byte) {} func GetSingleAddrPosts(ctx context.Context, addr [25]byte, newest bool, start time.Time) ([]*AddrPost, error) { - var startByte []byte + dbClient := db.GetShardClient(client.GenShardSource32(addr[:])) + prefix := client.NewPrefix(addr[:]) if !jutil.IsTimeZero(start) { - startByte = jutil.CombineBytes(addr[:], jutil.GetTimeByteNanoBig(start)) - } else { - startByte = addr[:] + prefix.Start = jutil.CombineBytes(addr[:], jutil.GetTimeByteNanoBig(start)) } - dbClient := client.NewClient(config.GetShardConfig(client.GenShardSource32(addr[:]), config.GetQueueShards()).GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrPost, - Prefixes: [][]byte{addr[:]}, - Max: client.ExLargeLimit, - Start: startByte, - Newest: newest, - Context: ctx, - }); err != nil { + var opts = []client.Option{client.OptionExLargeLimit(), client.NewOptionOrder(newest)} + err := dbClient.GetByPrefix(ctx, db.TopicMemoAddrPost, prefix, opts...) + if err != nil { return nil, fmt.Errorf("error getting db addr memo post by prefix; %w", err) } - var addrPosts []*AddrPost - for _, msg := range dbClient.Messages { - var addrPost = new(AddrPost) - db.Set(addrPost, msg) - addrPosts = append(addrPosts, addrPost) + var addrPosts = make([]*AddrPost, len(dbClient.Messages)) + for i := range dbClient.Messages { + addrPosts[i] = new(AddrPost) + db.Set(addrPosts[i], dbClient.Messages[i]) } return addrPosts, nil } func GetAddrPosts(ctx context.Context, addrs [][25]byte, newest bool) ([]*AddrPost, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range addrs { - shard := client.GenShardSource32(addrs[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], addrs[i][:]) + shardPrefixes := db.ShardPrefixesAddrs(addrs) + var opts = []client.Option{client.OptionExLargeLimit(), client.NewOptionOrder(newest)} + messages, err := db.GetByPrefixes(ctx, db.TopicMemoAddrPost, shardPrefixes, opts...) + if err != nil { + return nil, fmt.Errorf("error getting db addr memo post by prefix; %w", err) } - shardConfigs := config.GetQueueShards() - var addrPosts []*AddrPost - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, shardConfigs) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrPost, - Prefixes: prefixes, - Max: client.ExLargeLimit, - Newest: newest, - Context: ctx, - }); err != nil { - return nil, fmt.Errorf("error getting db addr memo post by prefix; %w", err) - } - for _, msg := range dbClient.Messages { - var addrPost = new(AddrPost) - db.Set(addrPost, msg) - addrPosts = append(addrPosts, addrPost) - } + var addrPosts = make([]*AddrPost, len(messages)) + for i := range messages { + addrPosts[i] = new(AddrPost) + db.Set(addrPosts[i], messages[i]) } return addrPosts, nil } diff --git a/db/item/memo/addr_profile.go b/db/item/memo/addr_profile.go index 4ea49b1e..26eb3218 100644 --- a/db/item/memo/addr_profile.go +++ b/db/item/memo/addr_profile.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" "time" ) @@ -52,30 +51,16 @@ func (p *AddrProfile) Deserialize(data []byte) { } func GetAddrProfiles(ctx context.Context, addrs [][25]byte) ([]*AddrProfile, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range addrs { - shard := db.GetShardIdFromByte32(addrs[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], addrs[i][:]) + shardPrefixes := db.ShardPrefixesAddrs(addrs) + var opts = []client.Option{client.OptionSinglePrefixLimit(), client.OptionNewest()} + messages, err := db.GetByPrefixes(ctx, db.TopicMemoAddrProfile, shardPrefixes, opts...) + if err != nil { + return nil, fmt.Errorf("error getting db addr memo profiles by prefix; %w", err) } - shardConfigs := config.GetQueueShards() - var addrProfiles []*AddrProfile - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, shardConfigs) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrProfile, - Prefixes: prefixes, - Max: 1, - Newest: true, - Context: ctx, - }); err != nil { - return nil, fmt.Errorf("error getting db addr memo profiles by prefix; %w", err) - } - for _, msg := range dbClient.Messages { - var addrProfile = new(AddrProfile) - db.Set(addrProfile, msg) - addrProfiles = append(addrProfiles, addrProfile) - } + var addrProfiles = make([]*AddrProfile, len(messages)) + for i := range messages { + addrProfiles[i] = new(AddrProfile) + db.Set(addrProfiles[i], messages[i]) } return addrProfiles, nil } diff --git a/db/item/memo/addr_profile_pic.go b/db/item/memo/addr_profile_pic.go index b9cb5326..12b7db26 100644 --- a/db/item/memo/addr_profile_pic.go +++ b/db/item/memo/addr_profile_pic.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" "time" ) @@ -52,30 +51,16 @@ func (p *AddrProfilePic) Deserialize(data []byte) { } func GetAddrProfilePics(ctx context.Context, addrs [][25]byte) ([]*AddrProfilePic, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range addrs { - shard := db.GetShardIdFromByte32(addrs[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], addrs[i][:]) + shardPrefixes := db.ShardPrefixesAddrs(addrs) + var opts = []client.Option{client.OptionSinglePrefixLimit(), client.OptionNewest()} + messages, err := db.GetByPrefixes(ctx, db.TopicMemoAddrProfilePic, shardPrefixes, opts...) + if err != nil { + return nil, fmt.Errorf("error getting db addr memo profile pics by prefix; %w", err) } - shardConfigs := config.GetQueueShards() - var addrProfilePics []*AddrProfilePic - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, shardConfigs) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrProfilePic, - Prefixes: prefixes, - Max: 1, - Newest: true, - Context: ctx, - }); err != nil { - return nil, fmt.Errorf("error getting db addr memo profile pics by prefix; %w", err) - } - for _, msg := range dbClient.Messages { - var addrProfilePic = new(AddrProfilePic) - db.Set(addrProfilePic, msg) - addrProfilePics = append(addrProfilePics, addrProfilePic) - } + var addrProfilePics = make([]*AddrProfilePic, len(messages)) + for i := range messages { + addrProfilePics[i] = new(AddrProfilePic) + db.Set(addrProfilePics[i], messages[i]) } return addrProfilePics, nil } diff --git a/db/item/memo/addr_room_follow.go b/db/item/memo/addr_room_follow.go index aadaadf5..740c5bb1 100644 --- a/db/item/memo/addr_room_follow.go +++ b/db/item/memo/addr_room_follow.go @@ -64,29 +64,14 @@ func (f *AddrRoomFollow) Deserialize(data []byte) { } func GetAddrRoomFollows(ctx context.Context, addrs [][25]byte) ([]*AddrRoomFollow, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range addrs { - shard := client.GenShardSource32(addrs[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], addrs[i][:]) + messages, err := db.GetByPrefixes(ctx, db.TopicMemoAddrRoomFollow, db.ShardPrefixesAddrs(addrs)) + if err != nil { + return nil, fmt.Errorf("error getting db memo addr room follow by prefix; %w", err) } - shardConfigs := config.GetQueueShards() - var addrFollows []*AddrRoomFollow - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, shardConfigs) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrRoomFollow, - Prefixes: prefixes, - Max: client.ExLargeLimit, - Context: ctx, - }); err != nil { - return nil, fmt.Errorf("error getting db memo addr room follow by prefix; %w", err) - } - for _, msg := range dbClient.Messages { - var addrFollow = new(AddrRoomFollow) - db.Set(addrFollow, msg) - addrFollows = append(addrFollows, addrFollow) - } + var addrFollows = make([]*AddrRoomFollow, len(messages)) + for i := range messages { + addrFollows[i] = new(AddrRoomFollow) + db.Set(addrFollows[i], messages[i]) } return addrFollows, nil } From 0b9d5064375ff92f842eab470cf827a5909a340f Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Wed, 31 Jul 2024 21:13:34 -0700 Subject: [PATCH 21/23] Get specific items by uids to continue removing GetWOpts usage. --- db/client/client.go | 26 ++++++++++++++++++++++++++ db/item/db/item.go | 26 +++++++++++++++++++------- db/item/memo/post.go | 27 +++++++-------------------- db/item/memo/post_child.go | 27 +++++++-------------------- 4 files changed, 59 insertions(+), 47 deletions(-) diff --git a/db/client/client.go b/db/client/client.go index fbcc5a73..c10909f2 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -180,6 +180,32 @@ func (s *Client) GetNext(ctx context.Context, topic string, start []byte) error return nil } +func (s *Client) GetSpecific(ctx context.Context, topic string, uids [][]byte) error { + if err := s.SetConn(); err != nil { + return fmt.Errorf("error setting connection; %w", err) + } + c := queue_pb.NewQueueClient(s.conn) + ctx, cancel := context.WithTimeout(ctx, DefaultGetTimeout) + defer cancel() + messages, err := c.GetMessages(ctx, &queue_pb.Request{ + Topic: topic, + Uids: uids, + }) + if err != nil { + return fmt.Errorf("error getting specific message rpc; %w", err) + } else if messages == nil { + return fmt.Errorf("error getting specific message rpc; %w", MessageNotSetError) + } + for _, message := range messages.Messages { + s.Messages = append(s.Messages, Message{ + Topic: message.Topic, + Uid: message.Uid, + Message: message.Message, + }) + } + return nil +} + type Opts struct { Topic string Start []byte diff --git a/db/item/db/item.go b/db/item/db/item.go index 8a9cdde4..575b7781 100644 --- a/db/item/db/item.go +++ b/db/item/db/item.go @@ -25,6 +25,23 @@ func GetItem(ctx context.Context, obj Object) error { return nil } +func ShardUidsTxHashes(txHashes [][32]byte) map[uint32][][]byte { + return ShardUids(hs.HashesToSlices(txHashes)) +} + +func ShardUidsAddrs(addrs [][25]byte) map[uint32][][]byte { + return ShardUids(wallet.AddrsToSlices(addrs)) +} + +func ShardUids(byteUids [][]byte) map[uint32][][]byte { + var shardUids = make(map[uint32][][]byte) + for _, uid := range byteUids { + shard := GetShardIdFromByte32(uid) + shardUids[shard] = append(shardUids[shard], uid) + } + return shardUids +} + func GetSpecific(ctx context.Context, topic string, shardUids map[uint32][][]byte) ([]client.Message, error) { wait := NewWait(len(shardUids)) var messages []client.Message @@ -32,8 +49,7 @@ func GetSpecific(ctx context.Context, topic string, shardUids map[uint32][][]byt go func(shard uint32, uids [][]byte) { defer wait.Group.Done() uids = jutil.RemoveDupesAndEmpties(uids) - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) + dbClient := GetShardClient(shard) for len(uids) > 0 { var uidsToUse [][]byte if len(uids) > client.HugeLimit { @@ -41,11 +57,7 @@ func GetSpecific(ctx context.Context, topic string, shardUids map[uint32][][]byt } else { uidsToUse, uids = uids, nil } - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: topic, - Uids: uidsToUse, - }); err != nil { + if err := dbClient.GetSpecific(ctx, topic, uidsToUse); err != nil { wait.AddError(fmt.Errorf("error getting client message get specific; %w", err)) return } diff --git a/db/item/memo/post.go b/db/item/memo/post.go index caf7651b..9797fc35 100644 --- a/db/item/memo/post.go +++ b/db/item/memo/post.go @@ -62,27 +62,14 @@ func GetPost(ctx context.Context, txHash [32]byte) (*Post, error) { } func GetPosts(ctx context.Context, txHashes [][32]byte) ([]*Post, error) { - var shardUids = make(map[uint32][][]byte) - for i := range txHashes { - shard := db.GetShardIdFromByte32(txHashes[i][:]) - shardUids[shard] = append(shardUids[shard], jutil.ByteReverse(txHashes[i][:])) + messages, err := db.GetSpecific(ctx, db.TopicMemoPost, db.ShardUidsTxHashes(txHashes)) + if err != nil { + return nil, fmt.Errorf("error getting client message memo posts; %w", err) } - var posts []*Post - for shard, uids := range shardUids { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: db.TopicMemoPost, - Uids: uids, - }); err != nil { - return nil, fmt.Errorf("error getting client message memo posts; %w", err) - } - for _, msg := range dbClient.Messages { - var post = new(Post) - db.Set(post, msg) - posts = append(posts, post) - } + var posts = make([]*Post, len(messages)) + for i := range messages { + posts[i] = new(Post) + db.Set(posts[i], messages[i]) } return posts, nil } diff --git a/db/item/memo/post_child.go b/db/item/memo/post_child.go index 00ac4af7..7cb59370 100644 --- a/db/item/memo/post_child.go +++ b/db/item/memo/post_child.go @@ -45,27 +45,14 @@ func (c *PostChild) Serialize() []byte { func (c *PostChild) Deserialize([]byte) {} func GetPostsChildren(ctx context.Context, postTxHashes [][32]byte) ([]*PostChild, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range postTxHashes { - shard := db.GetShardIdFromByte32(postTxHashes[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.ByteReverse(postTxHashes[i][:])) + messages, err := db.GetByPrefixes(ctx, db.TopicMemoPostChild, db.ShardPrefixesTxHashes(postTxHashes)) + if err != nil { + return nil, fmt.Errorf("error getting client message memo post children; %w", err) } - var postChildren []*PostChild - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: db.TopicMemoPostChild, - Prefixes: prefixes, - }); err != nil { - return nil, fmt.Errorf("error getting client message memo post children; %w", err) - } - for _, msg := range dbClient.Messages { - var postChild = new(PostChild) - db.Set(postChild, msg) - postChildren = append(postChildren, postChild) - } + var postChildren = make([]*PostChild, len(messages)) + for i := range messages { + postChildren[i] = new(PostChild) + db.Set(postChildren[i], messages[i]) } return postChildren, nil } From 4304569d8c3f81d3629788e36490985f5d68a67d Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Thu, 1 Aug 2024 18:44:36 -0700 Subject: [PATCH 22/23] Finish converting uses of GetWOpts. --- admin/server/node/history.go | 2 +- admin/server/node/peer_report.go | 2 +- cmd/maint/check_follows.go | 2 +- cmd/peer/list.go | 2 +- db/client/client.go | 111 +-------------------------- db/client/peer/list.go | 2 +- db/item/memo/post_like.go | 27 ++----- db/item/memo/post_parent.go | 44 +++-------- db/item/memo/post_room.go | 28 ++----- db/item/memo/room_follow.go | 11 +-- db/item/memo/room_post.go | 11 +-- db/item/memo/seen_post.go | 23 ++---- db/item/peer_connection.go | 37 ++++----- node/act/maint/check_follows.go | 13 ++-- node/act/maint/populate_seen_post.go | 8 +- node/group.go | 2 +- test/run/queue/get.go | 12 +-- 17 files changed, 75 insertions(+), 262 deletions(-) diff --git a/admin/server/node/history.go b/admin/server/node/history.go index dae92dfe..63c754a3 100644 --- a/admin/server/node/history.go +++ b/admin/server/node/history.go @@ -23,7 +23,7 @@ var historyRoute = admin.Route{ var shard uint32 PeerConnectionsLoop: for { - peerConnections, err := item.GetPeerConnections(item.PeerConnectionsRequest{ + peerConnections, err := item.GetPeerConnections(r.Request.Context(), item.PeerConnectionsRequest{ Shard: shard, StartId: startId, Ip: net.ParseIP(historyRequest.Ip), diff --git a/admin/server/node/peer_report.go b/admin/server/node/peer_report.go index da4cf469..f32e4643 100644 --- a/admin/server/node/peer_report.go +++ b/admin/server/node/peer_report.go @@ -45,7 +45,7 @@ var peerReportRoute = admin.Route{ var peerInfo = new(PeerInfo) for shard := uint32(0); shard < config.GetTotalShards(); shard++ { for startId := []byte{}; ; { - peerConnections, err := item.GetPeerConnections(item.PeerConnectionsRequest{ + peerConnections, err := item.GetPeerConnections(r.Request.Context(), item.PeerConnectionsRequest{ Shard: shard, StartId: startId, Max: client.LargeLimit, diff --git a/cmd/maint/check_follows.go b/cmd/maint/check_follows.go index 8798cb30..c9a6ff47 100644 --- a/cmd/maint/check_follows.go +++ b/cmd/maint/check_follows.go @@ -13,7 +13,7 @@ var checkFollowsCmd = &cobra.Command{ deleteItems, _ := c.Flags().GetBool(FlagDelete) checkFollows := maint.NewCheckFollows(deleteItems) log.Printf("Starting check follows (delete flag: %t)...\n", deleteItems) - if err := checkFollows.Check(); err != nil { + if err := checkFollows.Check(c.Context()); err != nil { log.Fatalf("error maint check follows; %v", err) } log.Printf("Checked follows: %d, bad: %d\n", checkFollows.Processed, checkFollows.BadFollows) diff --git a/cmd/peer/list.go b/cmd/peer/list.go index 6f85d7fe..49e18600 100644 --- a/cmd/peer/list.go +++ b/cmd/peer/list.go @@ -80,7 +80,7 @@ var listAttemptsCmd = &cobra.Command{ log.Fatalf("error parsing host ip") } port := jutil.GetUInt16FromString(portString) - lastPeerConnection, err := item.GetPeerConnectionLast(ip, port) + lastPeerConnection, err := item.GetPeerConnectionLast(cmd.Context(), ip, port) if err != nil { log.Fatalf("fatal error last peer connection; %v", err) } diff --git a/db/client/client.go b/db/client/client.go index c10909f2..a78b8012 100644 --- a/db/client/client.go +++ b/db/client/client.go @@ -216,120 +216,15 @@ type Opts struct { Context context.Context } -func (s *Client) GetWOpts(opts Opts) error { - var optGroups []Opts - if len(opts.Prefixes) > ExLargeLimit { - for i := 0; i < len(opts.Prefixes); i += ExLargeLimit { - end := i + ExLargeLimit - if end > len(opts.Prefixes) { - end = len(opts.Prefixes) - } - optGroups = append(optGroups, Opts{ - Topic: opts.Topic, - Start: opts.Start, - Prefixes: opts.Prefixes[i:end], - Max: opts.Max, - Uids: opts.Uids, - Newest: opts.Newest, - }) - } - } else if len(opts.Uids) > ExLargeLimit { - for i := 0; i < len(opts.Uids); i += ExLargeLimit { - end := i + ExLargeLimit - if end > len(opts.Uids) { - end = len(opts.Uids) - } - optGroups = append(optGroups, Opts{ - Topic: opts.Topic, - Prefixes: opts.Prefixes, - Start: opts.Start, - Max: opts.Max, - Uids: opts.Uids[i:end], - Newest: opts.Newest, - }) - } - } else { - optGroups = []Opts{opts} - } - if err := s.SetConn(); err != nil { - return fmt.Errorf("error setting connection; %w", err) - } - c := queue_pb.NewQueueClient(s.conn) - var bgCtx = opts.Context - if jutil.IsNil(bgCtx) { - bgCtx = context.Background() - } - ctx, cancel := context.WithTimeout(bgCtx, DefaultGetTimeout) - defer cancel() - s.Messages = nil - for _, optGroup := range optGroups { - message, err := c.GetMessages(ctx, &queue_pb.Request{ - Topic: optGroup.Topic, - Prefixes: optGroup.Prefixes, - Start: optGroup.Start, - Max: optGroup.Max, - Uids: optGroup.Uids, - Newest: optGroup.Newest, - }, grpc.MaxCallRecvMsgSize(MaxMessageSize)) - if err != nil { - return fmt.Errorf("error getting messages rpc; %w", err) - } - var messages = make([]Message, len(message.Messages)) - for i := range message.Messages { - messages[i] = Message{ - Topic: message.Messages[i].Topic, - Uid: message.Messages[i].Uid, - Message: message.Messages[i].Message, - } - } - s.Messages = append(s.Messages, messages...) - } - return nil -} - -func (s *Client) GetTopicList() error { - if err := s.SetConn(); err != nil { - return fmt.Errorf("error setting connection; %w", err) - } - c := queue_pb.NewQueueClient(s.conn) - ctx, cancel := context.WithTimeout(context.Background(), DefaultGetTimeout) - defer cancel() - topicList, err := c.GetTopicList(ctx, new(queue_pb.EmptyRequest)) - if err != nil { - return fmt.Errorf("error getting topic list; %w", err) - } - var topics = make([]Topic, len(topicList.Topics)) - for i := range topicList.Topics { - topics[i] = Topic{ - Name: topicList.Topics[i].Name, - Size: topicList.Topics[i].Count, - } - } - s.Topics = topics - return nil -} - func (s *Client) Listen(ctx context.Context, topic string, prefixes [][]byte) (chan *Message, error) { - messageChan, err := s.ListenOpts(Opts{ - Context: ctx, - Topic: topic, - Prefixes: prefixes, - }) - if err != nil { - return nil, fmt.Errorf("error getting message chan with opts; %w", err) - } - return messageChan, nil -} - -func (s *Client) ListenOpts(opts Opts) (chan *Message, error) { if err := s.SetConn(); err != nil { return nil, fmt.Errorf("error setting connection; %w", err) } c := queue_pb.NewQueueClient(s.conn) - ctx, cancel := context.WithTimeout(opts.Context, DefaultStreamTimeout) + ctx, cancel := context.WithTimeout(ctx, DefaultStreamTimeout) var request = &queue_pb.RequestStream{ - Topic: opts.Topic, - Prefixes: opts.Prefixes, + Topic: topic, + Prefixes: prefixes, } stream, err := c.GetStreamMessages(ctx, request) if err != nil { diff --git a/db/client/peer/list.go b/db/client/peer/list.go index c9201595..79e17136 100644 --- a/db/client/peer/list.go +++ b/db/client/peer/list.go @@ -44,7 +44,7 @@ func (l *List) GetPeers(filter string) error { Port: peers[i].Port, } } - peerConnectionLasts, err := item.GetPeerConnectionLasts(ipPorts) + peerConnectionLasts, err := item.GetPeerConnectionLasts(l.Context, ipPorts) if err != nil && !client.IsEntryNotFoundError(err) { return fmt.Errorf("error getting peer connection lasts for found peers; %w", err) } diff --git a/db/item/memo/post_like.go b/db/item/memo/post_like.go index 82fe3772..94d3c1d0 100644 --- a/db/item/memo/post_like.go +++ b/db/item/memo/post_like.go @@ -55,27 +55,14 @@ func (l *PostLike) Deserialize(data []byte) { } func GetPostLikes(ctx context.Context, postTxHashes [][32]byte) ([]*PostLike, error) { - var shardPrefixes = make(map[uint32][][]byte) - for i := range postTxHashes { - shard := db.GetShardIdFromByte32(postTxHashes[i][:]) - shardPrefixes[shard] = append(shardPrefixes[shard], jutil.ByteReverse(postTxHashes[i][:])) + messages, err := db.GetByPrefixes(ctx, db.TopicMemoPostLike, db.ShardPrefixesTxHashes(postTxHashes)) + if err != nil { + return nil, fmt.Errorf("error getting client message memo post likes; %w", err) } - var postLikes []*PostLike - for shard, prefixes := range shardPrefixes { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: db.TopicMemoPostLike, - Prefixes: prefixes, - }); err != nil { - return nil, fmt.Errorf("error getting client message memo post likes; %w", err) - } - for _, msg := range dbClient.Messages { - var postLike = new(PostLike) - db.Set(postLike, msg) - postLikes = append(postLikes, postLike) - } + var postLikes = make([]*PostLike, len(messages)) + for i := range messages { + postLikes[i] = new(PostLike) + db.Set(postLikes[i], messages[i]) } return postLikes, nil } diff --git a/db/item/memo/post_parent.go b/db/item/memo/post_parent.go index c473f47c..a548e99b 100644 --- a/db/item/memo/post_parent.go +++ b/db/item/memo/post_parent.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" ) type PostParent struct { @@ -46,45 +45,22 @@ func (p *PostParent) Deserialize(data []byte) { } func GetPostParent(ctx context.Context, postTxHash [32]byte) (*PostParent, error) { - shardConfig := config.GetShardConfig(db.GetShardIdFromByte32(postTxHash[:]), config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: db.TopicMemoPostParent, - Uids: [][]byte{jutil.ByteReverse(postTxHash[:])}, - }); err != nil { - return nil, fmt.Errorf("error getting client message memo post parents; %w", err) - } - if len(dbClient.Messages) == 0 { - return nil, nil + var postParent = &PostParent{PostTxHash: postTxHash} + if err := db.GetItem(ctx, postParent); err != nil { + return nil, fmt.Errorf("error getting client message memo post parent; %w", err) } - var postParent = new(PostParent) - db.Set(postParent, dbClient.Messages[0]) return postParent, nil } func GetPostParents(ctx context.Context, postTxHashes [][32]byte) ([]*PostParent, error) { - var shardUids = make(map[uint32][][]byte) - for i := range postTxHashes { - shard := db.GetShardIdFromByte32(postTxHashes[i][:]) - shardUids[shard] = append(shardUids[shard], jutil.ByteReverse(postTxHashes[i][:])) + messages, err := db.GetSpecific(ctx, db.TopicMemoPostParent, db.ShardUidsTxHashes(postTxHashes)) + if err != nil { + return nil, fmt.Errorf("error getting client message memo post parents; %w", err) } - var postParents []*PostParent - for shard, uids := range shardUids { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: db.TopicMemoPostParent, - Uids: uids, - }); err != nil { - return nil, fmt.Errorf("error getting client message memo post parents; %w", err) - } - for _, msg := range dbClient.Messages { - var postParent = new(PostParent) - db.Set(postParent, msg) - postParents = append(postParents, postParent) - } + var postParents = make([]*PostParent, len(messages)) + for i := range messages { + postParents[i] = new(PostParent) + db.Set(postParents[i], messages[i]) } return postParents, nil } diff --git a/db/item/memo/post_room.go b/db/item/memo/post_room.go index 66a60a99..96f2fb59 100644 --- a/db/item/memo/post_room.go +++ b/db/item/memo/post_room.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" ) type PostRoom struct { @@ -43,27 +42,14 @@ func (r *PostRoom) Deserialize(data []byte) { } func GetPostRooms(ctx context.Context, postTxHashes [][32]byte) ([]*PostRoom, error) { - var shardUids = make(map[uint32][][]byte) - for i := range postTxHashes { - shard := db.GetShardIdFromByte32(postTxHashes[i][:]) - shardUids[shard] = append(shardUids[shard], jutil.ByteReverse(postTxHashes[i][:])) + messages, err := db.GetSpecific(ctx, db.TopicMemoPostRoom, db.ShardUidsTxHashes(postTxHashes)) + if err != nil { + return nil, fmt.Errorf("error getting client message memo post rooms; %w", err) } - var postRooms []*PostRoom - for shard, uids := range shardUids { - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: db.TopicMemoPostRoom, - Uids: uids, - }); err != nil { - return nil, fmt.Errorf("error getting client message memo post rooms; %w", err) - } - for _, msg := range dbClient.Messages { - var postRoom = new(PostRoom) - db.Set(postRoom, msg) - postRooms = append(postRooms, postRoom) - } + var postRooms = make([]*PostRoom, len(messages)) + for i := range messages { + postRooms[i] = new(PostRoom) + db.Set(postRooms[i], messages[i]) } return postRooms, nil } diff --git a/db/item/memo/room_follow.go b/db/item/memo/room_follow.go index ce8b373e..d8d6f87f 100644 --- a/db/item/memo/room_follow.go +++ b/db/item/memo/room_follow.go @@ -7,7 +7,6 @@ import ( "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/ref/bitcoin/memo" - "github.com/memocash/index/ref/config" "time" ) @@ -65,14 +64,8 @@ func (f *RoomFollow) Deserialize(data []byte) { func GetRoomFollows(ctx context.Context, room string) ([]*RoomFollow, error) { roomHash := GetRoomHash(room) - shardConfig := config.GetShardConfig(client.GenShardSource32(roomHash), config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoRoomFollow, - Prefixes: [][]byte{roomHash}, - Max: client.ExLargeLimit, - Context: ctx, - }); err != nil { + dbClient := db.GetShardClient(client.GenShardSource32(roomHash)) + if err := dbClient.GetByPrefix(ctx, db.TopicMemoRoomFollow, client.NewPrefix(roomHash)); err != nil { return nil, fmt.Errorf("error getting db memo room follows; %w", err) } var roomFollows = make([]*RoomFollow, len(dbClient.Messages)) diff --git a/db/item/memo/room_post.go b/db/item/memo/room_post.go index ebe22ac3..1477a76e 100644 --- a/db/item/memo/room_post.go +++ b/db/item/memo/room_post.go @@ -56,15 +56,8 @@ func GetRoomHash(room string) []byte { func GetRoomPosts(ctx context.Context, room string) ([]*RoomPost, error) { roomHash := GetRoomHash(room) - shard := client.GenShardSource32(roomHash) - shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoRoomPost, - Prefixes: [][]byte{roomHash}, - Max: client.ExLargeLimit, - Context: ctx, - }); err != nil { + dbClient := db.GetShardClient(client.GenShardSource32(roomHash)) + if err := dbClient.GetByPrefix(ctx, db.TopicMemoRoomPost, client.NewPrefix(roomHash)); err != nil { return nil, fmt.Errorf("error getting db memo room posts; %w", err) } var roomPosts = make([]*RoomPost, len(dbClient.Messages)) diff --git a/db/item/memo/seen_post.go b/db/item/memo/seen_post.go index 6254734e..a74dc982 100644 --- a/db/item/memo/seen_post.go +++ b/db/item/memo/seen_post.go @@ -61,29 +61,22 @@ func (i *SeenPost) Serialize() []byte { func (i *SeenPost) Deserialize([]byte) {} func GetSeenPosts(ctx context.Context, start time.Time, startTxHash [32]byte, limit uint32) ([]*SeenPost, error) { - if limit == 0 { - limit = client.DefaultLimit - } else if limit > client.ExLargeLimit { + if limit > client.ExLargeLimit { limit = client.ExLargeLimit } - shardConfigs := config.GetQueueShards() + var options = []client.Option{client.OptionNewest()} + if limit > 0 { + options = append(options, client.NewOptionLimit(int(limit))) + } startShard := GetSeenPostShard32(start) var startByte []byte if !jutil.IsTimeZero(start) { startByte = jutil.CombineBytes(jutil.GetTimeByteNanoBig(start), jutil.ByteReverse(startTxHash[:])) } var allSeenPosts []*SeenPost - for i := range shardConfigs { - shardId := (startShard + uint32(i)) % uint32(len(shardConfigs)) - shardConfig := shardConfigs[shardId] - dbClient := client.NewClient(shardConfig.GetHost()) - if err := dbClient.GetWOpts(client.Opts{ - Context: ctx, - Topic: db.TopicMemoSeenPost, - Start: startByte, - Max: limit, - Newest: true, - }); err != nil { + for i := range config.GetQueueShards() { + dbClient := db.GetShardClient(startShard + uint32(i)) + if err := dbClient.GetByPrefix(ctx, db.TopicMemoSeenPost, client.NewStart(startByte), options...); err != nil { return nil, fmt.Errorf("error getting db seen posts; %w", err) } if len(dbClient.Messages) == 0 { diff --git a/db/item/peer_connection.go b/db/item/peer_connection.go index 368ab9df..6892575c 100644 --- a/db/item/peer_connection.go +++ b/db/item/peer_connection.go @@ -1,6 +1,7 @@ package item import ( + "context" "fmt" "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" @@ -82,22 +83,17 @@ func (r PeerConnectionsRequest) GetShard() uint32 { return r.Shard } -func GetPeerConnections(request PeerConnectionsRequest) ([]*PeerConnection, error) { - shardConfig := config.GetShardConfig(request.GetShard(), config.GetQueueShards()) - dbClient := client.NewClient(shardConfig.GetHost()) - var prefixes [][]byte +func GetPeerConnections(ctx context.Context, request PeerConnectionsRequest) ([]*PeerConnection, error) { + dbClient := db.GetShardClient(request.GetShard()) + var prefix client.Prefix if len(request.Ip) > 0 { - prefixes = [][]byte{jutil.CombineBytes( + prefix = client.NewPrefix(jutil.CombineBytes( jutil.BytePadPrefix(request.Ip, IpBytePadSize), jutil.GetUintData(uint(request.Port)), - )} + )) } - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicPeerConnection, - Start: request.StartId, - Prefixes: prefixes, - Max: request.Max, - }); err != nil { + prefix.Limit = request.Max + if err := dbClient.GetByPrefix(ctx, db.TopicPeerConnection, prefix); err != nil { return nil, fmt.Errorf("error getting peer connections from queue client; %w", err) } var peerConnections = make([]*PeerConnection, len(dbClient.Messages)) @@ -107,8 +103,8 @@ func GetPeerConnections(request PeerConnectionsRequest) ([]*PeerConnection, erro } return peerConnections, nil } -func GetPeerConnectionLast(ip []byte, port uint16) (*PeerConnection, error) { - peerConnections, err := GetPeerConnectionLasts([]IpPort{{ +func GetPeerConnectionLast(ctx context.Context, ip []byte, port uint16) (*PeerConnection, error) { + peerConnections, err := GetPeerConnectionLasts(ctx, []IpPort{{ Ip: ip, Port: port, }}) @@ -126,7 +122,7 @@ type IpPort struct { Port uint16 } -func GetPeerConnectionLasts(ipPorts []IpPort) ([]*PeerConnection, error) { +func GetPeerConnectionLasts(ctx context.Context, ipPorts []IpPort) ([]*PeerConnection, error) { if len(ipPorts) == 0 { return nil, nil } @@ -139,15 +135,12 @@ func GetPeerConnectionLasts(ipPorts []IpPort) ([]*PeerConnection, error) { for shard, ipPorts := range shardIpPorts { shardConfig := config.GetShardConfig(shard, config.GetQueueShards()) dbClient := client.NewClient(shardConfig.GetHost()) - var prefixes = make([][]byte, len(ipPorts)) + var prefixes = make([]client.Prefix, len(ipPorts)) for i := range ipPorts { - prefixes[i] = jutil.CombineBytes(jutil.BytePadPrefix(ipPorts[i].Ip, IpBytePadSize), jutil.GetUintData(uint(ipPorts[i].Port))) + prefixes[i] = client.NewPrefix(jutil.CombineBytes(jutil.BytePadPrefix(ipPorts[i].Ip, IpBytePadSize), jutil.GetUintData(uint(ipPorts[i].Port)))) } - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicPeerConnection, - Max: 1, - Prefixes: prefixes, - }); err != nil { + opt := client.OptionSinglePrefixLimit() + if err := dbClient.GetByPrefixes(ctx, db.TopicPeerConnection, prefixes, opt); err != nil { return nil, fmt.Errorf("error getting peer connection lasts: %d %d; %w", shard, len(ipPorts), err) } if len(dbClient.Messages) == 0 { diff --git a/node/act/maint/check_follows.go b/node/act/maint/check_follows.go index 8c4edae2..65354ff1 100644 --- a/node/act/maint/check_follows.go +++ b/node/act/maint/check_follows.go @@ -1,7 +1,9 @@ package maint import ( + "context" "fmt" + "github.com/jchavannes/jgo/jutil" "github.com/memocash/index/db/client" "github.com/memocash/index/db/item/db" "github.com/memocash/index/db/item/memo" @@ -14,16 +16,13 @@ type CheckFollows struct { BadFollows int } -func (c *CheckFollows) Check() error { +func (c *CheckFollows) Check(ctx context.Context) error { for _, shardConfig := range config.GetQueueShards() { dbClient := client.NewClient(shardConfig.GetHost()) var startUid []byte for { - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoAddrFollow, - Start: startUid, - Max: client.ExLargeLimit, - }); err != nil { + opt := client.OptionExLargeLimit() + if err := dbClient.GetByPrefix(ctx, db.TopicMemoAddrFollow, client.NewStart(startUid), opt); err != nil { return fmt.Errorf("error getting db memo follow by prefix; %w", err) } for _, msg := range dbClient.Messages { @@ -31,7 +30,7 @@ func (c *CheckFollows) Check() error { var addrMemoFollow = new(memo.AddrFollow) db.Set(addrMemoFollow, msg) startUid = addrMemoFollow.GetUid() - if len(addrMemoFollow.FollowAddr) == 0 { + if jutil.AllZeros(addrMemoFollow.FollowAddr[:]) { c.BadFollows++ if !c.Delete { continue diff --git a/node/act/maint/populate_seen_post.go b/node/act/maint/populate_seen_post.go index 11c6169e..2833cab1 100644 --- a/node/act/maint/populate_seen_post.go +++ b/node/act/maint/populate_seen_post.go @@ -27,12 +27,8 @@ func (p *PopulateSeenPost) Populate() error { var startTxHash [32]byte dbClient := client.NewClient(shardConfig.GetHost()) for { - if err := dbClient.GetWOpts(client.Opts{ - Topic: db.TopicMemoPost, - Start: startTxHash[:], - Max: client.HugeLimit, - Context: p.Ctx, - }); err != nil { + opt := client.OptionHugeLimit() + if err := dbClient.GetByPrefix(p.Ctx, db.TopicMemoPost, client.NewStart(startTxHash[:]), opt); err != nil { return fmt.Errorf("error getting memo posts for populate seen posts; %w", err) } var postTxHashes [][32]byte diff --git a/node/group.go b/node/group.go index ebb02973..7aef3a3c 100644 --- a/node/group.go +++ b/node/group.go @@ -44,7 +44,7 @@ func (g *Group) AddNextNode() error { log.Printf("newPeer: %x\n", newPeer.GetUid()) log.Printf("newPeer: %s:%d\n", net.IP(newPeer.Ip), newPeer.Port) g.LastPeerId = newPeer.GetUid() - peerConnection, err := item.GetPeerConnectionLast(newPeer.Ip, newPeer.Port) + peerConnection, err := item.GetPeerConnectionLast(g.Context, newPeer.Ip, newPeer.Port) if err != nil && !client.IsEntryNotFoundError(err) { return fmt.Errorf("error getting last peer connection for new peer; %w", err) } diff --git a/test/run/queue/get.go b/test/run/queue/get.go index 32460cfc..977b6bea 100644 --- a/test/run/queue/get.go +++ b/test/run/queue/get.go @@ -1,6 +1,7 @@ package queue import ( + "context" "fmt" "github.com/memocash/index/db/client" "github.com/memocash/index/ref/config" @@ -11,13 +12,14 @@ type Get struct { Items []Item } -func (r *Get) GetByPrefixes(topic string, prefixes [][]byte) error { +func (r *Get) GetByPrefixes(ctx context.Context, topic string, prefixes [][]byte) error { shardConfig := config.GetShardConfig(r.Shard, config.GetQueueShards()) db := client.NewClient(fmt.Sprintf("127.0.0.1:%d", shardConfig.Port)) - if err := db.GetWOpts(client.Opts{ - Topic: topic, - Prefixes: prefixes, - }); err != nil { + var clientPrefixes = make([]client.Prefix, len(prefixes)) + for i := range prefixes { + clientPrefixes[i] = client.NewPrefix(prefixes[i]) + } + if err := db.GetByPrefixes(ctx, topic, clientPrefixes); err != nil { return fmt.Errorf("error getting by prefixes using queue client; %w", err) } r.Items = make([]Item, len(db.Messages)) From cfe431ac7b18174064e2bc1e74c26e1d4b9a8f41 Mon Sep 17 00:00:00 2001 From: Jason Chavannes Date: Wed, 18 Feb 2026 22:36:10 -0800 Subject: [PATCH 23/23] Architecture diagram --- README.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/README.md b/README.md index 0eee3b4c..2974e10b 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,28 @@ go build ./index serve live ``` +## Architecture + +```mermaid +graph TD + BCH[BCH Node] <-->|P2P| Lead[Lead Processor] + Lead -->|Blocks/Txs| CS[Cluster Shards] + + subgraph Shard["Each Shard (0..N)"] + CS --> Queue[Queue Server] + Queue --> DB[(LevelDB)] + end + + GraphQL[GraphQL Server] -->|gRPC| Queue + Admin[Admin Server] -->|gRPC| Queue + Network[Network Server] -->|gRPC| Queue + + Client([Client]) -->|Query| GraphQL + Client -->|Manage| Admin + Client -->|Submit Tx| Broadcast[Broadcast Server] + Broadcast -->|Raw Tx| Lead +``` + ## Configuration Two options for setting config values.