From ba3cd85161e82c98ffd4059dade7564f0414c5a8 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Jun 2025 15:53:26 -0400 Subject: [PATCH 1/8] enforce limit on get nfts for user --- routes/nft.go | 27 ++++++++++++++++++++++++--- routes/user.go | 1 - 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/routes/nft.go b/routes/nft.go index 07cedbd2..b2bead01 100644 --- a/routes/nft.go +++ b/routes/nft.go @@ -930,7 +930,9 @@ type GetNFTsForUserRequest struct { ReaderPublicKeyBase58Check string `safeForLogging:"true"` IsForSale *bool `safeForLogging:"true"` // Ignored if IsForSale is provided - IsPending *bool `safeForLogging:"true"` + IsPending *bool `safeForLogging:"true"` + LastKeyHex string `safeForLogging:"true"` + Limit int `safeForLogging:"true"` } type NFTEntryAndPostEntryResponse struct { @@ -939,7 +941,8 @@ type NFTEntryAndPostEntryResponse struct { } type GetNFTsForUserResponse struct { - NFTsMap map[string]*NFTEntryAndPostEntryResponse + NFTsMap map[string]*NFTEntryAndPostEntryResponse + LastKeyHex string } func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) { @@ -969,6 +972,20 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) } } + limit := requestData.Limit + if limit <= 0 || limit > 100 { + limit = 100 + } + + lastKeyBytes := []byte{} + if requestData.LastKeyHex != "" { + lastKeyBytes, err = hex.DecodeString(requestData.LastKeyHex) + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("GetNFTsForUser: Problem decoding LastKeyHex: %v", err)) + return + } + } + // Get the NFT bid so we can do a more hardcore validation of the request data. utxoView, err := fes.backendServer.GetMempool().GetAugmentedUniversalView() if err != nil { @@ -987,7 +1004,7 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) readerPKID = readerPKIDEntry.PKID } - nftEntries := utxoView.GetNFTEntriesForPKID(pkid.PKID) + nftEntries, lastSeenKey := utxoView.GetNFTEntriesForPKID(pkid.PKID, limit, lastKeyBytes) filteredNFTEntries := []*lib.NFTEntry{} if requestData.IsForSale != nil { @@ -1041,6 +1058,10 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) fes._nftEntryToResponse(nftEntry, nil, utxoView, true, readerPKID)) } + if len(lastSeenKey) > 0 { + res.LastKeyHex = hex.EncodeToString(lastSeenKey) + } + if err = json.NewEncoder(ww).Encode(res); err != nil { _AddInternalServerError(ww, fmt.Sprintf("GetNFTsForUser: Problem serializing object to JSON: %v", err)) return diff --git a/routes/user.go b/routes/user.go index a62f23d2..9e15d030 100644 --- a/routes/user.go +++ b/routes/user.go @@ -2351,7 +2351,6 @@ func (fes *APIServer) GetNotifications(ww http.ResponseWriter, req *http.Request LastSeenIndex: lastSeenIndex, } if err = json.NewEncoder(ww).Encode(res); err != nil { - fmt.Printf("%#v\n", res) _AddBadRequestError(ww, fmt.Sprintf( "GetNotifications: Problem encoding response as JSON: %v", err)) return From c11047124c28e009e0e62ac02c45b8e3e334cac1 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Jun 2025 16:15:32 -0400 Subject: [PATCH 2/8] temporarily log error in gs get --- routes/global_state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/global_state.go b/routes/global_state.go index 3f761b3b..ddb03abe 100644 --- a/routes/global_state.go +++ b/routes/global_state.go @@ -964,7 +964,7 @@ func (gs *GlobalState) Get(key []byte) (value []byte, _err error) { "application/json", /*contentType*/ bytes.NewBuffer(json_data)) if err != nil { - return nil, fmt.Errorf("Get: Error processing remote request") + return nil, fmt.Errorf("Get: Error processing remote request: %v", err) } res := GetRemoteResponse{} From 671f1db7f7e6501034d75f79bc5954ddcf6b3c89 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Jun 2025 16:37:37 -0400 Subject: [PATCH 3/8] reuse shared client --- routes/global_state.go | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/routes/global_state.go b/routes/global_state.go index ddb03abe..e1f083b1 100644 --- a/routes/global_state.go +++ b/routes/global_state.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "strings" + "time" "github.com/deso-protocol/core/lib" @@ -948,23 +949,34 @@ func (gs *GlobalState) CreateGetRequest(key []byte) ( return url, json_data, nil } +var sharedClient = &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + }, +} + func (gs *GlobalState) Get(key []byte) (value []byte, _err error) { // If we have a remote node then use that node to fulfill this request. if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is currently annoying to test. - url, json_data, err := gs.CreateGetRequest(key) + url, jsonData, err := gs.CreateGetRequest(key) if err != nil { return nil, fmt.Errorf( "Get: Error constructing request: %v", err) } - resReturned, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("Get: Error creating new request: %v", err) + } + req.Header.Set("Content-Type", "application/json") + + resReturned, err := sharedClient.Do(req) if err != nil { - return nil, fmt.Errorf("Get: Error processing remote request: %v", err) + return nil, fmt.Errorf("Get: Error processing remote request: ") } res := GetRemoteResponse{} From 97c1fd28c3f819f71ea5a7388a4aa9dd5c15a090 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Jun 2025 16:50:15 -0400 Subject: [PATCH 4/8] remove err --- routes/global_state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/routes/global_state.go b/routes/global_state.go index e1f083b1..596442c4 100644 --- a/routes/global_state.go +++ b/routes/global_state.go @@ -970,7 +970,7 @@ func (gs *GlobalState) Get(key []byte) (value []byte, _err error) { req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) if err != nil { - return nil, fmt.Errorf("Get: Error creating new request: %v", err) + return nil, fmt.Errorf("Get: Error creating new request: ") } req.Header.Set("Content-Type", "application/json") From 11a3986583799fd0450ca73238322a30c26ea1e9 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Jun 2025 17:55:34 -0400 Subject: [PATCH 5/8] add todo --- routes/nft.go | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/nft.go b/routes/nft.go index b2bead01..bc7041e3 100644 --- a/routes/nft.go +++ b/routes/nft.go @@ -1004,6 +1004,7 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) readerPKID = readerPKIDEntry.PKID } + // TODO: move filtering of IsForSale and IsPending to GetNFTEntriesForPKID nftEntries, lastSeenKey := utxoView.GetNFTEntriesForPKID(pkid.PKID, limit, lastKeyBytes) filteredNFTEntries := []*lib.NFTEntry{} From ce1cf9d768248ba3d6c40639ee42ab6db7d108d6 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Tue, 24 Jun 2025 18:06:45 -0400 Subject: [PATCH 6/8] pass filter values to filter on db entries --- routes/nft.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/routes/nft.go b/routes/nft.go index bc7041e3..628b95c4 100644 --- a/routes/nft.go +++ b/routes/nft.go @@ -1005,7 +1005,8 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) } // TODO: move filtering of IsForSale and IsPending to GetNFTEntriesForPKID - nftEntries, lastSeenKey := utxoView.GetNFTEntriesForPKID(pkid.PKID, limit, lastKeyBytes) + nftEntries, lastSeenKey := utxoView.GetNFTEntriesForPKID( + pkid.PKID, limit, lastKeyBytes, requestData.IsForSale, requestData.IsPending) filteredNFTEntries := []*lib.NFTEntry{} if requestData.IsForSale != nil { From fb21e0161ff81591cbf462940cf08dcb056df8e7 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 25 Jun 2025 15:00:34 -0400 Subject: [PATCH 7/8] make all gs requests reuse client. json_data -> jsonData --- routes/global_state.go | 103 +++++++++++++++++++++-------------------- routes/server.go | 7 +++ 2 files changed, 60 insertions(+), 50 deletions(-) diff --git a/routes/global_state.go b/routes/global_state.go index 596442c4..713d33a8 100644 --- a/routes/global_state.go +++ b/routes/global_state.go @@ -7,10 +7,8 @@ import ( "io" "net/http" "strings" - "time" "github.com/deso-protocol/core/lib" - "github.com/dgraph-io/badger/v3" "github.com/nyaruka/phonenumbers" "github.com/pkg/errors" @@ -30,6 +28,7 @@ type GlobalState struct { GlobalStateRemoteNode string GlobalStateRemoteSecret string GlobalStateDB *badger.DB + SharedClient *http.Client } // GlobalStateRoutes returns the routes for managing global state. @@ -846,13 +845,13 @@ func (gs *GlobalState) PutRemote(ww http.ResponseWriter, rr *http.Request) { } func (gs *GlobalState) CreatePutRequest(key []byte, value []byte) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := PutRemoteRequest{ Key: key, Value: value, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("Put: Could not marshal JSON: %v", err) } @@ -861,7 +860,7 @@ func (gs *GlobalState) CreatePutRequest(key []byte, value []byte) ( gs.GlobalStateRemoteNode, RoutePathGlobalStatePutRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil + return url, jsonData, nil } func (gs *GlobalState) Put(key []byte, value []byte) error { @@ -869,16 +868,19 @@ func (gs *GlobalState) Put(key []byte, value []byte) error { if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is hard to exercise in a test. - url, json_data, err := gs.CreatePutRequest(key, value) + url, jsonData, err := gs.CreatePutRequest(key, value) if err != nil { return fmt.Errorf("Put: Error constructing request: %v", err) } - res, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("Put: Error creating new request: ") + } + req.Header.Set("Content-Type", "application/json") + + res, err := gs.SharedClient.Do(req) if err != nil { - return fmt.Errorf("Put: Error processing remote request") + return fmt.Errorf("Put: Error processing remote request: ") } res.Body.Close() @@ -932,12 +934,12 @@ func (gs *GlobalState) GetRemote(ww http.ResponseWriter, rr *http.Request) { } func (gs *GlobalState) CreateGetRequest(key []byte) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := GetRemoteRequest{ Key: key, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("Get: Could not marshal JSON: %v", err) } @@ -946,15 +948,7 @@ func (gs *GlobalState) CreateGetRequest(key []byte) ( gs.GlobalStateRemoteNode, RoutePathGlobalStateGetRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil -} - -var sharedClient = &http.Client{ - Transport: &http.Transport{ - MaxIdleConns: 100, - MaxIdleConnsPerHost: 100, - IdleConnTimeout: 90 * time.Second, - }, + return url, jsonData, nil } func (gs *GlobalState) Get(key []byte) (value []byte, _err error) { @@ -974,7 +968,7 @@ func (gs *GlobalState) Get(key []byte) (value []byte, _err error) { } req.Header.Set("Content-Type", "application/json") - resReturned, err := sharedClient.Do(req) + resReturned, err := gs.SharedClient.Do(req) if err != nil { return nil, fmt.Errorf("Get: Error processing remote request: ") } @@ -1047,12 +1041,12 @@ func (gs *GlobalState) BatchGetRemote(ww http.ResponseWriter, rr *http.Request) } func (gs *GlobalState) CreateBatchGetRequest(keyList [][]byte) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := BatchGetRemoteRequest{ KeyList: keyList, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("BatchGet: Could not marshal JSON: %v", err) } @@ -1061,7 +1055,7 @@ func (gs *GlobalState) CreateBatchGetRequest(keyList [][]byte) ( gs.GlobalStateRemoteNode, RoutePathGlobalStateBatchGetRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil + return url, jsonData, nil } func (gs *GlobalState) BatchGet(keyList [][]byte) (value [][]byte, _err error) { @@ -1069,18 +1063,21 @@ func (gs *GlobalState) BatchGet(keyList [][]byte) (value [][]byte, _err error) { if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is currently annoying to test. - url, json_data, err := gs.CreateBatchGetRequest(keyList) + url, jsonData, err := gs.CreateBatchGetRequest(keyList) if err != nil { return nil, fmt.Errorf( "BatchGet: Error constructing request: %v", err) } - resReturned, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) if err != nil { - return nil, fmt.Errorf("BatchGet: Error processing remote request") + return nil, fmt.Errorf("BatchGet: Error creating new request: ") + } + req.Header.Set("Content-Type", "application/json") + + resReturned, err := gs.SharedClient.Do(req) + if err != nil { + return nil, fmt.Errorf("BatchGet: Error processing remote request: ") } res := BatchGetRemoteResponse{} @@ -1125,12 +1122,12 @@ type DeleteRemoteResponse struct { } func (gs *GlobalState) CreateDeleteRequest(key []byte) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := DeleteRemoteRequest{ Key: key, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("Delete: Could not marshal JSON: %v", err) } @@ -1139,7 +1136,7 @@ func (gs *GlobalState) CreateDeleteRequest(key []byte) ( gs.GlobalStateRemoteNode, RoutePathGlobalStateDeleteRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil + return url, jsonData, nil } func (gs *GlobalState) DeleteRemote(ww http.ResponseWriter, rr *http.Request) { @@ -1171,17 +1168,20 @@ func (gs *GlobalState) Delete(key []byte) error { if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is currently annoying to test. - url, json_data, err := gs.CreateDeleteRequest(key) + url, jsonData, err := gs.CreateDeleteRequest(key) if err != nil { return fmt.Errorf("Delete: Could not construct request: %v", err) } - res, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("Delete: Error creating new request: ") + } + req.Header.Set("Content-Type", "application/json") + + res, err := gs.SharedClient.Do(req) if err != nil { - return fmt.Errorf("Delete: Error processing remote request") + return fmt.Errorf("Delete: Error processing remote request: ") } res.Body.Close() @@ -1214,7 +1214,7 @@ type SeekRemoteResponse struct { func (gs *GlobalState) CreateSeekRequest(startPrefix []byte, validForPrefix []byte, maxKeyLen int, numToFetch int, reverse bool, fetchValues bool) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := SeekRemoteRequest{ StartPrefix: startPrefix, @@ -1224,7 +1224,7 @@ func (gs *GlobalState) CreateSeekRequest(startPrefix []byte, validForPrefix []by Reverse: reverse, FetchValues: fetchValues, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("Seek: Could not marshal JSON: %v", err) } @@ -1233,7 +1233,7 @@ func (gs *GlobalState) CreateSeekRequest(startPrefix []byte, validForPrefix []by gs.GlobalStateRemoteNode, RoutePathGlobalStateSeekRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil + return url, jsonData, nil } func (gs *GlobalState) GlobalStateSeekRemote(ww http.ResponseWriter, rr *http.Request) { @@ -1278,7 +1278,7 @@ func (gs *GlobalState) Seek(startPrefix []byte, validForPrefix []byte, // If we have a remote node then use that node to fulfill this request. if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is currently annoying to test. - url, json_data, err := gs.CreateSeekRequest( + url, jsonData, err := gs.CreateSeekRequest( startPrefix, validForPrefix, maxKeyLen, @@ -1290,12 +1290,15 @@ func (gs *GlobalState) Seek(startPrefix []byte, validForPrefix []byte, "Seek: Error constructing request: %v", err) } - resReturned, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, nil, fmt.Errorf("Seek: Error creating new request: ") + } + req.Header.Set("Content-Type", "application/json") + + resReturned, err := gs.SharedClient.Do(req) if err != nil { - return nil, nil, fmt.Errorf("Seek: Error processing remote request") + return nil, nil, fmt.Errorf("Seek: Error processing remote request: ") } res := SeekRemoteResponse{} diff --git a/routes/server.go b/routes/server.go index a9734fa5..0034eb19 100644 --- a/routes/server.go +++ b/routes/server.go @@ -534,6 +534,13 @@ func NewAPIServer( GlobalStateRemoteSecret: config.GlobalStateRemoteSecret, GlobalStateRemoteNode: config.GlobalStateRemoteNode, GlobalStateDB: globalStateDB, + SharedClient: &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + }, + }, } if globalStateDB == nil && globalState.GlobalStateRemoteNode == "" { From 60c82068945e4201ddae87c0c1b5d42844f17f29 Mon Sep 17 00:00:00 2001 From: Lazy Nina Date: Wed, 25 Jun 2025 15:02:51 -0400 Subject: [PATCH 8/8] remove todo --- routes/nft.go | 1 - 1 file changed, 1 deletion(-) diff --git a/routes/nft.go b/routes/nft.go index 628b95c4..9e502064 100644 --- a/routes/nft.go +++ b/routes/nft.go @@ -1004,7 +1004,6 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) readerPKID = readerPKIDEntry.PKID } - // TODO: move filtering of IsForSale and IsPending to GetNFTEntriesForPKID nftEntries, lastSeenKey := utxoView.GetNFTEntriesForPKID( pkid.PKID, limit, lastKeyBytes, requestData.IsForSale, requestData.IsPending)