diff --git a/routes/global_state.go b/routes/global_state.go index 3f761b3b..713d33a8 100644 --- a/routes/global_state.go +++ b/routes/global_state.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/deso-protocol/core/lib" - "github.com/dgraph-io/badger/v3" "github.com/nyaruka/phonenumbers" "github.com/pkg/errors" @@ -29,6 +28,7 @@ type GlobalState struct { GlobalStateRemoteNode string GlobalStateRemoteSecret string GlobalStateDB *badger.DB + SharedClient *http.Client } // GlobalStateRoutes returns the routes for managing global state. @@ -845,13 +845,13 @@ func (gs *GlobalState) PutRemote(ww http.ResponseWriter, rr *http.Request) { } func (gs *GlobalState) CreatePutRequest(key []byte, value []byte) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := PutRemoteRequest{ Key: key, Value: value, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("Put: Could not marshal JSON: %v", err) } @@ -860,7 +860,7 @@ func (gs *GlobalState) CreatePutRequest(key []byte, value []byte) ( gs.GlobalStateRemoteNode, RoutePathGlobalStatePutRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil + return url, jsonData, nil } func (gs *GlobalState) Put(key []byte, value []byte) error { @@ -868,16 +868,19 @@ func (gs *GlobalState) Put(key []byte, value []byte) error { if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is hard to exercise in a test. - url, json_data, err := gs.CreatePutRequest(key, value) + url, jsonData, err := gs.CreatePutRequest(key, value) if err != nil { return fmt.Errorf("Put: Error constructing request: %v", err) } - res, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("Put: Error creating new request: ") + } + req.Header.Set("Content-Type", "application/json") + + res, err := gs.SharedClient.Do(req) if err != nil { - return fmt.Errorf("Put: Error processing remote request") + return fmt.Errorf("Put: Error processing remote request: ") } res.Body.Close() @@ -931,12 +934,12 @@ func (gs *GlobalState) GetRemote(ww http.ResponseWriter, rr *http.Request) { } func (gs *GlobalState) CreateGetRequest(key []byte) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := GetRemoteRequest{ Key: key, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("Get: Could not marshal JSON: %v", err) } @@ -945,7 +948,7 @@ func (gs *GlobalState) CreateGetRequest(key []byte) ( gs.GlobalStateRemoteNode, RoutePathGlobalStateGetRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil + return url, jsonData, nil } func (gs *GlobalState) Get(key []byte) (value []byte, _err error) { @@ -953,18 +956,21 @@ func (gs *GlobalState) Get(key []byte) (value []byte, _err error) { if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is currently annoying to test. - url, json_data, err := gs.CreateGetRequest(key) + url, jsonData, err := gs.CreateGetRequest(key) if err != nil { return nil, fmt.Errorf( "Get: Error constructing request: %v", err) } - resReturned, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) if err != nil { - return nil, fmt.Errorf("Get: Error processing remote request") + return nil, fmt.Errorf("Get: Error creating new request: ") + } + req.Header.Set("Content-Type", "application/json") + + resReturned, err := gs.SharedClient.Do(req) + if err != nil { + return nil, fmt.Errorf("Get: Error processing remote request: ") } res := GetRemoteResponse{} @@ -1035,12 +1041,12 @@ func (gs *GlobalState) BatchGetRemote(ww http.ResponseWriter, rr *http.Request) } func (gs *GlobalState) CreateBatchGetRequest(keyList [][]byte) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := BatchGetRemoteRequest{ KeyList: keyList, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("BatchGet: Could not marshal JSON: %v", err) } @@ -1049,7 +1055,7 @@ func (gs *GlobalState) CreateBatchGetRequest(keyList [][]byte) ( gs.GlobalStateRemoteNode, RoutePathGlobalStateBatchGetRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil + return url, jsonData, nil } func (gs *GlobalState) BatchGet(keyList [][]byte) (value [][]byte, _err error) { @@ -1057,18 +1063,21 @@ func (gs *GlobalState) BatchGet(keyList [][]byte) (value [][]byte, _err error) { if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is currently annoying to test. - url, json_data, err := gs.CreateBatchGetRequest(keyList) + url, jsonData, err := gs.CreateBatchGetRequest(keyList) if err != nil { return nil, fmt.Errorf( "BatchGet: Error constructing request: %v", err) } - resReturned, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, fmt.Errorf("BatchGet: Error creating new request: ") + } + req.Header.Set("Content-Type", "application/json") + + resReturned, err := gs.SharedClient.Do(req) if err != nil { - return nil, fmt.Errorf("BatchGet: Error processing remote request") + return nil, fmt.Errorf("BatchGet: Error processing remote request: ") } res := BatchGetRemoteResponse{} @@ -1113,12 +1122,12 @@ type DeleteRemoteResponse struct { } func (gs *GlobalState) CreateDeleteRequest(key []byte) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := DeleteRemoteRequest{ Key: key, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("Delete: Could not marshal JSON: %v", err) } @@ -1127,7 +1136,7 @@ func (gs *GlobalState) CreateDeleteRequest(key []byte) ( gs.GlobalStateRemoteNode, RoutePathGlobalStateDeleteRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil + return url, jsonData, nil } func (gs *GlobalState) DeleteRemote(ww http.ResponseWriter, rr *http.Request) { @@ -1159,17 +1168,20 @@ func (gs *GlobalState) Delete(key []byte) error { if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is currently annoying to test. - url, json_data, err := gs.CreateDeleteRequest(key) + url, jsonData, err := gs.CreateDeleteRequest(key) if err != nil { return fmt.Errorf("Delete: Could not construct request: %v", err) } - res, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return fmt.Errorf("Delete: Error creating new request: ") + } + req.Header.Set("Content-Type", "application/json") + + res, err := gs.SharedClient.Do(req) if err != nil { - return fmt.Errorf("Delete: Error processing remote request") + return fmt.Errorf("Delete: Error processing remote request: ") } res.Body.Close() @@ -1202,7 +1214,7 @@ type SeekRemoteResponse struct { func (gs *GlobalState) CreateSeekRequest(startPrefix []byte, validForPrefix []byte, maxKeyLen int, numToFetch int, reverse bool, fetchValues bool) ( - _url string, _json_data []byte, _err error) { + _url string, _jsonData []byte, _err error) { req := SeekRemoteRequest{ StartPrefix: startPrefix, @@ -1212,7 +1224,7 @@ func (gs *GlobalState) CreateSeekRequest(startPrefix []byte, validForPrefix []by Reverse: reverse, FetchValues: fetchValues, } - json_data, err := json.Marshal(req) + jsonData, err := json.Marshal(req) if err != nil { return "", nil, fmt.Errorf("Seek: Could not marshal JSON: %v", err) } @@ -1221,7 +1233,7 @@ func (gs *GlobalState) CreateSeekRequest(startPrefix []byte, validForPrefix []by gs.GlobalStateRemoteNode, RoutePathGlobalStateSeekRemote, GlobalStateSharedSecretParam, gs.GlobalStateRemoteSecret) - return url, json_data, nil + return url, jsonData, nil } func (gs *GlobalState) GlobalStateSeekRemote(ww http.ResponseWriter, rr *http.Request) { @@ -1266,7 +1278,7 @@ func (gs *GlobalState) Seek(startPrefix []byte, validForPrefix []byte, // If we have a remote node then use that node to fulfill this request. if gs.GlobalStateRemoteNode != "" { // TODO: This codepath is currently annoying to test. - url, json_data, err := gs.CreateSeekRequest( + url, jsonData, err := gs.CreateSeekRequest( startPrefix, validForPrefix, maxKeyLen, @@ -1278,12 +1290,15 @@ func (gs *GlobalState) Seek(startPrefix []byte, validForPrefix []byte, "Seek: Error constructing request: %v", err) } - resReturned, err := http.Post( - url, - "application/json", /*contentType*/ - bytes.NewBuffer(json_data)) + req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonData)) + if err != nil { + return nil, nil, fmt.Errorf("Seek: Error creating new request: ") + } + req.Header.Set("Content-Type", "application/json") + + resReturned, err := gs.SharedClient.Do(req) if err != nil { - return nil, nil, fmt.Errorf("Seek: Error processing remote request") + return nil, nil, fmt.Errorf("Seek: Error processing remote request: ") } res := SeekRemoteResponse{} diff --git a/routes/nft.go b/routes/nft.go index 07cedbd2..9e502064 100644 --- a/routes/nft.go +++ b/routes/nft.go @@ -930,7 +930,9 @@ type GetNFTsForUserRequest struct { ReaderPublicKeyBase58Check string `safeForLogging:"true"` IsForSale *bool `safeForLogging:"true"` // Ignored if IsForSale is provided - IsPending *bool `safeForLogging:"true"` + IsPending *bool `safeForLogging:"true"` + LastKeyHex string `safeForLogging:"true"` + Limit int `safeForLogging:"true"` } type NFTEntryAndPostEntryResponse struct { @@ -939,7 +941,8 @@ type NFTEntryAndPostEntryResponse struct { } type GetNFTsForUserResponse struct { - NFTsMap map[string]*NFTEntryAndPostEntryResponse + NFTsMap map[string]*NFTEntryAndPostEntryResponse + LastKeyHex string } func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) { @@ -969,6 +972,20 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) } } + limit := requestData.Limit + if limit <= 0 || limit > 100 { + limit = 100 + } + + lastKeyBytes := []byte{} + if requestData.LastKeyHex != "" { + lastKeyBytes, err = hex.DecodeString(requestData.LastKeyHex) + if err != nil { + _AddBadRequestError(ww, fmt.Sprintf("GetNFTsForUser: Problem decoding LastKeyHex: %v", err)) + return + } + } + // Get the NFT bid so we can do a more hardcore validation of the request data. utxoView, err := fes.backendServer.GetMempool().GetAugmentedUniversalView() if err != nil { @@ -987,7 +1004,8 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) readerPKID = readerPKIDEntry.PKID } - nftEntries := utxoView.GetNFTEntriesForPKID(pkid.PKID) + nftEntries, lastSeenKey := utxoView.GetNFTEntriesForPKID( + pkid.PKID, limit, lastKeyBytes, requestData.IsForSale, requestData.IsPending) filteredNFTEntries := []*lib.NFTEntry{} if requestData.IsForSale != nil { @@ -1041,6 +1059,10 @@ func (fes *APIServer) GetNFTsForUser(ww http.ResponseWriter, req *http.Request) fes._nftEntryToResponse(nftEntry, nil, utxoView, true, readerPKID)) } + if len(lastSeenKey) > 0 { + res.LastKeyHex = hex.EncodeToString(lastSeenKey) + } + if err = json.NewEncoder(ww).Encode(res); err != nil { _AddInternalServerError(ww, fmt.Sprintf("GetNFTsForUser: Problem serializing object to JSON: %v", err)) return diff --git a/routes/server.go b/routes/server.go index a9734fa5..0034eb19 100644 --- a/routes/server.go +++ b/routes/server.go @@ -534,6 +534,13 @@ func NewAPIServer( GlobalStateRemoteSecret: config.GlobalStateRemoteSecret, GlobalStateRemoteNode: config.GlobalStateRemoteNode, GlobalStateDB: globalStateDB, + SharedClient: &http.Client{ + Transport: &http.Transport{ + MaxIdleConns: 100, + MaxIdleConnsPerHost: 100, + IdleConnTimeout: 90 * time.Second, + }, + }, } if globalStateDB == nil && globalState.GlobalStateRemoteNode == "" { diff --git a/routes/user.go b/routes/user.go index a62f23d2..9e15d030 100644 --- a/routes/user.go +++ b/routes/user.go @@ -2351,7 +2351,6 @@ func (fes *APIServer) GetNotifications(ww http.ResponseWriter, req *http.Request LastSeenIndex: lastSeenIndex, } if err = json.NewEncoder(ww).Encode(res); err != nil { - fmt.Printf("%#v\n", res) _AddBadRequestError(ww, fmt.Sprintf( "GetNotifications: Problem encoding response as JSON: %v", err)) return