diff --git a/decoder/pokestop.go b/decoder/pokestop.go index ee7e966c..a2e4be8f 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1019,6 +1019,10 @@ func getFortIdFromContest(id string) string { } func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDetails, request *pogo.GetPokemonSizeLeaderboardEntryProto, contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) string { + if request == nil { + return "Request is not available" + } + fortId := getFortIdFromContest(request.GetContestId()) pokestopMutex, _ := pokestopStripedMutex.GetLock(fortId) diff --git a/decoder/scanarea.go b/decoder/scanarea.go index 9e7831ae..67959deb 100644 --- a/decoder/scanarea.go +++ b/decoder/scanarea.go @@ -20,14 +20,14 @@ type ScanParameters struct { ProactiveIVSwitchingToDB bool } -func FindScanConfiguration(scanContext string, lat, lon float64) ScanParameters { +func FindScanConfiguration(scanContext string, location geo.Location) ScanParameters { var areas []geo.AreaName areaLookedUp := false for _, rule := range config.Config.ScanRules { if len(rule.AreaNames) > 0 { if !areaLookedUp { - areas = MatchStatsGeofence(lat, lon) + areas = MatchStatsGeofence(location.Latitude, location.Longitude) areaLookedUp = true } if !geo.AreaMatchWithWildcards(areas, rule.AreaNames) { diff --git a/deviceList.go b/deviceList.go deleted file mode 100644 index 051142ae..00000000 --- a/deviceList.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "golbat/config" - "time" - - "github.com/jellydator/ttlcache/v3" -) - -type DeviceLocation struct { - Latitude float64 - Longitude float64 - LastUpdate int64 - ScanContext string -} - -var deviceLocation *ttlcache.Cache[string, DeviceLocation] - -func InitDeviceCache() { - deviceLocation = ttlcache.New[string, DeviceLocation]( - ttlcache.WithTTL[string, DeviceLocation](time.Hour * time.Duration(config.Config.Cleanup.DeviceHours)), - ) - go deviceLocation.Start() -} - -func UpdateDeviceLocation(deviceId string, lat, lon float64, scanContext string) { - deviceLocation.Set(deviceId, DeviceLocation{ - Latitude: lat, - Longitude: lon, - LastUpdate: time.Now().Unix(), - ScanContext: scanContext, - }, time.Hour*time.Duration(config.Config.Cleanup.DeviceHours)) -} - -type ApiDeviceLocation struct { - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - LastUpdate int64 `json:"last_update"` - ScanContext string `json:"scan_context"` -} - -func GetAllDevices() map[string]ApiDeviceLocation { - locations := map[string]ApiDeviceLocation{} - for _, key := range deviceLocation.Items() { - deviceLocation := key.Value() - locations[key.Key()] = ApiDeviceLocation{ - Latitude: deviceLocation.Latitude, - Longitude: deviceLocation.Longitude, - LastUpdate: deviceLocation.LastUpdate, - ScanContext: deviceLocation.ScanContext, - } - } - return locations -} diff --git a/device_tracker/device_tracker.go b/device_tracker/device_tracker.go new file mode 100644 index 00000000..01fb8bc8 --- /dev/null +++ b/device_tracker/device_tracker.go @@ -0,0 +1,62 @@ +package device_tracker + +import ( + "context" + "golbat/geo" + "time" + + "github.com/jellydator/ttlcache/v3" +) + +type DeviceLocation struct { + Latitude float64 + Longitude float64 + LastUpdate int64 + ScanContext string +} + +type DeviceTracker struct { + maxDeviceTTL time.Duration + deviceLocation *ttlcache.Cache[string, DeviceLocation] +} + +func (tracker *DeviceTracker) UpdateDeviceLocation(deviceId string, location geo.Location, scanContext string) { + if location.IsZero() || deviceId == "" { + return + } + tracker.deviceLocation.Set(deviceId, DeviceLocation{ + Latitude: location.Latitude, + Longitude: location.Longitude, + LastUpdate: time.Now().Unix(), + ScanContext: scanContext, + }, tracker.maxDeviceTTL) +} + +func (tracker *DeviceTracker) IterateDevices(yield func(string, DeviceLocation) bool) { + for _, key := range tracker.deviceLocation.Items() { + if !yield(key.Key(), key.Value()) { + return + } + } +} + +func (tracker *DeviceTracker) Run(ctx context.Context) { + ctx, cancelFn := context.WithCancel(ctx) + defer cancelFn() + go func() { + defer tracker.deviceLocation.Stop() + <-ctx.Done() + }() + tracker.deviceLocation.Start() +} + +func NewDeviceTracker(maxDeviceTTLHours int) *DeviceTracker { + maxDeviceTTL := time.Hour * time.Duration(maxDeviceTTLHours) + tracker := &DeviceTracker{ + maxDeviceTTL: maxDeviceTTL, + deviceLocation: ttlcache.New[string, DeviceLocation]( + ttlcache.WithTTL[string, DeviceLocation](maxDeviceTTL), + ), + } + return tracker +} diff --git a/geo/location.go b/geo/location.go index d81de99d..036d4263 100644 --- a/geo/location.go +++ b/geo/location.go @@ -7,8 +7,12 @@ import ( ) type Location struct { - Latitude float64 - Longitude float64 + Latitude float64 `json:"lat"` + Longitude float64 `json:"lon"` +} + +func (l Location) IsZero() bool { + return l.Latitude == 0 && l.Longitude == 0 } type Bbox struct { diff --git a/grpc_server_raw.go b/grpc_server_raw.go index 5f1e597b..ef277ff2 100644 --- a/grpc_server_raw.go +++ b/grpc_server_raw.go @@ -2,10 +2,10 @@ package main import ( "context" - "time" "golbat/config" pb "golbat/grpc" + "golbat/raw_decoder/grpc_raw_decoder" _ "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor "google.golang.org/grpc/metadata" @@ -14,10 +14,11 @@ import ( // server is used to implement helloworld.GreeterServer. type grpcRawServer struct { pb.UnimplementedRawProtoServer + + rawDecoder *grpc_raw_decoder.GRPCRawDecoder } func (s *grpcRawServer) SubmitRawProto(ctx context.Context, in *pb.RawProtoRequest) (*pb.RawProtoResponse, error) { - dataReceivedTimestamp := time.Now().UnixMilli() // Check for authorisation if config.Config.RawBearer != "" { md, _ := metadata.FromIncomingContext(ctx) @@ -26,64 +27,6 @@ func (s *grpcRawServer) SubmitRawProto(ctx context.Context, in *pb.RawProtoReque return &pb.RawProtoResponse{Message: "Incorrect authorisation received"}, nil } } - - uuid := in.DeviceId - account := in.Username - level := int(in.TrainerLevel) - scanContext := "" - if in.ScanContext != nil { - scanContext = *in.ScanContext - } - - if in.Timestamp > 0 { - dataReceivedTimestamp = in.Timestamp - } - - latTarget, lonTarget := float64(in.LatTarget), float64(in.LonTarget) - globalHaveAr := in.HaveAr - var protoData []ProtoData - - for _, v := range in.Contents { - inboundRawData := ProtoData{ - Method: int(v.Method), - Account: account, - Level: level, - ScanContext: scanContext, - Lat: latTarget, - Lon: lonTarget, - Data: v.ResponsePayload, - Request: v.RequestPayload, - Uuid: uuid, - HaveAr: func() *bool { - if v.HaveAr != nil { - return v.HaveAr - } - return globalHaveAr - }(), - TimestampMs: dataReceivedTimestamp, - } - - protoData = append(protoData, inboundRawData) - } - - // Process each proto in a packet in sequence, but in a go-routine - go func() { - timeout := 5 * time.Second - if config.Config.Tuning.ExtendedTimeout { - timeout = 30 * time.Second - } - - for _, entry := range protoData { - // provide independent cancellation contexts for each proto decode - ctx, cancel := context.WithTimeout(context.Background(), timeout) - decode(ctx, entry.Method, &entry) - cancel() - } - }() - - if latTarget != 0 && lonTarget != 0 && uuid != "" { - UpdateDeviceLocation(uuid, latTarget, lonTarget, scanContext) - } - + s.rawDecoder.DecodeRaw(ctx, in) return &pb.RawProtoResponse{Message: "Processed"}, nil } diff --git a/http_handler/device.go b/http_handler/device.go new file mode 100644 index 00000000..83136250 --- /dev/null +++ b/http_handler/device.go @@ -0,0 +1,22 @@ +package http_handler + +import ( + "net/http" + + "github.com/gin-gonic/gin" +) + +type DeviceLocation struct { + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + LastUpdate int64 `json:"last_update"` + ScanContext string `json:"scan_context"` +} + +func (h *HTTPHandler) GetAllDevices(ginContext *gin.Context) { + devices := map[string]DeviceLocation{} + for deviceId, location := range h.deviceTracker.IterateDevices { + devices[deviceId] = DeviceLocation(location) + } + ginContext.JSON(http.StatusOK, gin.H{"devices": devices}) +} diff --git a/http_handler/http_handler.go b/http_handler/http_handler.go new file mode 100644 index 00000000..ba3a0bc5 --- /dev/null +++ b/http_handler/http_handler.go @@ -0,0 +1,24 @@ +package http_handler + +import ( + "golbat/db" + "golbat/device_tracker" + "golbat/raw_decoder/http_raw_decoder" + "golbat/stats_collector" +) + +type HTTPHandler struct { + rawDecoder *http_raw_decoder.HTTPRawDecoder + dbDetails db.DbDetails + statsCollector stats_collector.StatsCollector + deviceTracker *device_tracker.DeviceTracker +} + +func NewHTTPHandler(rawDecoder *http_raw_decoder.HTTPRawDecoder, dbDetails db.DbDetails, statsCollector stats_collector.StatsCollector, deviceTracker *device_tracker.DeviceTracker) *HTTPHandler { + return &HTTPHandler{ + rawDecoder: rawDecoder, + dbDetails: dbDetails, + statsCollector: statsCollector, + deviceTracker: deviceTracker, + } +} diff --git a/http_handler/raw.go b/http_handler/raw.go new file mode 100644 index 00000000..85ebd78e --- /dev/null +++ b/http_handler/raw.go @@ -0,0 +1,58 @@ +package http_handler + +import ( + "context" + "io" + "net/http" + "time" + + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + + "golbat/config" +) + +func (h *HTTPHandler) Raw(c *gin.Context) { + w := c.Writer + r := c.Request + + statsCollector := h.statsCollector + + requestReceivedMs := time.Now().UnixMilli() + + authHeader := r.Header.Get("Authorization") + if config.Config.RawBearer != "" { + if authHeader != "Bearer "+config.Config.RawBearer { + statsCollector.IncRawRequests("error", "auth") + log.Errorf("Raw: Incorrect authorisation received (%s)", authHeader) + return + } + } + + body, err := io.ReadAll(io.LimitReader(r.Body, 5*1048576)) + if err != nil { + statsCollector.IncRawRequests("error", "io_error") + log.Errorf("Raw: Error (1) during HTTP receive %s", err) + return + } + + if err := r.Body.Close(); err != nil { + statsCollector.IncRawRequests("error", "io_close_error") + log.Errorf("Raw: Error (2) during HTTP receive %s", err) + return + } + + ctx := context.Background() + if err := h.rawDecoder.DecodeRaw(ctx, r.Header, body, requestReceivedMs); err != nil { + statsCollector.IncRawRequests("error", "decode") + userAgent := r.Header.Get("User-Agent") + log.Infof("Raw: Data could not be decoded. From User agent %s - Received data %s, err: %s", userAgent, body, err) + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(http.StatusUnprocessableEntity) + return + } + + statsCollector.IncRawRequests("ok", "") + w.Header().Set("Content-Type", "application/json; charset=UTF-8") + w.WriteHeader(http.StatusCreated) +} diff --git a/main.go b/main.go index e5588956..9c8e4157 100644 --- a/main.go +++ b/main.go @@ -6,7 +6,6 @@ import ( "net" "net/http" "net/http/pprof" - "strings" "sync" "time" _ "time/tzdata" @@ -17,17 +16,21 @@ import ( _ "github.com/golang-migrate/migrate/v4/database/mysql" _ "github.com/golang-migrate/migrate/v4/source/file" "github.com/jmoiron/sqlx" + "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus" ginlogrus "github.com/toorop/gin-logrus" "google.golang.org/grpc" - "google.golang.org/protobuf/proto" "golbat/config" db2 "golbat/db" "golbat/decoder" + "golbat/device_tracker" "golbat/external" - pb "golbat/grpc" - "golbat/pogo" + golbat_grpc "golbat/grpc" + "golbat/http_handler" + "golbat/proto_decoder" + "golbat/raw_decoder/grpc_raw_decoder" + "golbat/raw_decoder/http_raw_decoder" "golbat/stats_collector" "golbat/webhooks" ) @@ -170,11 +173,19 @@ func main() { } //} + deviceTracker := device_tracker.NewDeviceTracker(cfg.Cleanup.DeviceHours) + wg.Add(1) + go func() { + defer cancelFn() + defer wg.Done() + deviceTracker.Run(ctx) + }() + // Create the web server. gin.SetMode(gin.ReleaseMode) r := gin.New() if cfg.Logging.Debug { - r.Use(ginlogrus.Logger(log.StandardLogger())) + r.Use(ginlogrus.Logger(logrus.StandardLogger())) } else { r.Use(gin.Recovery()) } @@ -185,6 +196,21 @@ func main() { decoder.SetStatsCollector(statsCollector) db2.SetStatsCollector(statsCollector) + protoDecoder := func() *proto_decoder.ProtoDecoder { + timeout := 5 * time.Second + if cfg.Tuning.ExtendedTimeout { + timeout = 30 * time.Second + } + return proto_decoder.NewProtoDecoder(timeout, dbDetails, statsCollector, deviceTracker) + }() + + httpHandler := http_handler.NewHTTPHandler( + http_raw_decoder.NewHTTPRawDecoder(protoDecoder), + dbDetails, + statsCollector, + deviceTracker, + ) + // collect live stats when prometheus and liveStats are enabled if cfg.Prometheus.Enabled && cfg.Prometheus.LiveStats { go db2.PromLiveStatsUpdater(dbDetails, cfg.Prometheus.LiveStatsSleep) @@ -210,7 +236,6 @@ func main() { _ = decoder.WatchMasterFileData() } decoder.LoadStatsGeofences() - InitDeviceCache() wg.Add(1) go func() { @@ -267,18 +292,17 @@ func main() { go decoder.LoadAllGyms(dbDetails) } - // Start the GRPC receiver - if cfg.GrpcPort > 0 { log.Infof("Starting GRPC server on port %d", cfg.GrpcPort) + grpcRawDecoder := grpc_raw_decoder.NewGRPCRawDecoder(protoDecoder) go func() { lis, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.GrpcPort)) if err != nil { log.Fatalf("failed to listen: %v", err) } s := grpc.NewServer() - pb.RegisterRawProtoServer(s, &grpcRawServer{}) - pb.RegisterPokemonServer(s, &grpcPokemonServer{}) + golbat_grpc.RegisterRawProtoServer(s, &grpcRawServer{rawDecoder: grpcRawDecoder}) + golbat_grpc.RegisterPokemonServer(s, &grpcPokemonServer{}) log.Printf("grpc server listening at %v", lis.Addr()) if err := s.Serve(lis); err != nil { log.Fatalf("failed to serve: %v", err) @@ -286,7 +310,7 @@ func main() { }() } - r.POST("/raw", Raw) + r.POST("/raw", httpHandler.Raw) r.GET("/health", GetHealth) apiGroup := r.Group("/api", AuthRequired()) @@ -310,7 +334,7 @@ func main() { apiGroup.GET("/tappable/id/:tappable_id", GetTappable) - apiGroup.GET("/devices/all", GetDevices) + apiGroup.GET("/devices/all", httpHandler.GetAllDevices) debugGroup := r.Group("/debug") @@ -386,725 +410,3 @@ func main() { log.Info("Golbat exiting!") } - -func decode(ctx context.Context, method int, protoData *ProtoData) { - getMethodName := func(method int, trimString bool) string { - if val, ok := pogo.Method_name[int32(method)]; ok { - if trimString && strings.HasPrefix(val, "METHOD_") { - return strings.TrimPrefix(val, "METHOD_") - } - return val - } - return fmt.Sprintf("#%d", method) - } - - if method != int(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION) && protoData.Level < 30 { - statsCollector.IncDecodeMethods("error", "low_level", getMethodName(method, true)) - log.Debugf("Insufficient Level %d Did not process hook type %s", protoData.Level, pogo.Method(method)) - return - } - - processed := false - ignore := false - start := time.Now() - result := "" - - switch pogo.Method(method) { - case pogo.Method_METHOD_START_INCIDENT: - result = decodeStartIncident(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_INVASION_OPEN_COMBAT_SESSION: - if protoData.Request != nil { - result = decodeOpenInvasion(ctx, protoData.Request, protoData.Data) - processed = true - } - case pogo.Method_METHOD_FORT_DETAILS: - result = decodeFortDetails(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_GET_MAP_OBJECTS: - result = decodeGMO(ctx, protoData, getScanParameters(protoData)) - processed = true - case pogo.Method_METHOD_GYM_GET_INFO: - result = decodeGetGymInfo(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_ENCOUNTER: - if getScanParameters(protoData).ProcessPokemon { - result = decodeEncounter(ctx, protoData.Data, protoData.Account, protoData.TimestampMs) - } - processed = true - case pogo.Method_METHOD_DISK_ENCOUNTER: - result = decodeDiskEncounter(ctx, protoData.Data, protoData.Account) - processed = true - case pogo.Method_METHOD_FORT_SEARCH: - result = decodeQuest(ctx, protoData.Data, protoData.HaveAr) - processed = true - case pogo.Method_METHOD_GET_PLAYER: - ignore = true - case pogo.Method_METHOD_GET_HOLOHOLO_INVENTORY: - ignore = true - case pogo.Method_METHOD_CREATE_COMBAT_CHALLENGE: - ignore = true - case pogo.Method(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION): - if protoData.Request != nil { - result = decodeSocialActionWithRequest(protoData.Request, protoData.Data) - processed = true - } - case pogo.Method_METHOD_GET_MAP_FORTS: - result = decodeGetMapForts(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_GET_ROUTES: - result = decodeGetRoutes(protoData.Data) - processed = true - case pogo.Method_METHOD_GET_CONTEST_DATA: - if getScanParameters(protoData).ProcessPokestops { - // Request helps, but can be decoded without it - result = decodeGetContestData(ctx, protoData.Request, protoData.Data) - } - processed = true - case pogo.Method_METHOD_GET_POKEMON_SIZE_CONTEST_ENTRY: - // Request is essential to decode this - if protoData.Request != nil { - if getScanParameters(protoData).ProcessPokestops { - result = decodeGetPokemonSizeContestEntry(ctx, protoData.Request, protoData.Data) - } - processed = true - } - case pogo.Method_METHOD_GET_STATION_DETAILS: - if getScanParameters(protoData).ProcessStations { - // Request is essential to decode this - result = decodeGetStationDetails(ctx, protoData.Request, protoData.Data) - } - processed = true - case pogo.Method_METHOD_PROCESS_TAPPABLE: - if getScanParameters(protoData).ProcessTappables { - // Request is essential to decode this - result = decodeTappable(ctx, protoData.Request, protoData.Data, protoData.Account, protoData.TimestampMs) - } - processed = true - case pogo.Method_METHOD_GET_EVENT_RSVPS: - if getScanParameters(protoData).ProcessGyms { - result = decodeGetEventRsvp(ctx, protoData.Request, protoData.Data) - } - processed = true - case pogo.Method_METHOD_GET_EVENT_RSVP_COUNT: - if getScanParameters(protoData).ProcessGyms { - result = decodeGetEventRsvpCount(ctx, protoData.Data) - } - processed = true - default: - log.Debugf("Did not know hook type %s", pogo.Method(method)) - } - if !ignore { - elapsed := time.Since(start) - if processed == true { - statsCollector.IncDecodeMethods("ok", "", getMethodName(method, true)) - log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, result) - } else { - log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, "**Did not process**") - statsCollector.IncDecodeMethods("unprocessed", "", getMethodName(method, true)) - } - } -} - -func getScanParameters(protoData *ProtoData) decoder.ScanParameters { - return decoder.FindScanConfiguration(protoData.ScanContext, protoData.Lat, protoData.Lon) -} - -func decodeQuest(ctx context.Context, sDec []byte, haveAr *bool) string { - if haveAr == nil { - statsCollector.IncDecodeQuest("error", "missing_ar_info") - log.Infoln("Cannot determine AR quest - ignoring") - // We should either assume AR quest, or trace inventory like RDM probably - return "No AR quest info" - } - decodedQuest := &pogo.FortSearchOutProto{} - if err := proto.Unmarshal(sDec, decodedQuest); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeQuest("error", "parse") - return "Parse failure" - } - - if decodedQuest.Result != pogo.FortSearchOutProto_SUCCESS { - statsCollector.IncDecodeQuest("error", "non_success") - res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedQuest.Result, - pogo.FortSearchOutProto_Result_name[int32(decodedQuest.Result)]) - return res - } - - return decoder.UpdatePokestopWithQuest(ctx, dbDetails, decodedQuest, *haveAr) - -} - -func decodeSocialActionWithRequest(request []byte, payload []byte) string { - var proxyRequestProto pogo.ProxyRequestProto - - if err := proto.Unmarshal(request, &proxyRequestProto); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeSocialActionWithRequest("error", "request_parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - var proxyResponseProto pogo.ProxyResponseProto - - if err := proto.Unmarshal(payload, &proxyResponseProto); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeSocialActionWithRequest("error", "response_parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED && proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED_AND_REASSIGNED { - statsCollector.IncDecodeSocialActionWithRequest("error", "non_success") - return fmt.Sprintf("unsuccessful proxyResponseProto response %d %s", int(proxyResponseProto.Status), proxyResponseProto.Status) - } - - switch pogo.InternalSocialAction(proxyRequestProto.GetAction()) { - case pogo.InternalSocialAction_SOCIAL_ACTION_LIST_FRIEND_STATUS: - statsCollector.IncDecodeSocialActionWithRequest("ok", "list_friend_status") - return decodeGetFriendDetails(proxyResponseProto.Payload) - case pogo.InternalSocialAction_SOCIAL_ACTION_SEARCH_PLAYER: - statsCollector.IncDecodeSocialActionWithRequest("ok", "search_player") - return decodeSearchPlayer(&proxyRequestProto, proxyResponseProto.Payload) - - } - - statsCollector.IncDecodeSocialActionWithRequest("ok", "unknown") - return fmt.Sprintf("Did not process %s", pogo.InternalSocialAction(proxyRequestProto.GetAction()).String()) -} - -func decodeGetFriendDetails(payload []byte) string { - var getFriendDetailsOutProto pogo.InternalGetFriendDetailsOutProto - getFriendDetailsError := proto.Unmarshal(payload, &getFriendDetailsOutProto) - - if getFriendDetailsError != nil { - statsCollector.IncDecodeGetFriendDetails("error", "parse") - log.Errorf("Failed to parse %s", getFriendDetailsError) - return fmt.Sprintf("Failed to parse %s", getFriendDetailsError) - } - - if getFriendDetailsOutProto.GetResult() != pogo.InternalGetFriendDetailsOutProto_SUCCESS || getFriendDetailsOutProto.GetFriend() == nil { - statsCollector.IncDecodeGetFriendDetails("error", "non_success") - return fmt.Sprintf("unsuccessful get friends details") - } - - failures := 0 - - for _, friend := range getFriendDetailsOutProto.GetFriend() { - player := friend.GetPlayer() - - updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, "", player.GetPlayerId()) - if updatePlayerError != nil { - failures++ - } - } - - statsCollector.IncDecodeGetFriendDetails("ok", "") - return fmt.Sprintf("%d players decoded on %d", len(getFriendDetailsOutProto.GetFriend())-failures, len(getFriendDetailsOutProto.GetFriend())) -} - -func decodeSearchPlayer(proxyRequestProto *pogo.ProxyRequestProto, payload []byte) string { - var searchPlayerOutProto pogo.InternalSearchPlayerOutProto - searchPlayerOutError := proto.Unmarshal(payload, &searchPlayerOutProto) - - if searchPlayerOutError != nil { - log.Errorf("Failed to parse %s", searchPlayerOutError) - statsCollector.IncDecodeSearchPlayer("error", "parse") - return fmt.Sprintf("Failed to parse %s", searchPlayerOutError) - } - - if searchPlayerOutProto.GetResult() != pogo.InternalSearchPlayerOutProto_SUCCESS || searchPlayerOutProto.GetPlayer() == nil { - statsCollector.IncDecodeSearchPlayer("error", "non_success") - return fmt.Sprintf("unsuccessful search player response") - } - - var searchPlayerProto pogo.InternalSearchPlayerProto - searchPlayerError := proto.Unmarshal(proxyRequestProto.GetPayload(), &searchPlayerProto) - - if searchPlayerError != nil || searchPlayerProto.GetFriendCode() == "" { - statsCollector.IncDecodeSearchPlayer("error", "parse") - return fmt.Sprintf("Failed to parse %s", searchPlayerError) - } - - player := searchPlayerOutProto.GetPlayer() - updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, searchPlayerProto.GetFriendCode(), "") - if updatePlayerError != nil { - statsCollector.IncDecodeSearchPlayer("error", "update") - return fmt.Sprintf("Failed update player %s", updatePlayerError) - } - - statsCollector.IncDecodeSearchPlayer("ok", "") - return fmt.Sprintf("1 player decoded from SearchPlayerProto") -} - -func decodeFortDetails(ctx context.Context, sDec []byte) string { - decodedFort := &pogo.FortDetailsOutProto{} - if err := proto.Unmarshal(sDec, decodedFort); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeFortDetails("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - switch decodedFort.FortType { - case pogo.FortType_CHECKPOINT: - statsCollector.IncDecodeFortDetails("ok", "pokestop") - return decoder.UpdatePokestopRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) - case pogo.FortType_GYM: - statsCollector.IncDecodeFortDetails("ok", "gym") - return decoder.UpdateGymRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) - } - - statsCollector.IncDecodeFortDetails("ok", "unknown") - return "Unknown fort type" -} - -func decodeGetMapForts(ctx context.Context, sDec []byte) string { - decodedMapForts := &pogo.GetMapFortsOutProto{} - if err := proto.Unmarshal(sDec, decodedMapForts); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeGetMapForts("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedMapForts.Status != pogo.GetMapFortsOutProto_SUCCESS { - statsCollector.IncDecodeGetMapForts("error", "non_success") - res := fmt.Sprintf(`GetMapFortsOutProto: Ignored non-success value %d:%s`, decodedMapForts.Status, - pogo.GetMapFortsOutProto_Status_name[int32(decodedMapForts.Status)]) - return res - } - - statsCollector.IncDecodeGetMapForts("ok", "") - var outputString string - processedForts := 0 - - for _, fort := range decodedMapForts.Fort { - status, output := decoder.UpdateFortRecordWithGetMapFortsOutProto(ctx, dbDetails, fort) - if status { - processedForts += 1 - outputString += output + ", " - } - } - - if processedForts > 0 { - return fmt.Sprintf("Updated %d forts: %s", processedForts, outputString) - } - return "No forts updated" -} - -func decodeGetRoutes(payload []byte) string { - getRoutesOutProto := &pogo.GetRoutesOutProto{} - if err := proto.Unmarshal(payload, getRoutesOutProto); err != nil { - return fmt.Sprintf("failed to decode GetRoutesOutProto %s", err) - } - - if getRoutesOutProto.Status != pogo.GetRoutesOutProto_SUCCESS { - return fmt.Sprintf("GetRoutesOutProto: Ignored non-success value %d:%s", getRoutesOutProto.Status, getRoutesOutProto.Status.String()) - } - - decodeSuccesses := map[string]bool{} - decodeErrors := map[string]bool{} - - for _, routeMapCell := range getRoutesOutProto.GetRouteMapCell() { - for _, route := range routeMapCell.GetRoute() { - //TODO we need to check the repeated field, for now access last element - routeSubmissionStatus := route.RouteSubmissionStatus[len(route.RouteSubmissionStatus)-1] - if routeSubmissionStatus != nil && routeSubmissionStatus.Status != pogo.RouteSubmissionStatus_PUBLISHED { - log.Warnf("Non published Route found in GetRoutesOutProto, status: %s", routeSubmissionStatus.String()) - continue - } - decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(dbDetails, route) - if decodeError != nil { - if decodeErrors[route.Id] != true { - decodeErrors[route.Id] = true - } - log.Errorf("Failed to decode route %s", decodeError) - } else if decodeSuccesses[route.Id] != true { - decodeSuccesses[route.Id] = true - } - } - } - - return fmt.Sprintf( - "Decoded %d routes, failed to decode %d routes, from %d cells", - len(decodeSuccesses), - len(decodeErrors), - len(getRoutesOutProto.GetRouteMapCell()), - ) -} - -func decodeGetGymInfo(ctx context.Context, sDec []byte) string { - decodedGymInfo := &pogo.GymGetInfoOutProto{} - if err := proto.Unmarshal(sDec, decodedGymInfo); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeGetGymInfo("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedGymInfo.Result != pogo.GymGetInfoOutProto_SUCCESS { - statsCollector.IncDecodeGetGymInfo("error", "non_success") - res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedGymInfo.Result, - pogo.GymGetInfoOutProto_Result_name[int32(decodedGymInfo.Result)]) - return res - } - - statsCollector.IncDecodeGetGymInfo("ok", "") - return decoder.UpdateGymRecordWithGymInfoProto(ctx, dbDetails, decodedGymInfo) -} - -func decodeEncounter(ctx context.Context, sDec []byte, username string, timestampMs int64) string { - decodedEncounterInfo := &pogo.EncounterOutProto{} - if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeEncounter("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedEncounterInfo.Status != pogo.EncounterOutProto_ENCOUNTER_SUCCESS { - statsCollector.IncDecodeEncounter("error", "non_success") - res := fmt.Sprintf(`EncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Status, - pogo.EncounterOutProto_Status_name[int32(decodedEncounterInfo.Status)]) - return res - } - - statsCollector.IncDecodeEncounter("ok", "") - return decoder.UpdatePokemonRecordWithEncounterProto(ctx, dbDetails, decodedEncounterInfo, username, timestampMs) -} - -func decodeDiskEncounter(ctx context.Context, sDec []byte, username string) string { - decodedEncounterInfo := &pogo.DiskEncounterOutProto{} - if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeDiskEncounter("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedEncounterInfo.Result != pogo.DiskEncounterOutProto_SUCCESS { - statsCollector.IncDecodeDiskEncounter("error", "non_success") - res := fmt.Sprintf(`DiskEncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Result, - pogo.DiskEncounterOutProto_Result_name[int32(decodedEncounterInfo.Result)]) - return res - } - - statsCollector.IncDecodeDiskEncounter("ok", "") - return decoder.UpdatePokemonRecordWithDiskEncounterProto(ctx, dbDetails, decodedEncounterInfo, username) -} - -func decodeStartIncident(ctx context.Context, sDec []byte) string { - decodedIncident := &pogo.StartIncidentOutProto{} - if err := proto.Unmarshal(sDec, decodedIncident); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeStartIncident("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedIncident.Status != pogo.StartIncidentOutProto_SUCCESS { - statsCollector.IncDecodeStartIncident("error", "non_success") - res := fmt.Sprintf(`GiovanniOutProto: Ignored non-success value %d:%s`, decodedIncident.Status, - pogo.StartIncidentOutProto_Status_name[int32(decodedIncident.Status)]) - return res - } - - statsCollector.IncDecodeStartIncident("ok", "") - return decoder.ConfirmIncident(ctx, dbDetails, decodedIncident) -} - -func decodeOpenInvasion(ctx context.Context, request []byte, payload []byte) string { - decodeOpenInvasionRequest := &pogo.OpenInvasionCombatSessionProto{} - - if err := proto.Unmarshal(request, decodeOpenInvasionRequest); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeOpenInvasion("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - if decodeOpenInvasionRequest.IncidentLookup == nil { - return "Invalid OpenInvasionCombatSessionProto received" - } - - decodedOpenInvasionResponse := &pogo.OpenInvasionCombatSessionOutProto{} - if err := proto.Unmarshal(payload, decodedOpenInvasionResponse); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeOpenInvasion("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedOpenInvasionResponse.Status != pogo.InvasionStatus_SUCCESS { - statsCollector.IncDecodeOpenInvasion("error", "non_success") - res := fmt.Sprintf(`InvasionLineupOutProto: Ignored non-success value %d:%s`, decodedOpenInvasionResponse.Status, - pogo.InvasionStatus_Status_name[int32(decodedOpenInvasionResponse.Status)]) - return res - } - - statsCollector.IncDecodeOpenInvasion("ok", "") - return decoder.UpdateIncidentLineup(ctx, dbDetails, decodeOpenInvasionRequest, decodedOpenInvasionResponse) -} - -func decodeGMO(ctx context.Context, protoData *ProtoData, scanParameters decoder.ScanParameters) string { - decodedGmo := &pogo.GetMapObjectsOutProto{} - - if err := proto.Unmarshal(protoData.Data, decodedGmo); err != nil { - statsCollector.IncDecodeGMO("error", "parse") - log.Errorf("Failed to parse %s", err) - } - - if decodedGmo.Status != pogo.GetMapObjectsOutProto_SUCCESS { - statsCollector.IncDecodeGMO("error", "non_success") - res := fmt.Sprintf(`GetMapObjectsOutProto: Ignored non-success value %d:%s`, decodedGmo.Status, - pogo.GetMapObjectsOutProto_Status_name[int32(decodedGmo.Status)]) - return res - } - - var newForts []decoder.RawFortData - var newStations []decoder.RawStationData - var newWildPokemon []decoder.RawWildPokemonData - var newNearbyPokemon []decoder.RawNearbyPokemonData - var newMapPokemon []decoder.RawMapPokemonData - var newMapCells []uint64 - var cellsToBeCleaned []uint64 - - // track forts per cell for memory-based cleanup (only if tracker enabled) - cellForts := make(map[uint64]*decoder.CellFortsData) - - if len(decodedGmo.MapCell) == 0 { - return "Skipping GetMapObjectsOutProto: No map cells found" - } - for _, mapCell := range decodedGmo.MapCell { - if isCellNotEmpty(mapCell) { - newMapCells = append(newMapCells, mapCell.S2CellId) - if cellContainsForts(mapCell) { - cellsToBeCleaned = append(cellsToBeCleaned, mapCell.S2CellId) - // initialize cell forts tracking (only if tracker enabled) - cellForts[mapCell.S2CellId] = &decoder.CellFortsData{ - Pokestops: make([]string, 0), - Gyms: make([]string, 0), - Timestamp: mapCell.AsOfTimeMs, - } - } - } - for _, fort := range mapCell.Fort { - newForts = append(newForts, decoder.RawFortData{Cell: mapCell.S2CellId, Data: fort, Timestamp: mapCell.AsOfTimeMs}) - - // track fort by type for memory-based cleanup (only if tracker enabled) - if cf, ok := cellForts[mapCell.S2CellId]; ok { - switch fort.FortType { - case pogo.FortType_GYM: - cf.Gyms = append(cf.Gyms, fort.FortId) - case pogo.FortType_CHECKPOINT: - cf.Pokestops = append(cf.Pokestops, fort.FortId) - } - } - - if fort.ActivePokemon != nil { - newMapPokemon = append(newMapPokemon, decoder.RawMapPokemonData{Cell: mapCell.S2CellId, Data: fort.ActivePokemon, Timestamp: mapCell.AsOfTimeMs}) - } - } - for _, mon := range mapCell.WildPokemon { - newWildPokemon = append(newWildPokemon, decoder.RawWildPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) - } - for _, mon := range mapCell.NearbyPokemon { - newNearbyPokemon = append(newNearbyPokemon, decoder.RawNearbyPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) - } - for _, station := range mapCell.Stations { - newStations = append(newStations, decoder.RawStationData{Cell: mapCell.S2CellId, Data: station}) - } - } - - if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { - decoder.UpdateFortBatch(ctx, dbDetails, scanParameters, newForts) - } - var weatherUpdates []decoder.WeatherUpdate - if scanParameters.ProcessWeather { - weatherUpdates = decoder.UpdateClientWeatherBatch(ctx, dbDetails, decodedGmo.ClientWeather, decodedGmo.MapCell[0].AsOfTimeMs) - } - if scanParameters.ProcessPokemon { - decoder.UpdatePokemonBatch(ctx, dbDetails, scanParameters, newWildPokemon, newNearbyPokemon, newMapPokemon, decodedGmo.ClientWeather, protoData.Account) - if scanParameters.ProcessWeather && scanParameters.ProactiveIVSwitching { - for _, weatherUpdate := range weatherUpdates { - go func(weatherUpdate decoder.WeatherUpdate) { - decoder.ProactiveIVSwitchSem <- true - defer func() { <-decoder.ProactiveIVSwitchSem }() - decoder.ProactiveIVSwitch(ctx, dbDetails, weatherUpdate, scanParameters.ProactiveIVSwitchingToDB, decodedGmo.MapCell[0].AsOfTimeMs/1000) - }(weatherUpdate) - } - } - } - if scanParameters.ProcessStations { - decoder.UpdateStationBatch(ctx, dbDetails, scanParameters, newStations) - } - - if scanParameters.ProcessCells { - decoder.UpdateClientMapS2CellBatch(ctx, dbDetails, newMapCells) - } - - if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - decoder.CheckRemovedForts(ctx, dbDetails, cellsToBeCleaned, cellForts) - }() - } - - newFortsLen := len(newForts) - newStationsLen := len(newStations) - newWildPokemonLen := len(newWildPokemon) - newNearbyPokemonLen := len(newNearbyPokemon) - newMapPokemonLen := len(newMapPokemon) - newClientWeatherLen := len(decodedGmo.ClientWeather) - newMapCellsLen := len(newMapCells) - - statsCollector.IncDecodeGMO("ok", "") - statsCollector.AddDecodeGMOType("fort", float64(newFortsLen)) - statsCollector.AddDecodeGMOType("station", float64(newStationsLen)) - statsCollector.AddDecodeGMOType("wild_pokemon", float64(newWildPokemonLen)) - statsCollector.AddDecodeGMOType("nearby_pokemon", float64(newNearbyPokemonLen)) - statsCollector.AddDecodeGMOType("map_pokemon", float64(newMapPokemonLen)) - statsCollector.AddDecodeGMOType("weather", float64(newClientWeatherLen)) - statsCollector.AddDecodeGMOType("cell", float64(newMapCellsLen)) - - return fmt.Sprintf("%d cells containing %d forts %d stations %d mon %d nearby", newMapCellsLen, newFortsLen, newStationsLen, newWildPokemonLen, newNearbyPokemonLen) -} - -func isCellNotEmpty(mapCell *pogo.ClientMapCellProto) bool { - return len(mapCell.Stations) > 0 || len(mapCell.Fort) > 0 || len(mapCell.WildPokemon) > 0 || len(mapCell.NearbyPokemon) > 0 || len(mapCell.CatchablePokemon) > 0 -} - -func cellContainsForts(mapCell *pogo.ClientMapCellProto) bool { - return len(mapCell.Fort) > 0 -} - -func decodeGetContestData(ctx context.Context, request []byte, data []byte) string { - var decodedContestData pogo.GetContestDataOutProto - if err := proto.Unmarshal(data, &decodedContestData); err != nil { - log.Errorf("Failed to parse GetContestDataOutProto %s", err) - return fmt.Sprintf("Failed to parse GetContestDataOutProto %s", err) - } - - var decodedContestDataRequest pogo.GetContestDataProto - if request != nil { - if err := proto.Unmarshal(request, &decodedContestDataRequest); err != nil { - log.Errorf("Failed to parse GetContestDataProto %s", err) - return fmt.Sprintf("Failed to parse GetContestDataProto %s", err) - } - } - return decoder.UpdatePokestopWithContestData(ctx, dbDetails, &decodedContestDataRequest, &decodedContestData) -} - -func decodeGetPokemonSizeContestEntry(ctx context.Context, request []byte, data []byte) string { - var decodedPokemonSizeContestEntry pogo.GetPokemonSizeLeaderboardEntryOutProto - if err := proto.Unmarshal(data, &decodedPokemonSizeContestEntry); err != nil { - log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - } - - if decodedPokemonSizeContestEntry.Status != pogo.GetPokemonSizeLeaderboardEntryOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetPokemonSizeLeaderboardEntryOutProto non-success status %s", decodedPokemonSizeContestEntry.Status) - } - - var decodedPokemonSizeContestEntryRequest pogo.GetPokemonSizeLeaderboardEntryProto - if request != nil { - if err := proto.Unmarshal(request, &decodedPokemonSizeContestEntryRequest); err != nil { - log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - } - } - - return decoder.UpdatePokestopWithPokemonSizeContestEntry(ctx, dbDetails, &decodedPokemonSizeContestEntryRequest, &decodedPokemonSizeContestEntry) -} - -func decodeGetStationDetails(ctx context.Context, request []byte, data []byte) string { - var decodedGetStationDetails pogo.GetStationedPokemonDetailsOutProto - if err := proto.Unmarshal(data, &decodedGetStationDetails); err != nil { - log.Errorf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) - return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) - } - - var decodedGetStationDetailsRequest pogo.GetStationedPokemonDetailsProto - if request != nil { - if err := proto.Unmarshal(request, &decodedGetStationDetailsRequest); err != nil { - log.Errorf("Failed to parse GetStationedPokemonDetailsProto %s", err) - return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsProto %s", err) - } - } - - if decodedGetStationDetails.Result == pogo.GetStationedPokemonDetailsOutProto_STATION_NOT_FOUND { - // station without stationed pokemon found, therefore we need to reset the columns - return decoder.ResetStationedPokemonWithStationDetailsNotFound(ctx, dbDetails, &decodedGetStationDetailsRequest) - } else if decodedGetStationDetails.Result != pogo.GetStationedPokemonDetailsOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetStationedPokemonDetailsOutProto non-success status %s", decodedGetStationDetails.Result) - } - - return decoder.UpdateStationWithStationDetails(ctx, dbDetails, &decodedGetStationDetailsRequest, &decodedGetStationDetails) -} - -func decodeTappable(ctx context.Context, request, data []byte, username string, timestampMs int64) string { - var tappable pogo.ProcessTappableOutProto - if err := proto.Unmarshal(data, &tappable); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse ProcessTappableOutProto %s", err) - } - - var tappableRequest pogo.ProcessTappableProto - if request != nil { - if err := proto.Unmarshal(request, &tappableRequest); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse ProcessTappableProto %s", err) - } - } - - if tappable.Status != pogo.ProcessTappableOutProto_SUCCESS { - return fmt.Sprintf("Ignored ProcessTappableOutProto non-success status %s", tappable.Status) - } - var result string - if encounter := tappable.GetEncounter(); encounter != nil { - result = decoder.UpdatePokemonRecordWithTappableEncounter(ctx, dbDetails, &tappableRequest, encounter, username, timestampMs) - } - return result + " " + decoder.UpdateTappable(ctx, dbDetails, &tappableRequest, &tappable, timestampMs) -} - -func decodeGetEventRsvp(ctx context.Context, request []byte, data []byte) string { - var rsvp pogo.GetEventRsvpsOutProto - if err := proto.Unmarshal(data, &rsvp); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse GetEventRsvpsOutProto %s", err) - } - - var rsvpRequest pogo.GetEventRsvpsProto - if request != nil { - if err := proto.Unmarshal(request, &rsvpRequest); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse GetEventRsvpsProto %s", err) - } - } - - if rsvp.Status != pogo.GetEventRsvpsOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetEventRsvpsOutProto non-success status %s", rsvp.Status) - } - - switch op := rsvpRequest.EventDetails.(type) { - case *pogo.GetEventRsvpsProto_Raid: - return decoder.UpdateGymRecordWithRsvpProto(ctx, dbDetails, op.Raid, &rsvp) - case *pogo.GetEventRsvpsProto_GmaxBattle: - return "Unsupported GmaxBattle Rsvp received" - } - - return "Failed to parse GetEventRsvpsProto - unknown event type" -} - -func decodeGetEventRsvpCount(ctx context.Context, data []byte) string { - var rsvp pogo.GetEventRsvpCountOutProto - if err := proto.Unmarshal(data, &rsvp); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse GetEventRsvpCountOutProto %s", err) - } - - if rsvp.Status != pogo.GetEventRsvpCountOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetEventRsvpCountOutProto non-success status %s", rsvp.Status) - } - - var clearLocations []string - for _, rsvpDetails := range rsvp.RsvpDetails { - if rsvpDetails.MaybeCount == 0 && rsvpDetails.GoingCount == 0 { - clearLocations = append(clearLocations, rsvpDetails.LocationId) - decoder.ClearGymRsvp(ctx, dbDetails, rsvpDetails.LocationId) - } - } - - return "Cleared RSVP @ " + strings.Join(clearLocations, ", ") -} diff --git a/proto_decoder/decode_contests.go b/proto_decoder/decode_contests.go new file mode 100644 index 00000000..dc6bef6c --- /dev/null +++ b/proto_decoder/decode_contests.go @@ -0,0 +1,47 @@ +package proto_decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/decoder" + "golbat/pogo" + "golbat/raw_decoder" +) + +func (dec *ProtoDecoder) decodeGetContestData(ctx context.Context, pogoProto PogoProto) (bool, string) { + decodedContestData, err := DecodeResponseProto[pogo.GetContestDataOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse GetContestDataOutProto %s", err) + return true, fmt.Sprintf("Failed to parse GetContestDataOutProto %s", err) + } + + // Request helps, but can be decoded without it + decodedContestDataRequest, err := DecodeRequestProto[pogo.GetContestDataProto](pogoProto) + if err != nil && !raw_decoder.IsErrRequestProtoNotAvailable(err) { + log.Errorf("Failed to parse GetContestDataProto %s", err) + return true, fmt.Sprintf("Failed to parse GetContestDataProto %s", err) + } + return true, decoder.UpdatePokestopWithContestData(ctx, dec.dbDetails, decodedContestDataRequest, decodedContestData) +} + +func (dec *ProtoDecoder) decodeGetPokemonSizeContestEntry(ctx context.Context, pogoProto PogoProto) (bool, string) { + decodedPokemonSizeContestEntry, err := DecodeResponseProto[pogo.GetPokemonSizeLeaderboardEntryOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + return true, fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + } + + if decodedPokemonSizeContestEntry.Status != pogo.GetPokemonSizeLeaderboardEntryOutProto_SUCCESS { + return true, fmt.Sprintf("Ignored GetPokemonSizeLeaderboardEntryOutProto non-success status %s", decodedPokemonSizeContestEntry.Status) + } + + decodedPokemonSizeContestEntryRequest, err := DecodeRequestProto[pogo.GetPokemonSizeLeaderboardEntryProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + return true, fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + } + return true, decoder.UpdatePokestopWithPokemonSizeContestEntry(ctx, dec.dbDetails, decodedPokemonSizeContestEntryRequest, decodedPokemonSizeContestEntry) +} diff --git a/proto_decoder/decode_encounters.go b/proto_decoder/decode_encounters.go new file mode 100644 index 00000000..6b668281 --- /dev/null +++ b/proto_decoder/decode_encounters.go @@ -0,0 +1,54 @@ +package proto_decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/decoder" + "golbat/pogo" +) + +func (dec *ProtoDecoder) decodeEncounter(ctx context.Context, pogoProto PogoProto) (bool, string) { + scanParameters := pogoProto.GetScanParameters() + if !scanParameters.ProcessPokemon { + return true, "Pokemon processing disabled" + } + + decodedEncounterInfo, err := DecodeResponseProto[pogo.EncounterOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeEncounter("error", "parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + + if decodedEncounterInfo.Status != pogo.EncounterOutProto_ENCOUNTER_SUCCESS { + dec.statsCollector.IncDecodeEncounter("error", "non_success") + res := fmt.Sprintf(`EncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Status, + pogo.EncounterOutProto_Status_name[int32(decodedEncounterInfo.Status)]) + return true, res + } + + dec.statsCollector.IncDecodeEncounter("ok", "") + return true, decoder.UpdatePokemonRecordWithEncounterProto(ctx, dec.dbDetails, decodedEncounterInfo, pogoProto.GetAccount(), pogoProto.GetTimestampMs()) +} + +func (dec *ProtoDecoder) decodeDiskEncounter(ctx context.Context, pogoProto PogoProto) (bool, string) { + decodedEncounterInfo, err := DecodeResponseProto[pogo.DiskEncounterOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeDiskEncounter("error", "parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + + if decodedEncounterInfo.Result != pogo.DiskEncounterOutProto_SUCCESS { + dec.statsCollector.IncDecodeDiskEncounter("error", "non_success") + res := fmt.Sprintf(`DiskEncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Result, + pogo.DiskEncounterOutProto_Result_name[int32(decodedEncounterInfo.Result)]) + return true, res + } + + dec.statsCollector.IncDecodeDiskEncounter("ok", "") + return true, decoder.UpdatePokemonRecordWithDiskEncounterProto(ctx, dec.dbDetails, decodedEncounterInfo, pogoProto.GetAccount()) +} diff --git a/proto_decoder/decode_forts.go b/proto_decoder/decode_forts.go new file mode 100644 index 00000000..47581f96 --- /dev/null +++ b/proto_decoder/decode_forts.go @@ -0,0 +1,168 @@ +package proto_decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/decoder" + "golbat/pogo" +) + +func (dec *ProtoDecoder) decodeQuest(ctx context.Context, pogoProto PogoProto) (bool, string) { + haveAr := pogoProto.GetHaveAr() + if haveAr == nil { + dec.statsCollector.IncDecodeQuest("error", "missing_ar_info") + log.Infoln("Cannot determine AR quest - ignoring") + // We should either assume AR quest, or trace inventory like RDM probably + return true, "No AR quest info" + } + decodedQuest, err := DecodeResponseProto[pogo.FortSearchOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeQuest("error", "parse") + return true, "Parse failure" + } + + if decodedQuest.Result != pogo.FortSearchOutProto_SUCCESS { + dec.statsCollector.IncDecodeQuest("error", "non_success") + res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedQuest.Result, + pogo.FortSearchOutProto_Result_name[int32(decodedQuest.Result)]) + return true, res + } + + return true, decoder.UpdatePokestopWithQuest(ctx, dec.dbDetails, decodedQuest, *haveAr) +} + +func (dec *ProtoDecoder) decodeFortDetails(ctx context.Context, pogoProto PogoProto) (bool, string) { + decodedFort, err := DecodeResponseProto[pogo.FortDetailsOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeFortDetails("error", "parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + + switch decodedFort.FortType { + case pogo.FortType_CHECKPOINT: + dec.statsCollector.IncDecodeFortDetails("ok", "pokestop") + return true, decoder.UpdatePokestopRecordWithFortDetailsOutProto(ctx, dec.dbDetails, decodedFort) + case pogo.FortType_GYM: + dec.statsCollector.IncDecodeFortDetails("ok", "gym") + return true, decoder.UpdateGymRecordWithFortDetailsOutProto(ctx, dec.dbDetails, decodedFort) + } + + dec.statsCollector.IncDecodeFortDetails("ok", "unknown") + return true, "Unknown fort type" +} + +func (dec *ProtoDecoder) decodeGetMapForts(ctx context.Context, pogoProto PogoProto) (bool, string) { + decodedMapForts, err := DecodeResponseProto[pogo.GetMapFortsOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeGetMapForts("error", "parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + + if decodedMapForts.Status != pogo.GetMapFortsOutProto_SUCCESS { + dec.statsCollector.IncDecodeGetMapForts("error", "non_success") + res := fmt.Sprintf(`GetMapFortsOutProto: Ignored non-success value %d:%s`, decodedMapForts.Status, + pogo.GetMapFortsOutProto_Status_name[int32(decodedMapForts.Status)]) + return true, res + } + + dec.statsCollector.IncDecodeGetMapForts("ok", "") + var outputString string + processedForts := 0 + + for _, fort := range decodedMapForts.Fort { + status, output := decoder.UpdateFortRecordWithGetMapFortsOutProto(ctx, dec.dbDetails, fort) + if status { + processedForts += 1 + outputString += output + ", " + } + } + + if processedForts > 0 { + return true, fmt.Sprintf("Updated %d forts: %s", processedForts, outputString) + } + return true, "No forts updated" +} + +func (dec *ProtoDecoder) decodeGetGymInfo(ctx context.Context, pogoProto PogoProto) (bool, string) { + decodedGymInfo, err := DecodeResponseProto[pogo.GymGetInfoOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeGetGymInfo("error", "parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + + if decodedGymInfo.Result != pogo.GymGetInfoOutProto_SUCCESS { + dec.statsCollector.IncDecodeGetGymInfo("error", "non_success") + res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedGymInfo.Result, + pogo.GymGetInfoOutProto_Result_name[int32(decodedGymInfo.Result)]) + return true, res + } + + dec.statsCollector.IncDecodeGetGymInfo("ok", "") + return true, decoder.UpdateGymRecordWithGymInfoProto(ctx, dec.dbDetails, decodedGymInfo) +} + +func (dec *ProtoDecoder) decodeGetEventRsvp(ctx context.Context, pogoProto PogoProto) (bool, string) { + scanParameters := pogoProto.GetScanParameters() + if !scanParameters.ProcessGyms { + return true, "Gym processing disabled" + } + + rsvp, err := DecodeResponseProto[pogo.GetEventRsvpsOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + return true, fmt.Sprintf("Failed to parse GetEventRsvpsOutProto %s", err) + } + + rsvpRequest, err := DecodeRequestProto[pogo.GetEventRsvpsProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + return true, fmt.Sprintf("Failed to parse GetEventRsvpsProto %s", err) + } + + if rsvp.Status != pogo.GetEventRsvpsOutProto_SUCCESS { + return true, fmt.Sprintf("Ignored GetEventRsvpsOutProto non-success status %s", rsvp.Status) + } + + switch op := rsvpRequest.EventDetails.(type) { + case *pogo.GetEventRsvpsProto_Raid: + return true, decoder.UpdateGymRecordWithRsvpProto(ctx, dec.dbDetails, op.Raid, rsvp) + case *pogo.GetEventRsvpsProto_GmaxBattle: + return true, "Unsupported GmaxBattle Rsvp received" + } + + return true, "Failed to parse GetEventRsvpsProto - unknown event type" +} + +func (dec *ProtoDecoder) decodeGetEventRsvpCount(ctx context.Context, pogoProto PogoProto) (bool, string) { + scanParameters := pogoProto.GetScanParameters() + if !scanParameters.ProcessGyms { + return true, "Gym processing disabled" + } + + rsvp, err := DecodeResponseProto[pogo.GetEventRsvpCountOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + return true, fmt.Sprintf("Failed to parse GetEventRsvpCountOutProto %s", err) + } + + if rsvp.Status != pogo.GetEventRsvpCountOutProto_SUCCESS { + return true, fmt.Sprintf("Ignored GetEventRsvpCountOutProto non-success status %s", rsvp.Status) + } + + var clearLocations []string + for _, rsvpDetails := range rsvp.RsvpDetails { + if rsvpDetails.MaybeCount == 0 && rsvpDetails.GoingCount == 0 { + clearLocations = append(clearLocations, rsvpDetails.LocationId) + decoder.ClearGymRsvp(ctx, dec.dbDetails, rsvpDetails.LocationId) + } + } + + return true, "Cleared RSVP @ " + fmt.Sprint(clearLocations) +} diff --git a/proto_decoder/decode_gmos.go b/proto_decoder/decode_gmos.go new file mode 100644 index 00000000..4811bd7a --- /dev/null +++ b/proto_decoder/decode_gmos.go @@ -0,0 +1,137 @@ +package proto_decoder + +import ( + "context" + "fmt" + "time" + + log "github.com/sirupsen/logrus" + + "golbat/decoder" + "golbat/pogo" +) + +func (dec *ProtoDecoder) decodeGMO(ctx context.Context, pogoProto PogoProto) (bool, string) { + decodedGmo, err := DecodeResponseProto[pogo.GetMapObjectsOutProto](pogoProto) + if err != nil { + dec.statsCollector.IncDecodeGMO("error", "parse") + log.Errorf("Failed to parse %s", err) + } + + if decodedGmo.Status != pogo.GetMapObjectsOutProto_SUCCESS { + dec.statsCollector.IncDecodeGMO("error", "non_success") + res := fmt.Sprintf(`GetMapObjectsOutProto: Ignored non-success value %d:%s`, decodedGmo.Status, + pogo.GetMapObjectsOutProto_Status_name[int32(decodedGmo.Status)]) + return true, res + } + + var newForts []decoder.RawFortData + var newStations []decoder.RawStationData + var newWildPokemon []decoder.RawWildPokemonData + var newNearbyPokemon []decoder.RawNearbyPokemonData + var newMapPokemon []decoder.RawMapPokemonData + var newMapCells []uint64 + var cellsToBeCleaned []uint64 + + // track forts per cell for memory-based cleanup (only if tracker enabled) + cellForts := make(map[uint64]*decoder.CellFortsData) + + if len(decodedGmo.MapCell) == 0 { + return true, "Skipping GetMapObjectsOutProto: No map cells found" + } + for _, mapCell := range decodedGmo.MapCell { + if isCellNotEmpty(mapCell) { + newMapCells = append(newMapCells, mapCell.S2CellId) + if cellContainsForts(mapCell) { + cellsToBeCleaned = append(cellsToBeCleaned, mapCell.S2CellId) + // initialize cell forts tracking (only if tracker enabled) + cellForts[mapCell.S2CellId] = &decoder.CellFortsData{ + Pokestops: make([]string, 0), + Gyms: make([]string, 0), + Timestamp: mapCell.AsOfTimeMs, + } + } + } + for _, fort := range mapCell.Fort { + newForts = append(newForts, decoder.RawFortData{Cell: mapCell.S2CellId, Data: fort, Timestamp: mapCell.AsOfTimeMs}) + + // track fort by type for memory-based cleanup (only if tracker enabled) + if cf, ok := cellForts[mapCell.S2CellId]; ok { + switch fort.FortType { + case pogo.FortType_GYM: + cf.Gyms = append(cf.Gyms, fort.FortId) + case pogo.FortType_CHECKPOINT: + cf.Pokestops = append(cf.Pokestops, fort.FortId) + } + } + + if fort.ActivePokemon != nil { + newMapPokemon = append(newMapPokemon, decoder.RawMapPokemonData{Cell: mapCell.S2CellId, Data: fort.ActivePokemon, Timestamp: mapCell.AsOfTimeMs}) + } + } + for _, mon := range mapCell.WildPokemon { + newWildPokemon = append(newWildPokemon, decoder.RawWildPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) + } + for _, mon := range mapCell.NearbyPokemon { + newNearbyPokemon = append(newNearbyPokemon, decoder.RawNearbyPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) + } + for _, station := range mapCell.Stations { + newStations = append(newStations, decoder.RawStationData{Cell: mapCell.S2CellId, Data: station}) + } + } + + scanParameters := pogoProto.GetScanParameters() + if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { + decoder.UpdateFortBatch(ctx, dec.dbDetails, scanParameters, newForts) + } + var weatherUpdates []decoder.WeatherUpdate + if scanParameters.ProcessWeather { + weatherUpdates = decoder.UpdateClientWeatherBatch(ctx, dec.dbDetails, decodedGmo.ClientWeather, decodedGmo.MapCell[0].AsOfTimeMs) + } + if scanParameters.ProcessPokemon { + decoder.UpdatePokemonBatch(ctx, dec.dbDetails, scanParameters, newWildPokemon, newNearbyPokemon, newMapPokemon, decodedGmo.ClientWeather, pogoProto.GetAccount()) + if scanParameters.ProcessWeather && scanParameters.ProactiveIVSwitching { + for _, weatherUpdate := range weatherUpdates { + go func(weatherUpdate decoder.WeatherUpdate) { + decoder.ProactiveIVSwitchSem <- true + defer func() { <-decoder.ProactiveIVSwitchSem }() + decoder.ProactiveIVSwitch(ctx, dec.dbDetails, weatherUpdate, scanParameters.ProactiveIVSwitchingToDB, decodedGmo.MapCell[0].AsOfTimeMs/1000) + }(weatherUpdate) + } + } + } + if scanParameters.ProcessStations { + decoder.UpdateStationBatch(ctx, dec.dbDetails, scanParameters, newStations) + } + + if scanParameters.ProcessCells { + decoder.UpdateClientMapS2CellBatch(ctx, dec.dbDetails, newMapCells) + } + + if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + decoder.CheckRemovedForts(ctx, dec.dbDetails, cellsToBeCleaned, cellForts) + }() + } + + newFortsLen := len(newForts) + newStationsLen := len(newStations) + newWildPokemonLen := len(newWildPokemon) + newNearbyPokemonLen := len(newNearbyPokemon) + newMapPokemonLen := len(newMapPokemon) + newClientWeatherLen := len(decodedGmo.ClientWeather) + newMapCellsLen := len(newMapCells) + + dec.statsCollector.IncDecodeGMO("ok", "") + dec.statsCollector.AddDecodeGMOType("fort", float64(newFortsLen)) + dec.statsCollector.AddDecodeGMOType("station", float64(newStationsLen)) + dec.statsCollector.AddDecodeGMOType("wild_pokemon", float64(newWildPokemonLen)) + dec.statsCollector.AddDecodeGMOType("nearby_pokemon", float64(newNearbyPokemonLen)) + dec.statsCollector.AddDecodeGMOType("map_pokemon", float64(newMapPokemonLen)) + dec.statsCollector.AddDecodeGMOType("weather", float64(newClientWeatherLen)) + dec.statsCollector.AddDecodeGMOType("cell", float64(newMapCellsLen)) + + return true, fmt.Sprintf("%d cells containing %d forts %d stations %d mon %d nearby", newMapCellsLen, newFortsLen, newStationsLen, newWildPokemonLen, newNearbyPokemonLen) +} diff --git a/proto_decoder/decode_invasions.go b/proto_decoder/decode_invasions.go new file mode 100644 index 00000000..36a4f8e2 --- /dev/null +++ b/proto_decoder/decode_invasions.go @@ -0,0 +1,59 @@ +package proto_decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/decoder" + "golbat/pogo" +) + +func (dec *ProtoDecoder) decodeStartIncident(ctx context.Context, pogoProto PogoProto) (bool, string) { + decodedIncident, err := DecodeResponseProto[pogo.StartIncidentOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeStartIncident("error", "parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + + if decodedIncident.Status != pogo.StartIncidentOutProto_SUCCESS { + dec.statsCollector.IncDecodeStartIncident("error", "non_success") + res := fmt.Sprintf(`GiovanniOutProto: Ignored non-success value %d:%s`, decodedIncident.Status, + pogo.StartIncidentOutProto_Status_name[int32(decodedIncident.Status)]) + return true, res + } + + dec.statsCollector.IncDecodeStartIncident("ok", "") + return true, decoder.ConfirmIncident(ctx, dec.dbDetails, decodedIncident) +} + +func (dec *ProtoDecoder) decodeOpenInvasion(ctx context.Context, pogoProto PogoProto) (bool, string) { + decodeOpenInvasionRequest, err := DecodeRequestProto[pogo.OpenInvasionCombatSessionProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeOpenInvasion("error", "parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + if decodeOpenInvasionRequest.IncidentLookup == nil { + return true, "Invalid OpenInvasionCombatSessionProto received" + } + + decodedOpenInvasionResponse, err := DecodeResponseProto[pogo.OpenInvasionCombatSessionOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeOpenInvasion("error", "parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + + if decodedOpenInvasionResponse.Status != pogo.InvasionStatus_SUCCESS { + dec.statsCollector.IncDecodeOpenInvasion("error", "non_success") + res := fmt.Sprintf(`InvasionLineupOutProto: Ignored non-success value %d:%s`, decodedOpenInvasionResponse.Status, + pogo.InvasionStatus_Status_name[int32(decodedOpenInvasionResponse.Status)]) + return true, res + } + + dec.statsCollector.IncDecodeOpenInvasion("ok", "") + return true, decoder.UpdateIncidentLineup(ctx, dec.dbDetails, decodeOpenInvasionRequest, decodedOpenInvasionResponse) +} diff --git a/proto_decoder/decode_routes.go b/proto_decoder/decode_routes.go new file mode 100644 index 00000000..7e3d9564 --- /dev/null +++ b/proto_decoder/decode_routes.go @@ -0,0 +1,52 @@ +package proto_decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/decoder" + "golbat/pogo" +) + +func (dec *ProtoDecoder) decodeGetRoutes(ctx context.Context, pogoProto PogoProto) (bool, string) { + getRoutesOutProto, err := DecodeResponseProto[pogo.GetRoutesOutProto](pogoProto) + if err != nil { + return true, fmt.Sprintf("failed to decode GetRoutesOutProto %s", err) + } + + if getRoutesOutProto.Status != pogo.GetRoutesOutProto_SUCCESS { + return true, fmt.Sprintf("GetRoutesOutProto: Ignored non-success value %d:%s", getRoutesOutProto.Status, getRoutesOutProto.Status.String()) + } + + decodeSuccesses := map[string]bool{} + decodeErrors := map[string]bool{} + + for _, routeMapCell := range getRoutesOutProto.GetRouteMapCell() { + for _, route := range routeMapCell.GetRoute() { + //TODO we need to check the repeated field, for now access last element + routeSubmissionStatus := route.RouteSubmissionStatus[len(route.RouteSubmissionStatus)-1] + if routeSubmissionStatus != nil && routeSubmissionStatus.Status != pogo.RouteSubmissionStatus_PUBLISHED { + log.Warnf("Non published Route found in GetRoutesOutProto, status: %s", routeSubmissionStatus.String()) + continue + } + decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(dec.dbDetails, route) + if decodeError != nil { + if decodeErrors[route.Id] != true { + decodeErrors[route.Id] = true + } + log.Errorf("Failed to decode route %s", decodeError) + } else if decodeSuccesses[route.Id] != true { + decodeSuccesses[route.Id] = true + } + } + } + + return true, fmt.Sprintf( + "Decoded %d routes, failed to decode %d routes, from %d cells", + len(decodeSuccesses), + len(decodeErrors), + len(getRoutesOutProto.GetRouteMapCell()), + ) +} diff --git a/proto_decoder/decode_social.go b/proto_decoder/decode_social.go new file mode 100644 index 00000000..04d9dd3e --- /dev/null +++ b/proto_decoder/decode_social.go @@ -0,0 +1,110 @@ +package proto_decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" + + "golbat/decoder" + "golbat/pogo" +) + +func (dec *ProtoDecoder) decodeSocialActionWithRequest(ctx context.Context, pogoProto PogoProto) (bool, string) { + proxyRequestProto, err := DecodeRequestProto[pogo.ProxyRequestProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeSocialActionWithRequest("error", "request_parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + + proxyResponseProto, err := DecodeResponseProto[pogo.ProxyResponseProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + dec.statsCollector.IncDecodeSocialActionWithRequest("error", "response_parse") + return true, fmt.Sprintf("Failed to parse %s", err) + } + + if proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED && proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED_AND_REASSIGNED { + dec.statsCollector.IncDecodeSocialActionWithRequest("error", "non_success") + return true, fmt.Sprintf("unsuccessful proxyResponseProto response %d %s", int(proxyResponseProto.Status), proxyResponseProto.Status) + } + + switch pogo.InternalSocialAction(proxyRequestProto.GetAction()) { + case pogo.InternalSocialAction_SOCIAL_ACTION_LIST_FRIEND_STATUS: + dec.statsCollector.IncDecodeSocialActionWithRequest("ok", "list_friend_status") + return true, dec.decodeGetFriendDetails(proxyResponseProto.Payload) + case pogo.InternalSocialAction_SOCIAL_ACTION_SEARCH_PLAYER: + dec.statsCollector.IncDecodeSocialActionWithRequest("ok", "search_player") + return true, dec.decodeSearchPlayer(proxyRequestProto, proxyResponseProto.Payload) + + } + + dec.statsCollector.IncDecodeSocialActionWithRequest("ok", "unknown") + return true, fmt.Sprintf("Did not process %s", pogo.InternalSocialAction(proxyRequestProto.GetAction()).String()) +} + +func (dec *ProtoDecoder) decodeGetFriendDetails(payload []byte) string { + var getFriendDetailsOutProto pogo.InternalGetFriendDetailsOutProto + getFriendDetailsError := proto.Unmarshal(payload, &getFriendDetailsOutProto) + + if getFriendDetailsError != nil { + dec.statsCollector.IncDecodeGetFriendDetails("error", "parse") + log.Errorf("Failed to parse %s", getFriendDetailsError) + return fmt.Sprintf("Failed to parse %s", getFriendDetailsError) + } + + if getFriendDetailsOutProto.GetResult() != pogo.InternalGetFriendDetailsOutProto_SUCCESS || getFriendDetailsOutProto.GetFriend() == nil { + dec.statsCollector.IncDecodeGetFriendDetails("error", "non_success") + return fmt.Sprintf("unsuccessful get friends details") + } + + failures := 0 + + for _, friend := range getFriendDetailsOutProto.GetFriend() { + player := friend.GetPlayer() + + updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dec.dbDetails, player, player.PublicData, "", player.GetPlayerId()) + if updatePlayerError != nil { + failures++ + } + } + + dec.statsCollector.IncDecodeGetFriendDetails("ok", "") + return fmt.Sprintf("%d players decoded on %d", len(getFriendDetailsOutProto.GetFriend())-failures, len(getFriendDetailsOutProto.GetFriend())) +} + +func (dec *ProtoDecoder) decodeSearchPlayer(proxyRequestProto *pogo.ProxyRequestProto, payload []byte) string { + var searchPlayerOutProto pogo.InternalSearchPlayerOutProto + searchPlayerOutError := proto.Unmarshal(payload, &searchPlayerOutProto) + + if searchPlayerOutError != nil { + log.Errorf("Failed to parse %s", searchPlayerOutError) + dec.statsCollector.IncDecodeSearchPlayer("error", "parse") + return fmt.Sprintf("Failed to parse %s", searchPlayerOutError) + } + + if searchPlayerOutProto.GetResult() != pogo.InternalSearchPlayerOutProto_SUCCESS || searchPlayerOutProto.GetPlayer() == nil { + dec.statsCollector.IncDecodeSearchPlayer("error", "non_success") + return fmt.Sprintf("unsuccessful search player response") + } + + var searchPlayerProto pogo.InternalSearchPlayerProto + searchPlayerError := proto.Unmarshal(proxyRequestProto.GetPayload(), &searchPlayerProto) + + if searchPlayerError != nil || searchPlayerProto.GetFriendCode() == "" { + dec.statsCollector.IncDecodeSearchPlayer("error", "parse") + return fmt.Sprintf("Failed to parse %s", searchPlayerError) + } + + player := searchPlayerOutProto.GetPlayer() + updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dec.dbDetails, player, player.PublicData, searchPlayerProto.GetFriendCode(), "") + if updatePlayerError != nil { + dec.statsCollector.IncDecodeSearchPlayer("error", "update") + return fmt.Sprintf("Failed update player %s", updatePlayerError) + } + + dec.statsCollector.IncDecodeSearchPlayer("ok", "") + return fmt.Sprintf("1 player decoded from SearchPlayerProto") +} diff --git a/proto_decoder/decode_stations.go b/proto_decoder/decode_stations.go new file mode 100644 index 00000000..db23ad6b --- /dev/null +++ b/proto_decoder/decode_stations.go @@ -0,0 +1,39 @@ +package proto_decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/decoder" + "golbat/pogo" +) + +func (dec *ProtoDecoder) decodeGetStationDetails(ctx context.Context, pogoProto PogoProto) (bool, string) { + scanParameters := pogoProto.GetScanParameters() + if !scanParameters.ProcessStations { + return true, "Station processing disabled" + } + + decodedGetStationDetails, err := DecodeResponseProto[pogo.GetStationedPokemonDetailsOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) + return true, fmt.Sprintf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) + } + + decodedGetStationDetailsRequest, err := DecodeRequestProto[pogo.GetStationedPokemonDetailsProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse GetStationedPokemonDetailsProto %s", err) + return true, fmt.Sprintf("Failed to parse GetStationedPokemonDetailsProto %s", err) + } + + if decodedGetStationDetails.Result == pogo.GetStationedPokemonDetailsOutProto_STATION_NOT_FOUND { + // station without stationed pokemon found, therefore we need to reset the columns + return true, decoder.ResetStationedPokemonWithStationDetailsNotFound(ctx, dec.dbDetails, decodedGetStationDetailsRequest) + } else if decodedGetStationDetails.Result != pogo.GetStationedPokemonDetailsOutProto_SUCCESS { + return true, fmt.Sprintf("Ignored GetStationedPokemonDetailsOutProto non-success status %s", decodedGetStationDetails.Result) + } + + return true, decoder.UpdateStationWithStationDetails(ctx, dec.dbDetails, decodedGetStationDetailsRequest, decodedGetStationDetails) +} diff --git a/proto_decoder/decode_tappables.go b/proto_decoder/decode_tappables.go new file mode 100644 index 00000000..be21438d --- /dev/null +++ b/proto_decoder/decode_tappables.go @@ -0,0 +1,39 @@ +package proto_decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/decoder" + "golbat/pogo" +) + +func (dec *ProtoDecoder) decodeTappable(ctx context.Context, pogoProto PogoProto) (bool, string) { + scanParameters := pogoProto.GetScanParameters() + if !scanParameters.ProcessTappables { + return true, "Tappable processing disabled" + } + + tappable, err := DecodeResponseProto[pogo.ProcessTappableOutProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + return true, fmt.Sprintf("Failed to parse ProcessTappableOutProto %s", err) + } + + tappableRequest, err := DecodeRequestProto[pogo.ProcessTappableProto](pogoProto) + if err != nil { + log.Errorf("Failed to parse %s", err) + return true, fmt.Sprintf("Failed to parse ProcessTappableProto %s", err) + } + + if tappable.Status != pogo.ProcessTappableOutProto_SUCCESS { + return true, fmt.Sprintf("Ignored ProcessTappableOutProto non-success status %s", tappable.Status) + } + var result string + if encounter := tappable.GetEncounter(); encounter != nil { + result = decoder.UpdatePokemonRecordWithTappableEncounter(ctx, dec.dbDetails, tappableRequest, encounter, pogoProto.GetAccount(), pogoProto.GetTimestampMs()) + } + return true, result + " " + decoder.UpdateTappable(ctx, dec.dbDetails, tappableRequest, tappable, pogoProto.GetTimestampMs()) +} diff --git a/proto_decoder/helpers.go b/proto_decoder/helpers.go new file mode 100644 index 00000000..e3060b4e --- /dev/null +++ b/proto_decoder/helpers.go @@ -0,0 +1,64 @@ +package proto_decoder + +import ( + "fmt" + "golbat/pogo" + "strings" + + "google.golang.org/protobuf/proto" +) + +func getMethodName(method int, trimString bool) string { + if val, ok := pogo.Method_name[int32(method)]; ok { + if trimString && strings.HasPrefix(val, "METHOD_") { + return strings.TrimPrefix(val, "METHOD_") + } + return val + } + return fmt.Sprintf("#%d", method) +} + +func isCellNotEmpty(mapCell *pogo.ClientMapCellProto) bool { + return len(mapCell.Stations) > 0 || len(mapCell.Fort) > 0 || len(mapCell.WildPokemon) > 0 || len(mapCell.NearbyPokemon) > 0 || len(mapCell.CatchablePokemon) > 0 +} + +func cellContainsForts(mapCell *pogo.ClientMapCellProto) bool { + return len(mapCell.Fort) > 0 +} + +// ProtoMessagePtr is a generic type that must be a pointer and +// satisfy proto.Message interface +type ProtoMessagePtr[T any] interface { + proto.Message + *T +} + +// DecodeRequestProto will unmarshal bytes into a destination type T and +// return a pointer to T. Because of type inference, only T needs to be specified. +// E.g.: +// gmoReqProto, err := DecodeRequestProto[pogo.GetMapObjectsProto](pogoProto) +func DecodeRequestProto[T any, TP ProtoMessagePtr[T]](pogoProto PogoProto) (TP, error) { + var dest T + + ptr := TP(&dest) + err := pogoProto.DecodeRequest(ptr) + if err != nil { + return nil, err + } + return ptr, nil +} + +// DecodeResponeProto will unmarshal bytes into a destination type T and +// return a pointer to T. Because of type inference, only T needs to be specified. +// E.g.: +// gmoRespProto, err := DecodeResponseProto[pogo.GetMapObjectsOutProto](pogoProto) +func DecodeResponseProto[T any, TP ProtoMessagePtr[T]](pogoProto PogoProto) (TP, error) { + var dest T + + ptr := TP(&dest) + err := pogoProto.DecodeResponse(ptr) + if err != nil { + return nil, err + } + return ptr, nil +} diff --git a/proto_decoder/proto_decoder.go b/proto_decoder/proto_decoder.go new file mode 100644 index 00000000..1fba287a --- /dev/null +++ b/proto_decoder/proto_decoder.go @@ -0,0 +1,115 @@ +package proto_decoder + +import ( + "context" + "time" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/device_tracker" + "golbat/pogo" + "golbat/raw_decoder" + "golbat/stats_collector" +) + +type decodeProtoMethod struct { + Decode func(*ProtoDecoder, context.Context, PogoProto) (bool, string) + MinLevel int +} + +var decodeMethods = map[int]*decodeProtoMethod{ + int(pogo.Method_METHOD_START_INCIDENT): &decodeProtoMethod{(*ProtoDecoder).decodeStartIncident, 30}, + int(pogo.Method_METHOD_INVASION_OPEN_COMBAT_SESSION): &decodeProtoMethod{(*ProtoDecoder).decodeOpenInvasion, 30}, + int(pogo.Method_METHOD_FORT_DETAILS): &decodeProtoMethod{(*ProtoDecoder).decodeFortDetails, 30}, + int(pogo.Method_METHOD_GET_MAP_OBJECTS): &decodeProtoMethod{(*ProtoDecoder).decodeGMO, 30}, + int(pogo.Method_METHOD_GYM_GET_INFO): &decodeProtoMethod{(*ProtoDecoder).decodeGetGymInfo, 30}, + int(pogo.Method_METHOD_ENCOUNTER): &decodeProtoMethod{(*ProtoDecoder).decodeEncounter, 30}, + int(pogo.Method_METHOD_DISK_ENCOUNTER): &decodeProtoMethod{(*ProtoDecoder).decodeDiskEncounter, 30}, + int(pogo.Method_METHOD_FORT_SEARCH): &decodeProtoMethod{(*ProtoDecoder).decodeQuest, 10}, + int(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION): &decodeProtoMethod{(*ProtoDecoder).decodeSocialActionWithRequest, 0}, + int(pogo.Method_METHOD_GET_MAP_FORTS): &decodeProtoMethod{(*ProtoDecoder).decodeGetMapForts, 10}, + int(pogo.Method_METHOD_GET_ROUTES): &decodeProtoMethod{(*ProtoDecoder).decodeGetRoutes, 30}, + int(pogo.Method_METHOD_GET_CONTEST_DATA): &decodeProtoMethod{(*ProtoDecoder).decodeGetContestData, 10}, + int(pogo.Method_METHOD_GET_POKEMON_SIZE_CONTEST_ENTRY): &decodeProtoMethod{(*ProtoDecoder).decodeGetPokemonSizeContestEntry, 10}, + int(pogo.Method_METHOD_GET_STATION_DETAILS): &decodeProtoMethod{(*ProtoDecoder).decodeGetStationDetails, 10}, + int(pogo.Method_METHOD_PROCESS_TAPPABLE): &decodeProtoMethod{(*ProtoDecoder).decodeTappable, 30}, + int(pogo.Method_METHOD_GET_EVENT_RSVPS): &decodeProtoMethod{(*ProtoDecoder).decodeGetEventRsvp, 10}, + int(pogo.Method_METHOD_GET_EVENT_RSVP_COUNT): &decodeProtoMethod{(*ProtoDecoder).decodeGetEventRsvpCount, 10}, + // ignores + int(pogo.Method_METHOD_GET_PLAYER): nil, + int(pogo.Method_METHOD_GET_HOLOHOLO_INVENTORY): nil, + int(pogo.Method_METHOD_CREATE_COMBAT_CHALLENGE): nil, +} + +type ProtoDecoder struct { + decodeTimeout time.Duration + dbDetails db.DbDetails + statsCollector stats_collector.StatsCollector + deviceTracker *device_tracker.DeviceTracker +} + +func (dec *ProtoDecoder) decode(ctx context.Context, pogoProto raw_decoder.PogoProto) { + processed := false + ignore := false + start := time.Now() + result := "" + + method := pogoProto.GetMethod() + decodeMethod, ok := decodeMethods[method] + if ok { + if decodeMethod == nil { + // completely ignore + return + } + if level := pogoProto.GetLevel(); level < decodeMethod.MinLevel { + dec.statsCollector.IncDecodeMethods("error", "low_level", getMethodName(method, true)) + log.Debugf("Insufficient Level %d Did not process hook type %d(%s)", level, method, pogo.Method(method)) + return + } + processed, result = decodeMethod.Decode(dec, ctx, pogoProto) + } else { + log.Debugf("Did not know hook type %d(%s)", method, pogo.Method(method)) + } + + if !ignore { + statResult := "ok" + if !processed { + result = "**Did not process**" + statResult = "unprocessed" + } + dec.statsCollector.IncDecodeMethods(statResult, "", getMethodName(method, true)) + log.Debugf("%s/%s %s - %s - %s", + pogoProto.GetDeviceId(), pogoProto.GetAccount(), pogo.Method(method), + time.Since(start), result, + ) + } +} + +func (dec *ProtoDecoder) decodeGroup(ctx context.Context, protoGroup raw_decoder.PogoProtoGroup) { + for _, proto := range protoGroup { + // provide independent cancellation contexts for each proto decode + ctx, cancel := context.WithTimeout(ctx, dec.decodeTimeout) + dec.decode(ctx, proto) + cancel() + } +} + +func (dec *ProtoDecoder) DecodeGroup(ctx context.Context, protoGroup raw_decoder.PogoProtoGroup) { + l := len(protoGroup) + if l == 0 { + return + } + go dec.decodeGroup(ctx, protoGroup) + lastProto := protoGroup[l-1] + dec.deviceTracker.UpdateDeviceLocation(lastProto.GetDeviceId(), lastProto.GetLocation(), lastProto.GetScanContext()) +} + +func NewProtoDecoder(decodeTimeout time.Duration, dbDetails db.DbDetails, statsCollector stats_collector.StatsCollector, deviceTracker *device_tracker.DeviceTracker) *ProtoDecoder { + return &ProtoDecoder{ + decodeTimeout: decodeTimeout, + dbDetails: dbDetails, + statsCollector: statsCollector, + deviceTracker: deviceTracker, + } +} diff --git a/proto_decoder/types.go b/proto_decoder/types.go new file mode 100644 index 00000000..9e7fa977 --- /dev/null +++ b/proto_decoder/types.go @@ -0,0 +1,6 @@ +package proto_decoder + +import "golbat/raw_decoder" + +type PogoProtoGroup = raw_decoder.PogoProtoGroup +type PogoProto = raw_decoder.PogoProto diff --git a/raw_decoder/errors.go b/raw_decoder/errors.go new file mode 100644 index 00000000..48ec4a05 --- /dev/null +++ b/raw_decoder/errors.go @@ -0,0 +1,13 @@ +package raw_decoder + +import "errors" + +var errRequestProtoNotAvailable = errors.New("request proto not available") + +func NewErrRequestProtoNotAvailable() error { + return errRequestProtoNotAvailable +} + +func IsErrRequestProtoNotAvailable(err error) bool { + return errors.Is(err, errRequestProtoNotAvailable) +} diff --git a/raw_decoder/grpc_raw_decoder/grpc_raw_decoder.go b/raw_decoder/grpc_raw_decoder/grpc_raw_decoder.go new file mode 100644 index 00000000..134a9dcf --- /dev/null +++ b/raw_decoder/grpc_raw_decoder/grpc_raw_decoder.go @@ -0,0 +1,109 @@ +package grpc_raw_decoder + +import ( + "context" + "time" + + "google.golang.org/protobuf/proto" + + "golbat/decoder" + "golbat/geo" + "golbat/grpc" + "golbat/raw_decoder" +) + +type PogoProtoGroup = raw_decoder.PogoProtoGroup +type PogoProto = raw_decoder.PogoProto +type ProtoDecoder = raw_decoder.ProtoDecoder +type ScanMetadata = raw_decoder.ScanMetadata + +type GRPCPogoProto struct { + ScanMetadata + + method int + haveAr *bool + location geo.Location + + requestBytes []byte + responseBytes []byte +} + +func (pogoProto *GRPCPogoProto) GetMethod() int { + return pogoProto.method +} + +func (pogoProto *GRPCPogoProto) GetHaveAr() *bool { + return pogoProto.haveAr +} + +func (pogoProto *GRPCPogoProto) GetLocation() geo.Location { + return pogoProto.location +} + +func (pogoProto *GRPCPogoProto) HasRequest() bool { + return len(pogoProto.requestBytes) > 0 +} + +func (pogoProto *GRPCPogoProto) DecodeRequest(dest proto.Message) error { + if !pogoProto.HasRequest() { + return raw_decoder.NewErrRequestProtoNotAvailable() + } + return proto.Unmarshal(pogoProto.requestBytes, dest) +} + +func (pogoProto *GRPCPogoProto) DecodeResponse(dest proto.Message) error { + return proto.Unmarshal(pogoProto.responseBytes, dest) +} + +func (pogoProto *GRPCPogoProto) GetScanParameters() decoder.ScanParameters { + return decoder.FindScanConfiguration(pogoProto.ScanContext, pogoProto.location) +} + +type GRPCRawDecoder struct { + protoDecoder ProtoDecoder +} + +func (dec *GRPCRawDecoder) DecodeRaw(ctx context.Context, in *grpc.RawProtoRequest) { + metadata := ScanMetadata{ + DeviceId: in.DeviceId, + Account: in.Username, + Level: int(in.TrainerLevel), + TimestampMs: in.Timestamp, + } + if metadata.TimestampMs <= 0 { + metadata.TimestampMs = time.Now().UnixMilli() + } + + baseProto := GRPCPogoProto{ + ScanMetadata: metadata, + location: geo.Location{ + Latitude: float64(in.LatTarget), + Longitude: float64(in.LonTarget), + }, + haveAr: in.HaveAr, + } + + if in.ScanContext != nil { + baseProto.ScanContext = *in.ScanContext + } + + protoGroup := make(PogoProtoGroup, len(in.Contents)) + for protoGroupIdx, content := range in.Contents { + proto := baseProto + proto.method = int(content.Method) + proto.requestBytes = content.RequestPayload + proto.responseBytes = content.ResponsePayload + if haveAr := content.HaveAr; haveAr != nil { + proto.haveAr = haveAr + } + protoGroup[protoGroupIdx] = &proto + } + + dec.protoDecoder.DecodeGroup(ctx, protoGroup) +} + +func NewGRPCRawDecoder(protoDecoder ProtoDecoder) *GRPCRawDecoder { + return &GRPCRawDecoder{ + protoDecoder: protoDecoder, + } +} diff --git a/raw_decoder/http_raw_decoder/helpers.go b/raw_decoder/http_raw_decoder/helpers.go new file mode 100644 index 00000000..c38507ec --- /dev/null +++ b/raw_decoder/http_raw_decoder/helpers.go @@ -0,0 +1,41 @@ +package http_raw_decoder + +import ( + "golbat/pogo" + + log "github.com/sirupsen/logrus" +) + +func getValueFromMap(data map[string]interface{}, key1, key2 string) interface{} { + if v := data[key1]; v != nil { + return v + } + if v := data[key2]; v != nil { + return v + } + return nil +} + +func questsHeldHasARTask(quests_held any) *bool { + const ar_quest_id = int64(pogo.QuestType_QUEST_GEOTARGETED_AR_SCAN) + + quests_held_list, ok := quests_held.([]any) + if !ok { + log.Errorf("Raw: unexpected quests_held type in data: %T", quests_held) + return nil + } + for _, quest_id := range quests_held_list { + if quest_id_f, ok := quest_id.(float64); ok { + if int64(quest_id_f) == ar_quest_id { + res := true + return &res + } + continue + } + // quest_id is not float64? Treat the whole thing as unknown. + log.Errorf("Raw: unexpected quest_id type in quests_held: %T", quest_id) + return nil + } + res := false + return &res +} diff --git a/raw_decoder/http_raw_decoder/http_raw_decoder.go b/raw_decoder/http_raw_decoder/http_raw_decoder.go new file mode 100644 index 00000000..d2459e51 --- /dev/null +++ b/raw_decoder/http_raw_decoder/http_raw_decoder.go @@ -0,0 +1,274 @@ +package http_raw_decoder + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" + + "golbat/decoder" + "golbat/geo" + "golbat/raw_decoder" +) + +type PogoProtoGroup = raw_decoder.PogoProtoGroup +type PogoProto = raw_decoder.PogoProto +type ProtoDecoder = raw_decoder.ProtoDecoder +type ScanMetadata = raw_decoder.ScanMetadata + +type HTTPPogoProto struct { + ScanMetadata + + method int + haveAr *bool + location geo.Location + + base64Request string + base64Response string +} + +func (pogoProto *HTTPPogoProto) GetMethod() int { + return pogoProto.method +} + +func (pogoProto *HTTPPogoProto) GetHaveAr() *bool { + return pogoProto.haveAr +} + +func (pogoProto *HTTPPogoProto) GetLocation() geo.Location { + return pogoProto.location +} + +func (pogoProto *HTTPPogoProto) HasRequest() bool { + return len(pogoProto.base64Request) > 0 +} + +func (pogoProto *HTTPPogoProto) DecodeRequest(dest proto.Message) error { + if !pogoProto.HasRequest() { + return raw_decoder.NewErrRequestProtoNotAvailable() + } + requestBytes, err := base64.StdEncoding.DecodeString(pogoProto.base64Request) + if err != nil { + return fmt.Errorf("failed to base64 decode request proto: %w", err) + } + return proto.Unmarshal(requestBytes, dest) +} + +func (pogoProto *HTTPPogoProto) DecodeResponse(dest proto.Message) error { + responseBytes, err := base64.StdEncoding.DecodeString(pogoProto.base64Response) + if err != nil { + return fmt.Errorf("failed to base64 decode response proto: %w", err) + } + return proto.Unmarshal(responseBytes, dest) +} + +func (pogoProto *HTTPPogoProto) GetScanParameters() decoder.ScanParameters { + return decoder.FindScanConfiguration(pogoProto.ScanContext, pogoProto.location) +} + +type HTTPRawDecoder struct { + protoDecoder ProtoDecoder +} + +func (dec *HTTPRawDecoder) decodePogodroidRaw(ctx context.Context, origin string, body []byte) error { + var raw []map[string]interface{} + + if err := json.Unmarshal(body, &raw); err != nil { + return fmt.Errorf("failed to decode raw bytes: %w", err) + } + + if len(raw) == 0 { + return nil + } + + metadata := ScanMetadata{ + DeviceId: origin, + Account: "Pogodroid", + Level: 30, + } + + location := geo.Location{} + + protoGroup := make(PogoProtoGroup, len(raw)) + for protoGroupIdx, entry := range raw { + lat := entry["lat"] + lng := entry["lng"] + if lat != nil && lng != nil { + lat_f, _ := lat.(float64) + lng_f, _ := lng.(float64) + if lat_f != 0 && lng_f != 0 { + location = geo.Location{ + Latitude: lat_f, + Longitude: lng_f, + } + } + } + + pogoProto := &HTTPPogoProto{ + ScanMetadata: metadata, + location: location, + method: int(entry["type"].(float64)), + haveAr: func() *bool { + if v := entry["quests_held"]; v != nil { + return questsHeldHasARTask(v) + } + return nil + }(), + } + pogoProto.base64Response, _ = entry["payload"].(string) + + protoGroup[protoGroupIdx] = pogoProto + } + dec.protoDecoder.DecodeGroup(ctx, protoGroup) + return nil +} + +func (dec *HTTPRawDecoder) decodeRawEntry(ctx context.Context, raw map[string]interface{}, requestReceivedMs int64) error { + contents, ok := raw["contents"].([]any) + if !ok { + return errors.New("raw entry is missing 'contents'") + } + if len(contents) == 0 { + return nil + } + + baseProto := HTTPPogoProto{} + + { + metadata := &baseProto.ScanMetadata + metadata.Level = 30 + if v := raw["uuid"]; v != nil { + metadata.DeviceId, _ = v.(string) + } + if v := raw["username"]; v != nil { + metadata.Account, _ = v.(string) + } + if v := raw["trainerlvl"]; v != nil { + lvl, ok := v.(float64) + if ok { + metadata.Level = int(lvl) + } + } + if v := raw["scan_context"]; v != nil { + metadata.ScanContext, _ = v.(string) + } + if v := raw["timestamp_ms"]; v != nil { + metadata.TimestampMs, _ = v.(int64) + } + if metadata.TimestampMs <= 0 { + metadata.TimestampMs = requestReceivedMs + } + } + + if v := raw["have_ar"]; v != nil { + res, ok := v.(bool) + if ok { + baseProto.haveAr = &res + } + } + + if lat_target, lon_target := raw["lat_target"], raw["lon_target"]; lat_target != nil && lon_target != nil { + baseProto.location = geo.Location{ + Latitude: lat_target.(float64), + Longitude: lon_target.(float64), + } + } + + protoGroup := make(PogoProtoGroup, len(contents)) + protoGroupIdx := 0 + for _, v := range contents { + entry, ok := v.(map[string]any) + if !ok || entry == nil { + continue + } + + // Try to decode the payload automatically without requiring any knowledge of the + // provider type + b64data := getValueFromMap(entry, "data", "payload") + method := getValueFromMap(entry, "method", "type") + if method == nil || b64data == nil { + log.Errorf("Error decoding raw (no method or base64 data)") + continue + } + + proto := baseProto + proto.base64Response, _ = b64data.(string) + proto.method = func() int { + if res, ok := method.(float64); ok { + return int(res) + } + return 0 + }() + if request := entry["request"]; request != nil { + proto.base64Request, _ = request.(string) + } + if haveAr := entry["have_ar"]; haveAr != nil { + res, ok := haveAr.(bool) + if ok { + proto.haveAr = &res + } + } + protoGroup[protoGroupIdx] = &proto + protoGroupIdx++ + } + if protoGroupIdx == 0 { + return errors.New("all contents were missing method and/or base64 data") + } + dec.protoDecoder.DecodeGroup(ctx, protoGroup[:protoGroupIdx]) + return nil +} + +func (dec *HTTPRawDecoder) DecodeRaw(ctx context.Context, headers http.Header, body []byte, requestReceivedMs int64) error { + if len(body) == 0 { + return errors.New("raw request contains empty body") + } + + if origin := headers.Get("origin"); origin != "" { + if err := dec.decodePogodroidRaw(ctx, origin, body); err != nil { + return fmt.Errorf("failed to decode raw entry: %w", err) + } + } + + if body[0] == '[' { + var rawEntries []map[string]interface{} + if err := json.Unmarshal(body, &rawEntries); err != nil { + return fmt.Errorf("failed to decode json from array of raw entries: %w", err) + } + if len(rawEntries) == 0 { + return nil + } + decoded := false + for _, rawEntry := range rawEntries { + err := dec.decodeRawEntry(ctx, rawEntry, requestReceivedMs) + if err != nil { + log.Errorf("failed to decode raw entry: %v", err) + continue + } + decoded = true + } + if !decoded { + return errors.New("no valid entry in batched entries") + } + return nil + } + + var rawEntry map[string]interface{} + if err := json.Unmarshal(body, &rawEntry); err != nil { + return fmt.Errorf("failed to decode json from raw body: %w", err) + } + if err := dec.decodeRawEntry(ctx, rawEntry, requestReceivedMs); err != nil { + return fmt.Errorf("failed to decode raw entry: %w", err) + } + return nil +} + +func NewHTTPRawDecoder(protoDecoder ProtoDecoder) *HTTPRawDecoder { + return &HTTPRawDecoder{ + protoDecoder: protoDecoder, + } +} diff --git a/raw_decoder/types.go b/raw_decoder/types.go new file mode 100644 index 00000000..45cffa1b --- /dev/null +++ b/raw_decoder/types.go @@ -0,0 +1,66 @@ +package raw_decoder + +import ( + "context" + "golbat/decoder" + "golbat/geo" + + "google.golang.org/protobuf/proto" +) + +type ScanMetadata struct { + Account string + Level int + DeviceId string + ScanContext string + TimestampMs int64 +} + +func (metadata *ScanMetadata) GetAccount() string { + return metadata.Account +} + +func (metadata *ScanMetadata) GetLevel() int { + return metadata.Level +} + +func (metadata *ScanMetadata) GetDeviceId() string { + return metadata.DeviceId +} + +func (metadata *ScanMetadata) GetScanContext() string { + return metadata.ScanContext +} + +func (metadata *ScanMetadata) GetTimestampMs() int64 { + return metadata.TimestampMs +} + +type PogoProto interface { + GetAccount() string + GetLevel() int + GetDeviceId() string + GetScanContext() string + GetTimestampMs() int64 + GetMethod() int + GetHaveAr() *bool + GetLocation() geo.Location + GetScanParameters() decoder.ScanParameters + HasRequest() bool + DecodeRequest(dest proto.Message) error + DecodeResponse(dest proto.Message) error +} + +type PogoProtoGroup []PogoProto + +func (protoGroup PogoProtoGroup) GetLastLocation() geo.Location { + l := len(protoGroup) + if l == 0 { + return geo.Location{} + } + return protoGroup[l-1].GetLocation() +} + +type ProtoDecoder interface { + DecodeGroup(ctx context.Context, protoGroup PogoProtoGroup) +} diff --git a/routes.go b/routes.go index e68c16a6..41a4c942 100644 --- a/routes.go +++ b/routes.go @@ -2,10 +2,7 @@ package main import ( "context" - b64 "encoding/base64" - "encoding/json" "errors" - "io" "net/http" "strconv" "time" @@ -16,304 +13,8 @@ import ( "golbat/config" "golbat/decoder" "golbat/geo" - "golbat/pogo" ) -type ProtoData struct { - Method int - Data []byte - Request []byte - HaveAr *bool - Account string - Level int - Uuid string - ScanContext string - Lat float64 - Lon float64 - TimestampMs int64 -} - -type InboundRawData struct { - Base64Data string - Request string - Method int - HaveAr *bool -} - -func questsHeldHasARTask(quests_held any) *bool { - const ar_quest_id = int64(pogo.QuestType_QUEST_GEOTARGETED_AR_SCAN) - - quests_held_list, ok := quests_held.([]any) - if !ok { - log.Errorf("Raw: unexpected quests_held type in data: %T", quests_held) - return nil - } - for _, quest_id := range quests_held_list { - if quest_id_f, ok := quest_id.(float64); ok { - if int64(quest_id_f) == ar_quest_id { - res := true - return &res - } - continue - } - // quest_id is not float64? Treat the whole thing as unknown. - log.Errorf("Raw: unexpected quest_id type in quests_held: %T", quest_id) - return nil - } - res := false - return &res -} - -func Raw(c *gin.Context) { - var w http.ResponseWriter = c.Writer - var r *http.Request = c.Request - - dataReceivedTimestamp := time.Now().UnixMilli() - - authHeader := r.Header.Get("Authorization") - if config.Config.RawBearer != "" { - if authHeader != "Bearer "+config.Config.RawBearer { - statsCollector.IncRawRequests("error", "auth") - log.Errorf("Raw: Incorrect authorisation received (%s)", authHeader) - return - } - } - - body, err := io.ReadAll(io.LimitReader(r.Body, 5*1048576)) - if err != nil { - statsCollector.IncRawRequests("error", "io_error") - log.Errorf("Raw: Error (1) during HTTP receive %s", err) - return - } - if err := r.Body.Close(); err != nil { - statsCollector.IncRawRequests("error", "io_close_error") - log.Errorf("Raw: Error (2) during HTTP receive %s", err) - return - } - - decodeError := false - uuid := "" - account := "" - level := 30 - scanContext := "" - var latTarget, lonTarget float64 - var globalHaveAr *bool - var protoData []InboundRawData - - // Objective is to normalise incoming proto data. Unfortunately each provider seems - // to be just different enough that this ends up being a little bit more of a mess - // than I would like - - pogodroidHeader := r.Header.Get("origin") - userAgent := r.Header.Get("User-Agent") - - //log.Infof("Raw: Received data from %s", body) - //log.Infof("User agent is %s", userAgent) - - if pogodroidHeader != "" { - var raw []map[string]interface{} - if err := json.Unmarshal(body, &raw); err != nil { - decodeError = true - } else { - for _, entry := range raw { - if latTarget == 0 && lonTarget == 0 { - lat := entry["lat"] - lng := entry["lng"] - if lat != nil && lng != nil { - lat_f, _ := lat.(float64) - lng_f, _ := lng.(float64) - if lat_f != 0 && lng_f != 0 { - latTarget = lat_f - lonTarget = lng_f - } - } - } - protoData = append(protoData, InboundRawData{ - Base64Data: entry["payload"].(string), - Method: int(entry["type"].(float64)), - HaveAr: func() *bool { - if v := entry["quests_held"]; v != nil { - return questsHeldHasARTask(v) - } - return nil - }(), - }) - } - } - uuid = pogodroidHeader - account = "Pogodroid" - } else { - var raw map[string]interface{} - if err := json.Unmarshal(body, &raw); err != nil { - decodeError = true - } else { - if v := raw["have_ar"]; v != nil { - res, ok := v.(bool) - if ok { - globalHaveAr = &res - } - } - if v := raw["uuid"]; v != nil { - uuid, _ = v.(string) - } - if v := raw["username"]; v != nil { - account, _ = v.(string) - } - if v := raw["trainerlvl"]; v != nil { - lvl, ok := v.(float64) - if ok { - level = int(lvl) - } - } - if v := raw["scan_context"]; v != nil { - scanContext, _ = v.(string) - } - - if v := raw["lat_target"]; v != nil { - latTarget, _ = v.(float64) - } - - if v := raw["lon_target"]; v != nil { - lonTarget, _ = v.(float64) - } - - if v := raw["timestamp_ms"]; v != nil { - ts, _ := v.(int64) - if ts > 0 { - dataReceivedTimestamp = ts - } - } - - contents, ok := raw["contents"].([]interface{}) - if !ok { - decodeError = true - - } else { - - decodeAlternate := func(data map[string]interface{}, key1, key2 string) interface{} { - if v := data[key1]; v != nil { - return v - } - if v := data[key2]; v != nil { - return v - } - return nil - } - - for _, v := range contents { - entry := v.(map[string]interface{}) - // Try to decode the payload automatically without requiring any knowledge of the - // provider type - - b64data := decodeAlternate(entry, "data", "payload") - method := decodeAlternate(entry, "method", "type") - request := entry["request"] - haveAr := entry["have_ar"] - - if method == nil || b64data == nil { - log.Errorf("Error decoding raw") - continue - } - inboundRawData := InboundRawData{ - Base64Data: func() string { - if res, ok := b64data.(string); ok { - return res - } - return "" - }(), - Method: func() int { - if res, ok := method.(float64); ok { - return int(res) - } - - return 0 - }(), - Request: func() string { - if request != nil { - res, ok := request.(string) - if ok { - return res - } - } - return "" - }(), - HaveAr: func() *bool { - if haveAr != nil { - res, ok := haveAr.(bool) - if ok { - return &res - } - } - return nil - }(), - } - - protoData = append(protoData, inboundRawData) - } - } - } - } - - if decodeError == true { - statsCollector.IncRawRequests("error", "decode") - log.Infof("Raw: Data could not be decoded. From User agent %s - Received data %s", userAgent, body) - - w.Header().Set("Content-Type", "application/json; charset=UTF-8") - w.WriteHeader(http.StatusUnprocessableEntity) - return - } - - // Process each proto in a packet in sequence, but in a go-routine - go func() { - timeout := 5 * time.Second - if config.Config.Tuning.ExtendedTimeout { - timeout = 30 * time.Second - } - - for _, entry := range protoData { - method := entry.Method - payload := entry.Base64Data - request := entry.Request - - haveAr := globalHaveAr - if entry.HaveAr != nil { - haveAr = entry.HaveAr - } - - protoData := ProtoData{ - Account: account, - Level: level, - HaveAr: haveAr, - Uuid: uuid, - Lat: latTarget, - Lon: lonTarget, - ScanContext: scanContext, - TimestampMs: dataReceivedTimestamp, - } - protoData.Data, _ = b64.StdEncoding.DecodeString(payload) - if request != "" { - protoData.Request, _ = b64.StdEncoding.DecodeString(request) - } - - // provide independent cancellation contexts for each proto decode - ctx, cancel := context.WithTimeout(context.Background(), timeout) - decode(ctx, method, &protoData) - cancel() - } - }() - - if latTarget != 0 && lonTarget != 0 && uuid != "" { - UpdateDeviceLocation(uuid, latTarget, lonTarget, scanContext) - } - - statsCollector.IncRawRequests("ok", "") - w.Header().Set("Content-Type", "application/json; charset=UTF-8") - w.WriteHeader(http.StatusCreated) - //if err := json.NewEncoder(w).Encode(t); err != nil { - // panic(err) - //} -} - func AuthRequired() gin.HandlerFunc { return func(context *gin.Context) { if config.Config.ApiSecret != "" { @@ -735,7 +436,3 @@ func GetTappable(c *gin.Context) { c.JSON(http.StatusAccepted, tappable) } - -func GetDevices(c *gin.Context) { - c.JSON(http.StatusOK, gin.H{"devices": GetAllDevices()}) -}