Skip to content

Refactor raw decoding #168

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
comstud wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

Refactor raw decoding #168

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions decoder/pokestop.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1019,6 +1019,10 @@ func getFortIdFromContest(id string) string {
}

func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDetails, request *pogo.GetPokemonSizeLeaderboardEntryProto, contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) string {
if request == nil {
return "Request is not available"
}

fortId := getFortIdFromContest(request.GetContestId())

pokestopMutex, _ := pokestopStripedMutex.GetLock(fortId)
Expand Down
4 changes: 2 additions & 2 deletions decoder/scanarea.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,14 @@ type ScanParameters struct {
ProactiveIVSwitchingToDB bool
}

func FindScanConfiguration(scanContext string, lat, lon float64) ScanParameters {
func FindScanConfiguration(scanContext string, location geo.Location) ScanParameters {
var areas []geo.AreaName
areaLookedUp := false

for _, rule := range config.Config.ScanRules {
if len(rule.AreaNames) > 0 {
if !areaLookedUp {
areas = MatchStatsGeofence(lat, lon)
areas = MatchStatsGeofence(location.Latitude, location.Longitude)
areaLookedUp = true
}
if !geo.AreaMatchWithWildcards(areas, rule.AreaNames) {
Expand Down
54 changes: 0 additions & 54 deletions deviceList.go
View file
Open in desktop

This file was deleted.

62 changes: 62 additions & 0 deletions device_tracker/device_tracker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package device_tracker

import (
"context"
"golbat/geo"
"time"

"github.com/jellydator/ttlcache/v3"
)

type DeviceLocation struct {
Latitude float64
Longitude float64
LastUpdate int64
ScanContext string
}

type DeviceTracker struct {
maxDeviceTTL time.Duration
deviceLocation *ttlcache.Cache[string, DeviceLocation]
}

func (tracker *DeviceTracker) UpdateDeviceLocation(deviceId string, location geo.Location, scanContext string) {
if location.IsZero() || deviceId == "" {
return
}
tracker.deviceLocation.Set(deviceId, DeviceLocation{
Latitude: location.Latitude,
Longitude: location.Longitude,
LastUpdate: time.Now().Unix(),
ScanContext: scanContext,
}, tracker.maxDeviceTTL)
}

func (tracker *DeviceTracker) IterateDevices(yield func(string, DeviceLocation) bool) {
for _, key := range tracker.deviceLocation.Items() {
if !yield(key.Key(), key.Value()) {
return
}
}
}

func (tracker *DeviceTracker) Run(ctx context.Context) {
ctx, cancelFn := context.WithCancel(ctx)
defer cancelFn()
go func() {
defer tracker.deviceLocation.Stop()
<-ctx.Done()
}()
tracker.deviceLocation.Start()
}

func NewDeviceTracker(maxDeviceTTLHours int) *DeviceTracker {
maxDeviceTTL := time.Hour * time.Duration(maxDeviceTTLHours)
tracker := &DeviceTracker{
maxDeviceTTL: maxDeviceTTL,
deviceLocation: ttlcache.New[string, DeviceLocation](
ttlcache.WithTTL[string, DeviceLocation](maxDeviceTTL),
),
}
return tracker
}
8 changes: 6 additions & 2 deletions geo/location.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,12 @@ import (
)

type Location struct {
Latitude float64
Longitude float64
Latitude float64 `json:"lat"`
Longitude float64 `json:"lon"`
}

func (l Location) IsZero() bool {
return l.Latitude == 0 && l.Longitude == 0
}

type Bbox struct {
Expand Down
65 changes: 4 additions & 61 deletions grpc_server_raw.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@ package main

import (
"context"
"time"

"golbat/config"
pb "golbat/grpc"
"golbat/raw_decoder/grpc_raw_decoder"

_ "google.golang.org/grpc/encoding/gzip" // Install the gzip compressor
"google.golang.org/grpc/metadata"
Expand All @@ -14,10 +14,11 @@ import (
// server is used to implement helloworld.GreeterServer.
type grpcRawServer struct {
pb.UnimplementedRawProtoServer

rawDecoder *grpc_raw_decoder.GRPCRawDecoder
}

func (s *grpcRawServer) SubmitRawProto(ctx context.Context, in *pb.RawProtoRequest) (*pb.RawProtoResponse, error) {
dataReceivedTimestamp := time.Now().UnixMilli()
// Check for authorisation
if config.Config.RawBearer != "" {
md, _ := metadata.FromIncomingContext(ctx)
Expand All @@ -26,64 +27,6 @@ func (s *grpcRawServer) SubmitRawProto(ctx context.Context, in *pb.RawProtoReque
return &pb.RawProtoResponse{Message: "Incorrect authorisation received"}, nil
}
}

uuid := in.DeviceId
account := in.Username
level := int(in.TrainerLevel)
scanContext := ""
if in.ScanContext != nil {
scanContext = *in.ScanContext
}

if in.Timestamp > 0 {
dataReceivedTimestamp = in.Timestamp
}

latTarget, lonTarget := float64(in.LatTarget), float64(in.LonTarget)
globalHaveAr := in.HaveAr
var protoData []ProtoData

for _, v := range in.Contents {
inboundRawData := ProtoData{
Method: int(v.Method),
Account: account,
Level: level,
ScanContext: scanContext,
Lat: latTarget,
Lon: lonTarget,
Data: v.ResponsePayload,
Request: v.RequestPayload,
Uuid: uuid,
HaveAr: func() *bool {
if v.HaveAr != nil {
return v.HaveAr
}
return globalHaveAr
}(),
TimestampMs: dataReceivedTimestamp,
}

protoData = append(protoData, inboundRawData)
}

// Process each proto in a packet in sequence, but in a go-routine
go func() {
timeout := 5 * time.Second
if config.Config.Tuning.ExtendedTimeout {
timeout = 30 * time.Second
}

for _, entry := range protoData {
// provide independent cancellation contexts for each proto decode
ctx, cancel := context.WithTimeout(context.Background(), timeout)
decode(ctx, entry.Method, &entry)
cancel()
}
}()

if latTarget != 0 && lonTarget != 0 && uuid != "" {
UpdateDeviceLocation(uuid, latTarget, lonTarget, scanContext)
}

s.rawDecoder.DecodeRaw(ctx, in)
return &pb.RawProtoResponse{Message: "Processed"}, nil
}
22 changes: 22 additions & 0 deletions http_handler/device.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
package http_handler

import (
"net/http"

"github.com/gin-gonic/gin"
)

type DeviceLocation struct {
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
LastUpdate int64 `json:"last_update"`
ScanContext string `json:"scan_context"`
}

func (h *HTTPHandler) GetAllDevices(ginContext *gin.Context) {
devices := map[string]DeviceLocation{}
for deviceId, location := range h.deviceTracker.IterateDevices {
devices[deviceId] = DeviceLocation(location)
}
ginContext.JSON(http.StatusOK, gin.H{"devices": devices})
}
24 changes: 24 additions & 0 deletions http_handler/http_handler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
package http_handler

import (
"golbat/db"
"golbat/device_tracker"
"golbat/raw_decoder/http_raw_decoder"
"golbat/stats_collector"
)

type HTTPHandler struct {
rawDecoder *http_raw_decoder.HTTPRawDecoder
dbDetails db.DbDetails
statsCollector stats_collector.StatsCollector
deviceTracker *device_tracker.DeviceTracker
}

func NewHTTPHandler(rawDecoder *http_raw_decoder.HTTPRawDecoder, dbDetails db.DbDetails, statsCollector stats_collector.StatsCollector, deviceTracker *device_tracker.DeviceTracker) *HTTPHandler {
return &HTTPHandler{
rawDecoder: rawDecoder,
dbDetails: dbDetails,
statsCollector: statsCollector,
deviceTracker: deviceTracker,
}
}
58 changes: 58 additions & 0 deletions http_handler/raw.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
package http_handler

import (
"context"
"io"
"net/http"
"time"

"github.com/gin-gonic/gin"
log "github.com/sirupsen/logrus"

"golbat/config"
)

func (h *HTTPHandler) Raw(c *gin.Context) {
w := c.Writer
r := c.Request

statsCollector := h.statsCollector

requestReceivedMs := time.Now().UnixMilli()

authHeader := r.Header.Get("Authorization")
if config.Config.RawBearer != "" {
if authHeader != "Bearer "+config.Config.RawBearer {
statsCollector.IncRawRequests("error", "auth")
log.Errorf("Raw: Incorrect authorisation received (%s)", authHeader)
return
}
}

body, err := io.ReadAll(io.LimitReader(r.Body, 5*1048576))
if err != nil {
statsCollector.IncRawRequests("error", "io_error")
log.Errorf("Raw: Error (1) during HTTP receive %s", err)
return
}

if err := r.Body.Close(); err != nil {
statsCollector.IncRawRequests("error", "io_close_error")
log.Errorf("Raw: Error (2) during HTTP receive %s", err)
return
}

ctx := context.Background()
if err := h.rawDecoder.DecodeRaw(ctx, r.Header, body, requestReceivedMs); err != nil {
statsCollector.IncRawRequests("error", "decode")
userAgent := r.Header.Get("User-Agent")
log.Infof("Raw: Data could not be decoded. From User agent %s - Received data %s, err: %s", userAgent, body, err)
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.WriteHeader(http.StatusUnprocessableEntity)
return
}

statsCollector.IncRawRequests("ok", "")
w.Header().Set("Content-Type", "application/json; charset=UTF-8")
w.WriteHeader(http.StatusCreated)
}
Loading