From 215fcb2d1dff59d3cac40a6cf65938c40b61f235 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 25 Jan 2026 16:15:11 +0000 Subject: [PATCH 01/35] Webhooks to structures --- decoder/fort.go | 29 ++++--- decoder/gym.go | 126 +++++++++++++++++-------------- decoder/incident.go | 71 +++++++++++------- decoder/pokemon.go | 168 ++++++++++++++++++++++++----------------- decoder/pokestop.go | 179 +++++++++++++++++++++++++------------------- decoder/station.go | 68 +++++++++++------ decoder/weather.go | 51 +++++++++---- routes.go | 12 +-- 8 files changed, 421 insertions(+), 283 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index ed28030f..cb17aa73 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -27,6 +27,13 @@ type FortWebhook struct { Location Location `json:"location"` } +type FortChangeWebhook struct { + ChangeType string `json:"change_type"` + EditTypes []string `json:"edit_types,omitempty"` + Old *FortWebhook `json:"old,omitempty"` + New *FortWebhook `json:"new,omitempty"` +} + type FortChange string type FortType string @@ -129,17 +136,17 @@ func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []strin func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { if change == NEW { areas := MatchStatsGeofence(new.Location.Latitude, new.Location.Longitude) - hook := map[string]interface{}{ - "change_type": change.String(), - "new": new, + hook := FortChangeWebhook{ + ChangeType: change.String(), + New: new, } webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) statsCollector.UpdateFortCount(areas, new.Type, "addition") } else if change == REMOVAL { areas := MatchStatsGeofence(old.Location.Latitude, old.Location.Longitude) - hook := map[string]interface{}{ - "change_type": change.String(), - "old": old, + hook := FortChangeWebhook{ + ChangeType: change.String(), + Old: old, } webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) statsCollector.UpdateFortCount(areas, new.Type, "removal") @@ -181,11 +188,11 @@ func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { editTypes = append(editTypes, "location") } if len(editTypes) > 0 { - hook := map[string]interface{}{ - "change_type": change.String(), - "edit_types": editTypes, - "old": old, - "new": new, + hook := FortChangeWebhook{ + ChangeType: change.String(), + EditTypes: editTypes, + Old: old, + New: new, } webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) statsCollector.UpdateFortCount(areas, new.Type, "edit") diff --git a/decoder/gym.go b/decoder/gym.go index 4f1784fe..a1f3b06b 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -505,23 +505,37 @@ type GymDetailsWebhook struct { PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` ArScanEligible int64 `json:"ar_scan_eligible"` Defenders any `json:"defenders"` +} - //"id": id, - //"name": name ?? "Unknown", - //"url": url ?? "", - //"latitude": lat, - //"longitude": lon, - //"team": teamId ?? 0, - //"guard_pokemon_id": guardPokemonId ?? 0, - //"slots_available": availableSlots ?? 6, - //"ex_raid_eligible": exRaidEligible ?? 0, - //"in_battle": inBattle ?? false, - //"sponsor_id": sponsorId ?? 0, - //"partner_id": partnerId ?? 0, - //"power_up_points": powerUpPoints ?? 0, - //"power_up_level": powerUpLevel ?? 0, - //"power_up_end_timestamp": powerUpEndTimestamp ?? 0, - //"ar_scan_eligible": arScanEligible ?? 0 +type RaidWebhook struct { + GymId string `json:"gym_id"` + GymName string `json:"gym_name"` + GymUrl string `json:"gym_url"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + TeamId int64 `json:"team_id"` + Spawn int64 `json:"spawn"` + Start int64 `json:"start"` + End int64 `json:"end"` + Level int64 `json:"level"` + PokemonId int64 `json:"pokemon_id"` + Cp int64 `json:"cp"` + Gender int64 `json:"gender"` + Form int64 `json:"form"` + Alignment int64 `json:"alignment"` + Costume int64 `json:"costume"` + Evolution int64 `json:"evolution"` + Move1 int64 `json:"move_1"` + Move2 int64 `json:"move_2"` + ExRaidEligible int64 `json:"ex_raid_eligible"` + IsExclusive int64 `json:"is_exclusive"` + SponsorId int64 `json:"sponsor_id"` + PartnerId string `json:"partner_id"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + ArScanEligible int64 `json:"ar_scan_eligible"` + Rsvps json.RawMessage `json:"rsvps"` } func createGymFortWebhooks(oldGym *Gym, gym *Gym) { @@ -576,47 +590,45 @@ func createGymWebhooks(oldGym *Gym, gym *Gym, areas []geo.AreaName) { if (raidBattleTime > now && gym.RaidLevel.ValueOrZero() > 0) || (raidEndTime > now && gym.RaidPokemonId.ValueOrZero() > 0) { - raidHook := map[string]any{ - "gym_id": gym.Id, - "gym_name": func() string { - if !gym.Name.Valid { - return "Unknown" - } else { - return gym.Name.String - } - }(), - "gym_url": gym.Url.ValueOrZero(), - "latitude": gym.Lat, - "longitude": gym.Lon, - "team_id": gym.TeamId.ValueOrZero(), - "spawn": gym.RaidSpawnTimestamp.ValueOrZero(), - "start": gym.RaidBattleTimestamp.ValueOrZero(), - "end": gym.RaidEndTimestamp.ValueOrZero(), - "level": gym.RaidLevel.ValueOrZero(), - "pokemon_id": gym.RaidPokemonId.ValueOrZero(), - "cp": gym.RaidPokemonCp.ValueOrZero(), - "gender": gym.RaidPokemonGender.ValueOrZero(), - "form": gym.RaidPokemonForm.ValueOrZero(), - "alignment": gym.RaidPokemonAlignment.ValueOrZero(), - "costume": gym.RaidPokemonCostume.ValueOrZero(), - "evolution": gym.RaidPokemonEvolution.ValueOrZero(), - "move_1": gym.RaidPokemonMove1.ValueOrZero(), - "move_2": gym.RaidPokemonMove2.ValueOrZero(), - "ex_raid_eligible": gym.ExRaidEligible.ValueOrZero(), - "is_exclusive": gym.RaidIsExclusive.ValueOrZero(), - "sponsor_id": gym.SponsorId.ValueOrZero(), - "partner_id": gym.PartnerId.ValueOrZero(), - "power_up_points": gym.PowerUpPoints.ValueOrZero(), - "power_up_level": gym.PowerUpLevel.ValueOrZero(), - "power_up_end_timestamp": gym.PowerUpEndTimestamp.ValueOrZero(), - "ar_scan_eligible": gym.ArScanEligible.ValueOrZero(), - "rsvps": func() any { - if !gym.Rsvps.Valid { - return nil - } else { - return json.RawMessage(gym.Rsvps.ValueOrZero()) - } - }(), + gymName := "Unknown" + if gym.Name.Valid { + gymName = gym.Name.String + } + + var rsvps json.RawMessage + if gym.Rsvps.Valid { + rsvps = json.RawMessage(gym.Rsvps.ValueOrZero()) + } + + raidHook := RaidWebhook{ + GymId: gym.Id, + GymName: gymName, + GymUrl: gym.Url.ValueOrZero(), + Latitude: gym.Lat, + Longitude: gym.Lon, + TeamId: gym.TeamId.ValueOrZero(), + Spawn: gym.RaidSpawnTimestamp.ValueOrZero(), + Start: gym.RaidBattleTimestamp.ValueOrZero(), + End: gym.RaidEndTimestamp.ValueOrZero(), + Level: gym.RaidLevel.ValueOrZero(), + PokemonId: gym.RaidPokemonId.ValueOrZero(), + Cp: gym.RaidPokemonCp.ValueOrZero(), + Gender: gym.RaidPokemonGender.ValueOrZero(), + Form: gym.RaidPokemonForm.ValueOrZero(), + Alignment: gym.RaidPokemonAlignment.ValueOrZero(), + Costume: gym.RaidPokemonCostume.ValueOrZero(), + Evolution: gym.RaidPokemonEvolution.ValueOrZero(), + Move1: gym.RaidPokemonMove1.ValueOrZero(), + Move2: gym.RaidPokemonMove2.ValueOrZero(), + ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), + IsExclusive: gym.RaidIsExclusive.ValueOrZero(), + SponsorId: gym.SponsorId.ValueOrZero(), + PartnerId: gym.PartnerId.ValueOrZero(), + PowerUpPoints: gym.PowerUpPoints.ValueOrZero(), + PowerUpLevel: gym.PowerUpLevel.ValueOrZero(), + PowerUpEndTimestamp: gym.PowerUpEndTimestamp.ValueOrZero(), + ArScanEligible: gym.ArScanEligible.ValueOrZero(), + Rsvps: rsvps, } webhooksSender.AddMessage(webhooks.Raid, raidHook, areas) diff --git a/decoder/incident.go b/decoder/incident.go index 6d2a7dce..7e5fbde5 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -40,6 +40,26 @@ type webhookLineup struct { Form null.Int `json:"form"` } +type IncidentWebhook struct { + Id string `json:"id"` + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + PokestopName string `json:"pokestop_name"` + Url string `json:"url"` + Enabled bool `json:"enabled"` + Start int64 `json:"start"` + IncidentExpireTimestamp int64 `json:"incident_expire_timestamp"` + Expiration int64 `json:"expiration"` + DisplayType int16 `json:"display_type"` + Style int16 `json:"style"` + GruntType int16 `json:"grunt_type"` + Character int16 `json:"character"` + Updated int64 `json:"updated"` + Confirmed bool `json:"confirmed"` + Lineup []webhookLineup `json:"lineup"` +} + //-> `id` varchar(35) NOT NULL, //-> `pokestop_id` varchar(35) NOT NULL, //-> `start` int unsigned NOT NULL, @@ -160,34 +180,14 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, oldIncident *I stop = &Pokestop{} } - incidentHook := map[string]interface{}{ - "id": incident.Id, - "pokestop_id": incident.PokestopId, - "latitude": stop.Lat, - "longitude": stop.Lon, - "pokestop_name": func() string { - if stop.Name.Valid { - return stop.Name.String - } else { - return "Unknown" - } - }(), - "url": stop.Url.ValueOrZero(), - "enabled": stop.Enabled.ValueOrZero(), - "start": incident.StartTime, - "incident_expire_timestamp": incident.ExpirationTime, // deprecated, remove old key in the future - "expiration": incident.ExpirationTime, - "display_type": incident.DisplayType, - "style": incident.Style, - "grunt_type": incident.Character, // deprecated, remove old key in the future - "character": incident.Character, - "updated": incident.Updated, - "confirmed": incident.Confirmed, - "lineup": nil, + pokestopName := "Unknown" + if stop.Name.Valid { + pokestopName = stop.Name.String } + var lineup []webhookLineup if incident.Slot1PokemonId.Valid { - incidentHook["lineup"] = []webhookLineup{ + lineup = []webhookLineup{ { Slot: 1, PokemonId: incident.Slot1PokemonId, @@ -205,6 +205,27 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, oldIncident *I }, } } + + incidentHook := IncidentWebhook{ + Id: incident.Id, + PokestopId: incident.PokestopId, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Url: stop.Url.ValueOrZero(), + Enabled: stop.Enabled.ValueOrZero(), + Start: incident.StartTime, + IncidentExpireTimestamp: incident.ExpirationTime, + Expiration: incident.ExpirationTime, + DisplayType: incident.DisplayType, + Style: incident.Style, + GruntType: incident.Character, + Character: incident.Character, + Updated: incident.Updated, + Confirmed: incident.Confirmed, + Lineup: lineup, + } + areas := MatchStatsGeofence(stop.Lat, stop.Lon) webhooksSender.AddMessage(webhooks.Invasion, incidentHook, areas) statsCollector.UpdateIncidentCount(areas) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9244cc2b..27416d1c 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -412,81 +412,109 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } -func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, new *Pokemon, areas []geo.AreaName) { - //nullString := func (v null.Int) interface{} { - // if !v.Valid { - // return "null" - // } - // return v.ValueOrZero() - //} +type PokemonWebhook struct { + SpawnpointId string `json:"spawnpoint_id"` + PokestopId string `json:"pokestop_id"` + PokestopName *string `json:"pokestop_name"` + EncounterId string `json:"encounter_id"` + PokemonId int16 `json:"pokemon_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + DisappearTime int64 `json:"disappear_time"` + DisappearTimeVerified bool `json:"disappear_time_verified"` + FirstSeen int64 `json:"first_seen"` + LastModifiedTime null.Int `json:"last_modified_time"` + Gender null.Int `json:"gender"` + Cp null.Int `json:"cp"` + Form null.Int `json:"form"` + Costume null.Int `json:"costume"` + IndividualAttack null.Int `json:"individual_attack"` + IndividualDefense null.Int `json:"individual_defense"` + IndividualStamina null.Int `json:"individual_stamina"` + PokemonLevel null.Int `json:"pokemon_level"` + Move1 null.Int `json:"move_1"` + Move2 null.Int `json:"move_2"` + Weight null.Float `json:"weight"` + Size null.Int `json:"size"` + Height null.Float `json:"height"` + Weather null.Int `json:"weather"` + Capture1 float64 `json:"capture_1"` + Capture2 float64 `json:"capture_2"` + Capture3 float64 `json:"capture_3"` + Shiny null.Bool `json:"shiny"` + Username null.String `json:"username"` + DisplayPokemonId null.Int `json:"display_pokemon_id"` + IsEvent int8 `json:"is_event"` + SeenType null.String `json:"seen_type"` + Pvp json.RawMessage `json:"pvp"` +} +func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, new *Pokemon, areas []geo.AreaName) { if old == nil || old.PokemonId != new.PokemonId || old.Weather != new.Weather || old.Cp != new.Cp { - pokemonHook := map[string]interface{}{ - "spawnpoint_id": func() string { - if !new.SpawnId.Valid { - return "None" - } - return strconv.FormatInt(new.SpawnId.ValueOrZero(), 16) - }(), - "pokestop_id": func() string { - if !new.PokestopId.Valid { - return "None" - } else { - return new.PokestopId.ValueOrZero() - } - }(), - "pokestop_name": func() *string { - if !new.PokestopId.Valid { - return nil - } else { - pokestop, _ := GetPokestopRecord(ctx, db, new.PokestopId.String) - name := "Unknown" - if pokestop != nil { - name = pokestop.Name.ValueOrZero() - } - return &name - } - }(), - "encounter_id": strconv.FormatUint(new.Id, 10), - "pokemon_id": new.PokemonId, - "latitude": new.Lat, - "longitude": new.Lon, - "disappear_time": new.ExpireTimestamp.ValueOrZero(), - "disappear_time_verified": new.ExpireTimestampVerified, - "first_seen": new.FirstSeenTimestamp, - "last_modified_time": new.Updated, - "gender": new.Gender, - "cp": new.Cp, - "form": new.Form, - "costume": new.Costume, - "individual_attack": new.AtkIv, - "individual_defense": new.DefIv, - "individual_stamina": new.StaIv, - "pokemon_level": new.Level, - "move_1": new.Move1, - "move_2": new.Move2, - "weight": new.Weight, - "size": new.Size, - "height": new.Height, - "weather": new.Weather, - "capture_1": new.Capture1.ValueOrZero(), - "capture_2": new.Capture2.ValueOrZero(), - "capture_3": new.Capture3.ValueOrZero(), - "shiny": new.Shiny, - "username": new.Username, - "display_pokemon_id": new.DisplayPokemonId, - "is_event": new.IsEvent, - "seen_type": new.SeenType, - "pvp": func() interface{} { - if !new.Pvp.Valid { - return nil - } else { - return json.RawMessage(new.Pvp.ValueOrZero()) - } - }(), + + spawnpointId := "None" + if new.SpawnId.Valid { + spawnpointId = strconv.FormatInt(new.SpawnId.ValueOrZero(), 16) + } + + pokestopId := "None" + if new.PokestopId.Valid { + pokestopId = new.PokestopId.ValueOrZero() + } + + var pokestopName *string + if new.PokestopId.Valid { + pokestop, _ := GetPokestopRecord(ctx, db, new.PokestopId.String) + name := "Unknown" + if pokestop != nil { + name = pokestop.Name.ValueOrZero() + } + pokestopName = &name + } + + var pvp json.RawMessage + if new.Pvp.Valid { + pvp = json.RawMessage(new.Pvp.ValueOrZero()) + } + + pokemonHook := PokemonWebhook{ + SpawnpointId: spawnpointId, + PokestopId: pokestopId, + PokestopName: pokestopName, + EncounterId: strconv.FormatUint(new.Id, 10), + PokemonId: new.PokemonId, + Latitude: new.Lat, + Longitude: new.Lon, + DisappearTime: new.ExpireTimestamp.ValueOrZero(), + DisappearTimeVerified: new.ExpireTimestampVerified, + FirstSeen: new.FirstSeenTimestamp, + LastModifiedTime: new.Updated, + Gender: new.Gender, + Cp: new.Cp, + Form: new.Form, + Costume: new.Costume, + IndividualAttack: new.AtkIv, + IndividualDefense: new.DefIv, + IndividualStamina: new.StaIv, + PokemonLevel: new.Level, + Move1: new.Move1, + Move2: new.Move2, + Weight: new.Weight, + Size: new.Size, + Height: new.Height, + Weather: new.Weather, + Capture1: new.Capture1.ValueOrZero(), + Capture2: new.Capture2.ValueOrZero(), + Capture3: new.Capture3.ValueOrZero(), + Shiny: new.Shiny, + Username: new.Username, + DisplayPokemonId: new.DisplayPokemonId, + IsEvent: new.IsEvent, + SeenType: new.SeenType, + Pvp: pvp, } if new.AtkIv.Valid && new.DefIv.Valid && new.StaIv.Valid { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index ee7e966c..d6dbba86 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -108,6 +108,47 @@ type Pokestop struct { } +type QuestWebhook struct { + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + PokestopName string `json:"pokestop_name"` + Type null.Int `json:"type"` + Target null.Int `json:"target"` + Template null.String `json:"template"` + Title null.String `json:"title"` + Conditions json.RawMessage `json:"conditions"` + Rewards json.RawMessage `json:"rewards"` + Updated int64 `json:"updated"` + ArScanEligible int64 `json:"ar_scan_eligible"` + PokestopUrl string `json:"pokestop_url"` + WithAr bool `json:"with_ar"` +} + +type PokestopWebhook struct { + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + Url string `json:"url"` + LureExpiration int64 `json:"lure_expiration"` + LastModified int64 `json:"last_modified"` + Enabled bool `json:"enabled"` + LureId int16 `json:"lure_id"` + ArScanEligible int64 `json:"ar_scan_eligible"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + Updated int64 `json:"updated"` + ShowcaseFocus null.String `json:"showcase_focus"` + ShowcasePokemonId null.Int `json:"showcase_pokemon_id"` + ShowcasePokemonFormId null.Int `json:"showcase_pokemon_form_id"` + ShowcasePokemonTypeId null.Int `json:"showcase_pokemon_type_id"` + ShowcaseRankingStandard null.Int `json:"showcase_ranking_standard"` + ShowcaseExpiry null.Int `json:"showcase_expiry"` + ShowcaseRankings json.RawMessage `json:"showcase_rankings"` +} + func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, error) { stop := pokestopCache.Get(fortId) if stop != nil { @@ -663,92 +704,78 @@ func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { areas := MatchStatsGeofence(stop.Lat, stop.Lon) + pokestopName := "Unknown" + if stop.Name.Valid { + pokestopName = stop.Name.String + } + if stop.AlternativeQuestType.Valid && (oldStop == nil || stop.AlternativeQuestType != oldStop.AlternativeQuestType) { - questHook := map[string]any{ - "pokestop_id": stop.Id, - "latitude": stop.Lat, - "longitude": stop.Lon, - "pokestop_name": func() string { - if stop.Name.Valid { - return stop.Name.String - } else { - return "Unknown" - } - }(), - "type": stop.AlternativeQuestType, - "target": stop.AlternativeQuestTarget, - "template": stop.AlternativeQuestTemplate, - "title": stop.AlternativeQuestTitle, - "conditions": json.RawMessage(stop.AlternativeQuestConditions.ValueOrZero()), - "rewards": json.RawMessage(stop.AlternativeQuestRewards.ValueOrZero()), - "updated": stop.Updated, - "ar_scan_eligible": stop.ArScanEligible.ValueOrZero(), - "pokestop_url": stop.Url.ValueOrZero(), - "with_ar": false, + questHook := QuestWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Type: stop.AlternativeQuestType, + Target: stop.AlternativeQuestTarget, + Template: stop.AlternativeQuestTemplate, + Title: stop.AlternativeQuestTitle, + Conditions: json.RawMessage(stop.AlternativeQuestConditions.ValueOrZero()), + Rewards: json.RawMessage(stop.AlternativeQuestRewards.ValueOrZero()), + Updated: stop.Updated, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PokestopUrl: stop.Url.ValueOrZero(), + WithAr: false, } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } if stop.QuestType.Valid && (oldStop == nil || stop.QuestType != oldStop.QuestType) { - questHook := map[string]any{ - "pokestop_id": stop.Id, - "latitude": stop.Lat, - "longitude": stop.Lon, - "pokestop_name": func() string { - if stop.Name.Valid { - return stop.Name.String - } else { - return "Unknown" - } - }(), - "type": stop.QuestType, - "target": stop.QuestTarget, - "template": stop.QuestTemplate, - "title": stop.QuestTitle, - "conditions": json.RawMessage(stop.QuestConditions.ValueOrZero()), - "rewards": json.RawMessage(stop.QuestRewards.ValueOrZero()), - "updated": stop.Updated, - "ar_scan_eligible": stop.ArScanEligible.ValueOrZero(), - "pokestop_url": stop.Url.ValueOrZero(), - "with_ar": true, + questHook := QuestWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Type: stop.QuestType, + Target: stop.QuestTarget, + Template: stop.QuestTemplate, + Title: stop.QuestTitle, + Conditions: json.RawMessage(stop.QuestConditions.ValueOrZero()), + Rewards: json.RawMessage(stop.QuestRewards.ValueOrZero()), + Updated: stop.Updated, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PokestopUrl: stop.Url.ValueOrZero(), + WithAr: true, } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } if (oldStop == nil && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (oldStop != nil && ((stop.LureExpireTimestamp != oldStop.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != oldStop.PowerUpEndTimestamp)) { - pokestopHook := map[string]any{ - "pokestop_id": stop.Id, - "latitude": stop.Lat, - "longitude": stop.Lon, - "name": func() string { - if stop.Name.Valid { - return stop.Name.String - } else { - return "Unknown" - } - }(), - "url": stop.Url.ValueOrZero(), - "lure_expiration": stop.LureExpireTimestamp.ValueOrZero(), - "last_modified": stop.LastModifiedTimestamp.ValueOrZero(), - "enabled": stop.Enabled.ValueOrZero(), - "lure_id": stop.LureId, - "ar_scan_eligible": stop.ArScanEligible.ValueOrZero(), - "power_up_level": stop.PowerUpLevel.ValueOrZero(), - "power_up_points": stop.PowerUpPoints.ValueOrZero(), - "power_up_end_timestamp": stop.PowerUpPoints.ValueOrZero(), - "updated": stop.Updated, - "showcase_focus": stop.ShowcaseFocus, - "showcase_pokemon_id": stop.ShowcasePokemon, - "showcase_pokemon_form_id": stop.ShowcasePokemonForm, - "showcase_pokemon_type_id": stop.ShowcasePokemonType, - "showcase_ranking_standard": stop.ShowcaseRankingStandard, - "showcase_expiry": stop.ShowcaseExpiry, - "showcase_rankings": func() any { - if !stop.ShowcaseRankings.Valid { - return nil - } else { - return json.RawMessage(stop.ShowcaseRankings.ValueOrZero()) - } - }(), + var showcaseRankings json.RawMessage + if stop.ShowcaseRankings.Valid { + showcaseRankings = json.RawMessage(stop.ShowcaseRankings.ValueOrZero()) + } + + pokestopHook := PokestopWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + Name: pokestopName, + Url: stop.Url.ValueOrZero(), + LureExpiration: stop.LureExpireTimestamp.ValueOrZero(), + LastModified: stop.LastModifiedTimestamp.ValueOrZero(), + Enabled: stop.Enabled.ValueOrZero(), + LureId: stop.LureId, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PowerUpLevel: stop.PowerUpLevel.ValueOrZero(), + PowerUpPoints: stop.PowerUpPoints.ValueOrZero(), + PowerUpEndTimestamp: stop.PowerUpPoints.ValueOrZero(), + Updated: stop.Updated, + ShowcaseFocus: stop.ShowcaseFocus, + ShowcasePokemonId: stop.ShowcasePokemon, + ShowcasePokemonFormId: stop.ShowcasePokemonForm, + ShowcasePokemonTypeId: stop.ShowcasePokemonType, + ShowcaseRankingStandard: stop.ShowcaseRankingStandard, + ShowcaseExpiry: stop.ShowcaseExpiry, + ShowcaseRankings: showcaseRankings, } webhooksSender.AddMessage(webhooks.Pokestop, pokestopHook, areas) diff --git a/decoder/station.go b/decoder/station.go index 34b0de81..f02b46f1 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -49,6 +49,30 @@ type Station struct { StationedPokemon null.String `db:"stationed_pokemon"` } +type StationWebhook struct { + Id string `json:"id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + IsBattleAvailable bool `json:"is_battle_available"` + BattleLevel null.Int `json:"battle_level"` + BattleStart null.Int `json:"battle_start"` + BattleEnd null.Int `json:"battle_end"` + BattlePokemonId null.Int `json:"battle_pokemon_id"` + BattlePokemonForm null.Int `json:"battle_pokemon_form"` + BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` + BattlePokemonGender null.Int `json:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` + TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` + TotalStationedGmax null.Int `json:"total_stationed_gmax"` + Updated int64 `json:"updated"` +} + func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, error) { inMemoryStation := stationCache.Get(stationId) if inMemoryStation != nil { @@ -317,28 +341,28 @@ func createStationWebhooks(oldStation *Station, station *Station) { oldStation.BattlePokemonCostume != station.BattlePokemonCostume || oldStation.BattlePokemonGender != station.BattlePokemonGender || oldStation.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { - stationHook := map[string]any{ - "id": station.Id, - "latitude": station.Lat, - "longitude": station.Lon, - "name": station.Name, - "start_time": station.StartTime, - "end_time": station.EndTime, - "is_battle_available": station.IsBattleAvailable, - "battle_level": station.BattleLevel, - "battle_start": station.BattleStart, - "battle_end": station.BattleEnd, - "battle_pokemon_id": station.BattlePokemonId, - "battle_pokemon_form": station.BattlePokemonForm, - "battle_pokemon_costume": station.BattlePokemonCostume, - "battle_pokemon_gender": station.BattlePokemonGender, - "battle_pokemon_alignment": station.BattlePokemonAlignment, - "battle_pokemon_bread_mode": station.BattlePokemonBreadMode, - "battle_pokemon_move_1": station.BattlePokemonMove1, - "battle_pokemon_move_2": station.BattlePokemonMove2, - "total_stationed_pokemon": station.TotalStationedPokemon, - "total_stationed_gmax": station.TotalStationedGmax, - "updated": station.Updated, + stationHook := StationWebhook{ + Id: station.Id, + Latitude: station.Lat, + Longitude: station.Lon, + Name: station.Name, + StartTime: station.StartTime, + EndTime: station.EndTime, + IsBattleAvailable: station.IsBattleAvailable, + BattleLevel: station.BattleLevel, + BattleStart: station.BattleStart, + BattleEnd: station.BattleEnd, + BattlePokemonId: station.BattlePokemonId, + BattlePokemonForm: station.BattlePokemonForm, + BattlePokemonCostume: station.BattlePokemonCostume, + BattlePokemonGender: station.BattlePokemonGender, + BattlePokemonAlignment: station.BattlePokemonAlignment, + BattlePokemonBreadMode: station.BattlePokemonBreadMode, + BattlePokemonMove1: station.BattlePokemonMove1, + BattlePokemonMove2: station.BattlePokemonMove2, + TotalStationedPokemon: station.TotalStationedPokemon, + TotalStationedGmax: station.TotalStationedGmax, + Updated: station.Updated, } areas := MatchStatsGeofence(station.Lat, station.Lon) webhooksSender.AddMessage(webhooks.MaxBattle, stationHook, areas) diff --git a/decoder/weather.go b/decoder/weather.go index ae5aa573..3e29aa24 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -122,6 +122,24 @@ func hasChangesWeather(old *Weather, new *Weather) bool { !floatAlmostEqual(old.Longitude, new.Longitude, floatTolerance) } +type WeatherWebhook struct { + S2CellId int64 `json:"s2_cell_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Polygon [4][2]float64 `json:"polygon"` + GameplayCondition int64 `json:"gameplay_condition"` + WindDirection int64 `json:"wind_direction"` + CloudLevel int64 `json:"cloud_level"` + RainLevel int64 `json:"rain_level"` + WindLevel int64 `json:"wind_level"` + SnowLevel int64 `json:"snow_level"` + FogLevel int64 `json:"fog_level"` + SpecialEffectLevel int64 `json:"special_effect_level"` + Severity int64 `json:"severity"` + WarnWeather bool `json:"warn_weather"` + Updated int64 `json:"updated"` +} + func createWeatherWebhooks(oldWeather *Weather, weather *Weather) { if oldWeather == nil || oldWeather.GameplayCondition.ValueOrZero() != weather.GameplayCondition.ValueOrZero() || oldWeather.WarnWeather.ValueOrZero() != weather.WarnWeather.ValueOrZero() { @@ -133,22 +151,23 @@ func createWeatherWebhooks(oldWeather *Weather, weather *Weather) { latLng := s2.LatLngFromPoint(vertex) polygon[i] = [...]float64{latLng.Lat.Degrees(), latLng.Lng.Degrees()} } - weatherHook := map[string]interface{}{ - "s2_cell_id": weather.Id, - "latitude": weather.Latitude, - "longitude": weather.Longitude, - "polygon": polygon, - "gameplay_condition": weather.GameplayCondition.ValueOrZero(), - "wind_direction": weather.WindDirection.ValueOrZero(), - "cloud_level": weather.CloudLevel.ValueOrZero(), - "rain_level": weather.RainLevel.ValueOrZero(), - "wind_level": weather.WindLevel.ValueOrZero(), - "snow_level": weather.SnowLevel.ValueOrZero(), - "fog_level": weather.FogLevel.ValueOrZero(), - "special_effect_level": weather.SpecialEffectLevel.ValueOrZero(), - "severity": weather.Severity.ValueOrZero(), - "warn_weather": weather.WarnWeather.ValueOrZero(), - "updated": weather.UpdatedMs / 1000, + + weatherHook := WeatherWebhook{ + S2CellId: weather.Id, + Latitude: weather.Latitude, + Longitude: weather.Longitude, + Polygon: polygon, + GameplayCondition: weather.GameplayCondition.ValueOrZero(), + WindDirection: weather.WindDirection.ValueOrZero(), + CloudLevel: weather.CloudLevel.ValueOrZero(), + RainLevel: weather.RainLevel.ValueOrZero(), + WindLevel: weather.WindLevel.ValueOrZero(), + SnowLevel: weather.SnowLevel.ValueOrZero(), + FogLevel: weather.FogLevel.ValueOrZero(), + SpecialEffectLevel: weather.SpecialEffectLevel.ValueOrZero(), + Severity: weather.Severity.ValueOrZero(), + WarnWeather: weather.WarnWeather.ValueOrZero(), + Updated: weather.UpdatedMs / 1000, } areas := MatchStatsGeofence(weather.Latitude, weather.Longitude) webhooksSender.AddMessage(webhooks.Weather, weatherHook, areas) diff --git a/routes.go b/routes.go index e68c16a6..ca840480 100644 --- a/routes.go +++ b/routes.go @@ -40,6 +40,10 @@ type InboundRawData struct { HaveAr *bool } +type StatusResponse struct { + Status string `json:"status"` +} + func questsHeldHasARTask(quests_held any) *bool { const ar_quest_id = int64(pogo.QuestType_QUEST_GEOTARGETED_AR_SCAN) @@ -346,17 +350,13 @@ func ClearQuests(c *gin.Context) { decoder.ClearQuestsWithinGeofence(ctx, dbDetails, fence) log.Infof("Clear quest took %s", time.Since(startTime)) - c.JSON(http.StatusAccepted, map[string]interface{}{ - "status": "ok", - }) + c.JSON(http.StatusAccepted, StatusResponse{Status: "ok"}) } func ReloadGeojson(c *gin.Context) { decoder.ReloadGeofenceAndClearStats() - c.JSON(http.StatusAccepted, map[string]interface{}{ - "status": "ok", - }) + c.JSON(http.StatusAccepted, StatusResponse{Status: "ok"}) } func PokemonScan(c *gin.Context) { From f8b2eb151320ede613d51feef25530010f4cbfcc Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 11 Jan 2026 16:04:25 +0000 Subject: [PATCH 02/35] Replace per-pokemon goroutines with pending queue Instead of spawning a goroutine for each wild pokemon that sleeps waiting for an encounter, use a single pending queue with a background sweeper. This reduces memory pressure and goroutine overhead under high load. Co-Authored-By: Claude Opus 4.5 --- decoder/main.go | 33 +++----- decoder/pending_pokemon.go | 166 +++++++++++++++++++++++++++++++++++++ decoder/pokemon.go | 5 ++ main.go | 1 + 4 files changed, 184 insertions(+), 21 deletions(-) create mode 100644 decoder/pending_pokemon.go diff --git a/decoder/main.go b/decoder/main.go index 42ec4c1b..e8804edf 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -415,27 +415,18 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca } else { updateTime := wild.Timestamp / 1000 if pokemon.isNewRecord() || pokemon.wildSignificantUpdate(wild.Data, updateTime) { - go func(wildPokemon *pogo.WildPokemonProto, cellId int64, timestampMs int64) { - time.Sleep(15 * time.Second) - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - defer pokemonMutex.Unlock() - - ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) - defer cancel() - - if pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId); err != nil { - log.Errorf("getOrCreatePokemonRecord: %s", err) - } else { - // Update if there is still a change required & this update is the most recent - if pokemon.wildSignificantUpdate(wildPokemon, updateTime) && pokemon.Updated.ValueOrZero() < updateTime { - log.Debugf("DELAYED UPDATE: Updating pokemon %d from wild", encounterId) - - pokemon.updateFromWild(ctx, db, wildPokemon, cellId, weatherLookup, timestampMs, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, updateTime) - } - } - }(wild.Data, int64(wild.Cell), wild.Timestamp) + // Add to pending queue instead of spawning a goroutine + // The sweeper will process it after timeout if no encounter arrives + pending := &PendingPokemon{ + EncounterId: encounterId, + WildPokemon: wild.Data, + CellId: int64(wild.Cell), + TimestampMs: wild.Timestamp, + UpdateTime: updateTime, + WeatherLookup: weatherLookup, + Username: username, + } + pokemonPendingQueue.AddPending(pending) } } } diff --git a/decoder/pending_pokemon.go b/decoder/pending_pokemon.go new file mode 100644 index 00000000..4a22f070 --- /dev/null +++ b/decoder/pending_pokemon.go @@ -0,0 +1,166 @@ +package decoder + +import ( + "context" + "sync" + "time" + + "golbat/db" + "golbat/pogo" + + log "github.com/sirupsen/logrus" +) + +// PendingPokemon stores wild pokemon data awaiting a potential encounter +type PendingPokemon struct { + EncounterId uint64 + WildPokemon *pogo.WildPokemonProto + CellId int64 + TimestampMs int64 + UpdateTime int64 + WeatherLookup map[int64]pogo.GameplayWeatherProto_WeatherCondition + Username string + ReceivedAt time.Time +} + +// PokemonPendingQueue manages pokemon awaiting encounter data +type PokemonPendingQueue struct { + mu sync.RWMutex + pending map[uint64]*PendingPokemon + timeout time.Duration +} + +// NewPokemonPendingQueue creates a new pending queue with the specified timeout +func NewPokemonPendingQueue(timeout time.Duration) *PokemonPendingQueue { + return &PokemonPendingQueue{ + pending: make(map[uint64]*PendingPokemon), + timeout: timeout, + } +} + +// AddPending stores a wild pokemon awaiting encounter data. +// Returns true if the pokemon was added, false if it already exists. +func (q *PokemonPendingQueue) AddPending(p *PendingPokemon) bool { + q.mu.Lock() + defer q.mu.Unlock() + + // Only add if not already present (first sighting wins) + if _, exists := q.pending[p.EncounterId]; exists { + return false + } + + p.ReceivedAt = time.Now() + q.pending[p.EncounterId] = p + return true +} + +// TryComplete attempts to retrieve and remove a pending pokemon for an encounter. +// Returns the pending pokemon and true if found, nil and false otherwise. +func (q *PokemonPendingQueue) TryComplete(encounterId uint64) (*PendingPokemon, bool) { + q.mu.Lock() + defer q.mu.Unlock() + + p, exists := q.pending[encounterId] + if exists { + delete(q.pending, encounterId) + } + return p, exists +} + +// Remove removes a pending pokemon without processing it. +func (q *PokemonPendingQueue) Remove(encounterId uint64) { + q.mu.Lock() + delete(q.pending, encounterId) + q.mu.Unlock() +} + +// Size returns the current number of pending pokemon +func (q *PokemonPendingQueue) Size() int { + q.mu.RLock() + defer q.mu.RUnlock() + return len(q.pending) +} + +// collectExpired removes and returns all entries older than timeout +func (q *PokemonPendingQueue) collectExpired() []*PendingPokemon { + cutoff := time.Now().Add(-q.timeout) + + q.mu.Lock() + defer q.mu.Unlock() + + var expired []*PendingPokemon + for id, p := range q.pending { + if p.ReceivedAt.Before(cutoff) { + expired = append(expired, p) + delete(q.pending, id) + } + } + + return expired +} + +// StartSweeper starts a background goroutine that processes expired entries +func (q *PokemonPendingQueue) StartSweeper(ctx context.Context, interval time.Duration, dbDetails db.DbDetails) { + go func() { + ticker := time.NewTicker(interval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + log.Info("Pokemon pending queue sweeper stopped") + return + case <-ticker.C: + expired := q.collectExpired() + if len(expired) > 0 { + log.Debugf("Processing %d expired pending pokemon", len(expired)) + q.processExpired(ctx, dbDetails, expired) + } + } + } + }() +} + +// processExpired handles pokemon that didn't receive an encounter within the timeout +func (q *PokemonPendingQueue) processExpired(ctx context.Context, dbDetails db.DbDetails, expired []*PendingPokemon) { + for _, p := range expired { + pokemonMutex, _ := pokemonStripedMutex.GetLock(p.EncounterId) + pokemonMutex.Lock() + + processCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + + pokemon, err := getOrCreatePokemonRecord(processCtx, dbDetails, p.EncounterId) + if err != nil { + log.Errorf("getOrCreatePokemonRecord in sweeper: %s", err) + cancel() + pokemonMutex.Unlock() + continue + } + + // Update if there is still a change required & this update is the most recent + if pokemon.wildSignificantUpdate(p.WildPokemon, p.UpdateTime) && pokemon.Updated.ValueOrZero() < p.UpdateTime { + log.Debugf("DELAYED UPDATE: Updating pokemon %d from wild (sweeper)", p.EncounterId) + + pokemon.updateFromWild(processCtx, dbDetails, p.WildPokemon, p.CellId, p.WeatherLookup, p.TimestampMs, p.Username) + savePokemonRecordAsAtTime(processCtx, dbDetails, pokemon, false, true, true, p.UpdateTime) + } + + cancel() + pokemonMutex.Unlock() + } +} + +// Global pending queue instance +var pokemonPendingQueue *PokemonPendingQueue + +// InitPokemonPendingQueue initializes the global pending queue +func InitPokemonPendingQueue(ctx context.Context, dbDetails db.DbDetails, timeout time.Duration, sweepInterval time.Duration) { + pokemonPendingQueue = NewPokemonPendingQueue(timeout) + pokemonPendingQueue.StartSweeper(ctx, sweepInterval, dbDetails) + log.Infof("Pokemon pending queue started with %v timeout and %v sweep interval", timeout, sweepInterval) +} + +// GetPokemonPendingQueue returns the global pending queue instance +func GetPokemonPendingQueue() *PokemonPendingQueue { + return pokemonPendingQueue +} diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9244cc2b..33da3fa6 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -1400,6 +1400,11 @@ func UpdatePokemonRecordWithEncounterProto(ctx context.Context, db db.DbDetails, encounterId := encounter.Pokemon.EncounterId + // Remove from pending queue - encounter arrived so no need for delayed wild update + if pokemonPendingQueue != nil { + pokemonPendingQueue.Remove(encounterId) + } + pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) pokemonMutex.Lock() defer pokemonMutex.Unlock() diff --git a/main.go b/main.go index 02cae765..209599b8 100644 --- a/main.go +++ b/main.go @@ -210,6 +210,7 @@ func main() { _ = decoder.WatchMasterFileData() } decoder.LoadStatsGeofences() + decoder.InitPokemonPendingQueue(ctx, dbDetails, 30*time.Second, 5*time.Second) InitDeviceCache() wg.Add(1) From 22afd74fe3e6488f555ebe56fb3bcc30855b3178 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 11:39:38 +0000 Subject: [PATCH 03/35] Remove comment --- decoder/main.go | 1 - 1 file changed, 1 deletion(-) diff --git a/decoder/main.go b/decoder/main.go index e8804edf..4f7b9c38 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -415,7 +415,6 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca } else { updateTime := wild.Timestamp / 1000 if pokemon.isNewRecord() || pokemon.wildSignificantUpdate(wild.Data, updateTime) { - // Add to pending queue instead of spawning a goroutine // The sweeper will process it after timeout if no encounter arrives pending := &PendingPokemon{ EncounterId: encounterId, From 3de2ece4921b28498bbb454ef58e165124398c23 Mon Sep 17 00:00:00 2001 From: James Berry Date: Mon, 26 Jan 2026 18:06:02 +0000 Subject: [PATCH 04/35] Example dirty flag implementation --- decoder/fort.go | 16 +- decoder/main.go | 13 +- decoder/pokestop.go | 564 +++++++++++++++++++++++++++-------- decoder/pokestop_showcase.go | 18 +- 4 files changed, 459 insertions(+), 152 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index bc308a01..b4b8a08c 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -245,27 +245,27 @@ func (gym *Gym) copySharedFieldsFrom(pokestop *Pokestop) { // copySharedFieldsFrom copies shared fields from a gym to a pokestop during conversion func (stop *Pokestop) copySharedFieldsFrom(gym *Gym) { if gym.Name.Valid && !stop.Name.Valid { - stop.Name = gym.Name + stop.SetName(gym.Name) } if gym.Url.Valid && !stop.Url.Valid { - stop.Url = gym.Url + stop.SetUrl(gym.Url) } if gym.Description.Valid && !stop.Description.Valid { - stop.Description = gym.Description + stop.SetDescription(gym.Description) } if gym.PartnerId.Valid && !stop.PartnerId.Valid { - stop.PartnerId = gym.PartnerId + stop.SetPartnerId(gym.PartnerId) } if gym.ArScanEligible.Valid && !stop.ArScanEligible.Valid { - stop.ArScanEligible = gym.ArScanEligible + stop.SetArScanEligible(gym.ArScanEligible) } if gym.PowerUpLevel.Valid && !stop.PowerUpLevel.Valid { - stop.PowerUpLevel = gym.PowerUpLevel + stop.SetPowerUpLevel(gym.PowerUpLevel) } if gym.PowerUpPoints.Valid && !stop.PowerUpPoints.Valid { - stop.PowerUpPoints = gym.PowerUpPoints + stop.SetPowerUpPoints(gym.PowerUpPoints) } if gym.PowerUpEndTimestamp.Valid && !stop.PowerUpEndTimestamp.Valid { - stop.PowerUpEndTimestamp = gym.PowerUpEndTimestamp + stop.SetPowerUpEndTimestamp(gym.PowerUpEndTimestamp) } } diff --git a/decoder/main.go b/decoder/main.go index 42ec4c1b..9173ad5d 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -58,7 +58,7 @@ type webhooksSenderInterface interface { var webhooksSender webhooksSenderInterface var statsCollector stats_collector.StatsCollector -var pokestopCache *ttlcache.Cache[string, Pokestop] +var pokestopCache *ttlcache.Cache[string, *Pokestop] var gymCache *ttlcache.Cache[string, Gym] var stationCache *ttlcache.Cache[string, Station] var tappableCache *ttlcache.Cache[uint64, Tappable] @@ -118,8 +118,8 @@ func deletePokemonFromCache(key uint64) { } func initDataCache() { - pokestopCache = ttlcache.New[string, Pokestop]( - ttlcache.WithTTL[string, Pokestop](60 * time.Minute), + pokestopCache = ttlcache.New[string, *Pokestop]( + ttlcache.WithTTL[string, *Pokestop](60 * time.Minute), ) go pokestopCache.Start() @@ -297,14 +297,13 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa continue } - isNewPokestop := pokestop == nil - if isNewPokestop { - pokestop = &Pokestop{} + if pokestop == nil { + pokestop = &Pokestop{newRecord: true} } pokestop.updatePokestopFromFort(fort.Data, fort.Cell, fort.Timestamp/1000) // If this is a new pokestop, check if it was converted from a gym and copy shared fields - if isNewPokestop { + if pokestop.IsNewRecord() { gym, _ := GetGymRecord(ctx, db, fortId) if gym != nil { pokestop.copySharedFieldsFrom(gym) diff --git a/decoder/pokestop.go b/decoder/pokestop.go index ee7e966c..7be500ae 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -23,7 +23,6 @@ import ( ) // Pokestop struct. -// REMINDER! Keep hasChangesPokestop updated after making changes type Pokestop struct { Id string `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` @@ -68,6 +67,22 @@ type Pokestop struct { ShowcaseRankingStandard null.Int `db:"showcase_ranking_standard" json:"showcase_ranking_standard"` ShowcaseExpiry null.Int `db:"showcase_expiry" json:"showcase_expiry"` ShowcaseRankings null.String `db:"showcase_rankings" json:"showcase_rankings"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + // Old values for webhook comparison (populated when loading from cache/DB) + oldQuestType null.Int `db:"-" json:"-"` + oldAlternativeQuestType null.Int `db:"-" json:"-"` + oldLureExpireTimestamp null.Int `db:"-" json:"-"` + oldLureId int16 `db:"-" json:"-"` + oldPowerUpEndTimestamp null.Int `db:"-" json:"-"` + oldName null.String `db:"-" json:"-"` + oldUrl null.String `db:"-" json:"-"` + oldDescription null.String `db:"-" json:"-"` + oldLat float64 `db:"-" json:"-"` + oldLon float64 `db:"-" json:"-"` + //`id` varchar(35) NOT NULL, //`lat` double(18,14) NOT NULL, //`lon` double(18,14) NOT NULL, @@ -108,12 +123,339 @@ type Pokestop struct { } +// IsDirty returns true if any field has been modified +func (p *Pokestop) IsDirty() bool { + return p.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (p *Pokestop) ClearDirty() { + p.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (p *Pokestop) IsNewRecord() bool { + return p.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (p *Pokestop) snapshotOldValues() { + p.oldQuestType = p.QuestType + p.oldAlternativeQuestType = p.AlternativeQuestType + p.oldLureExpireTimestamp = p.LureExpireTimestamp + p.oldLureId = p.LureId + p.oldPowerUpEndTimestamp = p.PowerUpEndTimestamp + p.oldName = p.Name + p.oldUrl = p.Url + p.oldDescription = p.Description + p.oldLat = p.Lat + p.oldLon = p.Lon +} + +// --- Set methods with dirty tracking --- + +func (p *Pokestop) SetId(v string) { + if p.Id != v { + p.Id = v + p.dirty = true + } +} + +func (p *Pokestop) SetLat(v float64) { + if !floatAlmostEqual(p.Lat, v, floatTolerance) { + p.Lat = v + p.dirty = true + } +} + +func (p *Pokestop) SetLon(v float64) { + if !floatAlmostEqual(p.Lon, v, floatTolerance) { + p.Lon = v + p.dirty = true + } +} + +func (p *Pokestop) SetName(v null.String) { + if p.Name != v { + p.Name = v + p.dirty = true + } +} + +func (p *Pokestop) SetUrl(v null.String) { + if p.Url != v { + p.Url = v + p.dirty = true + } +} + +func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { + if p.LureExpireTimestamp != v { + p.LureExpireTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { + if p.LastModifiedTimestamp != v { + p.LastModifiedTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetEnabled(v null.Bool) { + if p.Enabled != v { + p.Enabled = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestType(v null.Int) { + if p.QuestType != v { + p.QuestType = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestTimestamp(v null.Int) { + if p.QuestTimestamp != v { + p.QuestTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestTarget(v null.Int) { + if p.QuestTarget != v { + p.QuestTarget = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestConditions(v null.String) { + if p.QuestConditions != v { + p.QuestConditions = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestRewards(v null.String) { + if p.QuestRewards != v { + p.QuestRewards = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestTemplate(v null.String) { + if p.QuestTemplate != v { + p.QuestTemplate = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestTitle(v null.String) { + if p.QuestTitle != v { + p.QuestTitle = v + p.dirty = true + } +} + +func (p *Pokestop) SetQuestExpiry(v null.Int) { + if p.QuestExpiry != v { + p.QuestExpiry = v + p.dirty = true + } +} + +func (p *Pokestop) SetCellId(v null.Int) { + if p.CellId != v { + p.CellId = v + p.dirty = true + } +} + +func (p *Pokestop) SetDeleted(v bool) { + if p.Deleted != v { + p.Deleted = v + p.dirty = true + } +} + +func (p *Pokestop) SetLureId(v int16) { + if p.LureId != v { + p.LureId = v + p.dirty = true + } +} + +func (p *Pokestop) SetFirstSeenTimestamp(v int16) { + if p.FirstSeenTimestamp != v { + p.FirstSeenTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetSponsorId(v null.Int) { + if p.SponsorId != v { + p.SponsorId = v + p.dirty = true + } +} + +func (p *Pokestop) SetPartnerId(v null.String) { + if p.PartnerId != v { + p.PartnerId = v + p.dirty = true + } +} + +func (p *Pokestop) SetArScanEligible(v null.Int) { + if p.ArScanEligible != v { + p.ArScanEligible = v + p.dirty = true + } +} + +func (p *Pokestop) SetPowerUpLevel(v null.Int) { + if p.PowerUpLevel != v { + p.PowerUpLevel = v + p.dirty = true + } +} + +func (p *Pokestop) SetPowerUpPoints(v null.Int) { + if p.PowerUpPoints != v { + p.PowerUpPoints = v + p.dirty = true + } +} + +func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { + if p.PowerUpEndTimestamp != v { + p.PowerUpEndTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestType(v null.Int) { + if p.AlternativeQuestType != v { + p.AlternativeQuestType = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { + if p.AlternativeQuestTimestamp != v { + p.AlternativeQuestTimestamp = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { + if p.AlternativeQuestTarget != v { + p.AlternativeQuestTarget = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { + if p.AlternativeQuestConditions != v { + p.AlternativeQuestConditions = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { + if p.AlternativeQuestRewards != v { + p.AlternativeQuestRewards = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { + if p.AlternativeQuestTemplate != v { + p.AlternativeQuestTemplate = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { + if p.AlternativeQuestTitle != v { + p.AlternativeQuestTitle = v + p.dirty = true + } +} + +func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { + if p.AlternativeQuestExpiry != v { + p.AlternativeQuestExpiry = v + p.dirty = true + } +} + +func (p *Pokestop) SetDescription(v null.String) { + if p.Description != v { + p.Description = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcaseFocus(v null.String) { + if p.ShowcaseFocus != v { + p.ShowcaseFocus = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcasePokemon(v null.Int) { + if p.ShowcasePokemon != v { + p.ShowcasePokemon = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { + if p.ShowcasePokemonForm != v { + p.ShowcasePokemonForm = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcasePokemonType(v null.Int) { + if p.ShowcasePokemonType != v { + p.ShowcasePokemonType = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { + if p.ShowcaseRankingStandard != v { + p.ShowcaseRankingStandard = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcaseExpiry(v null.Int) { + if p.ShowcaseExpiry != v { + p.ShowcaseExpiry = v + p.dirty = true + } +} + +func (p *Pokestop) SetShowcaseRankings(v null.String) { + if p.ShowcaseRankings != v { + p.ShowcaseRankings = v + p.dirty = true + } +} + func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, error) { stop := pokestopCache.Get(fortId) if stop != nil { - pokestop := stop.Value() //log.Debugf("GetPokestopRecord %s (from cache)", fortId) - return &pokestop, nil + pokestop := stop.Value() + pokestop.snapshotOldValues() // Snapshot for webhook comparison + return pokestop, nil } pokestop := Pokestop{} err := db.GeneralDb.GetContext(ctx, &pokestop, @@ -138,75 +480,33 @@ func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Po return nil, err } - pokestopCache.Set(fortId, pokestop, ttlcache.DefaultTTL) + pokestop.snapshotOldValues() // Snapshot for webhook comparison + pokestopCache.Set(fortId, &pokestop, ttlcache.DefaultTTL) if config.Config.TestFortInMemory { fortRtreeUpdatePokestopOnGet(&pokestop) } return &pokestop, nil } -// hasChangesPokestop compares two Pokestop structs -// Float tolerance: Lat, Lon -func hasChangesPokestop(old *Pokestop, new *Pokestop) bool { - return old.Id != new.Id || - old.Name != new.Name || - old.Url != new.Url || - old.LureExpireTimestamp != new.LureExpireTimestamp || - old.LastModifiedTimestamp != new.LastModifiedTimestamp || - old.Updated != new.Updated || - old.Enabled != new.Enabled || - old.QuestType != new.QuestType || - old.QuestTimestamp != new.QuestTimestamp || - old.QuestTarget != new.QuestTarget || - old.QuestConditions != new.QuestConditions || - old.QuestRewards != new.QuestRewards || - old.QuestTemplate != new.QuestTemplate || - old.QuestTitle != new.QuestTitle || - old.QuestExpiry != new.QuestExpiry || - old.CellId != new.CellId || - old.Deleted != new.Deleted || - old.LureId != new.LureId || - old.FirstSeenTimestamp != new.FirstSeenTimestamp || - old.SponsorId != new.SponsorId || - old.PartnerId != new.PartnerId || - old.ArScanEligible != new.ArScanEligible || - old.PowerUpLevel != new.PowerUpLevel || - old.PowerUpPoints != new.PowerUpPoints || - old.PowerUpEndTimestamp != new.PowerUpEndTimestamp || - old.AlternativeQuestType != new.AlternativeQuestType || - old.AlternativeQuestTimestamp != new.AlternativeQuestTimestamp || - old.AlternativeQuestTarget != new.AlternativeQuestTarget || - old.AlternativeQuestConditions != new.AlternativeQuestConditions || - old.AlternativeQuestRewards != new.AlternativeQuestRewards || - old.AlternativeQuestTemplate != new.AlternativeQuestTemplate || - old.AlternativeQuestTitle != new.AlternativeQuestTitle || - old.AlternativeQuestExpiry != new.AlternativeQuestExpiry || - old.Description != new.Description || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) || - old.ShowcaseRankingStandard != new.ShowcaseRankingStandard || - old.ShowcaseFocus != new.ShowcaseFocus || - old.ShowcaseRankings != new.ShowcaseRankings || - old.ShowcaseExpiry != new.ShowcaseExpiry -} - var LureTime int64 = 1800 func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, cellId uint64, now int64) *Pokestop { - stop.Id = fortData.FortId - stop.Lat = fortData.Latitude - stop.Lon = fortData.Longitude - - stop.PartnerId = null.NewString(fortData.PartnerId, fortData.PartnerId != "") - stop.SponsorId = null.IntFrom(int64(fortData.Sponsor)) - stop.Enabled = null.BoolFrom(fortData.Enabled) - stop.ArScanEligible = null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible)) - stop.PowerUpPoints = null.IntFrom(int64(fortData.PowerUpProgressPoints)) - stop.PowerUpLevel, stop.PowerUpEndTimestamp = calculatePowerUpPoints(fortData) + stop.SetId(fortData.FortId) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) + + stop.SetPartnerId(null.NewString(fortData.PartnerId, fortData.PartnerId != "")) + stop.SetSponsorId(null.IntFrom(int64(fortData.Sponsor))) + stop.SetEnabled(null.BoolFrom(fortData.Enabled)) + stop.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) + stop.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) + powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) + stop.SetPowerUpLevel(powerUpLevel) + stop.SetPowerUpEndTimestamp(powerUpEndTimestamp) // lasModifiedMs is also modified when incident happens lastModifiedTimestamp := fortData.LastModifiedMs / 1000 - stop.LastModifiedTimestamp = null.IntFrom(lastModifiedTimestamp) + stop.SetLastModifiedTimestamp(null.IntFrom(lastModifiedTimestamp)) if len(fortData.ActiveFortModifier) > 0 { lureId := int16(fortData.ActiveFortModifier[0]) @@ -214,8 +514,8 @@ func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, ce lureEnd := lastModifiedTimestamp + LureTime oldLureEnd := stop.LureExpireTimestamp.ValueOrZero() if stop.LureId != lureId { - stop.LureExpireTimestamp = null.IntFrom(lureEnd) - stop.LureId = lureId + stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) + stop.SetLureId(lureId) } else { // wait some time after lure end before a restart in case of timing issue if now > oldLureEnd+30 { @@ -223,19 +523,19 @@ func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, ce lureEnd += LureTime } // lure needs to be restarted - stop.LureExpireTimestamp = null.IntFrom(lureEnd) + stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) } } } } if fortData.ImageUrl != "" { - stop.Url = null.StringFrom(fortData.ImageUrl) + stop.SetUrl(null.StringFrom(fortData.ImageUrl)) } - stop.CellId = null.IntFrom(int64(cellId)) + stop.SetCellId(null.IntFrom(int64(cellId))) if stop.Deleted { - stop.Deleted = false + stop.SetDeleted(false) log.Warnf("Cleared Stop with id '%s' is found again in GMO, therefore un-deleted", stop.Id) // Restore in fort tracker if enabled if fortTracker != nil { @@ -514,41 +814,41 @@ func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOu } if !haveAr { - stop.AlternativeQuestType = null.IntFrom(questType) - stop.AlternativeQuestTarget = null.IntFrom(questTarget) - stop.AlternativeQuestTemplate = null.StringFrom(questTemplate) - stop.AlternativeQuestTitle = null.StringFrom(questTitle) - stop.AlternativeQuestConditions = null.StringFrom(string(questConditions)) - stop.AlternativeQuestRewards = null.StringFrom(string(questRewards)) - stop.AlternativeQuestTimestamp = null.IntFrom(questTimestamp) - stop.AlternativeQuestExpiry = questExpiry + stop.SetAlternativeQuestType(null.IntFrom(questType)) + stop.SetAlternativeQuestTarget(null.IntFrom(questTarget)) + stop.SetAlternativeQuestTemplate(null.StringFrom(questTemplate)) + stop.SetAlternativeQuestTitle(null.StringFrom(questTitle)) + stop.SetAlternativeQuestConditions(null.StringFrom(string(questConditions))) + stop.SetAlternativeQuestRewards(null.StringFrom(string(questRewards))) + stop.SetAlternativeQuestTimestamp(null.IntFrom(questTimestamp)) + stop.SetAlternativeQuestExpiry(questExpiry) } else { - stop.QuestType = null.IntFrom(questType) - stop.QuestTarget = null.IntFrom(questTarget) - stop.QuestTemplate = null.StringFrom(questTemplate) - stop.QuestTitle = null.StringFrom(questTitle) - stop.QuestConditions = null.StringFrom(string(questConditions)) - stop.QuestRewards = null.StringFrom(string(questRewards)) - stop.QuestTimestamp = null.IntFrom(questTimestamp) - stop.QuestExpiry = questExpiry + stop.SetQuestType(null.IntFrom(questType)) + stop.SetQuestTarget(null.IntFrom(questTarget)) + stop.SetQuestTemplate(null.StringFrom(questTemplate)) + stop.SetQuestTitle(null.StringFrom(questTitle)) + stop.SetQuestConditions(null.StringFrom(string(questConditions))) + stop.SetQuestRewards(null.StringFrom(string(questRewards))) + stop.SetQuestTimestamp(null.IntFrom(questTimestamp)) + stop.SetQuestExpiry(questExpiry) } return questTitle } func (stop *Pokestop) updatePokestopFromFortDetailsProto(fortData *pogo.FortDetailsOutProto) *Pokestop { - stop.Id = fortData.Id - stop.Lat = fortData.Latitude - stop.Lon = fortData.Longitude + stop.SetId(fortData.Id) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) if len(fortData.ImageUrl) > 0 { - stop.Url = null.StringFrom(fortData.ImageUrl[0]) + stop.SetUrl(null.StringFrom(fortData.ImageUrl[0])) } - stop.Name = null.StringFrom(fortData.Name) + stop.SetName(null.StringFrom(fortData.Name)) if fortData.Description == "" { - stop.Description = null.NewString("", false) + stop.SetDescription(null.NewString("", false)) } else { - stop.Description = null.StringFrom(fortData.Description) + stop.SetDescription(null.StringFrom(fortData.Description)) } if fortData.Modifier != nil && len(fortData.Modifier) > 0 { @@ -556,22 +856,22 @@ func (stop *Pokestop) updatePokestopFromFortDetailsProto(fortData *pogo.FortDeta lureId := int16(fortData.Modifier[0].ModifierType) lureExpiry := fortData.Modifier[0].ExpirationTimeMs / 1000 - stop.LureId = lureId - stop.LureExpireTimestamp = null.IntFrom(lureExpiry) + stop.SetLureId(lureId) + stop.SetLureExpireTimestamp(null.IntFrom(lureExpiry)) } return stop } func (stop *Pokestop) updatePokestopFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto) *Pokestop { - stop.Id = fortData.Id - stop.Lat = fortData.Latitude - stop.Lon = fortData.Longitude + stop.SetId(fortData.Id) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) if len(fortData.Image) > 0 { - stop.Url = null.StringFrom(fortData.Image[0].Url) + stop.SetUrl(null.StringFrom(fortData.Image[0].Url)) } - stop.Name = null.StringFrom(fortData.Name) + stop.SetName(null.StringFrom(fortData.Name)) if stop.Deleted { log.Debugf("Cleared Stop with id '%s' is found again in GMF, therefore kept deleted", stop.Id) } @@ -579,8 +879,8 @@ func (stop *Pokestop) updatePokestopFromGetMapFortsOutProto(fortData *pogo.GetMa } func (stop *Pokestop) updatePokestopFromGetContestDataOutProto(contest *pogo.ContestProto) { - stop.ShowcaseRankingStandard = null.IntFrom(int64(contest.GetMetric().GetRankingStandard())) - stop.ShowcaseExpiry = null.IntFrom(contest.GetSchedule().GetContestCycle().GetEndTimeMs() / 1000) + stop.SetShowcaseRankingStandard(null.IntFrom(int64(contest.GetMetric().GetRankingStandard()))) + stop.SetShowcaseExpiry(null.IntFrom(contest.GetSchedule().GetContestCycle().GetEndTimeMs() / 1000)) focusStore := createFocusStoreFromContestProto(contest) @@ -594,7 +894,7 @@ func (stop *Pokestop) updatePokestopFromGetContestDataOutProto(contest *pogo.Con if err != nil { log.Errorf("SHOWCASE: Stop '%s' - Focus '%v' marshalling failed: %s", stop.Id, focus, err) } - stop.ShowcaseFocus = null.StringFrom(string(jsonBytes)) + stop.SetShowcaseFocus(null.StringFrom(string(jsonBytes))) // still support old format - probably still required to filter in external tools stop.extractShowcasePokemonInfoDeprecated(key, focus) } @@ -646,24 +946,32 @@ func (stop *Pokestop) updatePokestopFromGetPokemonSizeContestEntryOutProto(conte } jsonString, _ := json.Marshal(j) - stop.ShowcaseRankings = null.StringFrom(string(jsonString)) + stop.SetShowcaseRankings(null.StringFrom(string(jsonString))) } -func createPokestopFortWebhooks(oldStop *Pokestop, stop *Pokestop) { +func createPokestopFortWebhooks(stop *Pokestop) { fort := InitWebHookFortFromPokestop(stop) - oldFort := InitWebHookFortFromPokestop(oldStop) - if oldStop == nil { - CreateFortWebHooks(oldFort, fort, NEW) + if stop.newRecord { + CreateFortWebHooks(nil, fort, NEW) } else { + // Build old fort from saved old values + oldFort := &FortWebhook{ + Type: POKESTOP.String(), + Id: stop.Id, + Name: stop.oldName.Ptr(), + ImageUrl: stop.oldUrl.Ptr(), + Description: stop.oldDescription.Ptr(), + Location: Location{Latitude: stop.oldLat, Longitude: stop.oldLon}, + } CreateFortWebHooks(oldFort, fort, EDIT) } } -func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { +func createPokestopWebhooks(stop *Pokestop) { areas := MatchStatsGeofence(stop.Lat, stop.Lon) - if stop.AlternativeQuestType.Valid && (oldStop == nil || stop.AlternativeQuestType != oldStop.AlternativeQuestType) { + if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldAlternativeQuestType) { questHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -689,7 +997,7 @@ func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } - if stop.QuestType.Valid && (oldStop == nil || stop.QuestType != oldStop.QuestType) { + if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldQuestType) { questHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -714,7 +1022,7 @@ func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } - if (oldStop == nil && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (oldStop != nil && ((stop.LureExpireTimestamp != oldStop.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != oldStop.PowerUpEndTimestamp)) { + if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldLureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldPowerUpEndTimestamp)) { pokestopHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -756,19 +1064,16 @@ func createPokestopWebhooks(oldStop *Pokestop, stop *Pokestop) { } func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop) { - oldPokestop, _ := GetPokestopRecord(ctx, db, pokestop.Id) now := time.Now().Unix() - if oldPokestop != nil && !hasChangesPokestop(oldPokestop, pokestop) { - if oldPokestop.Updated > now-900 { + if !pokestop.IsNewRecord() && !pokestop.IsDirty() { + if pokestop.Updated > now-900 { // if a pokestop is unchanged, but we did see it again after 15 minutes, then save again return } } pokestop.Updated = now - //log.Traceln(cmp.Diff(oldPokestop, pokestop)) - - if oldPokestop == nil { + if pokestop.IsNewRecord() { res, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO pokestop ( id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, @@ -800,6 +1105,7 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop } _ = res } else { + // Existing record - UPDATE res, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE pokestop SET lat = :lat, @@ -810,17 +1116,17 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop lure_expire_timestamp = :lure_expire_timestamp, last_modified_timestamp = :last_modified_timestamp, updated = :updated, - quest_type = :quest_type, - quest_timestamp = :quest_timestamp, - quest_target = :quest_target, - quest_conditions = :quest_conditions, - quest_rewards = :quest_rewards, - quest_template = :quest_template, + quest_type = :quest_type, + quest_timestamp = :quest_timestamp, + quest_target = :quest_target, + quest_conditions = :quest_conditions, + quest_rewards = :quest_rewards, + quest_template = :quest_template, quest_title = :quest_title, - alternative_quest_type = :alternative_quest_type, + alternative_quest_type = :alternative_quest_type, alternative_quest_timestamp = :alternative_quest_timestamp, - alternative_quest_target = :alternative_quest_target, - alternative_quest_conditions = :alternative_quest_conditions, + alternative_quest_target = :alternative_quest_target, + alternative_quest_conditions = :alternative_quest_conditions, alternative_quest_rewards = :alternative_quest_rewards, alternative_quest_template = :alternative_quest_template, alternative_quest_title = :alternative_quest_title, @@ -854,9 +1160,11 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop } _ = res } - pokestopCache.Set(pokestop.Id, *pokestop, ttlcache.DefaultTTL) - createPokestopWebhooks(oldPokestop, pokestop) - createPokestopFortWebhooks(oldPokestop, pokestop) + pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + pokestop.newRecord = false // After saving, it's no longer a new record + pokestop.ClearDirty() + createPokestopWebhooks(pokestop) + createPokestopFortWebhooks(pokestop) } func updatePokestopGetMapFortCache(pokestop *Pokestop) { @@ -881,7 +1189,7 @@ func UpdatePokestopRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDe } if pokestop == nil { - pokestop = &Pokestop{} + pokestop = &Pokestop{newRecord: true} } pokestop.updatePokestopFromFortDetailsProto(fort) @@ -913,7 +1221,7 @@ func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.F } if pokestop == nil { - pokestop = &Pokestop{} + pokestop = &Pokestop{newRecord: true} } questTitle := pokestop.updatePokestopFromQuestProto(quest, haveAr) diff --git a/decoder/pokestop_showcase.go b/decoder/pokestop_showcase.go index bf9de8a2..42876295 100644 --- a/decoder/pokestop_showcase.go +++ b/decoder/pokestop_showcase.go @@ -95,28 +95,28 @@ func createFocusStoreFromContestProto(contest *pogo.ContestProto) map[contestFoc func (stop *Pokestop) extractShowcasePokemonInfoDeprecated(key contestFocusType, focus map[string]any) { if key == focusPokemon { if pokemonID, ok := focus["pokemon_id"].(int32); ok { - stop.ShowcasePokemon = null.IntFrom(int64(pokemonID)) + stop.SetShowcasePokemon(null.IntFrom(int64(pokemonID))) } else { log.Warnf("SHOWCASE: Stop '%s' - Missing or invalid 'pokemon_id'", stop.Id) - stop.ShowcasePokemon = null.IntFromPtr(nil) + stop.SetShowcasePokemon(null.IntFromPtr(nil)) } if form, ok := focus["pokemon_form"].(int32); ok { - stop.ShowcasePokemonForm = null.IntFrom(int64(form)) + stop.SetShowcasePokemonForm(null.IntFrom(int64(form))) } else { - stop.ShowcasePokemonForm = null.IntFromPtr(nil) + stop.SetShowcasePokemonForm(null.IntFromPtr(nil)) } } else { - stop.ShowcasePokemon = null.IntFromPtr(nil) - stop.ShowcasePokemonForm = null.IntFromPtr(nil) + stop.SetShowcasePokemon(null.IntFromPtr(nil)) + stop.SetShowcasePokemonForm(null.IntFromPtr(nil)) } if key == focusPokemonType { if type1, ok := focus["pokemon_type_1"].(int32); ok { - stop.ShowcasePokemonType = null.IntFrom(int64(type1)) + stop.SetShowcasePokemonType(null.IntFrom(int64(type1))) } else { log.Warnf("SHOWCASE: Stop '%s' - Missing or invalid 'pokemon_type_1'", stop.Id) - stop.ShowcasePokemonType = null.IntFromPtr(nil) + stop.SetShowcasePokemonType(null.IntFromPtr(nil)) } if type2, ok := focus["pokemon_type_2"].(int32); ok { @@ -125,6 +125,6 @@ func (stop *Pokestop) extractShowcasePokemonInfoDeprecated(key contestFocusType, } } } else { - stop.ShowcasePokemonType = null.IntFromPtr(nil) + stop.SetShowcasePokemonType(null.IntFromPtr(nil)) } } From d23858962e3637146ca3edbd4046a53589012340 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 11:24:40 +0000 Subject: [PATCH 05/35] Update pokemon and gym to use new pattern --- decoder/api_pokemon.go | 4 +- decoder/api_pokemon_scan_v1.go | 2 +- decoder/api_pokemon_scan_v2.go | 2 +- decoder/api_pokemon_scan_v3.go | 2 +- decoder/fort.go | 16 +- decoder/gym.go | 620 +++++++++++++++++++++++++-------- decoder/main.go | 29 +- decoder/pokemon.go | 598 +++++++++++++++++++++---------- decoder/pokemonRtree.go | 13 +- decoder/pokestop.go | 141 ++++---- decoder/stats.go | 61 ++-- decoder/weather_iv.go | 4 +- 12 files changed, 1036 insertions(+), 456 deletions(-) diff --git a/decoder/api_pokemon.go b/decoder/api_pokemon.go index 0126cad4..c41393af 100644 --- a/decoder/api_pokemon.go +++ b/decoder/api_pokemon.go @@ -130,7 +130,7 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { if found { if pokemonCacheEntry := getPokemonFromCache(pokemonId); pokemonCacheEntry != nil { pokemon := pokemonCacheEntry.Value() - results = append(results, &pokemon) + results = append(results, pokemon) pokemonMatched++ if pokemonMatched > maxPokemon { @@ -153,7 +153,7 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { func GetOnePokemon(pokemonId uint64) *ApiPokemonResult { if item := getPokemonFromCache(pokemonId); item != nil { pokemon := item.Value() - apiPokemon := buildApiPokemonResult(&pokemon) + apiPokemon := buildApiPokemonResult(pokemon) return &apiPokemon } return nil diff --git a/decoder/api_pokemon_scan_v1.go b/decoder/api_pokemon_scan_v1.go index 737393ad..c4baf317 100644 --- a/decoder/api_pokemon_scan_v1.go +++ b/decoder/api_pokemon_scan_v1.go @@ -230,7 +230,7 @@ func GetPokemonInArea(retrieveParameters ApiPokemonScan) []*ApiPokemonResult { if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { pokemon := pokemonCacheEntry.Value() - apiPokemon := buildApiPokemonResult(&pokemon) + apiPokemon := buildApiPokemonResult(pokemon) results = append(results, &apiPokemon) } } diff --git a/decoder/api_pokemon_scan_v2.go b/decoder/api_pokemon_scan_v2.go index 878f3ab5..726d0f3f 100644 --- a/decoder/api_pokemon_scan_v2.go +++ b/decoder/api_pokemon_scan_v2.go @@ -110,7 +110,7 @@ func GetPokemonInArea2(retrieveParameters ApiPokemonScan2) []*ApiPokemonResult { continue } - apiPokemon := buildApiPokemonResult(&pokemon) + apiPokemon := buildApiPokemonResult(pokemon) results = append(results, &apiPokemon) } diff --git a/decoder/api_pokemon_scan_v3.go b/decoder/api_pokemon_scan_v3.go index 8c5ef4ad..a7c3b14d 100644 --- a/decoder/api_pokemon_scan_v3.go +++ b/decoder/api_pokemon_scan_v3.go @@ -118,7 +118,7 @@ func GetPokemonInArea3(retrieveParameters ApiPokemonScan3) *PokemonScan3Result { continue } - apiPokemon := buildApiPokemonResult(&pokemon) + apiPokemon := buildApiPokemonResult(pokemon) results = append(results, &apiPokemon) } diff --git a/decoder/fort.go b/decoder/fort.go index b4b8a08c..fe008724 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -217,28 +217,28 @@ func UpdateFortRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetail // copySharedFieldsFrom copies shared fields from a pokestop to a gym during conversion func (gym *Gym) copySharedFieldsFrom(pokestop *Pokestop) { if pokestop.Name.Valid && !gym.Name.Valid { - gym.Name = pokestop.Name + gym.SetName(pokestop.Name) } if pokestop.Url.Valid && !gym.Url.Valid { - gym.Url = pokestop.Url + gym.SetUrl(pokestop.Url) } if pokestop.Description.Valid && !gym.Description.Valid { - gym.Description = pokestop.Description + gym.SetDescription(pokestop.Description) } if pokestop.PartnerId.Valid && !gym.PartnerId.Valid { - gym.PartnerId = pokestop.PartnerId + gym.SetPartnerId(pokestop.PartnerId) } if pokestop.ArScanEligible.Valid && !gym.ArScanEligible.Valid { - gym.ArScanEligible = pokestop.ArScanEligible + gym.SetArScanEligible(pokestop.ArScanEligible) } if pokestop.PowerUpLevel.Valid && !gym.PowerUpLevel.Valid { - gym.PowerUpLevel = pokestop.PowerUpLevel + gym.SetPowerUpLevel(pokestop.PowerUpLevel) } if pokestop.PowerUpPoints.Valid && !gym.PowerUpPoints.Valid { - gym.PowerUpPoints = pokestop.PowerUpPoints + gym.SetPowerUpPoints(pokestop.PowerUpPoints) } if pokestop.PowerUpEndTimestamp.Valid && !gym.PowerUpEndTimestamp.Valid { - gym.PowerUpEndTimestamp = pokestop.PowerUpEndTimestamp + gym.SetPowerUpEndTimestamp(pokestop.PowerUpEndTimestamp) } } diff --git a/decoder/gym.go b/decoder/gym.go index 4f1784fe..a8b6541d 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -67,45 +67,67 @@ type Gym struct { Description null.String `db:"description" json:"description"` Defenders null.String `db:"defenders" json:"defenders"` Rsvps null.String `db:"rsvps" json:"rsvps"` - //`id` varchar(35) NOT NULL, - //`lat` double(18,14) NOT NULL, - //`lon` double(18,14) NOT NULL, - //`name` varchar(128) DEFAULT NULL, - //`url` varchar(200) DEFAULT NULL, - //`last_modified_timestamp` int unsigned DEFAULT NULL, - //`raid_end_timestamp` int unsigned DEFAULT NULL, - //`raid_spawn_timestamp` int unsigned DEFAULT NULL, - //`raid_battle_timestamp` int unsigned DEFAULT NULL, - //`updated` int unsigned NOT NULL, - //`raid_pokemon_id` smallint unsigned DEFAULT NULL, - //`guarding_pokemon_id` smallint unsigned DEFAULT NULL, - //`available_slots` smallint unsigned DEFAULT NULL, - //`availble_slots` smallint unsigned GENERATED ALWAYS AS (`available_slots`) VIRTUAL, - //`team_id` tinyint unsigned DEFAULT NULL, - //`raid_level` tinyint unsigned DEFAULT NULL, - //`enabled` tinyint unsigned DEFAULT NULL, - //`ex_raid_eligible` tinyint unsigned DEFAULT NULL, - //`in_battle` tinyint unsigned DEFAULT NULL, - //`raid_pokemon_move_1` smallint unsigned DEFAULT NULL, - //`raid_pokemon_move_2` smallint unsigned DEFAULT NULL, - //`raid_pokemon_form` smallint unsigned DEFAULT NULL, - //`raid_pokemon_cp` int unsigned DEFAULT NULL, - //`raid_is_exclusive` tinyint unsigned DEFAULT NULL, - //`cell_id` bigint unsigned DEFAULT NULL, - //`deleted` tinyint unsigned NOT NULL DEFAULT '0', - //`total_cp` int unsigned DEFAULT NULL, - //`first_seen_timestamp` int unsigned NOT NULL, - //`raid_pokemon_gender` tinyint unsigned DEFAULT NULL, - //`sponsor_id` smallint unsigned DEFAULT NULL, - //`partner_id` varchar(35) DEFAULT NULL, - //`raid_pokemon_costume` smallint unsigned DEFAULT NULL, - //`raid_pokemon_evolution` tinyint unsigned DEFAULT NULL, - //`ar_scan_eligible` tinyint unsigned DEFAULT NULL, - //`power_up_level` smallint unsigned DEFAULT NULL, - //`power_up_points` int unsigned DEFAULT NULL, - //`power_up_end_timestamp` int unsigned DEFAULT NULL, + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + oldValues GymOldValues `db:"-" json:"-"` // Old values for webhook comparison } +// GymOldValues holds old field values for webhook comparison (populated when loading from cache/DB) +type GymOldValues struct { + Name null.String + Url null.String + Description null.String + Lat float64 + Lon float64 + TeamId null.Int + AvailableSlots null.Int + RaidLevel null.Int + RaidPokemonId null.Int + RaidSpawnTimestamp null.Int + Rsvps null.String + InBattle null.Int +} + +//`id` varchar(35) NOT NULL, +//`lat` double(18,14) NOT NULL, +//`lon` double(18,14) NOT NULL, +//`name` varchar(128) DEFAULT NULL, +//`url` varchar(200) DEFAULT NULL, +//`last_modified_timestamp` int unsigned DEFAULT NULL, +//`raid_end_timestamp` int unsigned DEFAULT NULL, +//`raid_spawn_timestamp` int unsigned DEFAULT NULL, +//`raid_battle_timestamp` int unsigned DEFAULT NULL, +//`updated` int unsigned NOT NULL, +//`raid_pokemon_id` smallint unsigned DEFAULT NULL, +//`guarding_pokemon_id` smallint unsigned DEFAULT NULL, +//`available_slots` smallint unsigned DEFAULT NULL, +//`availble_slots` smallint unsigned GENERATED ALWAYS AS (`available_slots`) VIRTUAL, +//`team_id` tinyint unsigned DEFAULT NULL, +//`raid_level` tinyint unsigned DEFAULT NULL, +//`enabled` tinyint unsigned DEFAULT NULL, +//`ex_raid_eligible` tinyint unsigned DEFAULT NULL, +//`in_battle` tinyint unsigned DEFAULT NULL, +//`raid_pokemon_move_1` smallint unsigned DEFAULT NULL, +//`raid_pokemon_move_2` smallint unsigned DEFAULT NULL, +//`raid_pokemon_form` smallint unsigned DEFAULT NULL, +//`raid_pokemon_cp` int unsigned DEFAULT NULL, +//`raid_is_exclusive` tinyint unsigned DEFAULT NULL, +//`cell_id` bigint unsigned DEFAULT NULL, +//`deleted` tinyint unsigned NOT NULL DEFAULT '0', +//`total_cp` int unsigned DEFAULT NULL, +//`first_seen_timestamp` int unsigned NOT NULL, +//`raid_pokemon_gender` tinyint unsigned DEFAULT NULL, +//`sponsor_id` smallint unsigned DEFAULT NULL, +//`partner_id` varchar(35) DEFAULT NULL, +//`raid_pokemon_costume` smallint unsigned DEFAULT NULL, +//`raid_pokemon_evolution` tinyint unsigned DEFAULT NULL, +//`ar_scan_eligible` tinyint unsigned DEFAULT NULL, +//`power_up_level` smallint unsigned DEFAULT NULL, +//`power_up_points` int unsigned DEFAULT NULL, +//`power_up_end_timestamp` int unsigned DEFAULT NULL, + // //SELECT CONCAT("'", GROUP_CONCAT(column_name ORDER BY ordinal_position SEPARATOR "', '"), "'") AS columns //FROM information_schema.columns @@ -115,11 +137,323 @@ type Gym struct { //FROM information_schema.columns //WHERE table_schema = 'db_name' AND table_name = 'tbl_name' +// IsDirty returns true if any field has been modified +func (gym *Gym) IsDirty() bool { + return gym.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (gym *Gym) ClearDirty() { + gym.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (gym *Gym) IsNewRecord() bool { + return gym.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (gym *Gym) snapshotOldValues() { + gym.oldValues = GymOldValues{ + Name: gym.Name, + Url: gym.Url, + Description: gym.Description, + Lat: gym.Lat, + Lon: gym.Lon, + TeamId: gym.TeamId, + AvailableSlots: gym.AvailableSlots, + RaidLevel: gym.RaidLevel, + RaidPokemonId: gym.RaidPokemonId, + RaidSpawnTimestamp: gym.RaidSpawnTimestamp, + Rsvps: gym.Rsvps, + InBattle: gym.InBattle, + } +} + +// --- Set methods with dirty tracking --- + +func (gym *Gym) SetId(v string) { + if gym.Id != v { + gym.Id = v + gym.dirty = true + } +} + +func (gym *Gym) SetLat(v float64) { + if !floatAlmostEqual(gym.Lat, v, floatTolerance) { + gym.Lat = v + gym.dirty = true + } +} + +func (gym *Gym) SetLon(v float64) { + if !floatAlmostEqual(gym.Lon, v, floatTolerance) { + gym.Lon = v + gym.dirty = true + } +} + +func (gym *Gym) SetName(v null.String) { + if gym.Name != v { + gym.Name = v + gym.dirty = true + } +} + +func (gym *Gym) SetUrl(v null.String) { + if gym.Url != v { + gym.Url = v + gym.dirty = true + } +} + +func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { + if gym.LastModifiedTimestamp != v { + gym.LastModifiedTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidEndTimestamp(v null.Int) { + if gym.RaidEndTimestamp != v { + gym.RaidEndTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { + if gym.RaidSpawnTimestamp != v { + gym.RaidSpawnTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { + if gym.RaidBattleTimestamp != v { + gym.RaidBattleTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonId(v null.Int) { + if gym.RaidPokemonId != v { + gym.RaidPokemonId = v + gym.dirty = true + } +} + +func (gym *Gym) SetGuardingPokemonId(v null.Int) { + if gym.GuardingPokemonId != v { + gym.GuardingPokemonId = v + gym.dirty = true + } +} + +func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { + if gym.GuardingPokemonDisplay != v { + gym.GuardingPokemonDisplay = v + gym.dirty = true + } +} + +func (gym *Gym) SetAvailableSlots(v null.Int) { + if gym.AvailableSlots != v { + gym.AvailableSlots = v + gym.dirty = true + } +} + +func (gym *Gym) SetTeamId(v null.Int) { + if gym.TeamId != v { + gym.TeamId = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidLevel(v null.Int) { + if gym.RaidLevel != v { + gym.RaidLevel = v + gym.dirty = true + } +} + +func (gym *Gym) SetEnabled(v null.Int) { + if gym.Enabled != v { + gym.Enabled = v + gym.dirty = true + } +} + +func (gym *Gym) SetExRaidEligible(v null.Int) { + if gym.ExRaidEligible != v { + gym.ExRaidEligible = v + gym.dirty = true + } +} + +func (gym *Gym) SetInBattle(v null.Int) { + if gym.InBattle != v { + gym.InBattle = v + //Do not set to dirty, as don't trigger an update + //gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonMove1(v null.Int) { + if gym.RaidPokemonMove1 != v { + gym.RaidPokemonMove1 = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonMove2(v null.Int) { + if gym.RaidPokemonMove2 != v { + gym.RaidPokemonMove2 = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonForm(v null.Int) { + if gym.RaidPokemonForm != v { + gym.RaidPokemonForm = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { + if gym.RaidPokemonAlignment != v { + gym.RaidPokemonAlignment = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonCp(v null.Int) { + if gym.RaidPokemonCp != v { + gym.RaidPokemonCp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidIsExclusive(v null.Int) { + if gym.RaidIsExclusive != v { + gym.RaidIsExclusive = v + gym.dirty = true + } +} + +func (gym *Gym) SetCellId(v null.Int) { + if gym.CellId != v { + gym.CellId = v + gym.dirty = true + } +} + +func (gym *Gym) SetDeleted(v bool) { + if gym.Deleted != v { + gym.Deleted = v + gym.dirty = true + } +} + +func (gym *Gym) SetTotalCp(v null.Int) { + if gym.TotalCp != v { + gym.TotalCp = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonGender(v null.Int) { + if gym.RaidPokemonGender != v { + gym.RaidPokemonGender = v + gym.dirty = true + } +} + +func (gym *Gym) SetSponsorId(v null.Int) { + if gym.SponsorId != v { + gym.SponsorId = v + gym.dirty = true + } +} + +func (gym *Gym) SetPartnerId(v null.String) { + if gym.PartnerId != v { + gym.PartnerId = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonCostume(v null.Int) { + if gym.RaidPokemonCostume != v { + gym.RaidPokemonCostume = v + gym.dirty = true + } +} + +func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { + if gym.RaidPokemonEvolution != v { + gym.RaidPokemonEvolution = v + gym.dirty = true + } +} + +func (gym *Gym) SetArScanEligible(v null.Int) { + if gym.ArScanEligible != v { + gym.ArScanEligible = v + gym.dirty = true + } +} + +func (gym *Gym) SetPowerUpLevel(v null.Int) { + if gym.PowerUpLevel != v { + gym.PowerUpLevel = v + gym.dirty = true + } +} + +func (gym *Gym) SetPowerUpPoints(v null.Int) { + if gym.PowerUpPoints != v { + gym.PowerUpPoints = v + gym.dirty = true + } +} + +func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { + if gym.PowerUpEndTimestamp != v { + gym.PowerUpEndTimestamp = v + gym.dirty = true + } +} + +func (gym *Gym) SetDescription(v null.String) { + if gym.Description != v { + gym.Description = v + gym.dirty = true + } +} + +func (gym *Gym) SetDefenders(v null.String) { + if gym.Defenders != v { + gym.Defenders = v + //Do not set to dirty, as don't trigger an update + //gym.dirty = true + } +} + +func (gym *Gym) SetRsvps(v null.String) { + if gym.Rsvps != v { + gym.Rsvps = v + gym.dirty = true + } +} + func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, error) { inMemoryGym := gymCache.Get(fortId) if inMemoryGym != nil { gym := inMemoryGym.Value() - return &gym, nil + gym.snapshotOldValues() // Snapshot for webhook comparison + return gym, nil } gym := Gym{} err := db.GeneralDb.GetContext(ctx, &gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) @@ -133,7 +467,8 @@ func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, er return nil, err } - gymCache.Set(fortId, gym, ttlcache.DefaultTTL) + gym.snapshotOldValues() // Snapshot for webhook comparison + gymCache.Set(fortId, &gym, ttlcache.DefaultTTL) if config.Config.TestFortInMemory { fortRtreeUpdateGymOnGet(&gym) } @@ -183,13 +518,13 @@ func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64 Badge int `json:"badge,omitempty"` Background *int64 `json:"background,omitempty"` } - gym.Id = fortData.FortId - gym.Lat = fortData.Latitude //fmt.Sprintf("%f", fortData.Latitude) - gym.Lon = fortData.Longitude //fmt.Sprintf("%f", fortData.Longitude) - gym.Enabled = null.IntFrom(util.BoolToInt[int64](fortData.Enabled)) - gym.GuardingPokemonId = null.IntFrom(int64(fortData.GuardPokemonId)) + gym.SetId(fortData.FortId) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) + gym.SetEnabled(null.IntFrom(util.BoolToInt[int64](fortData.Enabled))) + gym.SetGuardingPokemonId(null.IntFrom(int64(fortData.GuardPokemonId))) if fortData.GuardPokemonDisplay == nil { - gym.GuardingPokemonDisplay = null.NewString("", false) + gym.SetGuardingPokemonDisplay(null.NewString("", false)) } else { display, _ := json.Marshal(pokemonDisplay{ Form: int(fortData.GuardPokemonDisplay.Form), @@ -202,90 +537,91 @@ func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64 Badge: int(fortData.GuardPokemonDisplay.PokemonBadge), Background: util.ExtractBackgroundFromDisplay(fortData.GuardPokemonDisplay), }) - gym.GuardingPokemonDisplay = null.StringFrom(string(display)) + gym.SetGuardingPokemonDisplay(null.StringFrom(string(display))) } - gym.TeamId = null.IntFrom(int64(fortData.Team)) + gym.SetTeamId(null.IntFrom(int64(fortData.Team))) if fortData.GymDisplay != nil { - gym.AvailableSlots = null.IntFrom(int64(fortData.GymDisplay.SlotsAvailable)) + gym.SetAvailableSlots(null.IntFrom(int64(fortData.GymDisplay.SlotsAvailable))) } else { - gym.AvailableSlots = null.IntFrom(6) // this may be an incorrect assumption + gym.SetAvailableSlots(null.IntFrom(6)) // this may be an incorrect assumption } - gym.LastModifiedTimestamp = null.IntFrom(fortData.LastModifiedMs / 1000) - gym.ExRaidEligible = null.IntFrom(util.BoolToInt[int64](fortData.IsExRaidEligible)) + gym.SetLastModifiedTimestamp(null.IntFrom(fortData.LastModifiedMs / 1000)) + gym.SetExRaidEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsExRaidEligible))) if fortData.ImageUrl != "" { - gym.Url = null.StringFrom(fortData.ImageUrl) + gym.SetUrl(null.StringFrom(fortData.ImageUrl)) } - gym.InBattle = null.IntFrom(util.BoolToInt[int64](fortData.IsInBattle)) - gym.ArScanEligible = null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible)) - gym.PowerUpPoints = null.IntFrom(int64(fortData.PowerUpProgressPoints)) + gym.SetInBattle(null.IntFrom(util.BoolToInt[int64](fortData.IsInBattle))) + gym.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) + gym.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) - gym.PowerUpLevel, gym.PowerUpEndTimestamp = calculatePowerUpPoints(fortData) + powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) + gym.SetPowerUpLevel(powerUpLevel) + gym.SetPowerUpEndTimestamp(powerUpEndTimestamp) if fortData.PartnerId == "" { - gym.PartnerId = null.NewString("", false) + gym.SetPartnerId(null.NewString("", false)) } else { - gym.PartnerId = null.StringFrom(fortData.PartnerId) + gym.SetPartnerId(null.StringFrom(fortData.PartnerId)) } if fortData.ImageUrl != "" { - gym.Url = null.StringFrom(fortData.ImageUrl) - + gym.SetUrl(null.StringFrom(fortData.ImageUrl)) } if fortData.Team == 0 { // check!! - gym.TotalCp = null.IntFrom(0) + gym.SetTotalCp(null.IntFrom(0)) } else { if fortData.GymDisplay != nil { totalCp := int64(fortData.GymDisplay.TotalGymCp) if gym.TotalCp.Int64-totalCp > 100 || totalCp-gym.TotalCp.Int64 > 100 { - gym.TotalCp = null.IntFrom(totalCp) + gym.SetTotalCp(null.IntFrom(totalCp)) } } else { - gym.TotalCp = null.IntFrom(0) + gym.SetTotalCp(null.IntFrom(0)) } } if fortData.RaidInfo != nil { - gym.RaidEndTimestamp = null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000) - gym.RaidSpawnTimestamp = null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000) + gym.SetRaidEndTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000)) + gym.SetRaidSpawnTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000)) raidBattleTimestamp := int64(fortData.RaidInfo.RaidBattleMs) / 1000 if gym.RaidBattleTimestamp.ValueOrZero() != raidBattleTimestamp { // We are reporting a new raid, clear rsvp data - gym.Rsvps = null.NewString("", false) + gym.SetRsvps(null.NewString("", false)) } - gym.RaidBattleTimestamp = null.IntFrom(raidBattleTimestamp) + gym.SetRaidBattleTimestamp(null.IntFrom(raidBattleTimestamp)) - gym.RaidLevel = null.IntFrom(int64(fortData.RaidInfo.RaidLevel)) + gym.SetRaidLevel(null.IntFrom(int64(fortData.RaidInfo.RaidLevel))) if fortData.RaidInfo.RaidPokemon != nil { - gym.RaidPokemonId = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonId)) - gym.RaidPokemonMove1 = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move1)) - gym.RaidPokemonMove2 = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move2)) - gym.RaidPokemonForm = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Form)) - gym.RaidPokemonAlignment = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Alignment)) - gym.RaidPokemonCp = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Cp)) - gym.RaidPokemonGender = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Gender)) - gym.RaidPokemonCostume = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Costume)) - gym.RaidPokemonEvolution = null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.CurrentTempEvolution)) + gym.SetRaidPokemonId(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonId))) + gym.SetRaidPokemonMove1(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move1))) + gym.SetRaidPokemonMove2(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move2))) + gym.SetRaidPokemonForm(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Form))) + gym.SetRaidPokemonAlignment(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Alignment))) + gym.SetRaidPokemonCp(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Cp))) + gym.SetRaidPokemonGender(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Gender))) + gym.SetRaidPokemonCostume(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Costume))) + gym.SetRaidPokemonEvolution(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.CurrentTempEvolution))) } else { - gym.RaidPokemonId = null.IntFrom(0) - gym.RaidPokemonMove1 = null.IntFrom(0) - gym.RaidPokemonMove2 = null.IntFrom(0) - gym.RaidPokemonForm = null.IntFrom(0) - gym.RaidPokemonAlignment = null.IntFrom(0) - gym.RaidPokemonCp = null.IntFrom(0) - gym.RaidPokemonGender = null.IntFrom(0) - gym.RaidPokemonCostume = null.IntFrom(0) - gym.RaidPokemonEvolution = null.IntFrom(0) + gym.SetRaidPokemonId(null.IntFrom(0)) + gym.SetRaidPokemonMove1(null.IntFrom(0)) + gym.SetRaidPokemonMove2(null.IntFrom(0)) + gym.SetRaidPokemonForm(null.IntFrom(0)) + gym.SetRaidPokemonAlignment(null.IntFrom(0)) + gym.SetRaidPokemonCp(null.IntFrom(0)) + gym.SetRaidPokemonGender(null.IntFrom(0)) + gym.SetRaidPokemonCostume(null.IntFrom(0)) + gym.SetRaidPokemonEvolution(null.IntFrom(0)) } - gym.RaidIsExclusive = null.IntFrom(0) //null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive)) + gym.SetRaidIsExclusive(null.IntFrom(0)) //null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive)) } - gym.CellId = null.IntFrom(int64(cellId)) + gym.SetCellId(null.IntFrom(int64(cellId))) if gym.Deleted { - gym.Deleted = false + gym.SetDeleted(false) log.Warnf("Cleared Gym with id '%s' is found again in GMO, therefore un-deleted", gym.Id) // Restore in fort tracker if enabled if fortTracker != nil { @@ -297,32 +633,32 @@ func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64 } func (gym *Gym) updateGymFromFortProto(fortData *pogo.FortDetailsOutProto) *Gym { - gym.Id = fortData.Id - gym.Lat = fortData.Latitude //fmt.Sprintf("%f", fortData.Latitude) - gym.Lon = fortData.Longitude //fmt.Sprintf("%f", fortData.Longitude) + gym.SetId(fortData.Id) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) if len(fortData.ImageUrl) > 0 { - gym.Url = null.StringFrom(fortData.ImageUrl[0]) + gym.SetUrl(null.StringFrom(fortData.ImageUrl[0])) } - gym.Name = null.StringFrom(fortData.Name) + gym.SetName(null.StringFrom(fortData.Name)) return gym } func (gym *Gym) updateGymFromGymInfoOutProto(gymData *pogo.GymGetInfoOutProto) *Gym { - gym.Id = gymData.GymStatusAndDefenders.PokemonFortProto.FortId - gym.Lat = gymData.GymStatusAndDefenders.PokemonFortProto.Latitude - gym.Lon = gymData.GymStatusAndDefenders.PokemonFortProto.Longitude + gym.SetId(gymData.GymStatusAndDefenders.PokemonFortProto.FortId) + gym.SetLat(gymData.GymStatusAndDefenders.PokemonFortProto.Latitude) + gym.SetLon(gymData.GymStatusAndDefenders.PokemonFortProto.Longitude) // This will have gym defenders in it... if len(gymData.Url) > 0 { - gym.Url = null.StringFrom(gymData.Url) + gym.SetUrl(null.StringFrom(gymData.Url)) } - gym.Name = null.StringFrom(gymData.Name) + gym.SetName(null.StringFrom(gymData.Name)) if gymData.Description == "" { - gym.Description = null.NewString("", false) + gym.SetDescription(null.NewString("", false)) } else { - gym.Description = null.StringFrom(gymData.Description) + gym.SetDescription(null.StringFrom(gymData.Description)) } type pokemonGymDefender struct { @@ -377,22 +713,22 @@ func (gym *Gym) updateGymFromGymInfoOutProto(gymData *pogo.GymGetInfoOutProto) * defenders = append(defenders, defender) } bDefenders, _ := json.Marshal(defenders) - gym.Defenders = null.StringFrom(string(bDefenders)) + gym.SetDefenders(null.StringFrom(string(bDefenders))) // log.Debugf("Gym %s defenders %s ", gym.Id, string(bDefenders)) return gym } func (gym *Gym) updateGymFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto, skipName bool) *Gym { - gym.Id = fortData.Id - gym.Lat = fortData.Latitude - gym.Lon = fortData.Longitude + gym.SetId(fortData.Id) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) if len(fortData.Image) > 0 { - gym.Url = null.StringFrom(fortData.Image[0].Url) + gym.SetUrl(null.StringFrom(fortData.Image[0].Url)) } if !skipName { - gym.Name = null.StringFrom(fortData.Name) + gym.SetName(null.StringFrom(fortData.Name)) } if gym.Deleted { @@ -422,14 +758,14 @@ func (gym *Gym) updateGymFromRsvpProto(fortData *pogo.GetEventRsvpsOutProto) *Gy } if len(timeslots) == 0 { - gym.Rsvps = null.NewString("", false) + gym.SetRsvps(null.NewString("", false)) } else { slices.SortFunc(timeslots, func(a, b rsvpTimeslot) int { return cmp.Compare(a.Timeslot, b.Timeslot) }) bRsvps, _ := json.Marshal(timeslots) - gym.Rsvps = null.StringFrom(string(bRsvps)) + gym.SetRsvps(null.StringFrom(string(bRsvps))) } return gym @@ -524,19 +860,27 @@ type GymDetailsWebhook struct { //"ar_scan_eligible": arScanEligible ?? 0 } -func createGymFortWebhooks(oldGym *Gym, gym *Gym) { +func createGymFortWebhooks(gym *Gym) { fort := InitWebHookFortFromGym(gym) - oldFort := InitWebHookFortFromGym(oldGym) - if oldGym == nil { - CreateFortWebHooks(oldFort, fort, NEW) + if gym.newRecord { + CreateFortWebHooks(nil, fort, NEW) } else { + // Build old fort from saved old values + oldFort := &FortWebhook{ + Type: GYM.String(), + Id: gym.Id, + Name: gym.oldValues.Name.Ptr(), + ImageUrl: gym.oldValues.Url.Ptr(), + Description: gym.oldValues.Description.Ptr(), + Location: Location{Latitude: gym.oldValues.Lat, Longitude: gym.oldValues.Lon}, + } CreateFortWebHooks(oldFort, fort, EDIT) } } -func createGymWebhooks(oldGym *Gym, gym *Gym, areas []geo.AreaName) { - if oldGym == nil || - (oldGym.AvailableSlots != gym.AvailableSlots || oldGym.TeamId != gym.TeamId || oldGym.InBattle != gym.InBattle) { +func createGymWebhooks(gym *Gym, areas []geo.AreaName) { + if gym.newRecord || + (gym.oldValues.AvailableSlots != gym.AvailableSlots || gym.oldValues.TeamId != gym.TeamId || gym.oldValues.InBattle != gym.InBattle) { gymDetails := GymDetailsWebhook{ Id: gym.Id, Name: gym.Name.ValueOrZero(), @@ -567,9 +911,9 @@ func createGymWebhooks(oldGym *Gym, gym *Gym, areas []geo.AreaName) { } if gym.RaidSpawnTimestamp.ValueOrZero() > 0 && - (oldGym == nil || oldGym.RaidLevel != gym.RaidLevel || - oldGym.RaidPokemonId != gym.RaidPokemonId || - oldGym.RaidSpawnTimestamp != gym.RaidSpawnTimestamp || oldGym.Rsvps != gym.Rsvps) { + (gym.newRecord || gym.oldValues.RaidLevel != gym.RaidLevel || + gym.oldValues.RaidPokemonId != gym.RaidPokemonId || + gym.oldValues.RaidSpawnTimestamp != gym.RaidSpawnTimestamp || gym.oldValues.Rsvps != gym.Rsvps) { raidBattleTime := gym.RaidBattleTimestamp.ValueOrZero() raidEndTime := gym.RaidEndTimestamp.ValueOrZero() now := time.Now().Unix() @@ -626,30 +970,16 @@ func createGymWebhooks(oldGym *Gym, gym *Gym, areas []geo.AreaName) { } func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { - oldGym, _ := GetGymRecord(ctx, db, gym.Id) - now := time.Now().Unix() - if oldGym != nil && !hasChangesGym(oldGym, gym) { - if oldGym.Updated > now-900 { - // if a gym is unchanged, and we are within 15 minutes don't make any changes - // however, gym battle toggle a chance to trigger a web hook and make sure we - // save defender changes to internal cache - - if hasInternalChangesGym(oldGym, gym) { - areas := MatchStatsGeofence(gym.Lat, gym.Lon) - createGymWebhooks(oldGym, gym, areas) - - gymCache.Set(gym.Id, *gym, ttlcache.DefaultTTL) - } - + if !gym.IsNewRecord() && !gym.IsDirty() { + if gym.Updated > now-900 { + // if a gym is unchanged, but we did see it again after 15 minutes, then save again return } } - gym.Updated = now - //log.Traceln(cmp.Diff(oldGym, gym)) - if oldGym == nil { + if gym.IsNewRecord() { res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) @@ -710,11 +1040,13 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { _, _ = res, err } - gymCache.Set(gym.Id, *gym, ttlcache.DefaultTTL) + gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) areas := MatchStatsGeofence(gym.Lat, gym.Lon) - createGymWebhooks(oldGym, gym, areas) - createGymFortWebhooks(oldGym, gym) - updateRaidStats(oldGym, gym, areas) + createGymWebhooks(gym, areas) + createGymFortWebhooks(gym) + updateRaidStats(gym, areas) + gym.newRecord = false // After saving, it's no longer a new record + gym.ClearDirty() } func updateGymGetMapFortCache(gym *Gym, skipName bool) { @@ -738,7 +1070,7 @@ func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails } if gym == nil { - gym = &Gym{} + gym = &Gym{newRecord: true} } gym.updateGymFromFortProto(fort) @@ -759,7 +1091,7 @@ func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymIn } if gym == nil { - gym = &Gym{} + gym = &Gym{newRecord: true} } gym.updateGymFromGymInfoOutProto(gymInfo) @@ -825,7 +1157,7 @@ func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { } if gym.Rsvps.Valid { - gym.Rsvps = null.NewString("", false) + gym.SetRsvps(null.NewString("", false)) saveGymRecord(ctx, db, gym) } diff --git a/decoder/main.go b/decoder/main.go index 9173ad5d..8f820679 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -59,14 +59,14 @@ type webhooksSenderInterface interface { var webhooksSender webhooksSenderInterface var statsCollector stats_collector.StatsCollector var pokestopCache *ttlcache.Cache[string, *Pokestop] -var gymCache *ttlcache.Cache[string, Gym] +var gymCache *ttlcache.Cache[string, *Gym] var stationCache *ttlcache.Cache[string, Station] var tappableCache *ttlcache.Cache[uint64, Tappable] var weatherCache *ttlcache.Cache[int64, Weather] var weatherConsensusCache *ttlcache.Cache[int64, *WeatherConsensusState] var s2CellCache *ttlcache.Cache[uint64, S2Cell] var spawnpointCache *ttlcache.Cache[int64, Spawnpoint] -var pokemonCache []*ttlcache.Cache[uint64, Pokemon] +var pokemonCache []*ttlcache.Cache[uint64, *Pokemon] var incidentCache *ttlcache.Cache[string, Incident] var playerCache *ttlcache.Cache[string, Player] var routeCache *ttlcache.Cache[string, Route] @@ -101,15 +101,15 @@ func (cl *gohbemLogger) Print(message string) { log.Info("Gohbem - ", message) } -func getPokemonCache(key uint64) *ttlcache.Cache[uint64, Pokemon] { +func getPokemonCache(key uint64) *ttlcache.Cache[uint64, *Pokemon] { return pokemonCache[key%uint64(len(pokemonCache))] } -func setPokemonCache(key uint64, value Pokemon, ttl time.Duration) { +func setPokemonCache(key uint64, value *Pokemon, ttl time.Duration) { getPokemonCache(key).Set(key, value, ttl) } -func getPokemonFromCache(key uint64) *ttlcache.Item[uint64, Pokemon] { +func getPokemonFromCache(key uint64) *ttlcache.Item[uint64, *Pokemon] { return getPokemonCache(key).Get(key) } @@ -123,8 +123,8 @@ func initDataCache() { ) go pokestopCache.Start() - gymCache = ttlcache.New[string, Gym]( - ttlcache.WithTTL[string, Gym](60 * time.Minute), + gymCache = ttlcache.New[string, *Gym]( + ttlcache.WithTTL[string, *Gym](60 * time.Minute), ) go gymCache.Start() @@ -160,11 +160,11 @@ func initDataCache() { // pokemon is the most active table. Use an array of caches to increase concurrency for querying ttlcache, which places a global lock for each Get/Set operation // Initialize pokemon cache array: by picking it to be nproc, we should expect ~nproc*(1-1/e) ~ 63% concurrency - pokemonCache = make([]*ttlcache.Cache[uint64, Pokemon], runtime.NumCPU()) + pokemonCache = make([]*ttlcache.Cache[uint64, *Pokemon], runtime.NumCPU()) for i := 0; i < len(pokemonCache); i++ { - pokemonCache[i] = ttlcache.New[uint64, Pokemon]( - ttlcache.WithTTL[uint64, Pokemon](60*time.Minute), - ttlcache.WithDisableTouchOnHit[uint64, Pokemon](), // Pokemon will last 60 mins from when we first see them not last see them + pokemonCache[i] = ttlcache.New[uint64, *Pokemon]( + ttlcache.WithTTL[uint64, *Pokemon](60*time.Minute), + ttlcache.WithDisableTouchOnHit[uint64, *Pokemon](), // Pokemon will last 60 mins from when we first see them not last see them ) go pokemonCache[i].Start() } @@ -353,15 +353,14 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa continue } - isNewGym := gym == nil - if isNewGym { - gym = &Gym{} + if gym == nil { + gym = &Gym{newRecord: true} } gym.updateGymFromFort(fort.Data, fort.Cell) // If this is a new gym, check if it was converted from a pokestop and copy shared fields - if isNewGym { + if gym.IsNewRecord() { pokestop, _ := GetPokestopRecord(ctx, db, fortId) if pokestop != nil { gym.copySharedFieldsFrom(pokestop) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9244cc2b..482a5901 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -76,6 +76,21 @@ type Pokemon struct { IsEvent int8 `db:"is_event" json:"is_event"` internal grpc.PokemonInternal + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + // Note: newRecord tracked via FirstSeenTimestamp == 0 (see isNewRecord method) + + oldValues PokemonOldValues `db:"-" json:"-"` // Old values for webhook comparison and stats +} + +// PokemonOldValues holds old field values for webhook comparison, stats, and R-tree updates +type PokemonOldValues struct { + PokemonId int16 + Weather null.Int + Cp null.Int + SeenType null.String + Lat float64 + Lon float64 } // @@ -131,12 +146,248 @@ type Pokemon struct { //KEY `ix_iv` (`iv`) //) +// IsDirty returns true if any field has been modified +func (pokemon *Pokemon) IsDirty() bool { + return pokemon.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (pokemon *Pokemon) ClearDirty() { + pokemon.dirty = false +} + +// snapshotOldValues saves current values for webhook comparison, stats, and R-tree updates +// Call this after loading from cache/DB but before modifications +func (pokemon *Pokemon) snapshotOldValues() { + pokemon.oldValues = PokemonOldValues{ + PokemonId: pokemon.PokemonId, + Weather: pokemon.Weather, + Cp: pokemon.Cp, + SeenType: pokemon.SeenType, + Lat: pokemon.Lat, + Lon: pokemon.Lon, + } +} + +// --- Set methods with dirty tracking --- + +func (pokemon *Pokemon) SetPokestopId(v null.String) { + if pokemon.PokestopId != v { + pokemon.PokestopId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetSpawnId(v null.Int) { + if pokemon.SpawnId != v { + pokemon.SpawnId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetLat(v float64) { + if !floatAlmostEqual(pokemon.Lat, v, floatTolerance) { + pokemon.Lat = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetLon(v float64) { + if !floatAlmostEqual(pokemon.Lon, v, floatTolerance) { + pokemon.Lon = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetPokemonId(v int16) { + if pokemon.PokemonId != v { + pokemon.PokemonId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetForm(v null.Int) { + if pokemon.Form != v { + pokemon.Form = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCostume(v null.Int) { + if pokemon.Costume != v { + pokemon.Costume = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetGender(v null.Int) { + if pokemon.Gender != v { + pokemon.Gender = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetWeather(v null.Int) { + if pokemon.Weather != v { + pokemon.Weather = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetIsStrong(v null.Bool) { + if pokemon.IsStrong != v { + pokemon.IsStrong = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetExpireTimestamp(v null.Int) { + if pokemon.ExpireTimestamp != v { + pokemon.ExpireTimestamp = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetExpireTimestampVerified(v bool) { + if pokemon.ExpireTimestampVerified != v { + pokemon.ExpireTimestampVerified = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetSeenType(v null.String) { + if pokemon.SeenType != v { + pokemon.SeenType = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetUsername(v null.String) { + if pokemon.Username != v { + pokemon.Username = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCellId(v null.Int) { + if pokemon.CellId != v { + pokemon.CellId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetIsEvent(v int8) { + if pokemon.IsEvent != v { + pokemon.IsEvent = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetShiny(v null.Bool) { + if pokemon.Shiny != v { + pokemon.Shiny = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCp(v null.Int) { + if pokemon.Cp != v { + pokemon.Cp = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetLevel(v null.Int) { + if pokemon.Level != v { + pokemon.Level = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetMove1(v null.Int) { + if pokemon.Move1 != v { + pokemon.Move1 = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetMove2(v null.Int) { + if pokemon.Move2 != v { + pokemon.Move2 = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetHeight(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Height, v, floatTolerance) { + pokemon.Height = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetWeight(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Weight, v, floatTolerance) { + pokemon.Weight = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetSize(v null.Int) { + if pokemon.Size != v { + pokemon.Size = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetIsDitto(v bool) { + if pokemon.IsDitto != v { + pokemon.IsDitto = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { + if pokemon.DisplayPokemonId != v { + pokemon.DisplayPokemonId = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetPvp(v null.String) { + if pokemon.Pvp != v { + pokemon.Pvp = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCapture1(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Capture1, v, floatTolerance) { + pokemon.Capture1 = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCapture2(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Capture2, v, floatTolerance) { + pokemon.Capture2 = v + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) SetCapture3(v null.Float) { + if !nullFloatAlmostEqual(pokemon.Capture3, v, floatTolerance) { + pokemon.Capture3 = v + pokemon.dirty = true + } +} + func getPokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, error) { if db.UsePokemonCache { inMemoryPokemon := getPokemonFromCache(encounterId) if inMemoryPokemon != nil { pokemon := inMemoryPokemon.Value() - return &pokemon, nil + pokemon.snapshotOldValues() // Snapshot for webhook comparison + return pokemon, nil } } if config.Config.PokemonMemoryOnly { @@ -160,8 +411,9 @@ func getPokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) return nil, err } + pokemon.snapshotOldValues() // Snapshot for webhook comparison if db.UsePokemonCache { - setPokemonCache(encounterId, pokemon, ttlcache.DefaultTTL) + setPokemonCache(encounterId, &pokemon, ttlcache.DefaultTTL) } pokemonRtreeUpdatePokemonOnGet(&pokemon) return &pokemon, nil @@ -174,7 +426,7 @@ func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId } pokemon = &Pokemon{Id: encounterId} if db.UsePokemonCache { - setPokemonCache(encounterId, *pokemon, ttlcache.DefaultTTL) + setPokemonCache(encounterId, pokemon, ttlcache.DefaultTTL) } return pokemon, nil } @@ -222,19 +474,12 @@ func hasChangesPokemon(old *Pokemon, new *Pokemon) bool { } func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Pokemon, isEncounter, writeDB, webhook bool, now int64) { - oldPokemon, _ := getPokemonRecord(ctx, db, pokemon.Id) - - if oldPokemon != nil && !hasChangesPokemon(oldPokemon, pokemon) { + if !pokemon.isNewRecord() && !pokemon.IsDirty() { return } - // Blank, non-persisted record are now inserted into the cache to save on DB calls - if oldPokemon != nil && oldPokemon.isNewRecord() { - oldPokemon = nil - } - // uncomment to debug excessive writes - //if oldPokemon != nil && oldPokemon.AtkIv == pokemon.AtkIv && oldPokemon.DefIv == pokemon.DefIv && oldPokemon.StaIv == pokemon.StaIv && oldPokemon.Level == pokemon.Level && oldPokemon.ExpireTimestampVerified == pokemon.ExpireTimestampVerified && oldPokemon.PokemonId == pokemon.PokemonId && oldPokemon.ExpireTimestamp == pokemon.ExpireTimestamp && oldPokemon.PokestopId == pokemon.PokestopId && math.Abs(pokemon.Lat-oldPokemon.Lat) < .000001 && math.Abs(pokemon.Lon-oldPokemon.Lon) < .000001 { + //if !pokemon.isNewRecord() && oldPokemon.AtkIv == pokemon.AtkIv && oldPokemon.DefIv == pokemon.DefIv && oldPokemon.StaIv == pokemon.StaIv && oldPokemon.Level == pokemon.Level && oldPokemon.ExpireTimestampVerified == pokemon.ExpireTimestampVerified && oldPokemon.PokemonId == pokemon.PokemonId && oldPokemon.ExpireTimestamp == pokemon.ExpireTimestamp && oldPokemon.PokestopId == pokemon.PokestopId && math.Abs(pokemon.Lat-oldPokemon.Lat) < .000001 && math.Abs(pokemon.Lon-oldPokemon.Lon) < .000001 { // log.Errorf("Why are we updating this? %s", cmp.Diff(oldPokemon, pokemon, cmp.Options{ // ignoreNearFloats, ignoreNearNullFloats, // cmpopts.IgnoreFields(Pokemon{}, "Username", "Iv", "Pvp"), @@ -246,18 +491,17 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } pokemon.Updated = null.IntFrom(now) - if oldPokemon == nil || oldPokemon.PokemonId != pokemon.PokemonId || oldPokemon.Cp != pokemon.Cp { + if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId || pokemon.oldValues.Cp != pokemon.Cp { pokemon.Changed = now } changePvpField := false var pvpResults map[string][]gohbem.PokemonEntry if ohbem != nil { - // Calculating PVP data - if pokemon.AtkIv.Valid && (oldPokemon == nil || oldPokemon.PokemonId != pokemon.PokemonId || - oldPokemon.Level != pokemon.Level || oldPokemon.Form != pokemon.Form || - oldPokemon.Costume != pokemon.Costume || oldPokemon.Gender != pokemon.Gender || - oldPokemon.Weather != pokemon.Weather) { + // Calculating PVP data - check for changes in pokemon properties that affect PVP rankings + // For new records, always calculate; for existing, check if relevant fields changed + shouldCalculatePvp := pokemon.AtkIv.Valid && (pokemon.isNewRecord() || pokemon.IsDirty()) + if shouldCalculatePvp { pvp, err := ohbem.QueryPvPRank(int(pokemon.PokemonId), int(pokemon.Form.ValueOrZero()), int(pokemon.Costume.ValueOrZero()), @@ -274,19 +518,13 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po pvpResults = pvp } } - if !pokemon.AtkIv.Valid && (oldPokemon == nil || oldPokemon.AtkIv.Valid) { + if !pokemon.AtkIv.Valid && pokemon.isNewRecord() { pokemon.Pvp = null.NewString("", false) changePvpField = true } } - var oldSeenType string - if oldPokemon == nil { - oldSeenType = "n/a" - } else { - oldSeenType = oldPokemon.SeenType.ValueOrZero() - } - log.Debugf("Updating pokemon [%d] from %s->%s", pokemon.Id, oldSeenType, pokemon.SeenType.ValueOrZero()) + log.Debugf("Updating pokemon [%d] to %s", pokemon.Id, pokemon.SeenType.ValueOrZero()) //log.Println(cmp.Diff(oldPokemon, pokemon)) if writeDB && !config.Config.PokemonMemoryOnly { @@ -306,7 +544,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po log.Errorf("[POKEMON] Failed to marshal internal data for %d, data may be lost: %s", pokemon.Id, err) } } - if oldPokemon == nil { + if pokemon.isNewRecord() { pvpField, pvpValue := "", "" if changePvpField { pvpField, pvpValue = "pvp, ", ":pvp, " @@ -388,31 +626,32 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } // Update pokemon rtree - if oldPokemon == nil { + if pokemon.isNewRecord() { + addPokemonToTree(pokemon) + } else if pokemon.Lat != pokemon.oldValues.Lat || pokemon.Lon != pokemon.oldValues.Lon { + // Position changed - update R-tree by removing from old position and adding to new + removePokemonFromTree(pokemon.Id, pokemon.oldValues.Lat, pokemon.oldValues.Lon) addPokemonToTree(pokemon) - } else { - if pokemon.Lat != oldPokemon.Lat || pokemon.Lon != oldPokemon.Lon { - removePokemonFromTree(oldPokemon) - addPokemonToTree(pokemon) - } } updatePokemonLookup(pokemon, changePvpField, pvpResults) areas := MatchStatsGeofence(pokemon.Lat, pokemon.Lon) if webhook { - createPokemonWebhooks(ctx, db, oldPokemon, pokemon, areas) + createPokemonWebhooks(ctx, db, pokemon, areas) } - updatePokemonStats(oldPokemon, pokemon, areas, now) + updatePokemonStats(pokemon, areas, now) + + pokemon.ClearDirty() pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache if db.UsePokemonCache { - setPokemonCache(pokemon.Id, *pokemon, pokemon.remainingDuration(now)) + setPokemonCache(pokemon.Id, pokemon, pokemon.remainingDuration(now)) } } -func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, new *Pokemon, areas []geo.AreaName) { +func createPokemonWebhooks(ctx context.Context, db db.DbDetails, pokemon *Pokemon, areas []geo.AreaName) { //nullString := func (v null.Int) interface{} { // if !v.Valid { // return "null" @@ -420,29 +659,29 @@ func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, n // return v.ValueOrZero() //} - if old == nil || - old.PokemonId != new.PokemonId || - old.Weather != new.Weather || - old.Cp != new.Cp { + if pokemon.isNewRecord() || + pokemon.oldValues.PokemonId != pokemon.PokemonId || + pokemon.oldValues.Weather != pokemon.Weather || + pokemon.oldValues.Cp != pokemon.Cp { pokemonHook := map[string]interface{}{ "spawnpoint_id": func() string { - if !new.SpawnId.Valid { + if !pokemon.SpawnId.Valid { return "None" } - return strconv.FormatInt(new.SpawnId.ValueOrZero(), 16) + return strconv.FormatInt(pokemon.SpawnId.ValueOrZero(), 16) }(), "pokestop_id": func() string { - if !new.PokestopId.Valid { + if !pokemon.PokestopId.Valid { return "None" } else { - return new.PokestopId.ValueOrZero() + return pokemon.PokestopId.ValueOrZero() } }(), "pokestop_name": func() *string { - if !new.PokestopId.Valid { + if !pokemon.PokestopId.Valid { return nil } else { - pokestop, _ := GetPokestopRecord(ctx, db, new.PokestopId.String) + pokestop, _ := GetPokestopRecord(ctx, db, pokemon.PokestopId.String) name := "Unknown" if pokestop != nil { name = pokestop.Name.ValueOrZero() @@ -450,46 +689,46 @@ func createPokemonWebhooks(ctx context.Context, db db.DbDetails, old *Pokemon, n return &name } }(), - "encounter_id": strconv.FormatUint(new.Id, 10), - "pokemon_id": new.PokemonId, - "latitude": new.Lat, - "longitude": new.Lon, - "disappear_time": new.ExpireTimestamp.ValueOrZero(), - "disappear_time_verified": new.ExpireTimestampVerified, - "first_seen": new.FirstSeenTimestamp, - "last_modified_time": new.Updated, - "gender": new.Gender, - "cp": new.Cp, - "form": new.Form, - "costume": new.Costume, - "individual_attack": new.AtkIv, - "individual_defense": new.DefIv, - "individual_stamina": new.StaIv, - "pokemon_level": new.Level, - "move_1": new.Move1, - "move_2": new.Move2, - "weight": new.Weight, - "size": new.Size, - "height": new.Height, - "weather": new.Weather, - "capture_1": new.Capture1.ValueOrZero(), - "capture_2": new.Capture2.ValueOrZero(), - "capture_3": new.Capture3.ValueOrZero(), - "shiny": new.Shiny, - "username": new.Username, - "display_pokemon_id": new.DisplayPokemonId, - "is_event": new.IsEvent, - "seen_type": new.SeenType, + "encounter_id": strconv.FormatUint(pokemon.Id, 10), + "pokemon_id": pokemon.PokemonId, + "latitude": pokemon.Lat, + "longitude": pokemon.Lon, + "disappear_time": pokemon.ExpireTimestamp.ValueOrZero(), + "disappear_time_verified": pokemon.ExpireTimestampVerified, + "first_seen": pokemon.FirstSeenTimestamp, + "last_modified_time": pokemon.Updated, + "gender": pokemon.Gender, + "cp": pokemon.Cp, + "form": pokemon.Form, + "costume": pokemon.Costume, + "individual_attack": pokemon.AtkIv, + "individual_defense": pokemon.DefIv, + "individual_stamina": pokemon.StaIv, + "pokemon_level": pokemon.Level, + "move_1": pokemon.Move1, + "move_2": pokemon.Move2, + "weight": pokemon.Weight, + "size": pokemon.Size, + "height": pokemon.Height, + "weather": pokemon.Weather, + "capture_1": pokemon.Capture1.ValueOrZero(), + "capture_2": pokemon.Capture2.ValueOrZero(), + "capture_3": pokemon.Capture3.ValueOrZero(), + "shiny": pokemon.Shiny, + "username": pokemon.Username, + "display_pokemon_id": pokemon.DisplayPokemonId, + "is_event": pokemon.IsEvent, + "seen_type": pokemon.SeenType, "pvp": func() interface{} { - if !new.Pvp.Valid { + if !pokemon.Pvp.Valid { return nil } else { - return json.RawMessage(new.Pvp.ValueOrZero()) + return json.RawMessage(pokemon.Pvp.ValueOrZero()) } }(), } - if new.AtkIv.Valid && new.DefIv.Valid && new.StaIv.Valid { + if pokemon.AtkIv.Valid && pokemon.DefIv.Valid && pokemon.StaIv.Valid { webhooksSender.AddMessage(webhooks.PokemonIV, pokemonHook, areas) } else { webhooksSender.AddMessage(webhooks.PokemonNoIV, pokemonHook, areas) @@ -557,14 +796,14 @@ func (pokemon *Pokemon) addWildPokemon(ctx context.Context, db db.DbDetails, wil if wildPokemon.EncounterId != pokemon.Id { panic("Unmatched EncounterId") } - pokemon.Lat = wildPokemon.Latitude - pokemon.Lon = wildPokemon.Longitude + pokemon.SetLat(wildPokemon.Latitude) + pokemon.SetLon(wildPokemon.Longitude) spawnId, err := strconv.ParseInt(wildPokemon.SpawnPointId, 16, 64) if err != nil { panic(err) } - pokemon.SpawnId = null.IntFrom(spawnId) + pokemon.SetSpawnId(null.IntFrom(spawnId)) pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, trustworthyTimestamp) pokemon.setPokemonDisplay(int16(wildPokemon.Pokemon.PokemonId), wildPokemon.Pokemon.PokemonDisplay) @@ -587,15 +826,15 @@ func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto } func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) switch pokemon.SeenType.ValueOrZero() { case "", SeenType_Cell, SeenType_NearbyStop: - pokemon.SeenType = null.StringFrom(SeenType_Wild) + pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) } pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs, true) pokemon.recomputeCpIfNeeded(ctx, db, weather) - pokemon.Username = null.StringFrom(username) - pokemon.CellId = null.IntFrom(cellId) + pokemon.SetUsername(null.StringFrom(username)) + pokemon.SetCellId(null.IntFrom(cellId)) } func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapPokemon *pogo.MapPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { @@ -605,7 +844,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP return } - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) pokemon.Id = mapPokemon.EncounterId @@ -616,10 +855,10 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP // Unrecognised pokestop return } - pokemon.PokestopId = null.StringFrom(pokestop.Id) - pokemon.Lat = pokestop.Lat - pokemon.Lon = pokestop.Lon - pokemon.SeenType = null.StringFrom(SeenType_LureWild) + pokemon.SetPokestopId(null.StringFrom(pokestop.Id)) + pokemon.SetLat(pokestop.Lat) + pokemon.SetLon(pokestop.Lon) + pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) if mapPokemon.PokemonDisplay != nil { pokemon.setPokemonDisplay(int16(mapPokemon.PokedexTypeId), mapPokemon.PokemonDisplay) @@ -630,34 +869,38 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP log.Warnf("[POKEMON] MapPokemonProto missing PokemonDisplay for %d", pokemon.Id) } if !pokemon.Username.Valid { - pokemon.Username = null.StringFrom(username) + pokemon.SetUsername(null.StringFrom(username)) } if mapPokemon.ExpirationTimeMs > 0 && !pokemon.ExpireTimestampVerified { - pokemon.ExpireTimestamp = null.IntFrom(mapPokemon.ExpirationTimeMs / 1000) - pokemon.ExpireTimestampVerified = true + pokemon.SetExpireTimestamp(null.IntFrom(mapPokemon.ExpirationTimeMs / 1000)) + pokemon.SetExpireTimestampVerified(true) // if we have cached an encounter for this pokemon, update the TTL. encounterCache.UpdateTTL(pokemon.Id, pokemon.remainingDuration(timestampMs/1000)) } else { - pokemon.ExpireTimestampVerified = false + pokemon.SetExpireTimestampVerified(false) } - pokemon.CellId = null.IntFrom(cellId) + pokemon.SetCellId(null.IntFrom(cellId)) } func (pokemon *Pokemon) calculateIv(a int64, d int64, s int64) { - pokemon.AtkIv = null.IntFrom(a) - pokemon.DefIv = null.IntFrom(d) - pokemon.StaIv = null.IntFrom(s) - pokemon.Iv = null.FloatFrom(float64(a+d+s) / .45) + if pokemon.AtkIv.ValueOrZero() != a || pokemon.DefIv.ValueOrZero() != d || pokemon.StaIv.ValueOrZero() != s || + !pokemon.AtkIv.Valid || !pokemon.DefIv.Valid || !pokemon.StaIv.Valid { + pokemon.AtkIv = null.IntFrom(a) + pokemon.DefIv = null.IntFrom(d) + pokemon.StaIv = null.IntFrom(s) + pokemon.Iv = null.FloatFrom(float64(a+d+s) / .45) + pokemon.dirty = true + } } func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, nearbyPokemon *pogo.NearbyPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) pokestopId := nearbyPokemon.FortId pokemon.setPokemonDisplay(int16(nearbyPokemon.PokedexNumber), nearbyPokemon.PokemonDisplay) pokemon.recomputeCpIfNeeded(ctx, db, weather) - pokemon.Username = null.StringFrom(username) + pokemon.SetUsername(null.StringFrom(username)) var lat, lon float64 overrideLatLon := pokemon.isNewRecord() @@ -675,8 +918,8 @@ func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, n // Unrecognised pokestop, rollback changes overrideLatLon = pokemon.isNewRecord() } else { - pokemon.SeenType = null.StringFrom(SeenType_NearbyStop) - pokemon.PokestopId = null.StringFrom(pokestopId) + pokemon.SetSeenType(null.StringFrom(SeenType_NearbyStop)) + pokemon.SetPokestopId(null.StringFrom(pokestopId)) lat, lon = pokestop.Lat, pokestop.Lon useCellLatLon = false } @@ -692,17 +935,18 @@ func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, n lat = s2cell.CapBound().RectBound().Center().Lat.Degrees() lon = s2cell.CapBound().RectBound().Center().Lng.Degrees() - pokemon.SeenType = null.StringFrom(SeenType_Cell) + pokemon.SetSeenType(null.StringFrom(SeenType_Cell)) } if overrideLatLon { - pokemon.Lat, pokemon.Lon = lat, lon + pokemon.SetLat(lat) + pokemon.SetLon(lon) } else { midpoint := s2.LatLngFromPoint(s2.Point{s2.PointFromLatLng(s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon)). Add(s2.PointFromLatLng(s2.LatLngFromDegrees(lat, lon)).Vector)}) - pokemon.Lat = midpoint.Lat.Degrees() - pokemon.Lon = midpoint.Lng.Degrees() + pokemon.SetLat(midpoint.Lat.Degrees()) + pokemon.SetLon(midpoint.Lng.Degrees()) } - pokemon.CellId = null.IntFrom(cellId) + pokemon.SetCellId(null.IntFrom(cellId)) pokemon.setUnknownTimestamp(timestampMs / 1000) } @@ -745,8 +989,8 @@ func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db if despawnOffset < 0 { despawnOffset += 3600 } - pokemon.ExpireTimestamp = null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset)) - pokemon.ExpireTimestampVerified = true + pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) + pokemon.SetExpireTimestampVerified(true) } else { pokemon.setUnknownTimestamp(timestampMs / 1000) } @@ -754,10 +998,10 @@ func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db func (pokemon *Pokemon) setUnknownTimestamp(now int64) { if !pokemon.ExpireTimestamp.Valid { - pokemon.ExpireTimestamp = null.IntFrom(now + 20*60) // should be configurable, add on 20min + pokemon.SetExpireTimestamp(null.IntFrom(now + 20*60)) // should be configurable, add on 20min } else { if pokemon.ExpireTimestamp.Int64 < now { - pokemon.ExpireTimestamp = null.IntFrom(now + 10*60) // should be configurable, add on 10min + pokemon.SetExpireTimestamp(null.IntFrom(now + 10*60)) // should be configurable, add on 10min } } } @@ -772,18 +1016,18 @@ func checkScans(old *grpc.PokemonScan, new *grpc.PokemonScan) error { func (pokemon *Pokemon) setDittoAttributes(mode string, isDitto bool, old, new *grpc.PokemonScan) { if isDitto { log.Debugf("[POKEMON] %d: %s Ditto found %s -> %s", pokemon.Id, mode, old, new) - pokemon.IsDitto = true - pokemon.DisplayPokemonId = null.IntFrom(int64(pokemon.PokemonId)) - pokemon.PokemonId = int16(pogo.HoloPokemonId_DITTO) + pokemon.SetIsDitto(true) + pokemon.SetDisplayPokemonId(null.IntFrom(int64(pokemon.PokemonId))) + pokemon.SetPokemonId(int16(pogo.HoloPokemonId_DITTO)) } else { log.Debugf("[POKEMON] %d: %s not Ditto found %s -> %s", pokemon.Id, mode, old, new) } } func (pokemon *Pokemon) resetDittoAttributes(mode string, old, aux, new *grpc.PokemonScan) (*grpc.PokemonScan, error) { log.Debugf("[POKEMON] %d: %s Ditto was reset %s (%s) -> %s", pokemon.Id, mode, old, aux, new) - pokemon.IsDitto = false - pokemon.DisplayPokemonId = null.NewInt(0, false) - pokemon.PokemonId = int16(pokemon.DisplayPokemonId.Int64) + pokemon.SetIsDitto(false) + pokemon.SetDisplayPokemonId(null.NewInt(0, false)) + pokemon.SetPokemonId(int16(pokemon.DisplayPokemonId.Int64)) return new, checkScans(old, new) } @@ -1085,6 +1329,9 @@ func (pokemon *Pokemon) detectDitto(scan *grpc.PokemonScan) (*grpc.PokemonScan, } func (pokemon *Pokemon) clearIv(cp bool) { + if pokemon.AtkIv.Valid || pokemon.DefIv.Valid || pokemon.StaIv.Valid || pokemon.Iv.Valid { + pokemon.dirty = true + } pokemon.AtkIv = null.NewInt(0, false) pokemon.DefIv = null.NewInt(0, false) pokemon.StaIv = null.NewInt(0, false) @@ -1092,25 +1339,25 @@ func (pokemon *Pokemon) clearIv(cp bool) { if cp { switch pokemon.SeenType.ValueOrZero() { case SeenType_LureEncounter: - pokemon.SeenType = null.StringFrom(SeenType_LureWild) + pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) case SeenType_Encounter: - pokemon.SeenType = null.StringFrom(SeenType_Wild) + pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) } - pokemon.Cp = null.NewInt(0, false) - pokemon.Pvp = null.NewString("", false) + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) } } // caller should setPokemonDisplay prior to calling this func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails, proto *pogo.PokemonProto, username string) { - pokemon.Username = null.StringFrom(username) - pokemon.Shiny = null.BoolFrom(proto.PokemonDisplay.Shiny) - pokemon.Cp = null.IntFrom(int64(proto.Cp)) - pokemon.Move1 = null.IntFrom(int64(proto.Move1)) - pokemon.Move2 = null.IntFrom(int64(proto.Move2)) - pokemon.Height = null.FloatFrom(float64(proto.HeightM)) - pokemon.Size = null.IntFrom(int64(proto.Size)) - pokemon.Weight = null.FloatFrom(float64(proto.WeightKg)) + pokemon.SetUsername(null.StringFrom(username)) + pokemon.SetShiny(null.BoolFrom(proto.PokemonDisplay.Shiny)) + pokemon.SetCp(null.IntFrom(int64(proto.Cp))) + pokemon.SetMove1(null.IntFrom(int64(proto.Move1))) + pokemon.SetMove2(null.IntFrom(int64(proto.Move2))) + pokemon.SetHeight(null.FloatFrom(float64(proto.HeightM))) + pokemon.SetSize(null.IntFrom(int64(proto.Size))) + pokemon.SetWeight(null.FloatFrom(float64(proto.WeightKg))) scan := grpc.PokemonScan{ Weather: int32(pokemon.Weather.Int64), @@ -1146,10 +1393,10 @@ func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails log.Errorf("[POKEMON] Unexpected %d: %s", pokemon.Id, err) } if caughtIv == nil { // this can only happen for a 0P Ditto - pokemon.Level = null.IntFrom(int64(scan.Level - 5)) + pokemon.SetLevel(null.IntFrom(int64(scan.Level - 5))) pokemon.clearIv(false) } else { - pokemon.Level = null.IntFrom(int64(caughtIv.Level)) + pokemon.SetLevel(null.IntFrom(int64(caughtIv.Level))) pokemon.calculateIv(int64(caughtIv.Attack), int64(caughtIv.Defense), int64(caughtIv.Stamina)) } if err == nil { @@ -1175,18 +1422,18 @@ func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails } func (pokemon *Pokemon) updatePokemonFromEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.EncounterOutProto, username string, timestampMs int64) { - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) pokemon.addWildPokemon(ctx, db, encounterData.Pokemon, timestampMs, false) // tappable encounter can also be available in seen as normal encounter once tapped if pokemon.isSeenFromTappable() { - pokemon.SeenType = null.StringFrom(SeenType_Encounter) + pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) } pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon.Pokemon, username) if pokemon.CellId.Valid == false { centerCoord := s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon) cellID := s2.CellIDFromLatLng(centerCoord).Parent(15) - pokemon.CellId = null.IntFrom(int64(cellID)) + pokemon.SetCellId(null.IntFrom(int64(cellID))) } } @@ -1195,37 +1442,37 @@ func (pokemon *Pokemon) isSeenFromTappable() bool { } func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.DiskEncounterOutProto, username string) { - pokemon.IsEvent = 0 + pokemon.SetIsEvent(0) pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) - pokemon.SeenType = null.StringFrom(SeenType_LureEncounter) + pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) } func (pokemon *Pokemon) updatePokemonFromTappableEncounterProto(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounterData *pogo.TappableEncounterProto, username string, timestampMs int64) { - pokemon.IsEvent = 0 - pokemon.Lat = request.LocationHintLat - pokemon.Lon = request.LocationHintLng + pokemon.SetIsEvent(0) + pokemon.SetLat(request.LocationHintLat) + pokemon.SetLon(request.LocationHintLng) if spawnpointId := request.GetLocation().GetSpawnpointId(); spawnpointId != "" { - pokemon.SeenType = null.StringFrom(SeenType_TappableEncounter) + pokemon.SetSeenType(null.StringFrom(SeenType_TappableEncounter)) spawnId, err := strconv.ParseInt(spawnpointId, 16, 64) if err != nil { panic(err) } - pokemon.SpawnId = null.IntFrom(spawnId) + pokemon.SetSpawnId(null.IntFrom(spawnId)) pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, false) } else if fortId := request.GetLocation().GetFortId(); fortId != "" { - pokemon.SeenType = null.StringFrom(SeenType_TappableLureEncounter) + pokemon.SetSeenType(null.StringFrom(SeenType_TappableLureEncounter)) - pokemon.PokestopId = null.StringFrom(fortId) + pokemon.SetPokestopId(null.StringFrom(fortId)) // we don't know any despawn times from lured/fort tappables - pokemon.ExpireTimestamp = null.IntFrom(int64(timestampMs)/1000 + int64(120)) - pokemon.ExpireTimestampVerified = false + pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) + pokemon.SetExpireTimestampVerified(false) } if !pokemon.Username.Valid { - pokemon.Username = null.StringFrom(username) + pokemon.SetUsername(null.StringFrom(username)) } pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) @@ -1248,29 +1495,29 @@ func (pokemon *Pokemon) setPokemonDisplay(pokemonId int16, display *pogo.Pokemon pokemon.Form.ValueOrZero(), pokemon.Costume.ValueOrZero(), pokemon.Gender.ValueOrZero(), pokemon.IsStrong.ValueOrZero(), pokemonId, display.Form, display.Costume, display.Gender, display.IsStrongPokemon) - pokemon.Weight = null.NewFloat(0, false) - pokemon.Height = null.NewFloat(0, false) - pokemon.Size = null.NewInt(0, false) - pokemon.Move1 = null.NewInt(0, false) - pokemon.Move2 = null.NewInt(0, false) - pokemon.Cp = null.NewInt(0, false) - pokemon.Shiny = null.NewBool(false, false) - pokemon.IsDitto = false - pokemon.DisplayPokemonId = null.NewInt(0, false) - pokemon.Pvp = null.NewString("", false) + pokemon.SetWeight(null.NewFloat(0, false)) + pokemon.SetHeight(null.NewFloat(0, false)) + pokemon.SetSize(null.NewInt(0, false)) + pokemon.SetMove1(null.NewInt(0, false)) + pokemon.SetMove2(null.NewInt(0, false)) + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetShiny(null.NewBool(false, false)) + pokemon.SetIsDitto(false) + pokemon.SetDisplayPokemonId(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) } } if pokemon.isNewRecord() || !pokemon.IsDitto { - pokemon.PokemonId = pokemonId + pokemon.SetPokemonId(pokemonId) } - pokemon.Gender = null.IntFrom(int64(display.Gender)) - pokemon.Form = null.IntFrom(int64(display.Form)) - pokemon.Costume = null.IntFrom(int64(display.Costume)) + pokemon.SetGender(null.IntFrom(int64(display.Gender))) + pokemon.SetForm(null.IntFrom(int64(display.Form))) + pokemon.SetCostume(null.IntFrom(int64(display.Costume))) if !pokemon.isNewRecord() { pokemon.repopulateIv(int64(display.WeatherBoostedCondition), display.IsStrongPokemon) } - pokemon.Weather = null.IntFrom(int64(display.WeatherBoostedCondition)) - pokemon.IsStrong = null.BoolFrom(display.IsStrongPokemon) + pokemon.SetWeather(null.IntFrom(int64(display.WeatherBoostedCondition))) + pokemon.SetIsStrong(null.BoolFrom(display.IsStrongPokemon)) } func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { @@ -1296,7 +1543,7 @@ func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { matchingScan, isBoostedMatches := pokemon.locateScan(isStrong, isBoosted) var oldAtk, oldDef, oldSta int64 if matchingScan == nil { - pokemon.Level = null.NewInt(0, false) + pokemon.SetLevel(null.NewInt(0, false)) pokemon.clearIv(true) } else { oldLevel := pokemon.Level.ValueOrZero() @@ -1309,29 +1556,30 @@ func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { oldDef = -1 oldSta = -1 } - pokemon.Level = null.IntFrom(int64(matchingScan.Level)) + newLevel := int64(matchingScan.Level) if isBoostedMatches || isStrong { // strong Pokemon IV is unaffected by weather pokemon.calculateIv(int64(matchingScan.Attack), int64(matchingScan.Defense), int64(matchingScan.Stamina)) switch pokemon.SeenType.ValueOrZero() { case SeenType_LureWild: - pokemon.SeenType = null.StringFrom(SeenType_LureEncounter) + pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) case SeenType_Wild: - pokemon.SeenType = null.StringFrom(SeenType_Encounter) + pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) } } else { pokemon.clearIv(true) } if !isBoostedMatches { if isBoosted { - pokemon.Level.Int64 += 5 + newLevel += 5 } else { - pokemon.Level.Int64 -= 5 + newLevel -= 5 } } - if pokemon.Level.Int64 != oldLevel || pokemon.AtkIv.Valid && + pokemon.SetLevel(null.IntFrom(newLevel)) + if newLevel != oldLevel || pokemon.AtkIv.Valid && (pokemon.AtkIv.Int64 != oldAtk || pokemon.DefIv.Int64 != oldDef || pokemon.StaIv.Int64 != oldSta) { - pokemon.Cp = null.NewInt(0, false) - pokemon.Pvp = null.NewString("", false) + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) } } } @@ -1387,7 +1635,7 @@ func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails float64(pokemon.Level.Int64)) } if err == nil { - pokemon.Cp = null.IntFrom(int64(cp)) + pokemon.SetCp(null.IntFrom(int64(cp))) } else { log.Warnf("Pokemon %d %d CP unset due to error %s", pokemon.Id, displayPokemon, err) } diff --git a/decoder/pokemonRtree.go b/decoder/pokemonRtree.go index 89e1edde..02037386 100644 --- a/decoder/pokemonRtree.go +++ b/decoder/pokemonRtree.go @@ -52,9 +52,9 @@ func initPokemonRtree() { // Set up OnEviction callbacks for each cache in the array for i := range pokemonCache { - pokemonCache[i].OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, Pokemon]) { - r := v.Value() - removePokemonFromTree(&r) + pokemonCache[i].OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, *Pokemon]) { + pokemon := v.Value() + removePokemonFromTree(pokemon.Id, pokemon.Lat, pokemon.Lon) // Rely on the pokemon pvp lookup caches to remove themselves rather than trying to synchronise }) } @@ -160,16 +160,15 @@ func addPokemonToTree(pokemon *Pokemon) { pokemonTreeMutex.Unlock() } -func removePokemonFromTree(pokemon *Pokemon) { - pokemonId := pokemon.Id +func removePokemonFromTree(pokemonId uint64, lat, lon float64) { pokemonTreeMutex.Lock() beforeLen := pokemonTree.Len() - pokemonTree.Delete([2]float64{pokemon.Lon, pokemon.Lat}, [2]float64{pokemon.Lon, pokemon.Lat}, pokemonId) + pokemonTree.Delete([2]float64{lon, lat}, [2]float64{lon, lat}, pokemonId) afterLen := pokemonTree.Len() pokemonTreeMutex.Unlock() pokemonLookupCache.Delete(pokemonId) if beforeLen != afterLen+1 { - log.Infof("PokemonRtree - UNEXPECTED removing %d, lat %f lon %f size %d->%d Map Len %d", pokemonId, pokemon.Lat, pokemon.Lon, beforeLen, afterLen, pokemonLookupCache.Size()) + log.Infof("PokemonRtree - UNEXPECTED removing %d, lat %f lon %f size %d->%d Map Len %d", pokemonId, lat, lon, beforeLen, afterLen, pokemonLookupCache.Size()) } } diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 7be500ae..face7098 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -71,57 +71,60 @@ type Pokestop struct { dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record - // Old values for webhook comparison (populated when loading from cache/DB) - oldQuestType null.Int `db:"-" json:"-"` - oldAlternativeQuestType null.Int `db:"-" json:"-"` - oldLureExpireTimestamp null.Int `db:"-" json:"-"` - oldLureId int16 `db:"-" json:"-"` - oldPowerUpEndTimestamp null.Int `db:"-" json:"-"` - oldName null.String `db:"-" json:"-"` - oldUrl null.String `db:"-" json:"-"` - oldDescription null.String `db:"-" json:"-"` - oldLat float64 `db:"-" json:"-"` - oldLon float64 `db:"-" json:"-"` - - //`id` varchar(35) NOT NULL, - //`lat` double(18,14) NOT NULL, - //`lon` double(18,14) NOT NULL, - //`name` varchar(128) DEFAULT NULL, - //`url` varchar(200) DEFAULT NULL, - //`lure_expire_timestamp` int unsigned DEFAULT NULL, - //`last_modified_timestamp` int unsigned DEFAULT NULL, - //`updated` int unsigned NOT NULL, - //`enabled` tinyint unsigned DEFAULT NULL, - //`quest_type` int unsigned DEFAULT NULL, - //`quest_timestamp` int unsigned DEFAULT NULL, - //`quest_target` smallint unsigned DEFAULT NULL, - //`quest_conditions` text, - //`quest_rewards` text, - //`quest_template` varchar(100) DEFAULT NULL, - //`quest_title` varchar(100) DEFAULT NULL, - //`quest_reward_type` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].type'),_utf8mb4'$[0]')) VIRTUAL, - //`quest_item_id` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.item_id'),_utf8mb4'$[0]')) VIRTUAL, - //`quest_reward_amount` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.amount'),_utf8mb4'$[0]')) VIRTUAL, - //`cell_id` bigint unsigned DEFAULT NULL, - //`deleted` tinyint unsigned NOT NULL DEFAULT '0', - //`lure_id` smallint DEFAULT '0', - //`first_seen_timestamp` int unsigned NOT NULL, - //`sponsor_id` smallint unsigned DEFAULT NULL, - //`partner_id` varchar(35) DEFAULT NULL, - //`quest_pokemon_id` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.pokemon_id'),_utf8mb4'$[0]')) VIRTUAL, - //`ar_scan_eligible` tinyint unsigned DEFAULT NULL, - //`power_up_level` smallint unsigned DEFAULT NULL, - //`power_up_points` int unsigned DEFAULT NULL, - //`power_up_end_timestamp` int unsigned DEFAULT NULL, - //`alternative_quest_type` int unsigned DEFAULT NULL, - //`alternative_quest_timestamp` int unsigned DEFAULT NULL, - //`alternative_quest_target` smallint unsigned DEFAULT NULL, - //`alternative_quest_conditions` text, - //`alternative_quest_rewards` text, - //`alternative_quest_template` varchar(100) DEFAULT NULL, - //`alternative_quest_title` varchar(100) DEFAULT NULL, - -} + oldValues PokestopOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// PokestopOldValues holds old field values for webhook comparison (populated when loading from cache/DB) +type PokestopOldValues struct { + QuestType null.Int + AlternativeQuestType null.Int + LureExpireTimestamp null.Int + LureId int16 + PowerUpEndTimestamp null.Int + Name null.String + Url null.String + Description null.String + Lat float64 + Lon float64 +} + +//`id` varchar(35) NOT NULL, +//`lat` double(18,14) NOT NULL, +//`lon` double(18,14) NOT NULL, +//`name` varchar(128) DEFAULT NULL, +//`url` varchar(200) DEFAULT NULL, +//`lure_expire_timestamp` int unsigned DEFAULT NULL, +//`last_modified_timestamp` int unsigned DEFAULT NULL, +//`updated` int unsigned NOT NULL, +//`enabled` tinyint unsigned DEFAULT NULL, +//`quest_type` int unsigned DEFAULT NULL, +//`quest_timestamp` int unsigned DEFAULT NULL, +//`quest_target` smallint unsigned DEFAULT NULL, +//`quest_conditions` text, +//`quest_rewards` text, +//`quest_template` varchar(100) DEFAULT NULL, +//`quest_title` varchar(100) DEFAULT NULL, +//`quest_reward_type` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].type'),_utf8mb4'$[0]')) VIRTUAL, +//`quest_item_id` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.item_id'),_utf8mb4'$[0]')) VIRTUAL, +//`quest_reward_amount` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.amount'),_utf8mb4'$[0]')) VIRTUAL, +//`cell_id` bigint unsigned DEFAULT NULL, +//`deleted` tinyint unsigned NOT NULL DEFAULT '0', +//`lure_id` smallint DEFAULT '0', +//`first_seen_timestamp` int unsigned NOT NULL, +//`sponsor_id` smallint unsigned DEFAULT NULL, +//`partner_id` varchar(35) DEFAULT NULL, +//`quest_pokemon_id` smallint unsigned GENERATED ALWAYS AS (json_extract(json_extract(`quest_rewards`,_utf8mb4'$[*].info.pokemon_id'),_utf8mb4'$[0]')) VIRTUAL, +//`ar_scan_eligible` tinyint unsigned DEFAULT NULL, +//`power_up_level` smallint unsigned DEFAULT NULL, +//`power_up_points` int unsigned DEFAULT NULL, +//`power_up_end_timestamp` int unsigned DEFAULT NULL, +//`alternative_quest_type` int unsigned DEFAULT NULL, +//`alternative_quest_timestamp` int unsigned DEFAULT NULL, +//`alternative_quest_target` smallint unsigned DEFAULT NULL, +//`alternative_quest_conditions` text, +//`alternative_quest_rewards` text, +//`alternative_quest_template` varchar(100) DEFAULT NULL, +//`alternative_quest_title` varchar(100) DEFAULT NULL, // IsDirty returns true if any field has been modified func (p *Pokestop) IsDirty() bool { @@ -141,16 +144,18 @@ func (p *Pokestop) IsNewRecord() bool { // snapshotOldValues saves current values for webhook comparison // Call this after loading from cache/DB but before modifications func (p *Pokestop) snapshotOldValues() { - p.oldQuestType = p.QuestType - p.oldAlternativeQuestType = p.AlternativeQuestType - p.oldLureExpireTimestamp = p.LureExpireTimestamp - p.oldLureId = p.LureId - p.oldPowerUpEndTimestamp = p.PowerUpEndTimestamp - p.oldName = p.Name - p.oldUrl = p.Url - p.oldDescription = p.Description - p.oldLat = p.Lat - p.oldLon = p.Lon + p.oldValues = PokestopOldValues{ + QuestType: p.QuestType, + AlternativeQuestType: p.AlternativeQuestType, + LureExpireTimestamp: p.LureExpireTimestamp, + LureId: p.LureId, + PowerUpEndTimestamp: p.PowerUpEndTimestamp, + Name: p.Name, + Url: p.Url, + Description: p.Description, + Lat: p.Lat, + Lon: p.Lon, + } } // --- Set methods with dirty tracking --- @@ -958,10 +963,10 @@ func createPokestopFortWebhooks(stop *Pokestop) { oldFort := &FortWebhook{ Type: POKESTOP.String(), Id: stop.Id, - Name: stop.oldName.Ptr(), - ImageUrl: stop.oldUrl.Ptr(), - Description: stop.oldDescription.Ptr(), - Location: Location{Latitude: stop.oldLat, Longitude: stop.oldLon}, + Name: stop.oldValues.Name.Ptr(), + ImageUrl: stop.oldValues.Url.Ptr(), + Description: stop.oldValues.Description.Ptr(), + Location: Location{Latitude: stop.oldValues.Lat, Longitude: stop.oldValues.Lon}, } CreateFortWebHooks(oldFort, fort, EDIT) } @@ -971,7 +976,7 @@ func createPokestopWebhooks(stop *Pokestop) { areas := MatchStatsGeofence(stop.Lat, stop.Lon) - if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldAlternativeQuestType) { + if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldValues.AlternativeQuestType) { questHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -997,7 +1002,7 @@ func createPokestopWebhooks(stop *Pokestop) { webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } - if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldQuestType) { + if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldValues.QuestType) { questHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, @@ -1022,7 +1027,7 @@ func createPokestopWebhooks(stop *Pokestop) { } webhooksSender.AddMessage(webhooks.Quest, questHook, areas) } - if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldLureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldPowerUpEndTimestamp)) { + if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldValues.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldValues.PowerUpEndTimestamp)) { pokestopHook := map[string]any{ "pokestop_id": stop.Id, "latitude": stop.Lat, diff --git a/decoder/stats.go b/decoder/stats.go index 86ea7956..ee7c1d36 100644 --- a/decoder/stats.go +++ b/decoder/stats.go @@ -241,7 +241,7 @@ func updateEncounterStats(pokemon *Pokemon) { } } -func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now int64) { +func updatePokemonStats(pokemon *Pokemon, areas []geo.AreaName, now int64) { if len(areas) == 0 { areas = []geo.AreaName{ { @@ -273,15 +273,12 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in populateEncounterCacheVal := func() { if encounterCacheVal == nil { - encounterCacheVal = encounterCache.GetOrCreate(new.Id) + encounterCacheVal = encounterCache.GetOrCreate(pokemon.Id) } } - currentSeenType := new.SeenType.ValueOrZero() - oldSeenType := "" - if old != nil { - oldSeenType = old.SeenType.ValueOrZero() - } + currentSeenType := pokemon.SeenType.ValueOrZero() + oldSeenType := pokemon.oldValues.SeenType.ValueOrZero() if currentSeenType != oldSeenType { if oldSeenType == "" || oldSeenType == SeenType_NearbyStop || oldSeenType == SeenType_Cell { @@ -291,7 +288,7 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in // transition to wild for the first time.. populateEncounterCacheVal() encounterCacheVal.FirstEncounter = 0 - encounterCacheVal.FirstWild = new.Updated.ValueOrZero() + encounterCacheVal.FirstWild = pokemon.Updated.ValueOrZero() // This will be put into the cache later. } @@ -305,7 +302,7 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in populateEncounterCacheVal() if encounterCacheVal.FirstEncounter == 0 { // This is first encounter - encounterCacheVal.FirstEncounter = new.Updated.ValueOrZero() + encounterCacheVal.FirstEncounter = pokemon.Updated.ValueOrZero() if encounterCacheVal.FirstWild > 0 { timeToEncounter = encounterCacheVal.FirstEncounter - encounterCacheVal.FirstWild @@ -313,8 +310,8 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in monsIvIncr = 1 - if new.ExpireTimestampVerified { - tth := new.ExpireTimestamp.ValueOrZero() - new.Updated.ValueOrZero() // relies on Updated being set + if pokemon.ExpireTimestampVerified { + tth := pokemon.ExpireTimestamp.ValueOrZero() - pokemon.Updated.ValueOrZero() // relies on Updated being set bucket = tth / (5 * 60) if bucket > 11 { bucket = 11 @@ -325,8 +322,8 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in unverifiedEncIncr = 1 } } else { - if new.ExpireTimestampVerified { - tth := new.ExpireTimestamp.ValueOrZero() - new.Updated.ValueOrZero() // relies on Updated being set + if pokemon.ExpireTimestampVerified { + tth := pokemon.ExpireTimestamp.ValueOrZero() - pokemon.Updated.ValueOrZero() // relies on Updated being set verifiedReEncounterIncr = 1 verifiedReEncSecTotalIncr = tth @@ -337,12 +334,12 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in // If we have a cache entry, it means we updated it. So now let's store it. if encounterCacheVal != nil { - encounterCache.Put(new.Id, encounterCacheVal, new.remainingDuration(now)) + encounterCache.Put(pokemon.Id, encounterCacheVal, pokemon.remainingDuration(now)) } if (currentSeenType == SeenType_Wild && oldSeenType == SeenType_Encounter) || (currentSeenType == SeenType_Encounter && oldSeenType == SeenType_Encounter && - new.PokemonId != old.PokemonId) { + pokemon.PokemonId != pokemon.oldValues.PokemonId) { // stats reset statsResetCountIncr = 1 } @@ -352,10 +349,10 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in var isHundo bool var isNundo bool - if new.Cp.Valid && new.AtkIv.Valid && new.DefIv.Valid && new.StaIv.Valid { - atk := new.AtkIv.ValueOrZero() - def := new.DefIv.ValueOrZero() - sta := new.StaIv.ValueOrZero() + if pokemon.Cp.Valid && pokemon.AtkIv.Valid && pokemon.DefIv.Valid && pokemon.StaIv.Valid { + atk := pokemon.AtkIv.ValueOrZero() + def := pokemon.DefIv.ValueOrZero() + sta := pokemon.StaIv.ValueOrZero() if atk == 15 && def == 15 && sta == 15 { isHundo = true } else if atk == 0 && def == 0 && sta == 0 { @@ -369,7 +366,7 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in // Count stats - if old == nil || old.Cp != new.Cp { // pokemon is new or CP has changed (encountered or re-encountered) + if pokemon.isNewRecord() || pokemon.oldValues.Cp != pokemon.Cp { // pokemon is new or CP has changed (encountered or re-encountered) if !locked { pokemonStatsLock.Lock() locked = true @@ -387,18 +384,18 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in pokemonCount[area] = countStats } - formId := int(new.Form.ValueOrZero()) - pf := pokemonForm{pokemonId: new.PokemonId, formId: formId} + formId := int(pokemon.Form.ValueOrZero()) + pf := pokemonForm{pokemonId: pokemon.PokemonId, formId: formId} - if old == nil || old.PokemonId != new.PokemonId { // pokemon is new or type has changed + if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId { // pokemon is new or type has changed countStats.count[pf]++ statsCollector.IncPokemonCountNew(fullAreaName) - if new.ExpireTimestampVerified { - statsCollector.UpdateVerifiedTtl(area, new.SeenType, new.ExpireTimestamp) + if pokemon.ExpireTimestampVerified { + statsCollector.UpdateVerifiedTtl(area, pokemon.SeenType, pokemon.ExpireTimestamp) } } - if new.Cp.Valid { + if pokemon.Cp.Valid { countStats.ivCount[pf]++ statsCollector.IncPokemonCountIv(fullAreaName) if isHundo { @@ -448,7 +445,7 @@ func updatePokemonStats(old *Pokemon, new *Pokemon, areas []geo.AreaName, now in } } -func updateRaidStats(old *Gym, new *Gym, areas []geo.AreaName) { +func updateRaidStats(gym *Gym, areas []geo.AreaName) { if len(areas) == 0 { areas = []geo.AreaName{{Parent: "unmatched", Name: "unmatched"}} } @@ -459,8 +456,8 @@ func updateRaidStats(old *Gym, new *Gym, areas []geo.AreaName) { for i := 0; i < len(areas); i++ { area := areas[i] - if new.RaidPokemonId.ValueOrZero() > 0 && - (old == nil || old.RaidPokemonId != new.RaidPokemonId || old.RaidEndTimestamp != new.RaidEndTimestamp) { + if gym.RaidPokemonId.ValueOrZero() > 0 && + (gym.newRecord || gym.oldValues.RaidPokemonId != gym.RaidPokemonId || gym.oldValues.RaidSpawnTimestamp != gym.RaidSpawnTimestamp) { if !locked { raidStatsLock.Lock() @@ -471,13 +468,13 @@ func updateRaidStats(old *Gym, new *Gym, areas []geo.AreaName) { raidCount[area] = make(map[int64]*areaRaidCountDetail) } countStats := raidCount[area] - raidLevel := new.RaidLevel.ValueOrZero() + raidLevel := gym.RaidLevel.ValueOrZero() if countStats[raidLevel] == nil { countStats[raidLevel] = &areaRaidCountDetail{count: make(map[pokemonForm]int)} } pf := pokemonForm{ - pokemonId: int16(new.RaidPokemonId.ValueOrZero()), - formId: int(new.RaidPokemonForm.ValueOrZero()), + pokemonId: int16(gym.RaidPokemonId.ValueOrZero()), + formId: int(gym.RaidPokemonForm.ValueOrZero()), } countStats[raidLevel].count[pf]++ } diff --git a/decoder/weather_iv.go b/decoder/weather_iv.go index 582e1760..8560c3ec 100644 --- a/decoder/weather_iv.go +++ b/decoder/weather_iv.go @@ -203,7 +203,7 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath pokemonLocked := 0 pokemonUpdated := 0 pokemonCpUpdated := 0 - var pokemon Pokemon + var pokemon *Pokemon pokemonTree2.Search([2]float64{cellLo.Lng.Degrees(), cellLo.Lat.Degrees()}, [2]float64{cellHi.Lng.Degrees(), cellHi.Lat.Degrees()}, func(min, max [2]float64, pokemonId uint64) bool { if !weatherCell.ContainsPoint(s2.PointFromLatLng(s2.LatLngFromDegrees(min[1], min[0]))) { return true @@ -237,7 +237,7 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath pokemon.recomputeCpIfNeeded(ctx, db, map[int64]pogo.GameplayWeatherProto_WeatherCondition{ weatherUpdate.S2CellId: pogo.GameplayWeatherProto_WeatherCondition(newWeather), }) - savePokemonRecordAsAtTime(ctx, db, &pokemon, false, toDB && pokemon.Cp.Valid, pokemon.Cp.Valid, timestamp) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, toDB && pokemon.Cp.Valid, pokemon.Cp.Valid, timestamp) pokemonUpdated++ if pokemon.Cp.Valid { pokemonCpUpdated++ From 80e1826311625d5c1fc5fd364dca8c58a58be65c Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 12:35:28 +0000 Subject: [PATCH 06/35] Update isNewRecord logic --- decoder/pokemon.go | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 482a5901..8c6c4d7f 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -77,8 +77,8 @@ type Pokemon struct { internal grpc.PokemonInternal - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - // Note: newRecord tracked via FirstSeenTimestamp == 0 (see isNewRecord method) + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` oldValues PokemonOldValues `db:"-" json:"-"` // Old values for webhook comparison and stats } @@ -424,7 +424,7 @@ func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId if pokemon != nil || err != nil { return pokemon, err } - pokemon = &Pokemon{Id: encounterId} + pokemon = &Pokemon{Id: encounterId, newRecord: true} if db.UsePokemonCache { setPokemonCache(encounterId, pokemon, ttlcache.DefaultTTL) } @@ -474,7 +474,7 @@ func hasChangesPokemon(old *Pokemon, new *Pokemon) bool { } func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Pokemon, isEncounter, writeDB, webhook bool, now int64) { - if !pokemon.isNewRecord() && !pokemon.IsDirty() { + if !pokemon.newRecord && !pokemon.IsDirty() { return } @@ -524,7 +524,14 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } - log.Debugf("Updating pokemon [%d] to %s", pokemon.Id, pokemon.SeenType.ValueOrZero()) + var oldSeenType string + if !pokemon.oldValues.SeenType.Valid { + oldSeenType = "n/a" + } else { + oldSeenType = pokemon.oldValues.SeenType.ValueOrZero() + } + + log.Debugf("Updating pokemon [%d] from %s->%s - newRecord: %t", pokemon.Id, oldSeenType, pokemon.SeenType.ValueOrZero(), pokemon.isNewRecord()) //log.Println(cmp.Diff(oldPokemon, pokemon)) if writeDB && !config.Config.PokemonMemoryOnly { @@ -568,7 +575,8 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po return } - _, _ = res, err + rows, rowsErr := res.RowsAffected() + log.Debugf("Inserting pokemon [%d] after insert res = %d %v", pokemon.Id, rows, rowsErr) } else { pvpUpdate := "" if changePvpField { @@ -642,6 +650,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } updatePokemonStats(pokemon, areas, now) + pokemon.newRecord = false // After saving, it's no longer a new record pokemon.ClearDirty() pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache @@ -778,7 +787,7 @@ func (pokemon *Pokemon) locateAllScans() (unboosted, boosted, strong *grpc.Pokem } func (pokemon *Pokemon) isNewRecord() bool { - return pokemon.FirstSeenTimestamp == 0 + return pokemon.newRecord } func (pokemon *Pokemon) remainingDuration(now int64) time.Duration { From 2b5b7fb14937b3464ee4b778f6e61c8c30706e3c Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 16:18:28 +0000 Subject: [PATCH 07/35] Additional types converted to new model --- decoder/gym.go | 2 +- decoder/incident.go | 233 +++++++++--- decoder/main.go | 76 ++-- decoder/player.go | 834 ++++++++++++++++++++++++++++++++---------- decoder/pokemon.go | 32 +- decoder/pokestop.go | 2 +- decoder/routes.go | 273 ++++++++++---- decoder/s2cell.go | 17 +- decoder/spawnpoint.go | 135 ++++--- decoder/station.go | 364 ++++++++++++++---- decoder/stats.go | 10 +- decoder/tappable.go | 164 ++++++--- decoder/weather.go | 219 ++++++++--- 13 files changed, 1782 insertions(+), 579 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index 9b101839..dba5fb34 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -1052,7 +1052,7 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { _, _ = res, err } - gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) + //gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) areas := MatchStatsGeofence(gym.Lat, gym.Lon) createGymWebhooks(gym, areas) createGymFortWebhooks(gym) diff --git a/decoder/incident.go b/decoder/incident.go index 7e5fbde5..dbc0b067 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -5,7 +5,6 @@ import ( "database/sql" "time" - "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" null "gopkg.in/guregu/null.v4" @@ -15,7 +14,7 @@ import ( ) // Incident struct. -// REMINDER! Keep hasChangesIncident updated after making changes +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Incident struct { Id string `db:"id"` PokestopId string `db:"pokestop_id"` @@ -32,6 +31,20 @@ type Incident struct { Slot2Form null.Int `db:"slot_2_form"` Slot3PokemonId null.Int `db:"slot_3_pokemon_id"` Slot3Form null.Int `db:"slot_3_form"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + oldValues IncidentOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// IncidentOldValues holds old field values for webhook comparison and stats +type IncidentOldValues struct { + StartTime int64 + ExpirationTime int64 + Character int16 + Confirmed bool + Slot1PokemonId null.Int } type webhookLineup struct { @@ -69,11 +82,139 @@ type IncidentWebhook struct { //-> `character` smallint unsigned NOT NULL, //-> `updated` int unsigned NOT NULL, +// IsDirty returns true if any field has been modified +func (incident *Incident) IsDirty() bool { + return incident.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (incident *Incident) ClearDirty() { + incident.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (incident *Incident) IsNewRecord() bool { + return incident.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (incident *Incident) snapshotOldValues() { + incident.oldValues = IncidentOldValues{ + StartTime: incident.StartTime, + ExpirationTime: incident.ExpirationTime, + Character: incident.Character, + Confirmed: incident.Confirmed, + Slot1PokemonId: incident.Slot1PokemonId, + } +} + +// --- Set methods with dirty tracking --- + +func (incident *Incident) SetId(v string) { + if incident.Id != v { + incident.Id = v + incident.dirty = true + } +} + +func (incident *Incident) SetPokestopId(v string) { + if incident.PokestopId != v { + incident.PokestopId = v + incident.dirty = true + } +} + +func (incident *Incident) SetStartTime(v int64) { + if incident.StartTime != v { + incident.StartTime = v + incident.dirty = true + } +} + +func (incident *Incident) SetExpirationTime(v int64) { + if incident.ExpirationTime != v { + incident.ExpirationTime = v + incident.dirty = true + } +} + +func (incident *Incident) SetDisplayType(v int16) { + if incident.DisplayType != v { + incident.DisplayType = v + incident.dirty = true + } +} + +func (incident *Incident) SetStyle(v int16) { + if incident.Style != v { + incident.Style = v + incident.dirty = true + } +} + +func (incident *Incident) SetCharacter(v int16) { + if incident.Character != v { + incident.Character = v + incident.dirty = true + } +} + +func (incident *Incident) SetConfirmed(v bool) { + if incident.Confirmed != v { + incident.Confirmed = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot1PokemonId(v null.Int) { + if incident.Slot1PokemonId != v { + incident.Slot1PokemonId = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot1Form(v null.Int) { + if incident.Slot1Form != v { + incident.Slot1Form = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot2PokemonId(v null.Int) { + if incident.Slot2PokemonId != v { + incident.Slot2PokemonId = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot2Form(v null.Int) { + if incident.Slot2Form != v { + incident.Slot2Form = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot3PokemonId(v null.Int) { + if incident.Slot3PokemonId != v { + incident.Slot3PokemonId = v + incident.dirty = true + } +} + +func (incident *Incident) SetSlot3Form(v null.Int) { + if incident.Slot3Form != v { + incident.Slot3Form = v + incident.dirty = true + } +} + func getIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, error) { inMemoryIncident := incidentCache.Get(incidentId) if inMemoryIncident != nil { incident := inMemoryIncident.Value() - return &incident, nil + incident.snapshotOldValues() + return incident, nil } incident := Incident{} @@ -90,44 +231,19 @@ func getIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string) return nil, err } - incidentCache.Set(incidentId, incident, ttlcache.DefaultTTL) + incident.snapshotOldValues() return &incident, nil } -// hasChangesIncident compares two Incident structs -func hasChangesIncident(old *Incident, new *Incident) bool { - return old.Id != new.Id || - old.PokestopId != new.PokestopId || - old.StartTime != new.StartTime || - old.ExpirationTime != new.ExpirationTime || - old.DisplayType != new.DisplayType || - old.Style != new.Style || - old.Character != new.Character || - old.Confirmed != new.Confirmed || - old.Updated != new.Updated || - old.Slot1PokemonId != new.Slot1PokemonId || - old.Slot1Form != new.Slot1Form || - old.Slot2PokemonId != new.Slot2PokemonId || - old.Slot2Form != new.Slot2Form || - old.Slot3PokemonId != new.Slot3PokemonId || - old.Slot3Form != new.Slot3Form - -} - func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident) { - oldIncident, _ := getIncidentRecord(ctx, db, incident.Id) - - if oldIncident != nil && !hasChangesIncident(oldIncident, incident) { + // Skip save if not dirty and not new + if !incident.IsDirty() && !incident.IsNewRecord() { return } - //log.Traceln(cmp.Diff(oldIncident, incident)) - incident.Updated = time.Now().Unix() - //log.Println(cmp.Diff(oldIncident, incident)) - - if oldIncident == nil { + if incident.IsNewRecord() { res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ "VALUES (:id, :pokestop_id, :start, :expiration, :display_type, :style, :character, :updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, :slot_2_form, :slot_3_pokemon_id, :slot_3_form)", incident) @@ -161,8 +277,7 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident _, _ = res, err } - incidentCache.Set(incident.Id, *incident, ttlcache.DefaultTTL) - createIncidentWebhooks(ctx, db, oldIncident, incident) + createIncidentWebhooks(ctx, db, incident) stop, _ := GetPokestopRecord(ctx, db, incident.PokestopId) if stop == nil { @@ -170,11 +285,18 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident } areas := MatchStatsGeofence(stop.Lat, stop.Lon) - updateIncidentStats(oldIncident, incident, areas) + updateIncidentStats(incident, areas) + + incident.ClearDirty() + incident.newRecord = false + //incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) } -func createIncidentWebhooks(ctx context.Context, db db.DbDetails, oldIncident *Incident, incident *Incident) { - if oldIncident == nil || (oldIncident.ExpirationTime != incident.ExpirationTime || oldIncident.Character != incident.Character || oldIncident.Confirmed != incident.Confirmed || oldIncident.Slot1PokemonId != incident.Slot1PokemonId) { +func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Incident) { + old := &incident.oldValues + isNew := incident.IsNewRecord() + + if isNew || (old.ExpirationTime != incident.ExpirationTime || old.Character != incident.Character || old.Confirmed != incident.Confirmed || old.Slot1PokemonId != incident.Slot1PokemonId) { stop, _ := GetPokestopRecord(ctx, db, incident.PokestopId) if stop == nil { stop = &Pokestop{} @@ -233,10 +355,10 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, oldIncident *I } func (incident *Incident) updateFromPokestopIncidentDisplay(pokestopDisplay *pogo.PokestopIncidentDisplayProto) { - incident.Id = pokestopDisplay.IncidentId - incident.StartTime = int64(pokestopDisplay.IncidentStartMs / 1000) - incident.ExpirationTime = int64(pokestopDisplay.IncidentExpirationMs / 1000) - incident.DisplayType = int16(pokestopDisplay.IncidentDisplayType) + incident.SetId(pokestopDisplay.IncidentId) + incident.SetStartTime(int64(pokestopDisplay.IncidentStartMs / 1000)) + incident.SetExpirationTime(int64(pokestopDisplay.IncidentExpirationMs / 1000)) + incident.SetDisplayType(int16(pokestopDisplay.IncidentDisplayType)) if (incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE)) && incident.Confirmed { log.Debugf("Incident has already been confirmed as a decoy: %s", incident.Id) return @@ -244,35 +366,36 @@ func (incident *Incident) updateFromPokestopIncidentDisplay(pokestopDisplay *pog characterDisplay := pokestopDisplay.GetCharacterDisplay() if characterDisplay != nil { // team := pokestopDisplay.Open - incident.Style = int16(characterDisplay.Style) - incident.Character = int16(characterDisplay.Character) + incident.SetStyle(int16(characterDisplay.Style)) + incident.SetCharacter(int16(characterDisplay.Character)) } else { - incident.Style, incident.Character = 0, 0 + incident.SetStyle(0) + incident.SetCharacter(0) } } func (incident *Incident) updateFromOpenInvasionCombatSessionOut(protoRes *pogo.OpenInvasionCombatSessionOutProto) { - incident.Slot1PokemonId = null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokedexId.Number()), true) - incident.Slot1Form = null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokemonDisplay.Form.Number()), true) + incident.SetSlot1PokemonId(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokedexId.Number()), true)) + incident.SetSlot1Form(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokemonDisplay.Form.Number()), true)) for i, pokemon := range protoRes.Combat.Opponent.ReservePokemon { if i == 0 { - incident.Slot2PokemonId = null.NewInt(int64(pokemon.PokedexId.Number()), true) - incident.Slot2Form = null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true) + incident.SetSlot2PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) + incident.SetSlot2Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) } else if i == 1 { - incident.Slot3PokemonId = null.NewInt(int64(pokemon.PokedexId.Number()), true) - incident.Slot3Form = null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true) + incident.SetSlot3PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) + incident.SetSlot3Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) } } - incident.Confirmed = true + incident.SetConfirmed(true) } func (incident *Incident) updateFromStartIncidentOut(proto *pogo.StartIncidentOutProto) { - incident.Character = int16(proto.GetIncident().GetStep()[0].GetPokestopDialogue().GetDialogueLine()[0].GetCharacter()) + incident.SetCharacter(int16(proto.GetIncident().GetStep()[0].GetPokestopDialogue().GetDialogueLine()[0].GetCharacter())) if incident.Character == int16(pogo.EnumWrapper_CHARACTER_GIOVANNI) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE) { - incident.Confirmed = true + incident.SetConfirmed(true) } - incident.StartTime = int64(proto.Incident.GetCompletionDisplay().GetIncidentStartMs() / 1000) - incident.ExpirationTime = int64(proto.Incident.GetCompletionDisplay().GetIncidentExpirationMs() / 1000) + incident.SetStartTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentStartMs() / 1000)) + incident.SetExpirationTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentExpirationMs() / 1000)) } diff --git a/decoder/main.go b/decoder/main.go index 9f3bdad5..e398e5e9 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -60,27 +60,27 @@ var webhooksSender webhooksSenderInterface var statsCollector stats_collector.StatsCollector var pokestopCache *ttlcache.Cache[string, *Pokestop] var gymCache *ttlcache.Cache[string, *Gym] -var stationCache *ttlcache.Cache[string, Station] -var tappableCache *ttlcache.Cache[uint64, Tappable] -var weatherCache *ttlcache.Cache[int64, Weather] +var stationCache *ttlcache.Cache[string, *Station] +var tappableCache *ttlcache.Cache[uint64, *Tappable] +var weatherCache *ttlcache.Cache[int64, *Weather] var weatherConsensusCache *ttlcache.Cache[int64, *WeatherConsensusState] -var s2CellCache *ttlcache.Cache[uint64, S2Cell] -var spawnpointCache *ttlcache.Cache[int64, Spawnpoint] +var s2CellCache *ttlcache.Cache[uint64, *S2Cell] +var spawnpointCache *ttlcache.Cache[int64, *Spawnpoint] var pokemonCache []*ttlcache.Cache[uint64, *Pokemon] -var incidentCache *ttlcache.Cache[string, Incident] -var playerCache *ttlcache.Cache[string, Player] -var routeCache *ttlcache.Cache[string, Route] +var incidentCache *ttlcache.Cache[string, *Incident] +var playerCache *ttlcache.Cache[string, *Player] +var routeCache *ttlcache.Cache[string, *Route] var diskEncounterCache *ttlcache.Cache[uint64, *pogo.DiskEncounterOutProto] var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto] -var gymStripedMutex = stripedmutex.New(128) -var pokestopStripedMutex = stripedmutex.New(128) -var stationStripedMutex = stripedmutex.New(128) +var gymStripedMutex = stripedmutex.New(1103) +var pokestopStripedMutex = stripedmutex.New(1103) +var stationStripedMutex = stripedmutex.New(1103) var tappableStripedMutex = intstripedmutex.New(563) -var incidentStripedMutex = stripedmutex.New(128) +var incidentStripedMutex = stripedmutex.New(157) var pokemonStripedMutex = intstripedmutex.New(1103) var weatherStripedMutex = intstripedmutex.New(157) -var routeStripedMutex = stripedmutex.New(128) +var routeStripedMutex = stripedmutex.New(157) var ProactiveIVSwitchSem chan bool @@ -128,18 +128,18 @@ func initDataCache() { ) go gymCache.Start() - stationCache = ttlcache.New[string, Station]( - ttlcache.WithTTL[string, Station](60 * time.Minute), + stationCache = ttlcache.New[string, *Station]( + ttlcache.WithTTL[string, *Station](60 * time.Minute), ) go stationCache.Start() - tappableCache = ttlcache.New[uint64, Tappable]( - ttlcache.WithTTL[uint64, Tappable](60 * time.Minute), + tappableCache = ttlcache.New[uint64, *Tappable]( + ttlcache.WithTTL[uint64, *Tappable](60 * time.Minute), ) go tappableCache.Start() - weatherCache = ttlcache.New[int64, Weather]( - ttlcache.WithTTL[int64, Weather](60 * time.Minute), + weatherCache = ttlcache.New[int64, *Weather]( + ttlcache.WithTTL[int64, *Weather](60 * time.Minute), ) go weatherCache.Start() @@ -148,13 +148,13 @@ func initDataCache() { ) go weatherConsensusCache.Start() - s2CellCache = ttlcache.New[uint64, S2Cell]( - ttlcache.WithTTL[uint64, S2Cell](60 * time.Minute), + s2CellCache = ttlcache.New[uint64, *S2Cell]( + ttlcache.WithTTL[uint64, *S2Cell](60 * time.Minute), ) go s2CellCache.Start() - spawnpointCache = ttlcache.New[int64, Spawnpoint]( - ttlcache.WithTTL[int64, Spawnpoint](60 * time.Minute), + spawnpointCache = ttlcache.New[int64, *Spawnpoint]( + ttlcache.WithTTL[int64, *Spawnpoint](60 * time.Minute), ) go spawnpointCache.Start() @@ -171,13 +171,13 @@ func initDataCache() { initPokemonRtree() initFortRtree() - incidentCache = ttlcache.New[string, Incident]( - ttlcache.WithTTL[string, Incident](60 * time.Minute), + incidentCache = ttlcache.New[string, *Incident]( + ttlcache.WithTTL[string, *Incident](60 * time.Minute), ) go incidentCache.Start() - playerCache = ttlcache.New[string, Player]( - ttlcache.WithTTL[string, Player](60 * time.Minute), + playerCache = ttlcache.New[string, *Player]( + ttlcache.WithTTL[string, *Player](60 * time.Minute), ) go playerCache.Start() @@ -193,8 +193,8 @@ func initDataCache() { ) go getMapFortsCache.Start() - routeCache = ttlcache.New[string, Route]( - ttlcache.WithTTL[string, Route](60 * time.Minute), + routeCache = ttlcache.New[string, *Route]( + ttlcache.WithTTL[string, *Route](60 * time.Minute), ) go routeCache.Start() } @@ -331,6 +331,7 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa if incident == nil { incident = &Incident{ PokestopId: fortId, + newRecord: true, } } incident.updateFromPokestopIncidentDisplay(incidentProto) @@ -385,7 +386,7 @@ func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters Sca continue } if station == nil { - station = &Station{} + station = &Station{newRecord: true} } station.updateFromStationProto(stationProto.Data, stationProto.Cell) saveStationRecord(ctx, db, station) @@ -440,8 +441,11 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca if err != nil { log.Printf("getOrCreatePokemonRecord: %s", err) } else { - pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) + updateTime := nearby.Timestamp / 1000 + if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { + pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) + } } pokemonMutex.Unlock() @@ -489,12 +493,12 @@ func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.Cl publishProto = weatherProto } if weather == nil { - weather = &Weather{} + weather = &Weather{newRecord: true} } weather.UpdatedMs = timestampMs - oldWeather := weather.updateWeatherFromClientWeatherProto(publishProto) + weather.updateWeatherFromClientWeatherProto(publishProto) saveWeatherRecord(ctx, db, weather) - if oldWeather != weather.GameplayCondition { + if weather.oldValues.GameplayCondition != weather.GameplayCondition { updates = append(updates, WeatherUpdate{ S2CellId: publishProto.S2CellId, NewWeather: int32(publishProto.GetGameplayWeather().GetGameplayCondition()), @@ -527,6 +531,7 @@ func UpdateIncidentLineup(ctx context.Context, db db.DbDetails, protoReq *pogo.O incident = &Incident{ Id: protoReq.IncidentLookup.IncidentId, PokestopId: protoReq.IncidentLookup.FortId, + newRecord: true, } } incident.updateFromOpenInvasionCombatSessionOut(protoRes) @@ -550,6 +555,7 @@ func ConfirmIncident(ctx context.Context, db db.DbDetails, proto *pogo.StartInci incident = &Incident{ Id: proto.Incident.IncidentId, PokestopId: proto.Incident.FortId, + newRecord: true, } } incident.updateFromStartIncidentOut(proto) diff --git a/decoder/player.go b/decoder/player.go index 6c359f1f..721f7d88 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -15,7 +15,7 @@ import ( ) // Player struct. Name is the primary key. -// REMINDER! Keep hasChangesPlayer updated after making changes +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Player struct { // Name is the primary key Name string `db:"name"` @@ -102,6 +102,535 @@ type Player struct { CaughtDragon null.Int `db:"caught_dragon"` CaughtDark null.Int `db:"caught_dark"` CaughtFairy null.Int `db:"caught_fairy"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record +} + +// IsDirty returns true if any field has been modified +func (p *Player) IsDirty() bool { + return p.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (p *Player) ClearDirty() { + p.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (p *Player) IsNewRecord() bool { + return p.newRecord +} + +// setFieldDirty marks the dirty flag. Used by reflection-based updates. +func (p *Player) setFieldDirty() { + p.dirty = true +} + +// --- Set methods with dirty tracking --- + +func (p *Player) SetFriendshipId(v null.String) { + if p.FriendshipId != v { + p.FriendshipId = v + p.dirty = true + } +} + +func (p *Player) SetFriendCode(v null.String) { + if p.FriendCode != v { + p.FriendCode = v + p.dirty = true + } +} + +func (p *Player) SetTeam(v null.Int) { + if p.Team != v { + p.Team = v + p.dirty = true + } +} + +func (p *Player) SetLevel(v null.Int) { + if p.Level != v { + p.Level = v + p.dirty = true + } +} + +func (p *Player) SetXp(v null.Int) { + if p.Xp != v { + p.Xp = v + p.dirty = true + } +} + +func (p *Player) SetBattlesWon(v null.Int) { + if p.BattlesWon != v { + p.BattlesWon = v + p.dirty = true + } +} + +func (p *Player) SetKmWalked(v null.Float) { + if !nullFloatAlmostEqual(p.KmWalked, v, 0.001) { + p.KmWalked = v + p.dirty = true + } +} + +func (p *Player) SetCaughtPokemon(v null.Int) { + if p.CaughtPokemon != v { + p.CaughtPokemon = v + p.dirty = true + } +} + +func (p *Player) SetGblRank(v null.Int) { + if p.GblRank != v { + p.GblRank = v + p.dirty = true + } +} + +func (p *Player) SetGblRating(v null.Int) { + if p.GblRating != v { + p.GblRating = v + p.dirty = true + } +} + +func (p *Player) SetEventBadges(v null.String) { + if p.EventBadges != v { + p.EventBadges = v + p.dirty = true + } +} + +func (p *Player) SetStopsSpun(v null.Int) { + if p.StopsSpun != v { + p.StopsSpun = v + p.dirty = true + } +} +func (p *Player) SetEvolved(v null.Int) { + if p.Evolved != v { + p.Evolved = v + p.dirty = true + } +} +func (p *Player) SetHatched(v null.Int) { + if p.Hatched != v { + p.Hatched = v + p.dirty = true + } +} +func (p *Player) SetQuests(v null.Int) { + if p.Quests != v { + p.Quests = v + p.dirty = true + } +} +func (p *Player) SetTrades(v null.Int) { + if p.Trades != v { + p.Trades = v + p.dirty = true + } +} +func (p *Player) SetPhotobombs(v null.Int) { + if p.Photobombs != v { + p.Photobombs = v + p.dirty = true + } +} +func (p *Player) SetPurified(v null.Int) { + if p.Purified != v { + p.Purified = v + p.dirty = true + } +} +func (p *Player) SetGruntsDefeated(v null.Int) { + if p.GruntsDefeated != v { + p.GruntsDefeated = v + p.dirty = true + } +} +func (p *Player) SetGymBattlesWon(v null.Int) { + if p.GymBattlesWon != v { + p.GymBattlesWon = v + p.dirty = true + } +} +func (p *Player) SetNormalRaidsWon(v null.Int) { + if p.NormalRaidsWon != v { + p.NormalRaidsWon = v + p.dirty = true + } +} +func (p *Player) SetLegendaryRaidsWon(v null.Int) { + if p.LegendaryRaidsWon != v { + p.LegendaryRaidsWon = v + p.dirty = true + } +} +func (p *Player) SetTrainingsWon(v null.Int) { + if p.TrainingsWon != v { + p.TrainingsWon = v + p.dirty = true + } +} +func (p *Player) SetBerriesFed(v null.Int) { + if p.BerriesFed != v { + p.BerriesFed = v + p.dirty = true + } +} +func (p *Player) SetHoursDefended(v null.Int) { + if p.HoursDefended != v { + p.HoursDefended = v + p.dirty = true + } +} +func (p *Player) SetBestFriends(v null.Int) { + if p.BestFriends != v { + p.BestFriends = v + p.dirty = true + } +} +func (p *Player) SetBestBuddies(v null.Int) { + if p.BestBuddies != v { + p.BestBuddies = v + p.dirty = true + } +} +func (p *Player) SetGiovanniDefeated(v null.Int) { + if p.GiovanniDefeated != v { + p.GiovanniDefeated = v + p.dirty = true + } +} +func (p *Player) SetMegaEvos(v null.Int) { + if p.MegaEvos != v { + p.MegaEvos = v + p.dirty = true + } +} +func (p *Player) SetCollectionsDone(v null.Int) { + if p.CollectionsDone != v { + p.CollectionsDone = v + p.dirty = true + } +} +func (p *Player) SetUniqueStopsSpun(v null.Int) { + if p.UniqueStopsSpun != v { + p.UniqueStopsSpun = v + p.dirty = true + } +} +func (p *Player) SetUniqueMegaEvos(v null.Int) { + if p.UniqueMegaEvos != v { + p.UniqueMegaEvos = v + p.dirty = true + } +} +func (p *Player) SetUniqueRaidBosses(v null.Int) { + if p.UniqueRaidBosses != v { + p.UniqueRaidBosses = v + p.dirty = true + } +} +func (p *Player) SetUniqueUnown(v null.Int) { + if p.UniqueUnown != v { + p.UniqueUnown = v + p.dirty = true + } +} +func (p *Player) SetSevenDayStreaks(v null.Int) { + if p.SevenDayStreaks != v { + p.SevenDayStreaks = v + p.dirty = true + } +} +func (p *Player) SetTradeKm(v null.Int) { + if p.TradeKm != v { + p.TradeKm = v + p.dirty = true + } +} +func (p *Player) SetRaidsWithFriends(v null.Int) { + if p.RaidsWithFriends != v { + p.RaidsWithFriends = v + p.dirty = true + } +} +func (p *Player) SetCaughtAtLure(v null.Int) { + if p.CaughtAtLure != v { + p.CaughtAtLure = v + p.dirty = true + } +} +func (p *Player) SetWayfarerAgreements(v null.Int) { + if p.WayfarerAgreements != v { + p.WayfarerAgreements = v + p.dirty = true + } +} +func (p *Player) SetTrainersReferred(v null.Int) { + if p.TrainersReferred != v { + p.TrainersReferred = v + p.dirty = true + } +} +func (p *Player) SetRaidAchievements(v null.Int) { + if p.RaidAchievements != v { + p.RaidAchievements = v + p.dirty = true + } +} +func (p *Player) SetXlKarps(v null.Int) { + if p.XlKarps != v { + p.XlKarps = v + p.dirty = true + } +} +func (p *Player) SetXsRats(v null.Int) { + if p.XsRats != v { + p.XsRats = v + p.dirty = true + } +} +func (p *Player) SetPikachuCaught(v null.Int) { + if p.PikachuCaught != v { + p.PikachuCaught = v + p.dirty = true + } +} +func (p *Player) SetLeagueGreatWon(v null.Int) { + if p.LeagueGreatWon != v { + p.LeagueGreatWon = v + p.dirty = true + } +} +func (p *Player) SetLeagueUltraWon(v null.Int) { + if p.LeagueUltraWon != v { + p.LeagueUltraWon = v + p.dirty = true + } +} +func (p *Player) SetLeagueMasterWon(v null.Int) { + if p.LeagueMasterWon != v { + p.LeagueMasterWon = v + p.dirty = true + } +} +func (p *Player) SetTinyPokemonCaught(v null.Int) { + if p.TinyPokemonCaught != v { + p.TinyPokemonCaught = v + p.dirty = true + } +} +func (p *Player) SetJumboPokemonCaught(v null.Int) { + if p.JumboPokemonCaught != v { + p.JumboPokemonCaught = v + p.dirty = true + } +} +func (p *Player) SetVivillon(v null.Int) { + if p.Vivillon != v { + p.Vivillon = v + p.dirty = true + } +} +func (p *Player) SetMaxSizeFirstPlace(v null.Int) { + if p.MaxSizeFirstPlace != v { + p.MaxSizeFirstPlace = v + p.dirty = true + } +} +func (p *Player) SetTotalRoutePlay(v null.Int) { + if p.TotalRoutePlay != v { + p.TotalRoutePlay = v + p.dirty = true + } +} +func (p *Player) SetPartiesCompleted(v null.Int) { + if p.PartiesCompleted != v { + p.PartiesCompleted = v + p.dirty = true + } +} +func (p *Player) SetEventCheckIns(v null.Int) { + if p.EventCheckIns != v { + p.EventCheckIns = v + p.dirty = true + } +} +func (p *Player) SetDexGen1(v null.Int) { + if p.DexGen1 != v { + p.DexGen1 = v + p.dirty = true + } +} +func (p *Player) SetDexGen2(v null.Int) { + if p.DexGen2 != v { + p.DexGen2 = v + p.dirty = true + } +} +func (p *Player) SetDexGen3(v null.Int) { + if p.DexGen3 != v { + p.DexGen3 = v + p.dirty = true + } +} +func (p *Player) SetDexGen4(v null.Int) { + if p.DexGen4 != v { + p.DexGen4 = v + p.dirty = true + } +} +func (p *Player) SetDexGen5(v null.Int) { + if p.DexGen5 != v { + p.DexGen5 = v + p.dirty = true + } +} +func (p *Player) SetDexGen6(v null.Int) { + if p.DexGen6 != v { + p.DexGen6 = v + p.dirty = true + } +} +func (p *Player) SetDexGen7(v null.Int) { + if p.DexGen7 != v { + p.DexGen7 = v + p.dirty = true + } +} +func (p *Player) SetDexGen8(v null.Int) { + if p.DexGen8 != v { + p.DexGen8 = v + p.dirty = true + } +} +func (p *Player) SetDexGen8A(v null.Int) { + if p.DexGen8A != v { + p.DexGen8A = v + p.dirty = true + } +} +func (p *Player) SetDexGen9(v null.Int) { + if p.DexGen9 != v { + p.DexGen9 = v + p.dirty = true + } +} +func (p *Player) SetCaughtNormal(v null.Int) { + if p.CaughtNormal != v { + p.CaughtNormal = v + p.dirty = true + } +} +func (p *Player) SetCaughtFighting(v null.Int) { + if p.CaughtFighting != v { + p.CaughtFighting = v + p.dirty = true + } +} +func (p *Player) SetCaughtFlying(v null.Int) { + if p.CaughtFlying != v { + p.CaughtFlying = v + p.dirty = true + } +} +func (p *Player) SetCaughtPoison(v null.Int) { + if p.CaughtPoison != v { + p.CaughtPoison = v + p.dirty = true + } +} +func (p *Player) SetCaughtGround(v null.Int) { + if p.CaughtGround != v { + p.CaughtGround = v + p.dirty = true + } +} +func (p *Player) SetCaughtRock(v null.Int) { + if p.CaughtRock != v { + p.CaughtRock = v + p.dirty = true + } +} +func (p *Player) SetCaughtBug(v null.Int) { + if p.CaughtBug != v { + p.CaughtBug = v + p.dirty = true + } +} +func (p *Player) SetCaughtGhost(v null.Int) { + if p.CaughtGhost != v { + p.CaughtGhost = v + p.dirty = true + } +} +func (p *Player) SetCaughtSteel(v null.Int) { + if p.CaughtSteel != v { + p.CaughtSteel = v + p.dirty = true + } +} +func (p *Player) SetCaughtFire(v null.Int) { + if p.CaughtFire != v { + p.CaughtFire = v + p.dirty = true + } +} +func (p *Player) SetCaughtWater(v null.Int) { + if p.CaughtWater != v { + p.CaughtWater = v + p.dirty = true + } +} +func (p *Player) SetCaughtGrass(v null.Int) { + if p.CaughtGrass != v { + p.CaughtGrass = v + p.dirty = true + } +} +func (p *Player) SetCaughtElectric(v null.Int) { + if p.CaughtElectric != v { + p.CaughtElectric = v + p.dirty = true + } +} +func (p *Player) SetCaughtPsychic(v null.Int) { + if p.CaughtPsychic != v { + p.CaughtPsychic = v + p.dirty = true + } +} +func (p *Player) SetCaughtIce(v null.Int) { + if p.CaughtIce != v { + p.CaughtIce = v + p.dirty = true + } +} +func (p *Player) SetCaughtDragon(v null.Int) { + if p.CaughtDragon != v { + p.CaughtDragon = v + p.dirty = true + } +} +func (p *Player) SetCaughtDark(v null.Int) { + if p.CaughtDark != v { + p.CaughtDark = v + p.dirty = true + } +} +func (p *Player) SetCaughtFairy(v null.Int) { + if p.CaughtFairy != v { + p.CaughtFairy = v + p.dirty = true + } } var badgeTypeToPlayerKey = map[pogo.HoloBadgeType]string{ @@ -194,7 +723,7 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo inMemoryPlayer := playerCache.Get(name) if inMemoryPlayer != nil { player := inMemoryPlayer.Value() - return &player, nil + return player, nil } player := Player{} @@ -202,7 +731,7 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo ` SELECT * FROM player - WHERE player.name = ? + WHERE player.name = ? `, name, ) @@ -213,7 +742,7 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo ` SELECT * FROM player - WHERE player.friendship_id = ? + WHERE player.friendship_id = ? `, friendshipId, ) @@ -223,7 +752,7 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo ` SELECT * FROM player - WHERE player.friend_code = ? + WHERE player.friend_code = ? `, friendCode, ) @@ -243,111 +772,19 @@ func getPlayerRecord(db db.DbDetails, name string, friendshipId string, friendCo return nil, err } - playerCache.Set(name, player, ttlcache.DefaultTTL) + playerCache.Set(name, &player, ttlcache.DefaultTTL) return &player, nil } -// hasChangesPlayer compares two Player structs -// Float tolerance: KmWalked = 0.001 -func hasChangesPlayer(old *Player, new *Player) bool { - return old.Name != new.Name || - old.FriendshipId != new.FriendshipId || - old.LastSeen != new.LastSeen || - old.FriendCode != new.FriendCode || - old.Team != new.Team || - old.Level != new.Level || - old.Xp != new.Xp || - old.BattlesWon != new.BattlesWon || - old.CaughtPokemon != new.CaughtPokemon || - old.GblRank != new.GblRank || - old.GblRating != new.GblRating || - old.EventBadges != new.EventBadges || - old.StopsSpun != new.StopsSpun || - old.Evolved != new.Evolved || - old.Hatched != new.Hatched || - old.Quests != new.Quests || - old.Trades != new.Trades || - old.Photobombs != new.Photobombs || - old.Purified != new.Purified || - old.GruntsDefeated != new.GruntsDefeated || - old.GymBattlesWon != new.GymBattlesWon || - old.NormalRaidsWon != new.NormalRaidsWon || - old.LegendaryRaidsWon != new.LegendaryRaidsWon || - old.TrainingsWon != new.TrainingsWon || - old.BerriesFed != new.BerriesFed || - old.HoursDefended != new.HoursDefended || - old.BestFriends != new.BestFriends || - old.BestBuddies != new.BestBuddies || - old.GiovanniDefeated != new.GiovanniDefeated || - old.MegaEvos != new.MegaEvos || - old.CollectionsDone != new.CollectionsDone || - old.UniqueStopsSpun != new.UniqueStopsSpun || - old.UniqueMegaEvos != new.UniqueMegaEvos || - old.UniqueRaidBosses != new.UniqueRaidBosses || - old.UniqueUnown != new.UniqueUnown || - old.SevenDayStreaks != new.SevenDayStreaks || - old.TradeKm != new.TradeKm || - old.RaidsWithFriends != new.RaidsWithFriends || - old.CaughtAtLure != new.CaughtAtLure || - old.WayfarerAgreements != new.WayfarerAgreements || - old.TrainersReferred != new.TrainersReferred || - old.RaidAchievements != new.RaidAchievements || - old.XlKarps != new.XlKarps || - old.XsRats != new.XsRats || - old.PikachuCaught != new.PikachuCaught || - old.LeagueGreatWon != new.LeagueGreatWon || - old.LeagueUltraWon != new.LeagueUltraWon || - old.LeagueMasterWon != new.LeagueMasterWon || - old.TinyPokemonCaught != new.TinyPokemonCaught || - old.JumboPokemonCaught != new.JumboPokemonCaught || - old.Vivillon != new.Vivillon || - old.MaxSizeFirstPlace != new.MaxSizeFirstPlace || - old.TotalRoutePlay != new.TotalRoutePlay || - old.PartiesCompleted != new.PartiesCompleted || - old.EventCheckIns != new.EventCheckIns || - old.DexGen1 != new.DexGen1 || - old.DexGen2 != new.DexGen2 || - old.DexGen3 != new.DexGen3 || - old.DexGen4 != new.DexGen4 || - old.DexGen5 != new.DexGen5 || - old.DexGen6 != new.DexGen6 || - old.DexGen7 != new.DexGen7 || - old.DexGen8 != new.DexGen8 || - old.DexGen8A != new.DexGen8A || - old.DexGen9 != new.DexGen9 || - old.CaughtNormal != new.CaughtNormal || - old.CaughtFighting != new.CaughtFighting || - old.CaughtFlying != new.CaughtFlying || - old.CaughtPoison != new.CaughtPoison || - old.CaughtGround != new.CaughtGround || - old.CaughtRock != new.CaughtRock || - old.CaughtBug != new.CaughtBug || - old.CaughtGhost != new.CaughtGhost || - old.CaughtSteel != new.CaughtSteel || - old.CaughtFire != new.CaughtFire || - old.CaughtWater != new.CaughtWater || - old.CaughtGrass != new.CaughtGrass || - old.CaughtElectric != new.CaughtElectric || - old.CaughtPsychic != new.CaughtPsychic || - old.CaughtIce != new.CaughtIce || - old.CaughtDragon != new.CaughtDragon || - old.CaughtDark != new.CaughtDark || - old.CaughtFairy != new.CaughtFairy || - !nullFloatAlmostEqual(old.KmWalked, new.KmWalked, 0.001) -} - func savePlayerRecord(db db.DbDetails, player *Player) { - oldPlayer, _ := getPlayerRecord(db, player.Name, player.FriendshipId.String, player.FriendCode.String) - - if oldPlayer != nil && !hasChangesPlayer(oldPlayer, player) { + // Skip save if not dirty and not new + if !player.IsDirty() && !player.IsNewRecord() { return } - //log.Traceln(cmp.Diff(oldPlayer, player, transformNullFloats, ignoreApproxFloats)) - player.LastSeen = time.Now().Unix() - if oldPlayer == nil { + if player.IsNewRecord() { _, err := db.GeneralDb.NamedExec( ` INSERT INTO player (name, friendship_id, friend_code, last_seen, team, level, xp, battles_won, km_walked, caught_pokemon, gbl_rank, gbl_rating, @@ -384,88 +821,88 @@ func savePlayerRecord(db db.DbDetails, player *Player) { } else { _, err := db.GeneralDb.NamedExec( `UPDATE player SET - friendship_id = :friendship_id, - last_seen = :last_seen, - team = :team, - level = :level, - xp = :xp, - battles_won = :battles_won, - km_walked = :km_walked, - caught_pokemon = :caught_pokemon, - gbl_rank = :gbl_rank, - gbl_rating = :gbl_rating, - event_badges = :event_badges, - stops_spun = :stops_spun, - evolved = :evolved, - hatched = :hatched, - quests = :quests, - trades = :trades, - photobombs = :photobombs, - purified = :purified, - grunts_defeated = :grunts_defeated, - gym_battles_won = :gym_battles_won, - normal_raids_won = :normal_raids_won, - legendary_raids_won = :legendary_raids_won, - trainings_won = :trainings_won, - berries_fed = :berries_fed, - hours_defended = :hours_defended, - best_friends = :best_friends, - best_buddies = :best_buddies, - giovanni_defeated = :giovanni_defeated, - mega_evos = :mega_evos, - collections_done = :collections_done, - unique_stops_spun = :unique_stops_spun, - unique_mega_evos = :unique_mega_evos, - unique_raid_bosses = :unique_raid_bosses, - unique_unown = :unique_unown, - seven_day_streaks = :seven_day_streaks, - trade_km = :trade_km, - raids_with_friends = :raids_with_friends, - caught_at_lure = :caught_at_lure, - wayfarer_agreements = :wayfarer_agreements, - trainers_referred = :trainers_referred, - raid_achievements = :raid_achievements, - xl_karps = :xl_karps, - xs_rats = :xs_rats, - pikachu_caught = :pikachu_caught, - league_great_won = :league_great_won, - league_ultra_won = :league_ultra_won, - league_master_won = :league_master_won, - tiny_pokemon_caught = :tiny_pokemon_caught, - jumbo_pokemon_caught = :jumbo_pokemon_caught, - vivillon = :vivillon, + friendship_id = :friendship_id, + last_seen = :last_seen, + team = :team, + level = :level, + xp = :xp, + battles_won = :battles_won, + km_walked = :km_walked, + caught_pokemon = :caught_pokemon, + gbl_rank = :gbl_rank, + gbl_rating = :gbl_rating, + event_badges = :event_badges, + stops_spun = :stops_spun, + evolved = :evolved, + hatched = :hatched, + quests = :quests, + trades = :trades, + photobombs = :photobombs, + purified = :purified, + grunts_defeated = :grunts_defeated, + gym_battles_won = :gym_battles_won, + normal_raids_won = :normal_raids_won, + legendary_raids_won = :legendary_raids_won, + trainings_won = :trainings_won, + berries_fed = :berries_fed, + hours_defended = :hours_defended, + best_friends = :best_friends, + best_buddies = :best_buddies, + giovanni_defeated = :giovanni_defeated, + mega_evos = :mega_evos, + collections_done = :collections_done, + unique_stops_spun = :unique_stops_spun, + unique_mega_evos = :unique_mega_evos, + unique_raid_bosses = :unique_raid_bosses, + unique_unown = :unique_unown, + seven_day_streaks = :seven_day_streaks, + trade_km = :trade_km, + raids_with_friends = :raids_with_friends, + caught_at_lure = :caught_at_lure, + wayfarer_agreements = :wayfarer_agreements, + trainers_referred = :trainers_referred, + raid_achievements = :raid_achievements, + xl_karps = :xl_karps, + xs_rats = :xs_rats, + pikachu_caught = :pikachu_caught, + league_great_won = :league_great_won, + league_ultra_won = :league_ultra_won, + league_master_won = :league_master_won, + tiny_pokemon_caught = :tiny_pokemon_caught, + jumbo_pokemon_caught = :jumbo_pokemon_caught, + vivillon = :vivillon, showcase_max_size_first_place = :showcase_max_size_first_place, total_route_play = :total_route_play, parties_completed = :parties_completed, - event_check_ins = :event_check_ins, - dex_gen1 = :dex_gen1, - dex_gen2 = :dex_gen2, - dex_gen3 = :dex_gen3, - dex_gen4 = :dex_gen4, - dex_gen5 = :dex_gen5, - dex_gen6 = :dex_gen6, - dex_gen7 = :dex_gen7, - dex_gen8 = :dex_gen8, - dex_gen8a = :dex_gen8a, + event_check_ins = :event_check_ins, + dex_gen1 = :dex_gen1, + dex_gen2 = :dex_gen2, + dex_gen3 = :dex_gen3, + dex_gen4 = :dex_gen4, + dex_gen5 = :dex_gen5, + dex_gen6 = :dex_gen6, + dex_gen7 = :dex_gen7, + dex_gen8 = :dex_gen8, + dex_gen8a = :dex_gen8a, dex_gen9 = :dex_gen9, - caught_normal = :caught_normal, - caught_fighting = :caught_fighting, - caught_flying = :caught_flying, - caught_poison = :caught_poison, - caught_ground = :caught_ground, - caught_rock = :caught_rock, - caught_bug = :caught_bug, - caught_ghost = :caught_ghost, - caught_steel = :caught_steel, - caught_fire = :caught_fire, - caught_water = :caught_water, - caught_grass = :caught_grass, - caught_electric = :caught_electric, - caught_psychic = :caught_psychic, - caught_ice = :caught_ice, - caught_dragon = :caught_dragon, - caught_dark = :caught_dark, - caught_fairy = :caught_fairy + caught_normal = :caught_normal, + caught_fighting = :caught_fighting, + caught_flying = :caught_flying, + caught_poison = :caught_poison, + caught_ground = :caught_ground, + caught_rock = :caught_rock, + caught_bug = :caught_bug, + caught_ghost = :caught_ghost, + caught_steel = :caught_steel, + caught_fire = :caught_fire, + caught_water = :caught_water, + caught_grass = :caught_grass, + caught_electric = :caught_electric, + caught_psychic = :caught_psychic, + caught_ice = :caught_ice, + caught_dragon = :caught_dragon, + caught_dark = :caught_dark, + caught_fairy = :caught_fairy WHERE name = :name`, player, ) @@ -476,19 +913,21 @@ func savePlayerRecord(db db.DbDetails, player *Player) { } } - playerCache.Set(player.Name, *player, ttlcache.DefaultTTL) + player.ClearDirty() + player.newRecord = false + //playerCache.Set(player.Name, player, ttlcache.DefaultTTL) } func (player *Player) updateFromPublicProfile(publicProfile *pogo.PlayerPublicProfileProto) { - player.Name = publicProfile.GetName() - player.Team = null.IntFrom(int64(publicProfile.GetTeam())) - player.Level = null.IntFrom(int64(publicProfile.GetLevel())) - player.Xp = null.IntFrom(publicProfile.GetExperience()) - player.BattlesWon = null.IntFrom(int64(publicProfile.GetBattlesWon())) - player.KmWalked = null.FloatFrom(float64(publicProfile.GetKmWalked())) - player.CaughtPokemon = null.IntFrom(int64(publicProfile.GetCaughtPokemon())) - player.GblRank = null.IntFrom(int64(publicProfile.GetCombatRank())) - player.GblRating = null.IntFrom(int64(publicProfile.GetCombatRating())) + player.Name = publicProfile.GetName() // Name is primary key, don't track as dirty + player.SetTeam(null.IntFrom(int64(publicProfile.GetTeam()))) + player.SetLevel(null.IntFrom(int64(publicProfile.GetLevel()))) + player.SetXp(null.IntFrom(publicProfile.GetExperience())) + player.SetBattlesWon(null.IntFrom(int64(publicProfile.GetBattlesWon()))) + player.SetKmWalked(null.FloatFrom(float64(publicProfile.GetKmWalked()))) + player.SetCaughtPokemon(null.IntFrom(int64(publicProfile.GetCaughtPokemon()))) + player.SetGblRank(null.IntFrom(int64(publicProfile.GetCombatRank()))) + player.SetGblRating(null.IntFrom(int64(publicProfile.GetCombatRating()))) eventBadges := "" @@ -514,13 +953,15 @@ func (player *Player) updateFromPublicProfile(publicProfile *pogo.PlayerPublicPr field := reflect.ValueOf(player).Elem().FieldByName(playerKey) if field.IsValid() && field.CanSet() { - field.Set(reflect.ValueOf(newValue)) + oldValue := field.Interface().(null.Int) + if oldValue != newValue { + field.Set(reflect.ValueOf(newValue)) + player.setFieldDirty() + } } } - if eventBadges != "" { - player.EventBadges = null.StringFrom(eventBadges) - } + player.SetEventBadges(null.StringFrom(eventBadges)) } func UpdatePlayerRecordWithPlayerSummary(db db.DbDetails, playerSummary *pogo.InternalPlayerSummaryProto, publicProfile *pogo.PlayerPublicProfileProto, friendCode string, friendshipId string) error { @@ -531,15 +972,16 @@ func UpdatePlayerRecordWithPlayerSummary(db db.DbDetails, playerSummary *pogo.In if player == nil { player = &Player{ - Name: playerSummary.GetCodename(), + Name: playerSummary.GetCodename(), + newRecord: true, } } if player.FriendshipId.IsZero() && friendshipId != "" { - player.FriendshipId = null.StringFrom(friendshipId) + player.SetFriendshipId(null.StringFrom(friendshipId)) } if player.FriendCode.IsZero() && friendCode != "" { - player.FriendCode = null.StringFrom(friendCode) + player.SetFriendCode(null.StringFrom(friendCode)) } player.updateFromPublicProfile(publicProfile) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9d60f1f0..d18c6d16 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -265,7 +265,7 @@ func (pokemon *Pokemon) SetSeenType(v null.String) { func (pokemon *Pokemon) SetUsername(v null.String) { if pokemon.Username != v { pokemon.Username = v - pokemon.dirty = true + //pokemon.dirty = true } } @@ -862,6 +862,36 @@ func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) } +// wildSignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and +// should be written. +func (pokemon *Pokemon) nearbySignificantUpdate(wildPokemon *pogo.NearbyPokemonProto, time int64) bool { + pokemonDisplay := wildPokemon.PokemonDisplay + // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired + + pokemonChanged := pokemon.PokemonId != int16(pokemonDisplay.DisplayId) || + pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || + pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || + pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || + pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) + + if pokemonChanged { + return true + } + + hasExpired := (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) + + if hasExpired { + return true + } + + if pokemon.SeenType.ValueOrZero() == SeenType_Cell { + return true + } + + // if it's at a nearby stop, or encounter and no other details have changed update is not worthwhile + return false +} + func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { pokemon.SetIsEvent(0) switch pokemon.SeenType.ValueOrZero() { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 4aa1c736..c42200fc 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1192,7 +1192,7 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop } _ = res } - pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + //pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) pokestop.newRecord = false // After saving, it's no longer a new record pokestop.ClearDirty() createPokestopWebhooks(pokestop) diff --git a/decoder/routes.go b/decoder/routes.go index 892413ac..eb7565be 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -4,16 +4,19 @@ import ( "database/sql" "encoding/json" "fmt" + "time" + "golbat/db" "golbat/pogo" "golbat/util" - "time" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) +// Route struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Route struct { Id string `db:"id"` Name string `db:"name"` @@ -37,13 +40,173 @@ type Route struct { Updated int64 `db:"updated"` Version int64 `db:"version"` Waypoints string `db:"waypoints"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record +} + +// IsDirty returns true if any field has been modified +func (r *Route) IsDirty() bool { + return r.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (r *Route) ClearDirty() { + r.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (r *Route) IsNewRecord() bool { + return r.newRecord +} + +// --- Set methods with dirty tracking --- + +func (r *Route) SetName(v string) { + if r.Name != v { + r.Name = v + r.dirty = true + } +} + +func (r *Route) SetShortcode(v string) { + if r.Shortcode != v { + r.Shortcode = v + r.dirty = true + } +} + +func (r *Route) SetDescription(v string) { + if r.Description != v { + r.Description = v + r.dirty = true + } +} + +func (r *Route) SetDistanceMeters(v int64) { + if r.DistanceMeters != v { + r.DistanceMeters = v + r.dirty = true + } +} + +func (r *Route) SetDurationSeconds(v int64) { + if r.DurationSeconds != v { + r.DurationSeconds = v + r.dirty = true + } +} + +func (r *Route) SetEndFortId(v string) { + if r.EndFortId != v { + r.EndFortId = v + r.dirty = true + } +} + +func (r *Route) SetEndImage(v string) { + if r.EndImage != v { + r.EndImage = v + r.dirty = true + } +} + +func (r *Route) SetEndLat(v float64) { + if !floatAlmostEqual(r.EndLat, v, floatTolerance) { + r.EndLat = v + r.dirty = true + } +} + +func (r *Route) SetEndLon(v float64) { + if !floatAlmostEqual(r.EndLon, v, floatTolerance) { + r.EndLon = v + r.dirty = true + } +} + +func (r *Route) SetImage(v string) { + if r.Image != v { + r.Image = v + r.dirty = true + } +} + +func (r *Route) SetImageBorderColor(v string) { + if r.ImageBorderColor != v { + r.ImageBorderColor = v + r.dirty = true + } +} + +func (r *Route) SetReversible(v bool) { + if r.Reversible != v { + r.Reversible = v + r.dirty = true + } +} + +func (r *Route) SetStartFortId(v string) { + if r.StartFortId != v { + r.StartFortId = v + r.dirty = true + } +} + +func (r *Route) SetStartImage(v string) { + if r.StartImage != v { + r.StartImage = v + r.dirty = true + } +} + +func (r *Route) SetStartLat(v float64) { + if !floatAlmostEqual(r.StartLat, v, floatTolerance) { + r.StartLat = v + r.dirty = true + } +} + +func (r *Route) SetStartLon(v float64) { + if !floatAlmostEqual(r.StartLon, v, floatTolerance) { + r.StartLon = v + r.dirty = true + } +} + +func (r *Route) SetTags(v null.String) { + if r.Tags != v { + r.Tags = v + r.dirty = true + } +} + +func (r *Route) SetType(v int8) { + if r.Type != v { + r.Type = v + r.dirty = true + } +} + +func (r *Route) SetVersion(v int64) { + if r.Version != v { + r.Version = v + r.dirty = true + } +} + +func (r *Route) SetWaypoints(v string) { + if r.Waypoints != v { + r.Waypoints = v + r.dirty = true + } } func getRouteRecord(db db.DbDetails, id string) (*Route, error) { inMemoryRoute := routeCache.Get(id) if inMemoryRoute != nil { route := inMemoryRoute.Value() - return &route, nil + return route, nil } route := Route{} @@ -62,61 +225,40 @@ func getRouteRecord(db db.DbDetails, id string) (*Route, error) { return nil, err } - routeCache.Set(id, route, ttlcache.DefaultTTL) + routeCache.Set(id, &route, ttlcache.DefaultTTL) return &route, nil } -// hasChangesRoute compares two Route structs -func hasChangesRoute(old *Route, new *Route) bool { - return old.Name != new.Name || - old.Shortcode != new.Shortcode || - old.Description != new.Description || - old.DistanceMeters != new.DistanceMeters || - old.DurationSeconds != new.DurationSeconds || - old.EndFortId != new.EndFortId || - !floatAlmostEqual(old.EndLat, new.EndLat, floatTolerance) || - !floatAlmostEqual(old.EndLon, new.EndLon, floatTolerance) || - old.Image != new.Image || - old.ImageBorderColor != new.ImageBorderColor || - old.Reversible != new.Reversible || - old.StartFortId != new.StartFortId || - !floatAlmostEqual(old.StartLat, new.StartLat, floatTolerance) || - !floatAlmostEqual(old.StartLon, new.StartLon, floatTolerance) || - old.Tags != new.Tags || - old.Type != new.Type || - old.Version != new.Version || - old.Waypoints != new.Waypoints -} - func saveRouteRecord(db db.DbDetails, route *Route) error { - oldRoute, _ := getRouteRecord(db, route.Id) - - if oldRoute != nil && !hasChangesRoute(oldRoute, route) { - if oldRoute.Updated > time.Now().Unix()-900 { + // Skip save if not dirty and not new, unless 15-minute debounce expired + if !route.IsDirty() && !route.IsNewRecord() { + if route.Updated > time.Now().Unix()-900 { // if a route is unchanged, but we did see it again after 15 minutes, then save again return nil } } - if oldRoute == nil { + route.Updated = time.Now().Unix() + + if route.IsNewRecord() { _, err := db.GeneralDb.NamedExec( ` INSERT INTO route ( id, name, shortcode, description, distance_meters, duration_seconds, end_fort_id, end_image, - end_lat, end_lon, image, image_border_color, - reversible, start_fort_id, start_image, - start_lat, start_lon, tags, type, + end_lat, end_lon, image, image_border_color, + reversible, start_fort_id, start_image, + start_lat, start_lon, tags, type, updated, version, waypoints ) VALUES ( :id, :name, :shortcode, :description, :distance_meters, :duration_seconds, :end_fort_id, - :end_image, :end_lat, :end_lon, :image, - :image_border_color, :reversible, - :start_fort_id, :start_image, :start_lat, - :start_lon, :tags, :type, :updated, + :end_image, :end_lat, :end_lon, :image, + :image_border_color, :reversible, + :start_fort_id, :start_image, :start_lat, + :start_lon, :tags, :type, :updated, :version, :waypoints ) `, @@ -162,47 +304,49 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { } } - routeCache.Set(route.Id, *route, ttlcache.DefaultTTL) + route.ClearDirty() + route.newRecord = false + //routeCache.Set(route.Id, route, ttlcache.DefaultTTL) return nil } func (route *Route) updateFromSharedRouteProto(sharedRouteProto *pogo.SharedRouteProto) { - route.Name = sharedRouteProto.GetName() + route.SetName(sharedRouteProto.GetName()) if sharedRouteProto.GetShortCode() != "" { - route.Shortcode = sharedRouteProto.GetShortCode() + route.SetShortcode(sharedRouteProto.GetShortCode()) } - route.Description = sharedRouteProto.GetDescription() + description := sharedRouteProto.GetDescription() // NOTE: Some descriptions have more than 255 runes, which won't fit in our // varchar(255). - if truncateStr, truncated := util.TruncateUTF8(route.Description, 255); truncated { + if truncateStr, truncated := util.TruncateUTF8(description, 255); truncated { log.Warnf("truncating description for route id '%s'. Orig description: %s", route.Id, - route.Description, + description, ) - route.Description = truncateStr - } - route.DistanceMeters = sharedRouteProto.GetRouteDistanceMeters() - route.DurationSeconds = sharedRouteProto.GetRouteDurationSeconds() - route.EndFortId = sharedRouteProto.GetEndPoi().GetAnchor().GetFortId() - route.EndImage = sharedRouteProto.GetEndPoi().GetImageUrl() - route.EndLat = sharedRouteProto.GetEndPoi().GetAnchor().GetLatDegrees() - route.EndLon = sharedRouteProto.GetEndPoi().GetAnchor().GetLngDegrees() - route.Image = sharedRouteProto.GetImage().GetImageUrl() - route.ImageBorderColor = sharedRouteProto.GetImage().GetBorderColorHex() - route.Reversible = sharedRouteProto.GetReversible() - route.StartFortId = sharedRouteProto.GetStartPoi().GetAnchor().GetFortId() - route.StartImage = sharedRouteProto.GetStartPoi().GetImageUrl() - route.StartLat = sharedRouteProto.GetStartPoi().GetAnchor().GetLatDegrees() - route.StartLon = sharedRouteProto.GetStartPoi().GetAnchor().GetLngDegrees() - route.Type = int8(sharedRouteProto.GetType()) - route.Updated = time.Now().Unix() - route.Version = sharedRouteProto.GetVersion() + description = truncateStr + } + route.SetDescription(description) + route.SetDistanceMeters(sharedRouteProto.GetRouteDistanceMeters()) + route.SetDurationSeconds(sharedRouteProto.GetRouteDurationSeconds()) + route.SetEndFortId(sharedRouteProto.GetEndPoi().GetAnchor().GetFortId()) + route.SetEndImage(sharedRouteProto.GetEndPoi().GetImageUrl()) + route.SetEndLat(sharedRouteProto.GetEndPoi().GetAnchor().GetLatDegrees()) + route.SetEndLon(sharedRouteProto.GetEndPoi().GetAnchor().GetLngDegrees()) + route.SetImage(sharedRouteProto.GetImage().GetImageUrl()) + route.SetImageBorderColor(sharedRouteProto.GetImage().GetBorderColorHex()) + route.SetReversible(sharedRouteProto.GetReversible()) + route.SetStartFortId(sharedRouteProto.GetStartPoi().GetAnchor().GetFortId()) + route.SetStartImage(sharedRouteProto.GetStartPoi().GetImageUrl()) + route.SetStartLat(sharedRouteProto.GetStartPoi().GetAnchor().GetLatDegrees()) + route.SetStartLon(sharedRouteProto.GetStartPoi().GetAnchor().GetLngDegrees()) + route.SetType(int8(sharedRouteProto.GetType())) + route.SetVersion(sharedRouteProto.GetVersion()) waypoints, _ := json.Marshal(sharedRouteProto.GetWaypoints()) - route.Waypoints = string(waypoints) + route.SetWaypoints(string(waypoints)) if len(sharedRouteProto.GetTags()) > 0 { tags, _ := json.Marshal(sharedRouteProto.GetTags()) - route.Tags = null.StringFrom(string(tags)) + route.SetTags(null.StringFrom(string(tags))) } } @@ -218,7 +362,8 @@ func UpdateRouteRecordWithSharedRouteProto(db db.DbDetails, sharedRouteProto *po if route == nil { route = &Route{ - Id: sharedRouteProto.GetId(), + Id: sharedRouteProto.GetId(), + newRecord: true, } } diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 506eb952..07c9c311 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -2,11 +2,11 @@ package decoder import ( "context" - "golbat/db" "time" + "golbat/db" + "github.com/golang/geo/s2" - "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -30,11 +30,11 @@ type S2Cell struct { func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { now := time.Now().Unix() - outputCellIds := []S2Cell{} + var outputCellIds []*S2Cell // prepare list of cells to update for _, cellId := range cellIds { - var s2Cell = S2Cell{} + var s2Cell *S2Cell if c := s2CellCache.Get(cellId); c != nil { cachedCell := c.Value() @@ -44,6 +44,7 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { s2Cell = cachedCell } else { mapS2Cell := s2.CellFromCellID(s2.CellID(cellId)) + s2Cell = &S2Cell{} s2Cell.Id = cellId s2Cell.Latitude = mapS2Cell.CapBound().RectBound().Center().Lat.Degrees() s2Cell.Longitude = mapS2Cell.CapBound().RectBound().Center().Lng.Degrees() @@ -71,8 +72,8 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { return } - // set cache - for _, cellId := range outputCellIds { - s2CellCache.Set(cellId.Id, cellId, ttlcache.DefaultTTL) - } + // since cache is now a pointer, ttl will already have been updated + //for _, cellId := range outputCellIds { + // s2CellCache.Set(cellId.Id, cellId, ttlcache.DefaultTTL) + //} } diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 5867daa1..e4a70fcb 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -14,7 +14,7 @@ import ( ) // Spawnpoint struct. -// REMINDER! Keep hasChangesSpawnpoint updated after making changes +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Spawnpoint struct { Id int64 `db:"id"` Lat float64 `db:"lat"` @@ -22,6 +22,9 @@ type Spawnpoint struct { Updated int64 `db:"updated"` LastSeen int64 `db:"last_seen"` DespawnSec null.Int `db:"despawn_sec"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record } //CREATE TABLE `spawnpoint` ( @@ -37,11 +40,75 @@ type Spawnpoint struct { //KEY `ix_last_seen` (`last_seen`) //) +// IsDirty returns true if any field has been modified +func (s *Spawnpoint) IsDirty() bool { + return s.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (s *Spawnpoint) ClearDirty() { + s.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (s *Spawnpoint) IsNewRecord() bool { + return s.newRecord +} + +// --- Set methods with dirty tracking --- + +func (s *Spawnpoint) SetLat(v float64) { + if !floatAlmostEqual(s.Lat, v, floatTolerance) { + s.Lat = v + s.dirty = true + } +} + +func (s *Spawnpoint) SetLon(v float64) { + if !floatAlmostEqual(s.Lon, v, floatTolerance) { + s.Lon = v + s.dirty = true + } +} + +// SetDespawnSec sets despawn_sec with 2-second tolerance logic +func (s *Spawnpoint) SetDespawnSec(v null.Int) { + // Handle validity changes + if (s.DespawnSec.Valid && !v.Valid) || (!s.DespawnSec.Valid && v.Valid) { + s.DespawnSec = v + s.dirty = true + return + } + + // Both invalid - no change + if !s.DespawnSec.Valid && !v.Valid { + return + } + + // Both valid - check with tolerance + oldVal := s.DespawnSec.Int64 + newVal := v.Int64 + + // Handle wraparound at hour boundary (0/3600) + if oldVal <= 1 && newVal >= 3598 { + return + } + if newVal <= 1 && oldVal >= 3598 { + return + } + + // Allow 2-second tolerance for despawn time + if Abs(oldVal-newVal) > 2 { + s.DespawnSec = v + s.dirty = true + } +} + func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int64) (*Spawnpoint, error) { inMemorySpawnpoint := spawnpointCache.Get(spawnpointId) if inMemorySpawnpoint != nil { spawnpoint := inMemorySpawnpoint.Value() - return &spawnpoint, nil + return spawnpoint, nil } spawnpoint := Spawnpoint{} @@ -56,7 +123,6 @@ func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int6 return &Spawnpoint{Id: spawnpointId}, err } - spawnpointCache.Set(spawnpointId, spawnpoint, ttlcache.DefaultTTL) return &spawnpoint, nil } @@ -67,31 +133,6 @@ func Abs(x int64) int64 { return x } -func hasChangesSpawnpoint(old *Spawnpoint, new *Spawnpoint) bool { - if !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) || - (old.DespawnSec.Valid && !new.DespawnSec.Valid) || - (!old.DespawnSec.Valid && new.DespawnSec.Valid) { - return true - } - if !old.DespawnSec.Valid && !new.DespawnSec.Valid { - return false - } - - // Ignore small movements in despawn time - oldDespawnSec := old.DespawnSec.Int64 - newDespawnSec := new.DespawnSec.Int64 - - if oldDespawnSec <= 1 && newDespawnSec >= 3598 { - return false - } - if newDespawnSec <= 1 && oldDespawnSec >= 3598 { - return false - } - - return Abs(old.DespawnSec.Int64-new.DespawnSec.Int64) > 2 -} - func spawnpointUpdateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, timestampMs int64) { spawnId, err := strconv.ParseInt(wildPokemon.SpawnPointId, 16, 64) if err != nil { @@ -103,22 +144,25 @@ func spawnpointUpdateFromWild(ctx context.Context, db db.DbDetails, wildPokemon date := time.Unix(expireTimeStamp, 0) secondOfHour := date.Second() + date.Minute()*60 - spawnpoint := Spawnpoint{ - Id: spawnId, - Lat: wildPokemon.Latitude, - Lon: wildPokemon.Longitude, - DespawnSec: null.IntFrom(int64(secondOfHour)), + + spawnpoint, _ := getSpawnpointRecord(ctx, db, spawnId) + if spawnpoint == nil { + spawnpoint = &Spawnpoint{Id: spawnId, newRecord: true} } - spawnpointUpdate(ctx, db, &spawnpoint) + spawnpoint.SetLat(wildPokemon.Latitude) + spawnpoint.SetLon(wildPokemon.Longitude) + spawnpoint.SetDespawnSec(null.IntFrom(int64(secondOfHour))) + spawnpointUpdate(ctx, db, spawnpoint) } else { spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) if spawnPoint == nil { - spawnpoint := Spawnpoint{ - Id: spawnId, - Lat: wildPokemon.Latitude, - Lon: wildPokemon.Longitude, + spawnpoint := &Spawnpoint{ + Id: spawnId, + Lat: wildPokemon.Latitude, + Lon: wildPokemon.Longitude, + newRecord: true, } - spawnpointUpdate(ctx, db, &spawnpoint) + spawnpointUpdate(ctx, db, spawnpoint) } else { spawnpointSeen(ctx, db, spawnId) } @@ -126,14 +170,11 @@ func spawnpointUpdateFromWild(ctx context.Context, db db.DbDetails, wildPokemon } func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoint) { - oldSpawnpoint, _ := getSpawnpointRecord(ctx, db, spawnpoint.Id) - - if oldSpawnpoint != nil && !hasChangesSpawnpoint(oldSpawnpoint, spawnpoint) { + // Skip save if not dirty and not new + if !spawnpoint.IsDirty() && !spawnpoint.IsNewRecord() { return } - //log.Println(cmp.Diff(oldSpawnpoint, spawnpoint)) - spawnpoint.Updated = time.Now().Unix() // ensure future updates are set correctly spawnpoint.LastSeen = time.Now().Unix() // ensure future updates are set correctly @@ -152,7 +193,9 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi return } - spawnpointCache.Set(spawnpoint.Id, *spawnpoint, ttlcache.DefaultTTL) + spawnpoint.ClearDirty() + spawnpoint.newRecord = false + spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) } func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { @@ -176,6 +219,6 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { log.Printf("Error updating spawnpoint last seen %s", err) return } - spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) + // Cache already contains a pointer, no need to update } } diff --git a/decoder/station.go b/decoder/station.go index f02b46f1..ccbaef43 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -6,17 +6,19 @@ import ( "encoding/json" "errors" "fmt" + "time" + "golbat/db" "golbat/pogo" "golbat/util" "golbat/webhooks" - "time" - "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) +// Station struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Station struct { Id string `db:"id"` Lat float64 `db:"lat"` @@ -47,6 +49,235 @@ type Station struct { TotalStationedPokemon null.Int `db:"total_stationed_pokemon"` TotalStationedGmax null.Int `db:"total_stationed_gmax"` StationedPokemon null.String `db:"stationed_pokemon"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + oldValues StationOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// StationOldValues holds old field values for webhook comparison +type StationOldValues struct { + EndTime int64 + BattleEnd null.Int + BattlePokemonId null.Int + BattlePokemonForm null.Int + BattlePokemonCostume null.Int + BattlePokemonGender null.Int + BattlePokemonBreadMode null.Int +} + +// IsDirty returns true if any field has been modified +func (station *Station) IsDirty() bool { + return station.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (station *Station) ClearDirty() { + station.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (station *Station) IsNewRecord() bool { + return station.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (station *Station) snapshotOldValues() { + station.oldValues = StationOldValues{ + EndTime: station.EndTime, + BattleEnd: station.BattleEnd, + BattlePokemonId: station.BattlePokemonId, + BattlePokemonForm: station.BattlePokemonForm, + BattlePokemonCostume: station.BattlePokemonCostume, + BattlePokemonGender: station.BattlePokemonGender, + BattlePokemonBreadMode: station.BattlePokemonBreadMode, + } +} + +// --- Set methods with dirty tracking --- + +func (station *Station) SetId(v string) { + if station.Id != v { + station.Id = v + station.dirty = true + } +} + +func (station *Station) SetLat(v float64) { + if !floatAlmostEqual(station.Lat, v, floatTolerance) { + station.Lat = v + station.dirty = true + } +} + +func (station *Station) SetLon(v float64) { + if !floatAlmostEqual(station.Lon, v, floatTolerance) { + station.Lon = v + station.dirty = true + } +} + +func (station *Station) SetName(v string) { + if station.Name != v { + station.Name = v + station.dirty = true + } +} + +func (station *Station) SetCellId(v int64) { + if station.CellId != v { + station.CellId = v + station.dirty = true + } +} + +func (station *Station) SetStartTime(v int64) { + if station.StartTime != v { + station.StartTime = v + station.dirty = true + } +} + +func (station *Station) SetEndTime(v int64) { + if station.EndTime != v { + station.EndTime = v + station.dirty = true + } +} + +func (station *Station) SetCooldownComplete(v int64) { + if station.CooldownComplete != v { + station.CooldownComplete = v + station.dirty = true + } +} + +func (station *Station) SetIsBattleAvailable(v bool) { + if station.IsBattleAvailable != v { + station.IsBattleAvailable = v + station.dirty = true + } +} + +func (station *Station) SetIsInactive(v bool) { + if station.IsInactive != v { + station.IsInactive = v + station.dirty = true + } +} + +func (station *Station) SetBattleLevel(v null.Int) { + if station.BattleLevel != v { + station.BattleLevel = v + station.dirty = true + } +} + +func (station *Station) SetBattleStart(v null.Int) { + if station.BattleStart != v { + station.BattleStart = v + station.dirty = true + } +} + +func (station *Station) SetBattleEnd(v null.Int) { + if station.BattleEnd != v { + station.BattleEnd = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonId(v null.Int) { + if station.BattlePokemonId != v { + station.BattlePokemonId = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonForm(v null.Int) { + if station.BattlePokemonForm != v { + station.BattlePokemonForm = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonCostume(v null.Int) { + if station.BattlePokemonCostume != v { + station.BattlePokemonCostume = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonGender(v null.Int) { + if station.BattlePokemonGender != v { + station.BattlePokemonGender = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonAlignment(v null.Int) { + if station.BattlePokemonAlignment != v { + station.BattlePokemonAlignment = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonBreadMode(v null.Int) { + if station.BattlePokemonBreadMode != v { + station.BattlePokemonBreadMode = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonMove1(v null.Int) { + if station.BattlePokemonMove1 != v { + station.BattlePokemonMove1 = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonMove2(v null.Int) { + if station.BattlePokemonMove2 != v { + station.BattlePokemonMove2 = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonStamina(v null.Int) { + if station.BattlePokemonStamina != v { + station.BattlePokemonStamina = v + station.dirty = true + } +} + +func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { + if !nullFloatAlmostEqual(station.BattlePokemonCpMultiplier, v, floatTolerance) { + station.BattlePokemonCpMultiplier = v + station.dirty = true + } +} + +func (station *Station) SetTotalStationedPokemon(v null.Int) { + if station.TotalStationedPokemon != v { + station.TotalStationedPokemon = v + station.dirty = true + } +} + +func (station *Station) SetTotalStationedGmax(v null.Int) { + if station.TotalStationedGmax != v { + station.TotalStationedGmax = v + station.dirty = true + } +} + +func (station *Station) SetStationedPokemon(v null.String) { + if station.StationedPokemon != v { + station.StationedPokemon = v + station.dirty = true + } } type StationWebhook struct { @@ -77,7 +308,8 @@ func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (* inMemoryStation := stationCache.Get(stationId) if inMemoryStation != nil { station := inMemoryStation.Value() - return &station, nil + station.snapshotOldValues() + return station, nil } station := Station{} err := db.GeneralDb.GetContext(ctx, &station, @@ -94,23 +326,23 @@ func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (* if err != nil { return nil, err } + station.snapshotOldValues() return &station, nil } func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { - oldStation, _ := getStationRecord(ctx, db, station.Id) now := time.Now().Unix() - if oldStation != nil && !hasChangesStation(oldStation, station) { - if oldStation.Updated > now-900 { - // if a gym is unchanged, but we did see it again after 15 minutes, then save again + + // Skip save if not dirty and was updated recently (15-min debounce) + if !station.IsDirty() && !station.IsNewRecord() { + if station.Updated > now-900 { return } } station.Updated = now - //log.Traceln(cmp.Diff(oldStation, station)) - if oldStation == nil { + if station.IsNewRecord() { res, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO station (id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon) @@ -163,41 +395,15 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { _, _ = res, err } - stationCache.Set(station.Id, *station, ttlcache.DefaultTTL) - createStationWebhooks(oldStation, station) - -} - -// hasChangesStation compares two Station structs -// Float tolerance: Lat, Lon -func hasChangesStation(old *Station, new *Station) bool { - return old.Id != new.Id || - old.Name != new.Name || - old.StartTime != new.StartTime || - old.EndTime != new.EndTime || - old.StationedPokemon != new.StationedPokemon || - old.CooldownComplete != new.CooldownComplete || - old.IsBattleAvailable != new.IsBattleAvailable || - old.BattleLevel != new.BattleLevel || - old.BattleStart != new.BattleStart || - old.BattleEnd != new.BattleEnd || - old.BattlePokemonId != new.BattlePokemonId || - old.BattlePokemonForm != new.BattlePokemonForm || - old.BattlePokemonCostume != new.BattlePokemonCostume || - old.BattlePokemonGender != new.BattlePokemonGender || - old.BattlePokemonAlignment != new.BattlePokemonAlignment || - old.BattlePokemonBreadMode != new.BattlePokemonBreadMode || - old.BattlePokemonMove1 != new.BattlePokemonMove1 || - old.BattlePokemonMove2 != new.BattlePokemonMove2 || - old.BattlePokemonStamina != new.BattlePokemonStamina || - old.BattlePokemonCpMultiplier != new.BattlePokemonCpMultiplier || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) + station.ClearDirty() + station.newRecord = false + //stationCache.Set(station.Id, station, ttlcache.DefaultTTL) + createStationWebhooks(station) } func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, cellId uint64) *Station { - station.Id = stationProto.Id - station.Name = stationProto.Name + station.SetId(stationProto.Id) + name := stationProto.Name // NOTE: Some names have more than 255 runes, which won't fit in our // varchar(255). if truncateStr, truncated := util.TruncateUTF8(stationProto.Name, 255); truncated { @@ -205,35 +411,36 @@ func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, stationProto.Id, stationProto.Name, ) - station.Name = truncateStr - } - station.Lat = stationProto.Lat - station.Lon = stationProto.Lng - station.StartTime = stationProto.StartTimeMs / 1000 - station.EndTime = stationProto.EndTimeMs / 1000 - station.CooldownComplete = stationProto.CooldownCompleteMs - station.IsBattleAvailable = stationProto.IsBreadBattleAvailable + name = truncateStr + } + station.SetName(name) + station.SetLat(stationProto.Lat) + station.SetLon(stationProto.Lng) + station.SetStartTime(stationProto.StartTimeMs / 1000) + station.SetEndTime(stationProto.EndTimeMs / 1000) + station.SetCooldownComplete(stationProto.CooldownCompleteMs) + station.SetIsBattleAvailable(stationProto.IsBreadBattleAvailable) if battleDetails := stationProto.BattleDetails; battleDetails != nil { - station.BattleLevel = null.IntFrom(int64(battleDetails.BattleLevel)) - station.BattleStart = null.IntFrom(battleDetails.BattleWindowStartMs / 1000) - station.BattleEnd = null.IntFrom(battleDetails.BattleWindowEndMs / 1000) + station.SetBattleLevel(null.IntFrom(int64(battleDetails.BattleLevel))) + station.SetBattleStart(null.IntFrom(battleDetails.BattleWindowStartMs / 1000)) + station.SetBattleEnd(null.IntFrom(battleDetails.BattleWindowEndMs / 1000)) if pokemon := battleDetails.BattlePokemon; pokemon != nil { - station.BattlePokemonId = null.IntFrom(int64(pokemon.PokemonId)) - station.BattlePokemonMove1 = null.IntFrom(int64(pokemon.Move1)) - station.BattlePokemonMove2 = null.IntFrom(int64(pokemon.Move2)) - station.BattlePokemonForm = null.IntFrom(int64(pokemon.PokemonDisplay.Form)) - station.BattlePokemonCostume = null.IntFrom(int64(pokemon.PokemonDisplay.Costume)) - station.BattlePokemonGender = null.IntFrom(int64(pokemon.PokemonDisplay.Gender)) - station.BattlePokemonAlignment = null.IntFrom(int64(pokemon.PokemonDisplay.Alignment)) - station.BattlePokemonBreadMode = null.IntFrom(int64(pokemon.PokemonDisplay.BreadModeEnum)) - station.BattlePokemonStamina = null.IntFrom(int64(pokemon.Stamina)) - station.BattlePokemonCpMultiplier = null.FloatFrom(float64(pokemon.CpMultiplier)) + station.SetBattlePokemonId(null.IntFrom(int64(pokemon.PokemonId))) + station.SetBattlePokemonMove1(null.IntFrom(int64(pokemon.Move1))) + station.SetBattlePokemonMove2(null.IntFrom(int64(pokemon.Move2))) + station.SetBattlePokemonForm(null.IntFrom(int64(pokemon.PokemonDisplay.Form))) + station.SetBattlePokemonCostume(null.IntFrom(int64(pokemon.PokemonDisplay.Costume))) + station.SetBattlePokemonGender(null.IntFrom(int64(pokemon.PokemonDisplay.Gender))) + station.SetBattlePokemonAlignment(null.IntFrom(int64(pokemon.PokemonDisplay.Alignment))) + station.SetBattlePokemonBreadMode(null.IntFrom(int64(pokemon.PokemonDisplay.BreadModeEnum))) + station.SetBattlePokemonStamina(null.IntFrom(int64(pokemon.Stamina))) + station.SetBattlePokemonCpMultiplier(null.FloatFrom(float64(pokemon.CpMultiplier))) if rewardPokemon := battleDetails.RewardPokemon; rewardPokemon != nil && pokemon.PokemonId != rewardPokemon.PokemonId { log.Infof("[DYNAMAX] Pokemon reward differs from battle: Battle %v - Reward %v", pokemon, rewardPokemon) } } } - station.CellId = int64(cellId) + station.SetCellId(int64(cellId)) return station } @@ -275,17 +482,17 @@ func (station *Station) updateFromGetStationedPokemonDetailsOutProto(stationProt } } jsonString, _ := json.Marshal(stationedPokemon) - station.StationedPokemon = null.StringFrom(string(jsonString)) - station.TotalStationedPokemon = null.IntFrom(int64(stationProto.TotalNumStationedPokemon)) - station.TotalStationedGmax = null.IntFrom(stationedGmax) + station.SetStationedPokemon(null.StringFrom(string(jsonString))) + station.SetTotalStationedPokemon(null.IntFrom(int64(stationProto.TotalNumStationedPokemon))) + station.SetTotalStationedGmax(null.IntFrom(stationedGmax)) return station } func (station *Station) resetStationedPokemonFromStationDetailsNotFound() *Station { jsonString, _ := json.Marshal([]string{}) - station.StationedPokemon = null.StringFrom(string(jsonString)) - station.TotalStationedPokemon = null.IntFrom(0) - station.TotalStationedGmax = null.IntFrom(0) + station.SetStationedPokemon(null.StringFrom(string(jsonString))) + station.SetTotalStationedPokemon(null.IntFrom(0)) + station.SetTotalStationedGmax(null.IntFrom(0)) return station } @@ -333,14 +540,17 @@ func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, reque return fmt.Sprintf("StationedPokemonDetails %s", stationId) } -func createStationWebhooks(oldStation *Station, station *Station) { - if oldStation == nil || station.BattlePokemonId.Valid && (oldStation.EndTime != station.EndTime || - oldStation.BattleEnd != station.BattleEnd || - oldStation.BattlePokemonId != station.BattlePokemonId || - oldStation.BattlePokemonForm != station.BattlePokemonForm || - oldStation.BattlePokemonCostume != station.BattlePokemonCostume || - oldStation.BattlePokemonGender != station.BattlePokemonGender || - oldStation.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { +func createStationWebhooks(station *Station) { + old := &station.oldValues + isNew := station.IsNewRecord() + + if isNew || station.BattlePokemonId.Valid && (old.EndTime != station.EndTime || + old.BattleEnd != station.BattleEnd || + old.BattlePokemonId != station.BattlePokemonId || + old.BattlePokemonForm != station.BattlePokemonForm || + old.BattlePokemonCostume != station.BattlePokemonCostume || + old.BattlePokemonGender != station.BattlePokemonGender || + old.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { stationHook := StationWebhook{ Id: station.Id, Latitude: station.Lat, diff --git a/decoder/stats.go b/decoder/stats.go index ee7c1d36..d5312bf9 100644 --- a/decoder/stats.go +++ b/decoder/stats.go @@ -485,7 +485,7 @@ func updateRaidStats(gym *Gym, areas []geo.AreaName) { } } -func updateIncidentStats(old *Incident, new *Incident, areas []geo.AreaName) { +func updateIncidentStats(incident *Incident, areas []geo.AreaName) { if len(areas) == 0 { areas = []geo.AreaName{ { @@ -501,13 +501,15 @@ func updateIncidentStats(old *Incident, new *Incident, areas []geo.AreaName) { }) locked := false + old := &incident.oldValues + isNew := incident.IsNewRecord() // Loop though all areas for i := 0; i < len(areas); i++ { area := areas[i] // Check if StartTime has changed, then we can assume a new Incident has appeared. - if old == nil || old.StartTime != new.StartTime { + if isNew || old.StartTime != incident.StartTime { if !locked { incidentStatsLock.Lock() @@ -521,8 +523,8 @@ func updateIncidentStats(old *Incident, new *Incident, areas []geo.AreaName) { } // Exclude Kecleon, Showcases and other UNSET characters for invasionStats. - if new.Character != 0 { - invasionStats.count[new.Character]++ + if incident.Character != 0 { + invasionStats.count[incident.Character]++ } } } diff --git a/decoder/tappable.go b/decoder/tappable.go index bbca89a4..95c8da4b 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -5,16 +5,18 @@ import ( "database/sql" "errors" "fmt" - "golbat/db" - "golbat/pogo" "strconv" "time" - "github.com/jellydator/ttlcache/v3" + "golbat/db" + "golbat/pogo" + log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) +// Tappable struct. +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Tappable struct { Id uint64 `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` @@ -28,39 +30,129 @@ type Tappable struct { ExpireTimestamp null.Int `db:"expire_timestamp" json:"expire_timestamp"` ExpireTimestampVerified bool `db:"expire_timestamp_verified" json:"expire_timestamp_verified"` Updated int64 `db:"updated" json:"updated"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record +} + +// IsDirty returns true if any field has been modified +func (ta *Tappable) IsDirty() bool { + return ta.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (ta *Tappable) ClearDirty() { + ta.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (ta *Tappable) IsNewRecord() bool { + return ta.newRecord +} + +// --- Set methods with dirty tracking --- + +func (ta *Tappable) SetLat(v float64) { + if !floatAlmostEqual(ta.Lat, v, floatTolerance) { + ta.Lat = v + ta.dirty = true + } +} + +func (ta *Tappable) SetLon(v float64) { + if !floatAlmostEqual(ta.Lon, v, floatTolerance) { + ta.Lon = v + ta.dirty = true + } +} + +func (ta *Tappable) SetFortId(v null.String) { + if ta.FortId != v { + ta.FortId = v + ta.dirty = true + } +} + +func (ta *Tappable) SetSpawnId(v null.Int) { + if ta.SpawnId != v { + ta.SpawnId = v + ta.dirty = true + } +} + +func (ta *Tappable) SetType(v string) { + if ta.Type != v { + ta.Type = v + ta.dirty = true + } +} + +func (ta *Tappable) SetEncounter(v null.Int) { + if ta.Encounter != v { + ta.Encounter = v + ta.dirty = true + } +} + +func (ta *Tappable) SetItemId(v null.Int) { + if ta.ItemId != v { + ta.ItemId = v + ta.dirty = true + } +} + +func (ta *Tappable) SetCount(v null.Int) { + if ta.Count != v { + ta.Count = v + ta.dirty = true + } +} + +func (ta *Tappable) SetExpireTimestamp(v null.Int) { + if ta.ExpireTimestamp != v { + ta.ExpireTimestamp = v + ta.dirty = true + } +} + +func (ta *Tappable) SetExpireTimestampVerified(v bool) { + if ta.ExpireTimestampVerified != v { + ta.ExpireTimestampVerified = v + ta.dirty = true + } } func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.DbDetails, tappable *pogo.ProcessTappableOutProto, request *pogo.ProcessTappableProto, timestampMs int64) { // update from request - ta.Id = request.EncounterId + ta.Id = request.EncounterId // Id is primary key, don't track as dirty location := request.GetLocation() if spawnPointId := location.GetSpawnpointId(); spawnPointId != "" { spawnId, err := strconv.ParseInt(spawnPointId, 16, 64) if err != nil { panic(err) } - ta.SpawnId = null.IntFrom(spawnId) + ta.SetSpawnId(null.IntFrom(spawnId)) } if fortId := location.GetFortId(); fortId != "" { - ta.FortId = null.StringFrom(fortId) + ta.SetFortId(null.StringFrom(fortId)) } - ta.Type = request.TappableTypeId - ta.Lat = request.LocationHintLat - ta.Lon = request.LocationHintLng + ta.SetType(request.TappableTypeId) + ta.SetLat(request.LocationHintLat) + ta.SetLon(request.LocationHintLng) ta.setExpireTimestamp(ctx, db, timestampMs) // update from tappable if encounter := tappable.GetEncounter(); encounter != nil { // tappable is a Pokèmon, encounter is sent in a separate proto // we store this to link tappable with Pokèmon from encounter proto - ta.Encounter = null.IntFrom(int64(encounter.Pokemon.PokemonId)) + ta.SetEncounter(null.IntFrom(int64(encounter.Pokemon.PokemonId))) } else if reward := tappable.GetReward(); reward != nil { for _, lootProto := range reward { for _, itemProto := range lootProto.GetLootItem() { switch t := itemProto.Type.(type) { case *pogo.LootItemProto_Item: - ta.ItemId = null.IntFrom(int64(t.Item)) - ta.Count = null.IntFrom(int64(itemProto.Count)) + ta.SetItemId(null.IntFrom(int64(t.Item))) + ta.SetCount(null.IntFrom(int64(itemProto.Count))) case *pogo.LootItemProto_Stardust: log.Warnf("[TAPPABLE] Reward is Stardust: %t", t.Stardust) case *pogo.LootItemProto_Pokecoin: @@ -96,7 +188,7 @@ func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.Db } func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, timestampMs int64) { - ta.ExpireTimestampVerified = false + ta.SetExpireTimestampVerified(false) if spawnId := ta.SpawnId.ValueOrZero(); spawnId != 0 { spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) if spawnPoint != nil && spawnPoint.DespawnSec.Valid { @@ -109,23 +201,23 @@ func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, tim if despawnOffset < 0 { despawnOffset += 3600 } - ta.ExpireTimestamp = null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset)) - ta.ExpireTimestampVerified = true + ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) + ta.SetExpireTimestampVerified(true) } else { ta.setUnknownTimestamp(timestampMs / 1000) } } else if fortId := ta.FortId.ValueOrZero(); fortId != "" { // we don't know any despawn times from lured/fort tappables - ta.ExpireTimestamp = null.IntFrom(int64(timestampMs)/1000 + int64(120)) + ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) } } func (ta *Tappable) setUnknownTimestamp(now int64) { if !ta.ExpireTimestamp.Valid { - ta.ExpireTimestamp = null.IntFrom(now + 20*60) + ta.SetExpireTimestamp(null.IntFrom(now + 20*60)) } else { if ta.ExpireTimestamp.Int64 < now { - ta.ExpireTimestamp = null.IntFrom(now + 10*60) + ta.SetExpireTimestamp(null.IntFrom(now + 10*60)) } } } @@ -134,12 +226,12 @@ func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappab inMemoryTappable := tappableCache.Get(id) if inMemoryTappable != nil { tappable := inMemoryTappable.Value() - return &tappable, nil + return tappable, nil } tappable := Tappable{} err := db.GeneralDb.GetContext(ctx, &tappable, `SELECT id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated - FROM tappable + FROM tappable WHERE id = ?`, strconv.FormatUint(id, 10)) statsCollector.IncDbQuery("select tappable", err) if errors.Is(err, sql.ErrNoRows) { @@ -153,13 +245,15 @@ func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappab } func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tappable) { - oldTappable, _ := GetTappableRecord(ctx, details, tappable.Id) - now := time.Now().Unix() - if oldTappable != nil && !hasChangesTappable(oldTappable, tappable) { + // Skip save if not dirty and not new + if !tappable.IsDirty() && !tappable.IsNewRecord() { return } + + now := time.Now().Unix() tappable.Updated = now - if oldTappable == nil { + + if tappable.IsNewRecord() { res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` INSERT INTO tappable ( id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated @@ -184,7 +278,7 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap pokemon_id = :pokemon_id, item_id = :item_id, count = :count, - expire_timestamp = :expire_timestamp, + expire_timestamp = :expire_timestamp, expire_timestamp_verified = :expire_timestamp_verified, updated = :updated WHERE id = "%d" @@ -196,21 +290,9 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap } _ = res } - tappableCache.Set(tappable.Id, *tappable, ttlcache.DefaultTTL) -} - -func hasChangesTappable(old *Tappable, new *Tappable) bool { - return old.Id != new.Id || - old.FortId != new.FortId || - old.SpawnId != new.SpawnId || - old.Type != new.Type || - old.Encounter != new.Encounter || - old.ItemId != new.ItemId || - old.Count != new.Count || - old.ExpireTimestamp != new.ExpireTimestamp || - old.ExpireTimestampVerified != new.ExpireTimestampVerified || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) + tappable.ClearDirty() + tappable.newRecord = false + //tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) } func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { @@ -226,7 +308,7 @@ func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessT } if tappable == nil { - tappable = &Tappable{} + tappable = &Tappable{newRecord: true} } tappable.updateFromProcessTappableProto(ctx, db, tappableDetails, request, timestampMs) diff --git a/decoder/weather.go b/decoder/weather.go index 3e29aa24..8381de26 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -3,18 +3,18 @@ package decoder import ( "context" "database/sql" + "golbat/db" "golbat/pogo" "golbat/webhooks" "github.com/golang/geo/s2" - "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) // Weather struct. -// REMINDER! Keep hasChangesWeather updated after making changes +// REMINDER! Dirty flag pattern - use setter methods to modify fields type Weather struct { Id int64 `db:"id"` Latitude float64 `db:"latitude"` @@ -31,6 +31,17 @@ type Weather struct { Severity null.Int `db:"severity"` WarnWeather null.Bool `db:"warn_weather"` UpdatedMs int64 `db:"updated"` + + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + + oldValues WeatherOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// WeatherOldValues holds old field values for webhook comparison +type WeatherOldValues struct { + GameplayCondition null.Int + WarnWeather null.Bool } // CREATE TABLE `weather` ( @@ -52,11 +63,136 @@ type Weather struct { // PRIMARY KEY (`id`) //) +// IsDirty returns true if any field has been modified +func (weather *Weather) IsDirty() bool { + return weather.dirty +} + +// ClearDirty resets the dirty flag (call after saving to DB) +func (weather *Weather) ClearDirty() { + weather.dirty = false +} + +// IsNewRecord returns true if this is a new record (not yet in DB) +func (weather *Weather) IsNewRecord() bool { + return weather.newRecord +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (weather *Weather) snapshotOldValues() { + weather.oldValues = WeatherOldValues{ + GameplayCondition: weather.GameplayCondition, + WarnWeather: weather.WarnWeather, + } +} + +// --- Set methods with dirty tracking --- + +func (weather *Weather) SetId(v int64) { + if weather.Id != v { + weather.Id = v + weather.dirty = true + } +} + +func (weather *Weather) SetLatitude(v float64) { + if !floatAlmostEqual(weather.Latitude, v, floatTolerance) { + weather.Latitude = v + weather.dirty = true + } +} + +func (weather *Weather) SetLongitude(v float64) { + if !floatAlmostEqual(weather.Longitude, v, floatTolerance) { + weather.Longitude = v + weather.dirty = true + } +} + +func (weather *Weather) SetLevel(v null.Int) { + if weather.Level != v { + weather.Level = v + weather.dirty = true + } +} + +func (weather *Weather) SetGameplayCondition(v null.Int) { + if weather.GameplayCondition != v { + weather.GameplayCondition = v + weather.dirty = true + } +} + +func (weather *Weather) SetWindDirection(v null.Int) { + if weather.WindDirection != v { + weather.WindDirection = v + weather.dirty = true + } +} + +func (weather *Weather) SetCloudLevel(v null.Int) { + if weather.CloudLevel != v { + weather.CloudLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetRainLevel(v null.Int) { + if weather.RainLevel != v { + weather.RainLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetWindLevel(v null.Int) { + if weather.WindLevel != v { + weather.WindLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetSnowLevel(v null.Int) { + if weather.SnowLevel != v { + weather.SnowLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetFogLevel(v null.Int) { + if weather.FogLevel != v { + weather.FogLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetSpecialEffectLevel(v null.Int) { + if weather.SpecialEffectLevel != v { + weather.SpecialEffectLevel = v + weather.dirty = true + } +} + +func (weather *Weather) SetSeverity(v null.Int) { + if weather.Severity != v { + weather.Severity = v + weather.dirty = true + } +} + +func (weather *Weather) SetWarnWeather(v null.Bool) { + if weather.WarnWeather != v { + weather.WarnWeather = v + weather.dirty = true + } +} + func getWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, error) { inMemoryWeather := weatherCache.Get(weatherId) if inMemoryWeather != nil { weather := inMemoryWeather.Value() - return &weather, nil + weather.snapshotOldValues() + return weather, nil } weather := Weather{} @@ -72,7 +208,7 @@ func getWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*W } weather.UpdatedMs *= 1000 - weatherCache.Set(weatherId, weather, ttlcache.DefaultTTL) + weather.snapshotOldValues() return &weather, nil } @@ -80,46 +216,24 @@ func weatherCellIdFromLatLon(lat, lon float64) int64 { return int64(s2.CellIDFromLatLng(s2.LatLngFromDegrees(lat, lon)).Parent(10)) } -func (weather *Weather) updateWeatherFromClientWeatherProto(clientWeather *pogo.ClientWeatherProto) (oldGameplayCondition null.Int) { - oldGameplayCondition = weather.GameplayCondition - weather.Id = clientWeather.S2CellId +func (weather *Weather) updateWeatherFromClientWeatherProto(clientWeather *pogo.ClientWeatherProto) { + weather.SetId(clientWeather.S2CellId) s2cell := s2.CellFromCellID(s2.CellID(clientWeather.S2CellId)) - weather.Latitude = s2cell.CapBound().RectBound().Center().Lat.Degrees() - weather.Longitude = s2cell.CapBound().RectBound().Center().Lng.Degrees() - weather.Level = null.IntFrom(int64(s2cell.Level())) - weather.GameplayCondition = null.IntFrom(int64(clientWeather.GameplayWeather.GameplayCondition)) - weather.WindDirection = null.IntFrom(int64(clientWeather.DisplayWeather.WindDirection)) - weather.CloudLevel = null.IntFrom(int64(clientWeather.DisplayWeather.CloudLevel)) - weather.RainLevel = null.IntFrom(int64(clientWeather.DisplayWeather.RainLevel)) - weather.WindLevel = null.IntFrom(int64(clientWeather.DisplayWeather.WindLevel)) - weather.SnowLevel = null.IntFrom(int64(clientWeather.DisplayWeather.SnowLevel)) - weather.FogLevel = null.IntFrom(int64(clientWeather.DisplayWeather.FogLevel)) - weather.SpecialEffectLevel = null.IntFrom(int64(clientWeather.DisplayWeather.SpecialEffectLevel)) + weather.SetLatitude(s2cell.CapBound().RectBound().Center().Lat.Degrees()) + weather.SetLongitude(s2cell.CapBound().RectBound().Center().Lng.Degrees()) + weather.SetLevel(null.IntFrom(int64(s2cell.Level()))) + weather.SetGameplayCondition(null.IntFrom(int64(clientWeather.GameplayWeather.GameplayCondition))) + weather.SetWindDirection(null.IntFrom(int64(clientWeather.DisplayWeather.WindDirection))) + weather.SetCloudLevel(null.IntFrom(int64(clientWeather.DisplayWeather.CloudLevel))) + weather.SetRainLevel(null.IntFrom(int64(clientWeather.DisplayWeather.RainLevel))) + weather.SetWindLevel(null.IntFrom(int64(clientWeather.DisplayWeather.WindLevel))) + weather.SetSnowLevel(null.IntFrom(int64(clientWeather.DisplayWeather.SnowLevel))) + weather.SetFogLevel(null.IntFrom(int64(clientWeather.DisplayWeather.FogLevel))) + weather.SetSpecialEffectLevel(null.IntFrom(int64(clientWeather.DisplayWeather.SpecialEffectLevel))) for _, alert := range clientWeather.Alerts { - weather.Severity = null.IntFrom(int64(alert.Severity)) - weather.WarnWeather = null.BoolFrom(alert.WarnWeather) - } - return -} - -// hasChangesWeather compares two Weather structs -// Float tolerance: Latitude, Longitude -func hasChangesWeather(old *Weather, new *Weather) bool { - return old.Id != new.Id || - old.Level != new.Level || - old.GameplayCondition != new.GameplayCondition || - old.WindDirection != new.WindDirection || - old.CloudLevel != new.CloudLevel || - old.RainLevel != new.RainLevel || - old.WindLevel != new.WindLevel || - old.SnowLevel != new.SnowLevel || - old.FogLevel != new.FogLevel || - old.SpecialEffectLevel != new.SpecialEffectLevel || - old.Severity != new.Severity || - old.WarnWeather != new.WarnWeather || - old.UpdatedMs != new.UpdatedMs || - !floatAlmostEqual(old.Latitude, new.Latitude, floatTolerance) || - !floatAlmostEqual(old.Longitude, new.Longitude, floatTolerance) + weather.SetSeverity(null.IntFrom(int64(alert.Severity))) + weather.SetWarnWeather(null.BoolFrom(alert.WarnWeather)) + } } type WeatherWebhook struct { @@ -140,9 +254,12 @@ type WeatherWebhook struct { Updated int64 `json:"updated"` } -func createWeatherWebhooks(oldWeather *Weather, weather *Weather) { - if oldWeather == nil || oldWeather.GameplayCondition.ValueOrZero() != weather.GameplayCondition.ValueOrZero() || - oldWeather.WarnWeather.ValueOrZero() != weather.WarnWeather.ValueOrZero() { +func createWeatherWebhooks(weather *Weather) { + old := &weather.oldValues + isNew := weather.IsNewRecord() + + if isNew || old.GameplayCondition.ValueOrZero() != weather.GameplayCondition.ValueOrZero() || + old.WarnWeather.ValueOrZero() != weather.WarnWeather.ValueOrZero() { s2cell := s2.CellFromCellID(s2.CellID(weather.Id)) var polygon [4][2]float64 @@ -175,12 +292,12 @@ func createWeatherWebhooks(oldWeather *Weather, weather *Weather) { } func saveWeatherRecord(ctx context.Context, db db.DbDetails, weather *Weather) { - oldWeather, _ := getWeatherRecord(ctx, db, weather.Id) - if oldWeather != nil && !hasChangesWeather(oldWeather, weather) { + // Skip save if not dirty and not new + if !weather.IsDirty() && !weather.IsNewRecord() { return } - if oldWeather == nil { + if weather.IsNewRecord() { res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO weather ("+ "id, latitude, longitude, level, gameplay_condition, wind_direction, cloud_level, rain_level, "+ @@ -221,6 +338,8 @@ func saveWeatherRecord(ctx context.Context, db db.DbDetails, weather *Weather) { } _ = res } - weatherCache.Set(weather.Id, *weather, ttlcache.DefaultTTL) - createWeatherWebhooks(oldWeather, weather) + createWeatherWebhooks(weather) + weather.ClearDirty() + weather.newRecord = false + //weatherCache.Set(weather.Id, weather, ttlcache.DefaultTTL) } From a985123b8e6707c8e7903040a4ca36fe01a17023 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 16:57:04 +0000 Subject: [PATCH 08/35] Generic sharding introduced for pokestop/gym --- decoder/main.go | 55 +++++++++---------- decoder/pokemonRtree.go | 14 ++--- decoder/sharded_cache.go | 116 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 37 deletions(-) create mode 100644 decoder/sharded_cache.go diff --git a/decoder/main.go b/decoder/main.go index e398e5e9..6877aee4 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -58,15 +58,15 @@ type webhooksSenderInterface interface { var webhooksSender webhooksSenderInterface var statsCollector stats_collector.StatsCollector -var pokestopCache *ttlcache.Cache[string, *Pokestop] +var pokestopCache *ShardedCache[string, *Pokestop] var gymCache *ttlcache.Cache[string, *Gym] var stationCache *ttlcache.Cache[string, *Station] var tappableCache *ttlcache.Cache[uint64, *Tappable] var weatherCache *ttlcache.Cache[int64, *Weather] var weatherConsensusCache *ttlcache.Cache[int64, *WeatherConsensusState] var s2CellCache *ttlcache.Cache[uint64, *S2Cell] -var spawnpointCache *ttlcache.Cache[int64, *Spawnpoint] -var pokemonCache []*ttlcache.Cache[uint64, *Pokemon] +var spawnpointCache *ShardedCache[int64, *Spawnpoint] +var pokemonCache *ShardedCache[uint64, *Pokemon] var incidentCache *ttlcache.Cache[string, *Incident] var playerCache *ttlcache.Cache[string, *Player] var routeCache *ttlcache.Cache[string, *Route] @@ -101,27 +101,25 @@ func (cl *gohbemLogger) Print(message string) { log.Info("Gohbem - ", message) } -func getPokemonCache(key uint64) *ttlcache.Cache[uint64, *Pokemon] { - return pokemonCache[key%uint64(len(pokemonCache))] -} - func setPokemonCache(key uint64, value *Pokemon, ttl time.Duration) { - getPokemonCache(key).Set(key, value, ttl) + pokemonCache.Set(key, value, ttl) } func getPokemonFromCache(key uint64) *ttlcache.Item[uint64, *Pokemon] { - return getPokemonCache(key).Get(key) + return pokemonCache.Get(key) } func deletePokemonFromCache(key uint64) { - getPokemonCache(key).Delete(key) + pokemonCache.Delete(key) } func initDataCache() { - pokestopCache = ttlcache.New[string, *Pokestop]( - ttlcache.WithTTL[string, *Pokestop](60 * time.Minute), - ) - go pokestopCache.Start() + // Sharded caches for high-concurrency tables + pokestopCache = NewShardedCache(ShardedCacheConfig[string, *Pokestop]{ + NumShards: runtime.NumCPU(), + TTL: 60 * time.Minute, + KeyToShard: StringKeyToShard, + }) gymCache = ttlcache.New[string, *Gym]( ttlcache.WithTTL[string, *Gym](60 * time.Minute), @@ -153,21 +151,20 @@ func initDataCache() { ) go s2CellCache.Start() - spawnpointCache = ttlcache.New[int64, *Spawnpoint]( - ttlcache.WithTTL[int64, *Spawnpoint](60 * time.Minute), - ) - go spawnpointCache.Start() - - // pokemon is the most active table. Use an array of caches to increase concurrency for querying ttlcache, which places a global lock for each Get/Set operation - // Initialize pokemon cache array: by picking it to be nproc, we should expect ~nproc*(1-1/e) ~ 63% concurrency - pokemonCache = make([]*ttlcache.Cache[uint64, *Pokemon], runtime.NumCPU()) - for i := 0; i < len(pokemonCache); i++ { - pokemonCache[i] = ttlcache.New[uint64, *Pokemon]( - ttlcache.WithTTL[uint64, *Pokemon](60*time.Minute), - ttlcache.WithDisableTouchOnHit[uint64, *Pokemon](), // Pokemon will last 60 mins from when we first see them not last see them - ) - go pokemonCache[i].Start() - } + spawnpointCache = NewShardedCache(ShardedCacheConfig[int64, *Spawnpoint]{ + NumShards: runtime.NumCPU(), + TTL: 60 * time.Minute, + KeyToShard: Int64KeyToShard, + }) + + // Pokemon cache: sharded for high concurrency + // By picking NumShards to be nproc, we should expect ~nproc*(1-1/e) ~ 63% concurrency + pokemonCache = NewShardedCache(ShardedCacheConfig[uint64, *Pokemon]{ + NumShards: runtime.NumCPU(), + TTL: 60 * time.Minute, + KeyToShard: Uint64KeyToShard, + DisableTouchOnHit: true, // Pokemon will last 60 mins from when we first see them not last see them + }) initPokemonRtree() initFortRtree() diff --git a/decoder/pokemonRtree.go b/decoder/pokemonRtree.go index 02037386..48aadc01 100644 --- a/decoder/pokemonRtree.go +++ b/decoder/pokemonRtree.go @@ -50,14 +50,12 @@ var pokemonTree rtree.RTreeG[uint64] func initPokemonRtree() { pokemonLookupCache = xsync.NewMapOf[uint64, PokemonLookupCacheItem]() - // Set up OnEviction callbacks for each cache in the array - for i := range pokemonCache { - pokemonCache[i].OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, *Pokemon]) { - pokemon := v.Value() - removePokemonFromTree(pokemon.Id, pokemon.Lat, pokemon.Lon) - // Rely on the pokemon pvp lookup caches to remove themselves rather than trying to synchronise - }) - } + // Set up OnEviction callback on all shards + pokemonCache.OnEviction(func(ctx context.Context, ev ttlcache.EvictionReason, v *ttlcache.Item[uint64, *Pokemon]) { + pokemon := v.Value() + removePokemonFromTree(pokemon.Id, pokemon.Lat, pokemon.Lon) + // Rely on the pokemon pvp lookup caches to remove themselves rather than trying to synchronise + }) } func pokemonRtreeUpdatePokemonOnGet(pokemon *Pokemon) { diff --git a/decoder/sharded_cache.go b/decoder/sharded_cache.go new file mode 100644 index 00000000..b0bdecab --- /dev/null +++ b/decoder/sharded_cache.go @@ -0,0 +1,116 @@ +package decoder + +import ( + "context" + "hash/fnv" + "time" + + "github.com/jellydator/ttlcache/v3" +) + +// ShardedCache is a generic sharded cache for improved concurrency. +// It distributes entries across multiple ttlcache instances to reduce lock contention. +type ShardedCache[K comparable, V any] struct { + shards []*ttlcache.Cache[K, V] + keyToShard func(K) uint64 +} + +// ShardedCacheConfig holds configuration for creating a ShardedCache +type ShardedCacheConfig[K comparable, V any] struct { + NumShards int + TTL time.Duration + KeyToShard func(K) uint64 + DisableTouchOnHit bool +} + +// NewShardedCache creates a new sharded cache with the given configuration. +// The keyToShard function converts keys to uint64 for shard selection. +func NewShardedCache[K comparable, V any](config ShardedCacheConfig[K, V]) *ShardedCache[K, V] { + sc := &ShardedCache[K, V]{ + shards: make([]*ttlcache.Cache[K, V], config.NumShards), + keyToShard: config.KeyToShard, + } + + for i := 0; i < config.NumShards; i++ { + opts := []ttlcache.Option[K, V]{ + ttlcache.WithTTL[K, V](config.TTL), + } + if config.DisableTouchOnHit { + opts = append(opts, ttlcache.WithDisableTouchOnHit[K, V]()) + } + sc.shards[i] = ttlcache.New[K, V](opts...) + go sc.shards[i].Start() + } + + return sc +} + +// getShard returns the cache shard for the given key +func (sc *ShardedCache[K, V]) getShard(key K) *ttlcache.Cache[K, V] { + return sc.shards[sc.keyToShard(key)%uint64(len(sc.shards))] +} + +// Get retrieves an item from the appropriate shard +func (sc *ShardedCache[K, V]) Get(key K) *ttlcache.Item[K, V] { + return sc.getShard(key).Get(key) +} + +// Set stores an item in the appropriate shard +func (sc *ShardedCache[K, V]) Set(key K, value V, ttl time.Duration) { + sc.getShard(key).Set(key, value, ttl) +} + +// Delete removes an item from the appropriate shard +func (sc *ShardedCache[K, V]) Delete(key K) { + sc.getShard(key).Delete(key) +} + +// Range iterates over all items in all shards. +// The callback should return true to continue iteration or false to stop. +func (sc *ShardedCache[K, V]) Range(fn func(*ttlcache.Item[K, V]) bool) { + for _, shard := range sc.shards { + shard.Range(fn) + } +} + +// OnEviction sets an eviction callback on all shards +func (sc *ShardedCache[K, V]) OnEviction(fn func(context.Context, ttlcache.EvictionReason, *ttlcache.Item[K, V])) { + for _, shard := range sc.shards { + shard.OnEviction(fn) + } +} + +// Len returns the total number of items across all shards +func (sc *ShardedCache[K, V]) Len() int { + total := 0 + for _, shard := range sc.shards { + total += shard.Len() + } + return total +} + +// DeleteAll removes all items from all shards +func (sc *ShardedCache[K, V]) DeleteAll() { + for _, shard := range sc.shards { + shard.DeleteAll() + } +} + +// --- Key conversion helpers --- + +// Uint64KeyToShard is the identity function for uint64 keys +func Uint64KeyToShard(key uint64) uint64 { + return key +} + +// Int64KeyToShard converts int64 keys to uint64 for sharding +func Int64KeyToShard(key int64) uint64 { + return uint64(key) +} + +// StringKeyToShard hashes string keys to uint64 for sharding using FNV-1a +func StringKeyToShard(key string) uint64 { + h := fnv.New64a() + h.Write([]byte(key)) + return h.Sum64() +} From bb8210cba40177064e2ac07c332f314889349aac Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 21:12:36 +0000 Subject: [PATCH 09/35] Database change tracing --- decoder/db_debug.go | 21 ++++++ decoder/db_debug_off.go | 14 ++++ decoder/gym.go | 131 ++++++++++++++++++++++++++++++++++- decoder/player.go | 6 +- decoder/pokemon.go | 101 ++++++++++++++++++++++++++- decoder/pokestop.go | 146 +++++++++++++++++++++++++++++++++++++++- decoder/routes.go | 80 ++++++++++++++++++++-- decoder/s2cell.go | 13 ++++ decoder/spawnpoint.go | 11 +-- decoder/station.go | 99 +++++++++++++++++++++++++-- decoder/tappable.go | 51 ++++++++++++-- decoder/weather.go | 7 +- 12 files changed, 652 insertions(+), 28 deletions(-) create mode 100644 decoder/db_debug.go create mode 100644 decoder/db_debug_off.go diff --git a/decoder/db_debug.go b/decoder/db_debug.go new file mode 100644 index 00000000..89498a3f --- /dev/null +++ b/decoder/db_debug.go @@ -0,0 +1,21 @@ +//go:build dbdebug + +package decoder + +import ( + "strings" + + log "github.com/sirupsen/logrus" +) + +// dbDebugEnabled is true when built with -tags dbdebug +const dbDebugEnabled = true + +// dbDebugLog logs a database operation with changed fields +func dbDebugLog(reason, entityType, id string, changedFields []string) { + fields := "" + if len(changedFields) > 0 { + fields = " changed=[" + strings.Join(changedFields, ", ") + "]" + } + log.Debugf("[DB_%s] %s id=%s%s", reason, entityType, id, fields) +} diff --git a/decoder/db_debug_off.go b/decoder/db_debug_off.go new file mode 100644 index 00000000..b5a32b77 --- /dev/null +++ b/decoder/db_debug_off.go @@ -0,0 +1,14 @@ +//go:build !dbdebug + +package decoder + +// dbDebugEnabled is false when dbdebug build tag is not set. +// The compiler will eliminate dead code in if statements checking this const. +const dbDebugEnabled = false + +// dbDebugLog is a no-op stub when dbdebug build tag is not set. +// This function is never called at runtime due to const-folding of dbDebugEnabled. +func dbDebugLog(reason, entityType, id string, changedFields []string) { + // No-op: this function exists only to satisfy the compiler. + // It will never be called because dbDebugEnabled is false. +} diff --git a/decoder/gym.go b/decoder/gym.go index dba5fb34..b3bdcaf6 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -68,8 +68,9 @@ type Gym struct { Defenders null.String `db:"defenders" json:"defenders"` Rsvps null.String `db:"rsvps" json:"rsvps"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues GymOldValues `db:"-" json:"-"` // Old values for webhook comparison } @@ -177,6 +178,9 @@ func (gym *Gym) SetId(v string) { if gym.Id != v { gym.Id = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Id") + } } } @@ -184,6 +188,9 @@ func (gym *Gym) SetLat(v float64) { if !floatAlmostEqual(gym.Lat, v, floatTolerance) { gym.Lat = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Lat") + } } } @@ -191,6 +198,9 @@ func (gym *Gym) SetLon(v float64) { if !floatAlmostEqual(gym.Lon, v, floatTolerance) { gym.Lon = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Lon") + } } } @@ -198,6 +208,9 @@ func (gym *Gym) SetName(v null.String) { if gym.Name != v { gym.Name = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Name") + } } } @@ -205,6 +218,9 @@ func (gym *Gym) SetUrl(v null.String) { if gym.Url != v { gym.Url = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Url") + } } } @@ -212,6 +228,9 @@ func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { if gym.LastModifiedTimestamp != v { gym.LastModifiedTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "LastModifiedTimestamp") + } } } @@ -219,6 +238,9 @@ func (gym *Gym) SetRaidEndTimestamp(v null.Int) { if gym.RaidEndTimestamp != v { gym.RaidEndTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidEndTimestamp") + } } } @@ -226,6 +248,9 @@ func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { if gym.RaidSpawnTimestamp != v { gym.RaidSpawnTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidSpawnTimestamp") + } } } @@ -233,6 +258,9 @@ func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { if gym.RaidBattleTimestamp != v { gym.RaidBattleTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidBattleTimestamp") + } } } @@ -240,6 +268,9 @@ func (gym *Gym) SetRaidPokemonId(v null.Int) { if gym.RaidPokemonId != v { gym.RaidPokemonId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonId") + } } } @@ -247,6 +278,9 @@ func (gym *Gym) SetGuardingPokemonId(v null.Int) { if gym.GuardingPokemonId != v { gym.GuardingPokemonId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "GuardingPokemonId") + } } } @@ -254,6 +288,9 @@ func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { if gym.GuardingPokemonDisplay != v { gym.GuardingPokemonDisplay = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "GuardingPokemonDisplay") + } } } @@ -261,6 +298,9 @@ func (gym *Gym) SetAvailableSlots(v null.Int) { if gym.AvailableSlots != v { gym.AvailableSlots = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "AvailableSlots") + } } } @@ -268,6 +308,9 @@ func (gym *Gym) SetTeamId(v null.Int) { if gym.TeamId != v { gym.TeamId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "TeamId") + } } } @@ -275,6 +318,9 @@ func (gym *Gym) SetRaidLevel(v null.Int) { if gym.RaidLevel != v { gym.RaidLevel = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidLevel") + } } } @@ -282,6 +328,9 @@ func (gym *Gym) SetEnabled(v null.Int) { if gym.Enabled != v { gym.Enabled = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Enabled") + } } } @@ -289,6 +338,9 @@ func (gym *Gym) SetExRaidEligible(v null.Int) { if gym.ExRaidEligible != v { gym.ExRaidEligible = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "ExRaidEligible") + } } } @@ -304,6 +356,9 @@ func (gym *Gym) SetRaidPokemonMove1(v null.Int) { if gym.RaidPokemonMove1 != v { gym.RaidPokemonMove1 = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonMove1") + } } } @@ -311,6 +366,9 @@ func (gym *Gym) SetRaidPokemonMove2(v null.Int) { if gym.RaidPokemonMove2 != v { gym.RaidPokemonMove2 = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonMove2") + } } } @@ -318,6 +376,9 @@ func (gym *Gym) SetRaidPokemonForm(v null.Int) { if gym.RaidPokemonForm != v { gym.RaidPokemonForm = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonForm") + } } } @@ -325,6 +386,9 @@ func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { if gym.RaidPokemonAlignment != v { gym.RaidPokemonAlignment = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonAlignment") + } } } @@ -332,6 +396,9 @@ func (gym *Gym) SetRaidPokemonCp(v null.Int) { if gym.RaidPokemonCp != v { gym.RaidPokemonCp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonCp") + } } } @@ -339,6 +406,9 @@ func (gym *Gym) SetRaidIsExclusive(v null.Int) { if gym.RaidIsExclusive != v { gym.RaidIsExclusive = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidIsExclusive") + } } } @@ -346,6 +416,9 @@ func (gym *Gym) SetCellId(v null.Int) { if gym.CellId != v { gym.CellId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "CellId") + } } } @@ -353,6 +426,9 @@ func (gym *Gym) SetDeleted(v bool) { if gym.Deleted != v { gym.Deleted = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Deleted") + } } } @@ -360,6 +436,9 @@ func (gym *Gym) SetTotalCp(v null.Int) { if gym.TotalCp != v { gym.TotalCp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "TotalCp") + } } } @@ -367,6 +446,9 @@ func (gym *Gym) SetRaidPokemonGender(v null.Int) { if gym.RaidPokemonGender != v { gym.RaidPokemonGender = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonGender") + } } } @@ -374,6 +456,9 @@ func (gym *Gym) SetSponsorId(v null.Int) { if gym.SponsorId != v { gym.SponsorId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "SponsorId") + } } } @@ -381,6 +466,9 @@ func (gym *Gym) SetPartnerId(v null.String) { if gym.PartnerId != v { gym.PartnerId = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "PartnerId") + } } } @@ -388,6 +476,9 @@ func (gym *Gym) SetRaidPokemonCostume(v null.Int) { if gym.RaidPokemonCostume != v { gym.RaidPokemonCostume = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonCostume") + } } } @@ -395,6 +486,9 @@ func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { if gym.RaidPokemonEvolution != v { gym.RaidPokemonEvolution = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "RaidPokemonEvolution") + } } } @@ -402,6 +496,9 @@ func (gym *Gym) SetArScanEligible(v null.Int) { if gym.ArScanEligible != v { gym.ArScanEligible = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "ArScanEligible") + } } } @@ -409,6 +506,9 @@ func (gym *Gym) SetPowerUpLevel(v null.Int) { if gym.PowerUpLevel != v { gym.PowerUpLevel = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "PowerUpLevel") + } } } @@ -416,6 +516,9 @@ func (gym *Gym) SetPowerUpPoints(v null.Int) { if gym.PowerUpPoints != v { gym.PowerUpPoints = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "PowerUpPoints") + } } } @@ -423,6 +526,9 @@ func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { if gym.PowerUpEndTimestamp != v { gym.PowerUpEndTimestamp = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "PowerUpEndTimestamp") + } } } @@ -430,6 +536,9 @@ func (gym *Gym) SetDescription(v null.String) { if gym.Description != v { gym.Description = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Description") + } } } @@ -445,6 +554,9 @@ func (gym *Gym) SetRsvps(v null.String) { if gym.Rsvps != v { gym.Rsvps = v gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Rsvps") + } } } @@ -992,6 +1104,9 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { gym.Updated = now if gym.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) @@ -1003,6 +1118,9 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { _, _ = res, err } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ "lat = :lat, "+ "lon = :lon, "+ @@ -1057,8 +1175,15 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { createGymWebhooks(gym, areas) createGymFortWebhooks(gym) updateRaidStats(gym, areas) - gym.newRecord = false // After saving, it's no longer a new record + if dbDebugEnabled { + gym.changedFields = gym.changedFields[:0] + } + if gym.IsNewRecord() { + gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) + gym.newRecord = false + } gym.ClearDirty() + } func updateGymGetMapFortCache(gym *Gym, skipName bool) { diff --git a/decoder/player.go b/decoder/player.go index 721f7d88..0478ae75 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -914,8 +914,10 @@ func savePlayerRecord(db db.DbDetails, player *Player) { } player.ClearDirty() - player.newRecord = false - //playerCache.Set(player.Name, player, ttlcache.DefaultTTL) + if player.IsNewRecord() { + player.newRecord = false + playerCache.Set(player.Name, player, ttlcache.DefaultTTL) + } } func (player *Player) updateFromPublicProfile(publicProfile *pogo.PlayerPublicProfileProto) { diff --git a/decoder/pokemon.go b/decoder/pokemon.go index d18c6d16..9399bd2a 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -77,8 +77,9 @@ type Pokemon struct { internal grpc.PokemonInternal - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues PokemonOldValues `db:"-" json:"-"` // Old values for webhook comparison and stats } @@ -175,6 +176,9 @@ func (pokemon *Pokemon) SetPokestopId(v null.String) { if pokemon.PokestopId != v { pokemon.PokestopId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "PokestopId") + } } } @@ -182,6 +186,9 @@ func (pokemon *Pokemon) SetSpawnId(v null.Int) { if pokemon.SpawnId != v { pokemon.SpawnId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "SpawnId") + } } } @@ -189,6 +196,9 @@ func (pokemon *Pokemon) SetLat(v float64) { if !floatAlmostEqual(pokemon.Lat, v, floatTolerance) { pokemon.Lat = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Lat") + } } } @@ -196,6 +206,9 @@ func (pokemon *Pokemon) SetLon(v float64) { if !floatAlmostEqual(pokemon.Lon, v, floatTolerance) { pokemon.Lon = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Lon") + } } } @@ -203,6 +216,9 @@ func (pokemon *Pokemon) SetPokemonId(v int16) { if pokemon.PokemonId != v { pokemon.PokemonId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "PokemonId") + } } } @@ -210,6 +226,9 @@ func (pokemon *Pokemon) SetForm(v null.Int) { if pokemon.Form != v { pokemon.Form = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Form") + } } } @@ -217,6 +236,9 @@ func (pokemon *Pokemon) SetCostume(v null.Int) { if pokemon.Costume != v { pokemon.Costume = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Costume") + } } } @@ -224,6 +246,9 @@ func (pokemon *Pokemon) SetGender(v null.Int) { if pokemon.Gender != v { pokemon.Gender = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Gender") + } } } @@ -231,6 +256,9 @@ func (pokemon *Pokemon) SetWeather(v null.Int) { if pokemon.Weather != v { pokemon.Weather = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Weather") + } } } @@ -238,6 +266,9 @@ func (pokemon *Pokemon) SetIsStrong(v null.Bool) { if pokemon.IsStrong != v { pokemon.IsStrong = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "IsStrong") + } } } @@ -245,6 +276,9 @@ func (pokemon *Pokemon) SetExpireTimestamp(v null.Int) { if pokemon.ExpireTimestamp != v { pokemon.ExpireTimestamp = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "ExpireTimestamp") + } } } @@ -252,6 +286,9 @@ func (pokemon *Pokemon) SetExpireTimestampVerified(v bool) { if pokemon.ExpireTimestampVerified != v { pokemon.ExpireTimestampVerified = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "ExpireTimestampVerified") + } } } @@ -259,6 +296,9 @@ func (pokemon *Pokemon) SetSeenType(v null.String) { if pokemon.SeenType != v { pokemon.SeenType = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "SeenType") + } } } @@ -273,6 +313,9 @@ func (pokemon *Pokemon) SetCellId(v null.Int) { if pokemon.CellId != v { pokemon.CellId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "CellId") + } } } @@ -280,6 +323,9 @@ func (pokemon *Pokemon) SetIsEvent(v int8) { if pokemon.IsEvent != v { pokemon.IsEvent = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "IsEvent") + } } } @@ -287,6 +333,9 @@ func (pokemon *Pokemon) SetShiny(v null.Bool) { if pokemon.Shiny != v { pokemon.Shiny = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Shiny") + } } } @@ -294,6 +343,9 @@ func (pokemon *Pokemon) SetCp(v null.Int) { if pokemon.Cp != v { pokemon.Cp = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Cp") + } } } @@ -301,6 +353,9 @@ func (pokemon *Pokemon) SetLevel(v null.Int) { if pokemon.Level != v { pokemon.Level = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Level") + } } } @@ -308,6 +363,9 @@ func (pokemon *Pokemon) SetMove1(v null.Int) { if pokemon.Move1 != v { pokemon.Move1 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Move1") + } } } @@ -315,6 +373,9 @@ func (pokemon *Pokemon) SetMove2(v null.Int) { if pokemon.Move2 != v { pokemon.Move2 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Move2") + } } } @@ -322,6 +383,9 @@ func (pokemon *Pokemon) SetHeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Height, v, floatTolerance) { pokemon.Height = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Height") + } } } @@ -329,6 +393,9 @@ func (pokemon *Pokemon) SetWeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Weight, v, floatTolerance) { pokemon.Weight = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Weight") + } } } @@ -336,6 +403,9 @@ func (pokemon *Pokemon) SetSize(v null.Int) { if pokemon.Size != v { pokemon.Size = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Size") + } } } @@ -343,6 +413,9 @@ func (pokemon *Pokemon) SetIsDitto(v bool) { if pokemon.IsDitto != v { pokemon.IsDitto = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "IsDitto") + } } } @@ -350,6 +423,9 @@ func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { if pokemon.DisplayPokemonId != v { pokemon.DisplayPokemonId = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "DisplayPokemonId") + } } } @@ -357,6 +433,9 @@ func (pokemon *Pokemon) SetPvp(v null.String) { if pokemon.Pvp != v { pokemon.Pvp = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Pvp") + } } } @@ -364,6 +443,9 @@ func (pokemon *Pokemon) SetCapture1(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture1, v, floatTolerance) { pokemon.Capture1 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Capture1") + } } } @@ -371,6 +453,9 @@ func (pokemon *Pokemon) SetCapture2(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture2, v, floatTolerance) { pokemon.Capture2 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Capture2") + } } } @@ -378,6 +463,9 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture3, v, floatTolerance) { pokemon.Capture3 = v pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Capture3") + } } } @@ -552,6 +640,9 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } } if pokemon.isNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } pvpField, pvpValue := "", "" if changePvpField { pvpField, pvpValue = "pvp, ", ":pvp, " @@ -578,6 +669,9 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po rows, rowsErr := res.RowsAffected() log.Debugf("Inserting pokemon [%d] after insert res = %d %v", pokemon.Id, rows, rowsErr) } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } pvpUpdate := "" if changePvpField { pvpUpdate = "pvp = :pvp, " @@ -650,6 +744,9 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po } updatePokemonStats(pokemon, areas, now) + if dbDebugEnabled { + pokemon.changedFields = pokemon.changedFields[:0] + } pokemon.newRecord = false // After saving, it's no longer a new record pokemon.ClearDirty() diff --git a/decoder/pokestop.go b/decoder/pokestop.go index c42200fc..deb829f6 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -68,8 +68,9 @@ type Pokestop struct { ShowcaseExpiry null.Int `db:"showcase_expiry" json:"showcase_expiry"` ShowcaseRankings null.String `db:"showcase_rankings" json:"showcase_rankings"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues PokestopOldValues `db:"-" json:"-"` // Old values for webhook comparison } @@ -164,6 +165,9 @@ func (p *Pokestop) SetId(v string) { if p.Id != v { p.Id = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Id") + } } } @@ -171,6 +175,9 @@ func (p *Pokestop) SetLat(v float64) { if !floatAlmostEqual(p.Lat, v, floatTolerance) { p.Lat = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Lat") + } } } @@ -178,6 +185,9 @@ func (p *Pokestop) SetLon(v float64) { if !floatAlmostEqual(p.Lon, v, floatTolerance) { p.Lon = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Lon") + } } } @@ -185,6 +195,9 @@ func (p *Pokestop) SetName(v null.String) { if p.Name != v { p.Name = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Name") + } } } @@ -192,6 +205,9 @@ func (p *Pokestop) SetUrl(v null.String) { if p.Url != v { p.Url = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Url") + } } } @@ -199,6 +215,9 @@ func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { if p.LureExpireTimestamp != v { p.LureExpireTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "LureExpireTimestamp") + } } } @@ -206,6 +225,9 @@ func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { if p.LastModifiedTimestamp != v { p.LastModifiedTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "LastModifiedTimestamp") + } } } @@ -213,6 +235,9 @@ func (p *Pokestop) SetEnabled(v null.Bool) { if p.Enabled != v { p.Enabled = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Enabled") + } } } @@ -220,6 +245,9 @@ func (p *Pokestop) SetQuestType(v null.Int) { if p.QuestType != v { p.QuestType = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestType") + } } } @@ -227,6 +255,9 @@ func (p *Pokestop) SetQuestTimestamp(v null.Int) { if p.QuestTimestamp != v { p.QuestTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestTimestamp") + } } } @@ -234,6 +265,9 @@ func (p *Pokestop) SetQuestTarget(v null.Int) { if p.QuestTarget != v { p.QuestTarget = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestTarget") + } } } @@ -241,6 +275,9 @@ func (p *Pokestop) SetQuestConditions(v null.String) { if p.QuestConditions != v { p.QuestConditions = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestConditions") + } } } @@ -248,6 +285,9 @@ func (p *Pokestop) SetQuestRewards(v null.String) { if p.QuestRewards != v { p.QuestRewards = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestRewards") + } } } @@ -255,6 +295,9 @@ func (p *Pokestop) SetQuestTemplate(v null.String) { if p.QuestTemplate != v { p.QuestTemplate = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestTemplate") + } } } @@ -262,6 +305,9 @@ func (p *Pokestop) SetQuestTitle(v null.String) { if p.QuestTitle != v { p.QuestTitle = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestTitle") + } } } @@ -269,6 +315,9 @@ func (p *Pokestop) SetQuestExpiry(v null.Int) { if p.QuestExpiry != v { p.QuestExpiry = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "QuestExpiry") + } } } @@ -276,6 +325,9 @@ func (p *Pokestop) SetCellId(v null.Int) { if p.CellId != v { p.CellId = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "CellId") + } } } @@ -283,6 +335,9 @@ func (p *Pokestop) SetDeleted(v bool) { if p.Deleted != v { p.Deleted = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Deleted") + } } } @@ -290,6 +345,9 @@ func (p *Pokestop) SetLureId(v int16) { if p.LureId != v { p.LureId = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "LureId") + } } } @@ -297,6 +355,9 @@ func (p *Pokestop) SetFirstSeenTimestamp(v int16) { if p.FirstSeenTimestamp != v { p.FirstSeenTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "FirstSeenTimestamp") + } } } @@ -304,6 +365,9 @@ func (p *Pokestop) SetSponsorId(v null.Int) { if p.SponsorId != v { p.SponsorId = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "SponsorId") + } } } @@ -311,6 +375,9 @@ func (p *Pokestop) SetPartnerId(v null.String) { if p.PartnerId != v { p.PartnerId = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "PartnerId") + } } } @@ -318,6 +385,9 @@ func (p *Pokestop) SetArScanEligible(v null.Int) { if p.ArScanEligible != v { p.ArScanEligible = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ArScanEligible") + } } } @@ -325,6 +395,9 @@ func (p *Pokestop) SetPowerUpLevel(v null.Int) { if p.PowerUpLevel != v { p.PowerUpLevel = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "PowerUpLevel") + } } } @@ -332,6 +405,9 @@ func (p *Pokestop) SetPowerUpPoints(v null.Int) { if p.PowerUpPoints != v { p.PowerUpPoints = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "PowerUpPoints") + } } } @@ -339,6 +415,9 @@ func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { if p.PowerUpEndTimestamp != v { p.PowerUpEndTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "PowerUpEndTimestamp") + } } } @@ -346,6 +425,9 @@ func (p *Pokestop) SetAlternativeQuestType(v null.Int) { if p.AlternativeQuestType != v { p.AlternativeQuestType = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestType") + } } } @@ -353,6 +435,9 @@ func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { if p.AlternativeQuestTimestamp != v { p.AlternativeQuestTimestamp = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestTimestamp") + } } } @@ -360,6 +445,9 @@ func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { if p.AlternativeQuestTarget != v { p.AlternativeQuestTarget = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestTarget") + } } } @@ -367,6 +455,9 @@ func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { if p.AlternativeQuestConditions != v { p.AlternativeQuestConditions = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestConditions") + } } } @@ -374,6 +465,9 @@ func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { if p.AlternativeQuestRewards != v { p.AlternativeQuestRewards = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestRewards") + } } } @@ -381,6 +475,9 @@ func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { if p.AlternativeQuestTemplate != v { p.AlternativeQuestTemplate = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestTemplate") + } } } @@ -388,6 +485,9 @@ func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { if p.AlternativeQuestTitle != v { p.AlternativeQuestTitle = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestTitle") + } } } @@ -395,6 +495,9 @@ func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { if p.AlternativeQuestExpiry != v { p.AlternativeQuestExpiry = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "AlternativeQuestExpiry") + } } } @@ -402,6 +505,9 @@ func (p *Pokestop) SetDescription(v null.String) { if p.Description != v { p.Description = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Description") + } } } @@ -409,6 +515,9 @@ func (p *Pokestop) SetShowcaseFocus(v null.String) { if p.ShowcaseFocus != v { p.ShowcaseFocus = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcaseFocus") + } } } @@ -416,6 +525,9 @@ func (p *Pokestop) SetShowcasePokemon(v null.Int) { if p.ShowcasePokemon != v { p.ShowcasePokemon = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcasePokemon") + } } } @@ -423,6 +535,9 @@ func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { if p.ShowcasePokemonForm != v { p.ShowcasePokemonForm = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcasePokemonForm") + } } } @@ -430,6 +545,9 @@ func (p *Pokestop) SetShowcasePokemonType(v null.Int) { if p.ShowcasePokemonType != v { p.ShowcasePokemonType = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcasePokemonType") + } } } @@ -437,6 +555,9 @@ func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { if p.ShowcaseRankingStandard != v { p.ShowcaseRankingStandard = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcaseRankingStandard") + } } } @@ -444,6 +565,9 @@ func (p *Pokestop) SetShowcaseExpiry(v null.Int) { if p.ShowcaseExpiry != v { p.ShowcaseExpiry = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcaseExpiry") + } } } @@ -451,6 +575,9 @@ func (p *Pokestop) SetShowcaseRankings(v null.String) { if p.ShowcaseRankings != v { p.ShowcaseRankings = v p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "ShowcaseRankings") + } } } @@ -1106,6 +1233,9 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop pokestop.Updated = now if pokestop.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Pokestop", pokestop.Id, pokestop.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO pokestop ( id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, @@ -1138,6 +1268,9 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop _ = res } else { // Existing record - UPDATE + if dbDebugEnabled { + dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE pokestop SET lat = :lat, @@ -1193,8 +1326,15 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop _ = res } //pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) - pokestop.newRecord = false // After saving, it's no longer a new record + if dbDebugEnabled { + pokestop.changedFields = pokestop.changedFields[:0] + } + if pokestop.IsNewRecord() { + pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + pokestop.newRecord = false + } pokestop.ClearDirty() + createPokestopWebhooks(pokestop) createPokestopFortWebhooks(pokestop) } diff --git a/decoder/routes.go b/decoder/routes.go index eb7565be..8407e963 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -41,8 +41,9 @@ type Route struct { Version int64 `db:"version"` Waypoints string `db:"waypoints"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) } // IsDirty returns true if any field has been modified @@ -66,6 +67,9 @@ func (r *Route) SetName(v string) { if r.Name != v { r.Name = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Name") + } } } @@ -73,6 +77,9 @@ func (r *Route) SetShortcode(v string) { if r.Shortcode != v { r.Shortcode = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Shortcode") + } } } @@ -80,6 +87,9 @@ func (r *Route) SetDescription(v string) { if r.Description != v { r.Description = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Description") + } } } @@ -87,6 +97,9 @@ func (r *Route) SetDistanceMeters(v int64) { if r.DistanceMeters != v { r.DistanceMeters = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "DistanceMeters") + } } } @@ -94,6 +107,9 @@ func (r *Route) SetDurationSeconds(v int64) { if r.DurationSeconds != v { r.DurationSeconds = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "DurationSeconds") + } } } @@ -101,6 +117,9 @@ func (r *Route) SetEndFortId(v string) { if r.EndFortId != v { r.EndFortId = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "EndFortId") + } } } @@ -108,6 +127,9 @@ func (r *Route) SetEndImage(v string) { if r.EndImage != v { r.EndImage = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "EndImage") + } } } @@ -115,6 +137,9 @@ func (r *Route) SetEndLat(v float64) { if !floatAlmostEqual(r.EndLat, v, floatTolerance) { r.EndLat = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "EndLat") + } } } @@ -122,6 +147,9 @@ func (r *Route) SetEndLon(v float64) { if !floatAlmostEqual(r.EndLon, v, floatTolerance) { r.EndLon = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "EndLon") + } } } @@ -129,6 +157,9 @@ func (r *Route) SetImage(v string) { if r.Image != v { r.Image = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Image") + } } } @@ -136,6 +167,9 @@ func (r *Route) SetImageBorderColor(v string) { if r.ImageBorderColor != v { r.ImageBorderColor = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "ImageBorderColor") + } } } @@ -143,6 +177,9 @@ func (r *Route) SetReversible(v bool) { if r.Reversible != v { r.Reversible = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Reversible") + } } } @@ -150,6 +187,9 @@ func (r *Route) SetStartFortId(v string) { if r.StartFortId != v { r.StartFortId = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "StartFortId") + } } } @@ -157,6 +197,9 @@ func (r *Route) SetStartImage(v string) { if r.StartImage != v { r.StartImage = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "StartImage") + } } } @@ -164,6 +207,9 @@ func (r *Route) SetStartLat(v float64) { if !floatAlmostEqual(r.StartLat, v, floatTolerance) { r.StartLat = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "StartLat") + } } } @@ -171,6 +217,9 @@ func (r *Route) SetStartLon(v float64) { if !floatAlmostEqual(r.StartLon, v, floatTolerance) { r.StartLon = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "StartLon") + } } } @@ -178,6 +227,9 @@ func (r *Route) SetTags(v null.String) { if r.Tags != v { r.Tags = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Tags") + } } } @@ -185,6 +237,9 @@ func (r *Route) SetType(v int8) { if r.Type != v { r.Type = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Type") + } } } @@ -192,6 +247,9 @@ func (r *Route) SetVersion(v int64) { if r.Version != v { r.Version = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Version") + } } } @@ -199,6 +257,9 @@ func (r *Route) SetWaypoints(v string) { if r.Waypoints != v { r.Waypoints = v r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Waypoints") + } } } @@ -241,6 +302,9 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { route.Updated = time.Now().Unix() if route.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Route", route.Id, route.changedFields) + } _, err := db.GeneralDb.NamedExec( ` INSERT INTO route ( @@ -270,6 +334,9 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { return fmt.Errorf("insert route error: %w", err) } } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) + } _, err := db.GeneralDb.NamedExec( ` UPDATE route SET @@ -304,9 +371,14 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { } } + if dbDebugEnabled { + route.changedFields = route.changedFields[:0] + } route.ClearDirty() - route.newRecord = false - //routeCache.Set(route.Id, route, ttlcache.DefaultTTL) + if route.IsNewRecord() { + routeCache.Set(route.Id, route, ttlcache.DefaultTTL) + route.newRecord = false + } return nil } diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 07c9c311..5dca4187 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -2,11 +2,14 @@ package decoder import ( "context" + "strconv" + "strings" "time" "golbat/db" "github.com/golang/geo/s2" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -49,6 +52,8 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { s2Cell.Latitude = mapS2Cell.CapBound().RectBound().Center().Lat.Degrees() s2Cell.Longitude = mapS2Cell.CapBound().RectBound().Center().Lng.Degrees() s2Cell.Level = null.IntFrom(int64(mapS2Cell.Level())) + + s2CellCache.Set(s2Cell.Id, s2Cell, ttlcache.DefaultTTL) } s2Cell.Updated = now @@ -59,6 +64,14 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { return } + if dbDebugEnabled { + var updatedCells []string + for _, s2cell := range outputCellIds { + updatedCells = append(updatedCells, strconv.FormatUint(s2cell.Id, 10)) + } + log.Debugf("[DB_S2CELL] Updated cells: %s", strings.Join(updatedCells, ",")) + } + // run bulk query _, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO s2cell (id, center_lat, center_lon, level, updated) diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index e4a70fcb..7639f4aa 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -3,11 +3,12 @@ package decoder import ( "context" "database/sql" - "golbat/db" - "golbat/pogo" "strconv" "time" + "golbat/db" + "golbat/pogo" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" @@ -194,8 +195,10 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi } spawnpoint.ClearDirty() - spawnpoint.newRecord = false - spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) + if spawnpoint.IsNewRecord() { + spawnpoint.newRecord = false + spawnpointCache.Set(spawnpoint.Id, spawnpoint, ttlcache.DefaultTTL) + } } func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { diff --git a/decoder/station.go b/decoder/station.go index ccbaef43..2dcb6815 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -13,6 +13,7 @@ import ( "golbat/util" "golbat/webhooks" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -50,8 +51,9 @@ type Station struct { TotalStationedGmax null.Int `db:"total_stationed_gmax"` StationedPokemon null.String `db:"stationed_pokemon"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues StationOldValues `db:"-" json:"-"` // Old values for webhook comparison } @@ -102,6 +104,9 @@ func (station *Station) SetId(v string) { if station.Id != v { station.Id = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Id") + } } } @@ -109,6 +114,9 @@ func (station *Station) SetLat(v float64) { if !floatAlmostEqual(station.Lat, v, floatTolerance) { station.Lat = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Lat") + } } } @@ -116,6 +124,9 @@ func (station *Station) SetLon(v float64) { if !floatAlmostEqual(station.Lon, v, floatTolerance) { station.Lon = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Lon") + } } } @@ -123,6 +134,9 @@ func (station *Station) SetName(v string) { if station.Name != v { station.Name = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Name") + } } } @@ -130,6 +144,9 @@ func (station *Station) SetCellId(v int64) { if station.CellId != v { station.CellId = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "CellId") + } } } @@ -137,6 +154,9 @@ func (station *Station) SetStartTime(v int64) { if station.StartTime != v { station.StartTime = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "StartTime") + } } } @@ -144,6 +164,9 @@ func (station *Station) SetEndTime(v int64) { if station.EndTime != v { station.EndTime = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "EndTime") + } } } @@ -151,6 +174,9 @@ func (station *Station) SetCooldownComplete(v int64) { if station.CooldownComplete != v { station.CooldownComplete = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "CooldownComplete") + } } } @@ -158,6 +184,9 @@ func (station *Station) SetIsBattleAvailable(v bool) { if station.IsBattleAvailable != v { station.IsBattleAvailable = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "IsBattleAvailable") + } } } @@ -165,6 +194,9 @@ func (station *Station) SetIsInactive(v bool) { if station.IsInactive != v { station.IsInactive = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "IsInactive") + } } } @@ -172,6 +204,9 @@ func (station *Station) SetBattleLevel(v null.Int) { if station.BattleLevel != v { station.BattleLevel = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattleLevel") + } } } @@ -179,6 +214,9 @@ func (station *Station) SetBattleStart(v null.Int) { if station.BattleStart != v { station.BattleStart = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattleStart") + } } } @@ -186,6 +224,9 @@ func (station *Station) SetBattleEnd(v null.Int) { if station.BattleEnd != v { station.BattleEnd = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattleEnd") + } } } @@ -193,6 +234,9 @@ func (station *Station) SetBattlePokemonId(v null.Int) { if station.BattlePokemonId != v { station.BattlePokemonId = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonId") + } } } @@ -200,6 +244,9 @@ func (station *Station) SetBattlePokemonForm(v null.Int) { if station.BattlePokemonForm != v { station.BattlePokemonForm = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonForm") + } } } @@ -207,6 +254,9 @@ func (station *Station) SetBattlePokemonCostume(v null.Int) { if station.BattlePokemonCostume != v { station.BattlePokemonCostume = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonCostume") + } } } @@ -214,6 +264,9 @@ func (station *Station) SetBattlePokemonGender(v null.Int) { if station.BattlePokemonGender != v { station.BattlePokemonGender = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonGender") + } } } @@ -221,6 +274,9 @@ func (station *Station) SetBattlePokemonAlignment(v null.Int) { if station.BattlePokemonAlignment != v { station.BattlePokemonAlignment = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonAlignment") + } } } @@ -228,6 +284,9 @@ func (station *Station) SetBattlePokemonBreadMode(v null.Int) { if station.BattlePokemonBreadMode != v { station.BattlePokemonBreadMode = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonBreadMode") + } } } @@ -235,6 +294,9 @@ func (station *Station) SetBattlePokemonMove1(v null.Int) { if station.BattlePokemonMove1 != v { station.BattlePokemonMove1 = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonMove1") + } } } @@ -242,6 +304,9 @@ func (station *Station) SetBattlePokemonMove2(v null.Int) { if station.BattlePokemonMove2 != v { station.BattlePokemonMove2 = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonMove2") + } } } @@ -249,6 +314,9 @@ func (station *Station) SetBattlePokemonStamina(v null.Int) { if station.BattlePokemonStamina != v { station.BattlePokemonStamina = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonStamina") + } } } @@ -256,6 +324,9 @@ func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { if !nullFloatAlmostEqual(station.BattlePokemonCpMultiplier, v, floatTolerance) { station.BattlePokemonCpMultiplier = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "BattlePokemonCpMultiplier") + } } } @@ -263,6 +334,9 @@ func (station *Station) SetTotalStationedPokemon(v null.Int) { if station.TotalStationedPokemon != v { station.TotalStationedPokemon = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "TotalStationedPokemon") + } } } @@ -270,6 +344,9 @@ func (station *Station) SetTotalStationedGmax(v null.Int) { if station.TotalStationedGmax != v { station.TotalStationedGmax = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "TotalStationedGmax") + } } } @@ -277,6 +354,9 @@ func (station *Station) SetStationedPokemon(v null.String) { if station.StationedPokemon != v { station.StationedPokemon = v station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "StationedPokemon") + } } } @@ -343,6 +423,9 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { station.Updated = now if station.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Station", station.Id, station.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO station (id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon) @@ -356,6 +439,9 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { } _, _ = res, err } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Station", station.Id, station.changedFields) + } res, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE station SET @@ -395,9 +481,14 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { _, _ = res, err } + if dbDebugEnabled { + station.changedFields = station.changedFields[:0] + } station.ClearDirty() - station.newRecord = false - //stationCache.Set(station.Id, station, ttlcache.DefaultTTL) + if station.IsNewRecord() { + stationCache.Set(station.Id, station, ttlcache.DefaultTTL) + station.newRecord = false + } createStationWebhooks(station) } diff --git a/decoder/tappable.go b/decoder/tappable.go index 95c8da4b..c497e06b 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -11,6 +11,7 @@ import ( "golbat/db" "golbat/pogo" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -31,8 +32,9 @@ type Tappable struct { ExpireTimestampVerified bool `db:"expire_timestamp_verified" json:"expire_timestamp_verified"` Updated int64 `db:"updated" json:"updated"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) } // IsDirty returns true if any field has been modified @@ -56,6 +58,9 @@ func (ta *Tappable) SetLat(v float64) { if !floatAlmostEqual(ta.Lat, v, floatTolerance) { ta.Lat = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Lat") + } } } @@ -63,6 +68,9 @@ func (ta *Tappable) SetLon(v float64) { if !floatAlmostEqual(ta.Lon, v, floatTolerance) { ta.Lon = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Lon") + } } } @@ -70,6 +78,9 @@ func (ta *Tappable) SetFortId(v null.String) { if ta.FortId != v { ta.FortId = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "FortId") + } } } @@ -77,6 +88,9 @@ func (ta *Tappable) SetSpawnId(v null.Int) { if ta.SpawnId != v { ta.SpawnId = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "SpawnId") + } } } @@ -84,6 +98,9 @@ func (ta *Tappable) SetType(v string) { if ta.Type != v { ta.Type = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Type") + } } } @@ -91,6 +108,9 @@ func (ta *Tappable) SetEncounter(v null.Int) { if ta.Encounter != v { ta.Encounter = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Encounter") + } } } @@ -98,6 +118,9 @@ func (ta *Tappable) SetItemId(v null.Int) { if ta.ItemId != v { ta.ItemId = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "ItemId") + } } } @@ -105,6 +128,9 @@ func (ta *Tappable) SetCount(v null.Int) { if ta.Count != v { ta.Count = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Count") + } } } @@ -112,6 +138,9 @@ func (ta *Tappable) SetExpireTimestamp(v null.Int) { if ta.ExpireTimestamp != v { ta.ExpireTimestamp = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "ExpireTimestamp") + } } } @@ -119,6 +148,9 @@ func (ta *Tappable) SetExpireTimestampVerified(v bool) { if ta.ExpireTimestampVerified != v { ta.ExpireTimestampVerified = v ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "ExpireTimestampVerified") + } } } @@ -254,6 +286,9 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap tappable.Updated = now if tappable.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) + } res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` INSERT INTO tappable ( id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated @@ -268,6 +303,9 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap } _ = res } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) + } res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` UPDATE tappable SET lat = :lat, @@ -290,9 +328,14 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap } _ = res } + if dbDebugEnabled { + tappable.changedFields = tappable.changedFields[:0] + } tappable.ClearDirty() - tappable.newRecord = false - //tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) + if tappable.IsNewRecord() { + tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) + tappable.newRecord = false + } } func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { diff --git a/decoder/weather.go b/decoder/weather.go index 8381de26..15001a5e 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -9,6 +9,7 @@ import ( "golbat/webhooks" "github.com/golang/geo/s2" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" ) @@ -340,6 +341,8 @@ func saveWeatherRecord(ctx context.Context, db db.DbDetails, weather *Weather) { } createWeatherWebhooks(weather) weather.ClearDirty() - weather.newRecord = false - //weatherCache.Set(weather.Id, weather, ttlcache.DefaultTTL) + if weather.IsNewRecord() { + weatherCache.Set(weather.Id, weather, ttlcache.DefaultTTL) + weather.newRecord = false + } } From 33e6e2e4acd2223a06723b907834bc27b4655074 Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 21:41:50 +0000 Subject: [PATCH 10/35] Add to cache on get --- decoder/spawnpoint.go | 2 ++ decoder/station.go | 1 + decoder/tappable.go | 2 ++ 3 files changed, 5 insertions(+) diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 7639f4aa..9d411a2f 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -124,6 +124,8 @@ func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int6 return &Spawnpoint{Id: spawnpointId}, err } + spawnpointCache.Set(spawnpointId, &spawnpoint, ttlcache.DefaultTTL) + return &spawnpoint, nil } diff --git a/decoder/station.go b/decoder/station.go index 2dcb6815..68a76ce5 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -406,6 +406,7 @@ func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (* if err != nil { return nil, err } + stationCache.Set(stationId, &station, ttlcache.DefaultTTL) station.snapshotOldValues() return &station, nil } diff --git a/decoder/tappable.go b/decoder/tappable.go index c497e06b..cf089a71 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -273,6 +273,8 @@ func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappab if err != nil { return nil, err } + + tappableCache.Set(id, &tappable, ttlcache.DefaultTTL) return &tappable, nil } From 7db7d36fcfe00bf733789d527bf9d3d68a5d368c Mon Sep 17 00:00:00 2001 From: James Berry Date: Wed, 28 Jan 2026 22:02:51 +0000 Subject: [PATCH 11/35] Add to cache on get incident --- decoder/incident.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/decoder/incident.go b/decoder/incident.go index dbc0b067..8eb65949 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -5,6 +5,7 @@ import ( "database/sql" "time" + "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" null "gopkg.in/guregu/null.v4" @@ -231,6 +232,7 @@ func getIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string) return nil, err } + incidentCache.Set(incidentId, &incident, ttlcache.DefaultTTL) incident.snapshotOldValues() return &incident, nil } From af8b3ef287914702d5e310526b0b30031793539b Mon Sep 17 00:00:00 2001 From: James Berry Date: Thu, 29 Jan 2026 07:11:36 +0000 Subject: [PATCH 12/35] Add weather to cache on read --- decoder/weather.go | 1 + 1 file changed, 1 insertion(+) diff --git a/decoder/weather.go b/decoder/weather.go index 15001a5e..7aab48a5 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -210,6 +210,7 @@ func getWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*W weather.UpdatedMs *= 1000 weather.snapshotOldValues() + weatherCache.Set(weatherId, &weather, ttlcache.DefaultTTL) return &weather, nil } From f89eab36d4bbc26c7e3809b43ef287828638439d Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 08:39:10 +0000 Subject: [PATCH 13/35] copilot comments --- decoder/gym.go | 142 +++++++++++++++++++++++--------------------- decoder/pokestop.go | 7 ++- decoder/station.go | 2 +- 3 files changed, 80 insertions(+), 71 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index b3bdcaf6..d89ff26a 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -68,7 +68,8 @@ type Gym struct { Defenders null.String `db:"defenders" json:"defenders"` Rsvps null.String `db:"rsvps" json:"rsvps"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving (to db) + internalDirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving (in memory only) newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) @@ -143,9 +144,15 @@ func (gym *Gym) IsDirty() bool { return gym.dirty } +// IsInternalDirty returns true if any field has been modified for in-memory +func (gym *Gym) IsInternalDirty() bool { + return gym.internalDirty +} + // ClearDirty resets the dirty flag (call after saving to DB) func (gym *Gym) ClearDirty() { gym.dirty = false + gym.internalDirty = false } // IsNewRecord returns true if this is a new record (not yet in DB) @@ -348,7 +355,7 @@ func (gym *Gym) SetInBattle(v null.Int) { if gym.InBattle != v { gym.InBattle = v //Do not set to dirty, as don't trigger an update - //gym.dirty = true + gym.internalDirty = true } } @@ -546,7 +553,7 @@ func (gym *Gym) SetDefenders(v null.String) { if gym.Defenders != v { gym.Defenders = v //Do not set to dirty, as don't trigger an update - //gym.dirty = true + gym.internalDirty = true } } @@ -1095,7 +1102,7 @@ func createGymWebhooks(gym *Gym, areas []geo.AreaName) { func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { now := time.Now().Unix() - if !gym.IsNewRecord() && !gym.IsDirty() { + if !gym.IsNewRecord() && !gym.IsDirty() && !gym.IsInternalDirty() { if gym.Updated > now-900 { // if a gym is unchanged, but we did see it again after 15 minutes, then save again return @@ -1103,71 +1110,73 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { } gym.Updated = now - if gym.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ - "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) + if gym.IsDirty() { + if gym.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ + "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) - statsCollector.IncDbQuery("insert gym", err) - if err != nil { - log.Errorf("insert gym: %s", err) - return - } + statsCollector.IncDbQuery("insert gym", err) + if err != nil { + log.Errorf("insert gym: %s", err) + return + } - _, _ = res, err - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ - "lat = :lat, "+ - "lon = :lon, "+ - "name = :name, "+ - "url = :url, "+ - "last_modified_timestamp = :last_modified_timestamp, "+ - "raid_end_timestamp = :raid_end_timestamp, "+ - "raid_spawn_timestamp = :raid_spawn_timestamp, "+ - "raid_battle_timestamp = :raid_battle_timestamp, "+ - "updated = :updated, "+ - "raid_pokemon_id = :raid_pokemon_id, "+ - "guarding_pokemon_id = :guarding_pokemon_id, "+ - "guarding_pokemon_display = :guarding_pokemon_display, "+ - "available_slots = :available_slots, "+ - "team_id = :team_id, "+ - "raid_level = :raid_level, "+ - "enabled = :enabled, "+ - "ex_raid_eligible = :ex_raid_eligible, "+ - "in_battle = :in_battle, "+ - "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ - "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ - "raid_pokemon_form = :raid_pokemon_form, "+ - "raid_pokemon_alignment = :raid_pokemon_alignment, "+ - "raid_pokemon_cp = :raid_pokemon_cp, "+ - "raid_is_exclusive = :raid_is_exclusive, "+ - "cell_id = :cell_id, "+ - "deleted = :deleted, "+ - "total_cp = :total_cp, "+ - "raid_pokemon_gender = :raid_pokemon_gender, "+ - "sponsor_id = :sponsor_id, "+ - "partner_id = :partner_id, "+ - "raid_pokemon_costume = :raid_pokemon_costume, "+ - "raid_pokemon_evolution = :raid_pokemon_evolution, "+ - "ar_scan_eligible = :ar_scan_eligible, "+ - "power_up_level = :power_up_level, "+ - "power_up_points = :power_up_points, "+ - "power_up_end_timestamp = :power_up_end_timestamp,"+ - "description = :description,"+ - "defenders = :defenders,"+ - "rsvps = :rsvps "+ - "WHERE id = :id", gym, - ) - statsCollector.IncDbQuery("update gym", err) - if err != nil { - log.Errorf("Update gym %s", err) + _, _ = res, err + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ + "lat = :lat, "+ + "lon = :lon, "+ + "name = :name, "+ + "url = :url, "+ + "last_modified_timestamp = :last_modified_timestamp, "+ + "raid_end_timestamp = :raid_end_timestamp, "+ + "raid_spawn_timestamp = :raid_spawn_timestamp, "+ + "raid_battle_timestamp = :raid_battle_timestamp, "+ + "updated = :updated, "+ + "raid_pokemon_id = :raid_pokemon_id, "+ + "guarding_pokemon_id = :guarding_pokemon_id, "+ + "guarding_pokemon_display = :guarding_pokemon_display, "+ + "available_slots = :available_slots, "+ + "team_id = :team_id, "+ + "raid_level = :raid_level, "+ + "enabled = :enabled, "+ + "ex_raid_eligible = :ex_raid_eligible, "+ + "in_battle = :in_battle, "+ + "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ + "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ + "raid_pokemon_form = :raid_pokemon_form, "+ + "raid_pokemon_alignment = :raid_pokemon_alignment, "+ + "raid_pokemon_cp = :raid_pokemon_cp, "+ + "raid_is_exclusive = :raid_is_exclusive, "+ + "cell_id = :cell_id, "+ + "deleted = :deleted, "+ + "total_cp = :total_cp, "+ + "raid_pokemon_gender = :raid_pokemon_gender, "+ + "sponsor_id = :sponsor_id, "+ + "partner_id = :partner_id, "+ + "raid_pokemon_costume = :raid_pokemon_costume, "+ + "raid_pokemon_evolution = :raid_pokemon_evolution, "+ + "ar_scan_eligible = :ar_scan_eligible, "+ + "power_up_level = :power_up_level, "+ + "power_up_points = :power_up_points, "+ + "power_up_end_timestamp = :power_up_end_timestamp,"+ + "description = :description,"+ + "defenders = :defenders,"+ + "rsvps = :rsvps "+ + "WHERE id = :id", gym, + ) + statsCollector.IncDbQuery("update gym", err) + if err != nil { + log.Errorf("Update gym %s", err) + } + _, _ = res, err } - _, _ = res, err } //gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) @@ -1183,7 +1192,6 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { gym.newRecord = false } gym.ClearDirty() - } func updateGymGetMapFortCache(gym *Gym, skipName bool) { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index deb829f6..b5139d98 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1207,7 +1207,7 @@ func createPokestopWebhooks(stop *Pokestop) { ArScanEligible: stop.ArScanEligible.ValueOrZero(), PowerUpLevel: stop.PowerUpLevel.ValueOrZero(), PowerUpPoints: stop.PowerUpPoints.ValueOrZero(), - PowerUpEndTimestamp: stop.PowerUpPoints.ValueOrZero(), + PowerUpEndTimestamp: stop.PowerUpEndTimestamp.ValueOrZero(), Updated: stop.Updated, ShowcaseFocus: stop.ShowcaseFocus, ShowcasePokemonId: stop.ShowcasePokemon, @@ -1329,14 +1329,15 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop if dbDebugEnabled { pokestop.changedFields = pokestop.changedFields[:0] } + + createPokestopWebhooks(pokestop) + createPokestopFortWebhooks(pokestop) if pokestop.IsNewRecord() { pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) pokestop.newRecord = false } pokestop.ClearDirty() - createPokestopWebhooks(pokestop) - createPokestopFortWebhooks(pokestop) } func updatePokestopGetMapFortCache(pokestop *Pokestop) { diff --git a/decoder/station.go b/decoder/station.go index 68a76ce5..029fe168 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -486,11 +486,11 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { station.changedFields = station.changedFields[:0] } station.ClearDirty() + createStationWebhooks(station) if station.IsNewRecord() { stationCache.Set(station.Id, station, ttlcache.DefaultTTL) station.newRecord = false } - createStationWebhooks(station) } func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, cellId uint64) *Station { From f4b37140da783757513edba2f2ebcfbfdff4f208 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 14:08:34 +0000 Subject: [PATCH 14/35] switch to internal pokemon lock --- decoder/api_pokemon.go | 42 ++++++---- decoder/api_pokemon_scan_v1.go | 6 +- decoder/api_pokemon_scan_v2.go | 40 +++++----- decoder/api_pokemon_scan_v3.go | 42 +++++----- decoder/main.go | 104 +++++++++++------------- decoder/pending_pokemon.go | 8 +- decoder/pokemon.go | 142 ++++++++++++++++++++++----------- decoder/sharded_cache.go | 10 +++ decoder/weather_iv.go | 20 ++--- 9 files changed, 230 insertions(+), 184 deletions(-) diff --git a/decoder/api_pokemon.go b/decoder/api_pokemon.go index c41393af..7e017fdd 100644 --- a/decoder/api_pokemon.go +++ b/decoder/api_pokemon.go @@ -83,9 +83,9 @@ func haversine(start, end geo.Location) float64 { return earthRadiusKm * c } -func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { +func SearchPokemon(request ApiPokemonSearch) ([]*ApiPokemonResult, error) { start := time.Now() - results := make([]*Pokemon, 0, request.Limit) + results := make([]uint64, 0, request.Limit) pokemonMatched := 0 if request.SearchIds == nil { @@ -109,6 +109,7 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { if maxDistance == 0 { maxDistance = 10 } + pokemonTree2.Nearby( rtree.BoxDist[float64, uint64]([2]float64{request.Center.Longitude, request.Center.Latitude}, [2]float64{request.Center.Longitude, request.Center.Latitude}, nil), func(min, max [2]float64, pokemonId uint64, dist float64) bool { @@ -128,15 +129,12 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { found := slices.Contains(request.SearchIds, pokemonLookupItem.PokemonLookup.PokemonId) if found { - if pokemonCacheEntry := getPokemonFromCache(pokemonId); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - results = append(results, pokemon) - pokemonMatched++ - - if pokemonMatched > maxPokemon { - log.Infof("SearchPokemon - result would exceed maximum size (%d), stopping scan", maxPokemon) - return false - } + results = append(results, pokemonId) + pokemonMatched++ + + if pokemonMatched > maxPokemon { + log.Infof("SearchPokemon - result would exceed maximum size (%d), stopping scan", maxPokemon) + return false } } @@ -145,15 +143,29 @@ func SearchPokemon(request ApiPokemonSearch) ([]*Pokemon, error) { ) log.Infof("SearchPokemon - scanned %d pokemon, total time %s, %d returned", pokemonScanned, time.Since(start), len(results)) - return results, nil + + apiResults := make([]*ApiPokemonResult, 0, len(results)) + + for _, encounterId := range results { + pokemon, unlock, _ := peekPokemonRecordReadOnly(encounterId) + if pokemon != nil { + apiPokemon := buildApiPokemonResult(pokemon) + apiResults = append(apiResults, &apiPokemon) + unlock() + } + } + pokemonMatched++ + + return apiResults, nil } // Get one result func GetOnePokemon(pokemonId uint64) *ApiPokemonResult { - if item := getPokemonFromCache(pokemonId); item != nil { - pokemon := item.Value() - apiPokemon := buildApiPokemonResult(pokemon) + item, unlock, _ := peekPokemonRecordReadOnly(pokemonId) + if item != nil { + apiPokemon := buildApiPokemonResult(item) + defer unlock() return &apiPokemon } return nil diff --git a/decoder/api_pokemon_scan_v1.go b/decoder/api_pokemon_scan_v1.go index c4baf317..a3de7d4b 100644 --- a/decoder/api_pokemon_scan_v1.go +++ b/decoder/api_pokemon_scan_v1.go @@ -227,10 +227,10 @@ func GetPokemonInArea(retrieveParameters ApiPokemonScan) []*ApiPokemonResult { results := make([]*ApiPokemonResult, 0, len(returnKeys)) for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { apiPokemon := buildApiPokemonResult(pokemon) + unlock() results = append(results, &apiPokemon) } } diff --git a/decoder/api_pokemon_scan_v2.go b/decoder/api_pokemon_scan_v2.go index 726d0f3f..ae029c90 100644 --- a/decoder/api_pokemon_scan_v2.go +++ b/decoder/api_pokemon_scan_v2.go @@ -103,16 +103,14 @@ func GetPokemonInArea2(retrieveParameters ApiPokemonScan2) []*ApiPokemonResult { startUnix := start.Unix() for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - - if pokemon.ExpireTimestamp.ValueOrZero() < startUnix { - continue + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { + if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { + apiPokemon := buildApiPokemonResult(pokemon) + results = append(results, &apiPokemon) } + unlock() - apiPokemon := buildApiPokemonResult(pokemon) - - results = append(results, &apiPokemon) } } @@ -185,22 +183,20 @@ func GrpcGetPokemonInArea2(retrieveParameters *pb.PokemonScanRequest) []*pb.Poke startUnix := start.Unix() for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - - if pokemon.ExpireTimestamp.ValueOrZero() < startUnix { - continue - } - - apiPokemon := pb.PokemonDetails{ - Id: pokemon.Id, - PokestopId: pokemon.PokestopId.Ptr(), - SpawnId: pokemon.SpawnId.Ptr(), - Lat: pokemon.Lat, - Lon: pokemon.Lon, + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { + if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { + apiPokemon := pb.PokemonDetails{ + Id: pokemon.Id, + PokestopId: pokemon.PokestopId.Ptr(), + SpawnId: pokemon.SpawnId.Ptr(), + Lat: pokemon.Lat, + Lon: pokemon.Lon, + } + results = append(results, &apiPokemon) } - results = append(results, &apiPokemon) + unlock() } } diff --git a/decoder/api_pokemon_scan_v3.go b/decoder/api_pokemon_scan_v3.go index a7c3b14d..6cb5e220 100644 --- a/decoder/api_pokemon_scan_v3.go +++ b/decoder/api_pokemon_scan_v3.go @@ -110,17 +110,13 @@ func GetPokemonInArea3(retrieveParameters ApiPokemonScan3) *PokemonScan3Result { startUnix := start.Unix() for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - - if pokemon.ExpireTimestamp.ValueOrZero() < startUnix { - examined-- - continue + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { + if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { + apiPokemon := buildApiPokemonResult(pokemon) + results = append(results, &apiPokemon) } - - apiPokemon := buildApiPokemonResult(pokemon) - - results = append(results, &apiPokemon) + unlock() } } @@ -204,22 +200,20 @@ func GrpcGetPokemonInArea3(retrieveParameters *pb.PokemonScanRequestV3) ([]*pb.P startUnix := start.Unix() for _, key := range returnKeys { - if pokemonCacheEntry := getPokemonFromCache(key); pokemonCacheEntry != nil { - pokemon := pokemonCacheEntry.Value() - - if pokemon.ExpireTimestamp.ValueOrZero() < startUnix { - continue - } - - apiPokemon := pb.PokemonDetails{ - Id: pokemon.Id, - PokestopId: pokemon.PokestopId.Ptr(), - SpawnId: pokemon.SpawnId.Ptr(), - Lat: pokemon.Lat, - Lon: pokemon.Lon, + pokemon, unlock, _ := peekPokemonRecordReadOnly(key) + if pokemon != nil { + if pokemon.ExpireTimestamp.ValueOrZero() > startUnix { + apiPokemon := pb.PokemonDetails{ + Id: pokemon.Id, + PokestopId: pokemon.PokestopId.Ptr(), + SpawnId: pokemon.SpawnId.Ptr(), + Lat: pokemon.Lat, + Lon: pokemon.Lon, + } + results = append(results, &apiPokemon) } - results = append(results, &apiPokemon) + unlock() } } diff --git a/decoder/main.go b/decoder/main.go index 6877aee4..897a7361 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -78,7 +78,6 @@ var pokestopStripedMutex = stripedmutex.New(1103) var stationStripedMutex = stripedmutex.New(1103) var tappableStripedMutex = intstripedmutex.New(563) var incidentStripedMutex = stripedmutex.New(157) -var pokemonStripedMutex = intstripedmutex.New(1103) var weatherStripedMutex = intstripedmutex.New(157) var routeStripedMutex = stripedmutex.New(157) @@ -101,18 +100,6 @@ func (cl *gohbemLogger) Print(message string) { log.Info("Gohbem - ", message) } -func setPokemonCache(key uint64, value *Pokemon, ttl time.Duration) { - pokemonCache.Set(key, value, ttl) -} - -func getPokemonFromCache(key uint64) *ttlcache.Item[uint64, *Pokemon] { - return pokemonCache.Get(key) -} - -func deletePokemonFromCache(key uint64) { - pokemonCache.Delete(key) -} - func initDataCache() { // Sharded caches for high-concurrency tables pokestopCache = NewShardedCache(ShardedCacheConfig[string, *Pokestop]{ @@ -399,76 +386,81 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca for _, wild := range wildPokemonList { encounterId := wild.Data.EncounterId - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() + // spawnpointUpdateFromWild doesn't need Pokemon lock spawnpointUpdateFromWild(ctx, db, wild.Data, wild.Timestamp) if scanParameters.ProcessWild { - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + // Use read-only getter - we're only checking if update is needed, then queuing + pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) if err != nil { - log.Errorf("getOrCreatePokemonRecord: %s", err) - } else { - updateTime := wild.Timestamp / 1000 - if pokemon.isNewRecord() || pokemon.wildSignificantUpdate(wild.Data, updateTime) { - // The sweeper will process it after timeout if no encounter arrives - pending := &PendingPokemon{ - EncounterId: encounterId, - WildPokemon: wild.Data, - CellId: int64(wild.Cell), - TimestampMs: wild.Timestamp, - UpdateTime: updateTime, - WeatherLookup: weatherLookup, - Username: username, - } - pokemonPendingQueue.AddPending(pending) + log.Errorf("getPokemonRecordReadOnly: %s", err) + continue + } + + updateTime := wild.Timestamp / 1000 + shouldQueue := pokemon == nil || pokemon.wildSignificantUpdate(wild.Data, updateTime) + + if unlock != nil { + unlock() + } + + if shouldQueue { + // The sweeper will process it after timeout if no encounter arrives + pending := &PendingPokemon{ + EncounterId: encounterId, + WildPokemon: wild.Data, + CellId: int64(wild.Cell), + TimestampMs: wild.Timestamp, + UpdateTime: updateTime, + WeatherLookup: weatherLookup, + Username: username, } + pokemonPendingQueue.AddPending(pending) } } - pokemonMutex.Unlock() } if scanParameters.ProcessNearby { for _, nearby := range nearbyPokemonList { encounterId := nearby.Data.EncounterId - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { log.Printf("getOrCreatePokemonRecord: %s", err) - } else { - updateTime := nearby.Timestamp / 1000 - if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { - pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) - } + continue + } + + updateTime := nearby.Timestamp / 1000 + if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { + pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) } - pokemonMutex.Unlock() + unlock() } } for _, mapPokemon := range mapPokemonList { encounterId := mapPokemon.Data.EncounterId - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { log.Printf("getOrCreatePokemonRecord: %s", err) - } else { - pokemon.updateFromMap(ctx, db, mapPokemon.Data, int64(mapPokemon.Cell), weatherLookup, mapPokemon.Timestamp, username) - storedDiskEncounter := diskEncounterCache.Get(encounterId) - if storedDiskEncounter != nil { - diskEncounter := storedDiskEncounter.Value() - diskEncounterCache.Delete(encounterId) - pokemon.updatePokemonFromDiskEncounterProto(ctx, db, diskEncounter, username) - //log.Infof("Processed stored disk encounter") - } - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, mapPokemon.Timestamp/1000) + continue } - pokemonMutex.Unlock() + + pokemon.updateFromMap(ctx, db, mapPokemon.Data, int64(mapPokemon.Cell), weatherLookup, mapPokemon.Timestamp, username) + storedDiskEncounter := diskEncounterCache.Get(encounterId) + if storedDiskEncounter != nil { + diskEncounter := storedDiskEncounter.Value() + diskEncounterCache.Delete(encounterId) + pokemon.updatePokemonFromDiskEncounterProto(ctx, db, diskEncounter, username) + //log.Infof("Processed stored disk encounter") + } + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, mapPokemon.Timestamp/1000) + + unlock() } } diff --git a/decoder/pending_pokemon.go b/decoder/pending_pokemon.go index 4a22f070..60c1153f 100644 --- a/decoder/pending_pokemon.go +++ b/decoder/pending_pokemon.go @@ -124,16 +124,12 @@ func (q *PokemonPendingQueue) StartSweeper(ctx context.Context, interval time.Du // processExpired handles pokemon that didn't receive an encounter within the timeout func (q *PokemonPendingQueue) processExpired(ctx context.Context, dbDetails db.DbDetails, expired []*PendingPokemon) { for _, p := range expired { - pokemonMutex, _ := pokemonStripedMutex.GetLock(p.EncounterId) - pokemonMutex.Lock() - processCtx, cancel := context.WithTimeout(ctx, 3*time.Second) - pokemon, err := getOrCreatePokemonRecord(processCtx, dbDetails, p.EncounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(processCtx, dbDetails, p.EncounterId) if err != nil { log.Errorf("getOrCreatePokemonRecord in sweeper: %s", err) cancel() - pokemonMutex.Unlock() continue } @@ -145,8 +141,8 @@ func (q *PokemonPendingQueue) processExpired(ctx context.Context, dbDetails db.D savePokemonRecordAsAtTime(processCtx, dbDetails, pokemon, false, true, true, p.UpdateTime) } + unlock() cancel() - pokemonMutex.Unlock() } } diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 9399bd2a..222b2fed 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -35,6 +35,8 @@ import ( // // FirstSeenTimestamp: This field is used in IsNewRecord. It should only be set in savePokemonRecord. type Pokemon struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id uint64 `db:"id" json:"id,string"` PokestopId null.String `db:"pokestop_id" json:"pokestop_id"` SpawnId null.Int `db:"spawn_id" json:"spawn_id"` @@ -170,6 +172,16 @@ func (pokemon *Pokemon) snapshotOldValues() { } } +// Lock acquires the Pokemon's mutex +func (pokemon *Pokemon) Lock() { + pokemon.mu.Lock() +} + +// Unlock releases the Pokemon's mutex +func (pokemon *Pokemon) Unlock() { + pokemon.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (pokemon *Pokemon) SetPokestopId(v null.String) { @@ -469,54 +481,90 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { } } -func getPokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, error) { - if db.UsePokemonCache { - inMemoryPokemon := getPokemonFromCache(encounterId) - if inMemoryPokemon != nil { - pokemon := inMemoryPokemon.Value() - pokemon.snapshotOldValues() // Snapshot for webhook comparison - return pokemon, nil - } +// peekPokemonRecordReadOnly acquires lock, does NOT take snapshot. +// Use for read-only checks which will not cause a backing database lookup +// Caller must use returned unlock function +func peekPokemonRecordReadOnly(encounterId uint64) (*Pokemon, func(), error) { + if item := pokemonCache.Get(encounterId); item != nil { + pokemon := item.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil } + + return nil, nil, nil +} + +// getPokemonRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks, but will cause a backing database lookup +// Caller MUST call returned unlock function. +func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + // If we are in-memory only, this is identical to peek if config.Config.PokemonMemoryOnly { - return nil, nil + return peekPokemonRecordReadOnly(encounterId) } - pokemon := Pokemon{} - err := db.PokemonDb.GetContext(ctx, &pokemon, + // Check cache first + if item := pokemonCache.Get(encounterId); item != nil { + pokemon := item.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil + } + + dbPokemon := Pokemon{} + err := db.PokemonDb.GetContext(ctx, &dbPokemon, "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) - statsCollector.IncDbQuery("select pokemon", err) - if err == sql.ErrNoRows { - return nil, nil + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil } - if err != nil { - return nil, err + return nil, nil, err } - pokemon.snapshotOldValues() // Snapshot for webhook comparison - if db.UsePokemonCache { - setPokemonCache(encounterId, &pokemon, ttlcache.DefaultTTL) + // Atomically cache the loaded Pokemon - if another goroutine raced us, + // we'll get their Pokemon and use that instead (ensuring same mutex) + pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + // Only called if key doesn't exist - our Pokemon wins + pokemonRtreeUpdatePokemonOnGet(&dbPokemon) + return &dbPokemon + }, ttlcache.DefaultTTL) + + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil +} + +// getPokemonRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Pokemon. +// Caller MUST call returned unlock function. +func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) + if err != nil || pokemon == nil { + return nil, nil, err } - pokemonRtreeUpdatePokemonOnGet(&pokemon) - return &pokemon, nil + pokemon.snapshotOldValues() + return pokemon, unlock, nil } -func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, error) { - pokemon, err := getPokemonRecord(ctx, db, encounterId) +// getOrCreatePokemonRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) if pokemon != nil || err != nil { - return pokemon, err + return pokemon, unlock, err } - pokemon = &Pokemon{Id: encounterId, newRecord: true} - if db.UsePokemonCache { - setPokemonCache(encounterId, pokemon, ttlcache.DefaultTTL) - } - return pokemon, nil + + // Create new Pokemon atomically - function only called if key doesn't exist + pokemon = pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + return &Pokemon{Id: encounterId, newRecord: true} + }, ttlcache.DefaultTTL) + + pokemon.Lock() + pokemon.snapshotOldValues() + return pokemon, func() { pokemon.Unlock() }, nil } // hasChangesPokemon compares two Pokemon structs @@ -662,7 +710,8 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po if err != nil { log.Errorf("insert pokemon: [%d] %s", pokemon.Id, err) log.Errorf("Full structure: %+v", pokemon) - deletePokemonFromCache(pokemon.Id) // Force reload of pokemon from database + pokemonCache.Delete(pokemon.Id) + // Force reload of pokemon from database return } @@ -718,7 +767,8 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po if err != nil { log.Errorf("Update pokemon [%d] %s", pokemon.Id, err) log.Errorf("Full structure: %+v", pokemon) - deletePokemonFromCache(pokemon.Id) // Force reload of pokemon from database + pokemonCache.Delete(pokemon.Id) + // Force reload of pokemon from database return } @@ -753,7 +803,7 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache if db.UsePokemonCache { - setPokemonCache(pokemon.Id, pokemon, pokemon.remainingDuration(now)) + pokemonCache.Set(pokemon.Id, pokemon, pokemon.remainingDuration(now)) } } @@ -1817,15 +1867,12 @@ func UpdatePokemonRecordWithEncounterProto(ctx context.Context, db db.DbDetails, pokemonPendingQueue.Remove(encounterId) } - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - defer pokemonMutex.Unlock() - - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { log.Errorf("Error pokemon [%d]: %s", encounterId, err) return fmt.Sprintf("Error finding pokemon %s", err) } + defer unlock() pokemon.updatePokemonFromEncounterProto(ctx, db, encounter, username, timestamp) savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, timestamp/1000) @@ -1843,21 +1890,22 @@ func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDeta encounterId := uint64(encounter.Pokemon.PokemonDisplay.DisplayId) - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - defer pokemonMutex.Unlock() - - pokemon, err := getPokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) if err != nil { log.Errorf("Error pokemon [%d]: %s", encounterId, err) return fmt.Sprintf("Error finding pokemon %s", err) } if pokemon == nil || pokemon.isNewRecord() { - // No pokemon found + // No pokemon found - unlock not set when pokemon is nil + if unlock != nil { + unlock() + } diskEncounterCache.Set(encounterId, encounter, ttlcache.DefaultTTL) return fmt.Sprintf("%d Disk encounter without previous GMO - Pokemon stored for later", encounterId) } + defer unlock() + pokemon.updatePokemonFromDiskEncounterProto(ctx, db, encounter, username) savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) // updateEncounterStats() should only be called for encounters, and called @@ -1870,15 +1918,13 @@ func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDeta func UpdatePokemonRecordWithTappableEncounter(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounter *pogo.TappableEncounterProto, username string, timestampMs int64) string { encounterId := request.GetEncounterId() - pokemonMutex, _ := pokemonStripedMutex.GetLock(encounterId) - pokemonMutex.Lock() - defer pokemonMutex.Unlock() - - pokemon, err := getOrCreatePokemonRecord(ctx, db, encounterId) + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) if err != nil { log.Errorf("Error pokemon [%d]: %s", encounterId, err) return fmt.Sprintf("Error finding pokemon %s", err) } + defer unlock() + pokemon.updatePokemonFromTappableEncounterProto(ctx, db, request, encounter, username, timestampMs) savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) // updateEncounterStats() should only be called for encounters, and called diff --git a/decoder/sharded_cache.go b/decoder/sharded_cache.go index b0bdecab..4ae9ea9f 100644 --- a/decoder/sharded_cache.go +++ b/decoder/sharded_cache.go @@ -96,6 +96,16 @@ func (sc *ShardedCache[K, V]) DeleteAll() { } } +// GetOrSetFunc atomically gets an existing item or creates and sets a new one. +// If key exists, returns existing value (createFunc NOT called). +// If key doesn't exist, calls createFunc to create value, sets it, and returns it. +// This prevents race conditions and avoids creating objects unnecessarily. +func (sc *ShardedCache[K, V]) GetOrSetFunc(key K, createFunc func() V, ttl time.Duration) V { + shard := sc.getShard(key) + item, _ := shard.GetOrSetFunc(key, createFunc, ttlcache.WithTTL[K, V](ttl)) + return item.Value() +} + // --- Key conversion helpers --- // Uint64KeyToShard is the identity function for uint64 keys diff --git a/decoder/weather_iv.go b/decoder/weather_iv.go index 8560c3ec..7a85d256 100644 --- a/decoder/weather_iv.go +++ b/decoder/weather_iv.go @@ -4,13 +4,14 @@ import ( "context" "encoding/json" "errors" - "golbat/db" - "golbat/pogo" "net/http" "os" "reflect" "time" + "golbat/db" + "golbat/pogo" + "github.com/golang/geo/s2" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" @@ -203,7 +204,7 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath pokemonLocked := 0 pokemonUpdated := 0 pokemonCpUpdated := 0 - var pokemon *Pokemon + //var pokemon *Pokemon pokemonTree2.Search([2]float64{cellLo.Lng.Degrees(), cellLo.Lat.Degrees()}, [2]float64{cellHi.Lng.Degrees(), cellHi.Lat.Degrees()}, func(min, max [2]float64, pokemonId uint64) bool { if !weatherCell.ContainsPoint(s2.PointFromLatLng(s2.LatLngFromDegrees(min[1], min[0]))) { return true @@ -224,13 +225,12 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath if int8(newWeather) == pokemonLookup.PokemonLookup.Weather { return true } - pokemonMutex, _ := pokemonStripedMutex.GetLock(pokemonId) - pokemonMutex.Lock() - pokemonLocked++ - pokemonEntry := getPokemonFromCache(pokemonId) - if pokemonEntry != nil { - pokemon = pokemonEntry.Value() + + pokemon, unlock, _ := peekPokemonRecordReadOnly(pokemonId) + if pokemon != nil { + pokemonLocked++ if pokemonLookup.PokemonLookup.PokemonId == pokemon.PokemonId && (pokemon.IsDitto || int64(pokemonLookup.PokemonLookup.Form) == pokemon.Form.ValueOrZero()) && int64(newWeather) != pokemon.Weather.ValueOrZero() && pokemon.ExpireTimestamp.ValueOrZero() >= startUnix && pokemon.Updated.ValueOrZero() < timestamp { + pokemon.snapshotOldValues() pokemon.repopulateIv(int64(newWeather), pokemon.IsStrong.ValueOrZero()) if !pokemon.Cp.Valid { pokemon.Weather = null.IntFrom(int64(newWeather)) @@ -244,8 +244,8 @@ func ProactiveIVSwitch(ctx context.Context, db db.DbDetails, weatherUpdate Weath } } } + unlock() } - pokemonMutex.Unlock() return true }) if pokemonCpUpdated > 0 { From 9660a71d4e07b04544a2303a1a2fcf4039c62700 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 14:51:49 +0000 Subject: [PATCH 15/35] optimise locking in getOrCreatePokemonRecord --- decoder/pokemon.go | 48 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 222b2fed..5bb753bb 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -494,6 +494,18 @@ func peekPokemonRecordReadOnly(encounterId uint64) (*Pokemon, func(), error) { return nil, nil, nil } +func loadPokemonFromDatabase(ctx context.Context, db db.DbDetails, encounterId uint64, pokemon *Pokemon) error { + err := db.PokemonDb.GetContext(ctx, pokemon, + "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ + "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ + "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ + "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ + "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) + statsCollector.IncDbQuery("select pokemon", err) + + return err +} + // getPokemonRecordReadOnly acquires lock but does NOT take snapshot. // Use for read-only checks, but will cause a backing database lookup // Caller MUST call returned unlock function. @@ -511,13 +523,7 @@ func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId } dbPokemon := Pokemon{} - err := db.PokemonDb.GetContext(ctx, &dbPokemon, - "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ - "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ - "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ - "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ - "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) - statsCollector.IncDbQuery("select pokemon", err) + err := loadPokemonFromDatabase(ctx, db, encounterId, &dbPokemon) if errors.Is(err, sql.ErrNoRows) { return nil, nil, nil } @@ -552,17 +558,31 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId // getOrCreatePokemonRecord gets existing or creates new, locked with snapshot. // Caller MUST call returned unlock function. func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { - pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) - if pokemon != nil || err != nil { - return pokemon, unlock, err - } - // Create new Pokemon atomically - function only called if key doesn't exist - pokemon = pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { return &Pokemon{Id: encounterId, newRecord: true} }, ttlcache.DefaultTTL) - pokemon.Lock() + + if config.Config.PokemonMemoryOnly { + pokemon.snapshotOldValues() + return pokemon, func() { pokemon.Unlock() }, nil + } + + if pokemon.newRecord { + // We should attempt to load from database + err := loadPokemonFromDatabase(ctx, db, encounterId, pokemon) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + pokemon.Unlock() + return nil, nil, err + } + } else { + // We loaded + pokemon.newRecord = false + } + } + pokemon.snapshotOldValues() return pokemon, func() { pokemon.Unlock() }, nil } From 9e24751c2be0b072be1ad9e62821730e3cd252e3 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 15:03:54 +0000 Subject: [PATCH 16/35] optimise locking in getOrCreatePokemonRecord --- decoder/pokemon.go | 6 ++++-- decoder/sharded_cache.go | 2 +- go.mod | 2 +- go.sum | 2 ++ 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 5bb753bb..30789340 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -560,9 +560,10 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { // Create new Pokemon atomically - function only called if key doesn't exist pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - return &Pokemon{Id: encounterId, newRecord: true} + p := &Pokemon{Id: encounterId, newRecord: true} + p.Lock() + return p }, ttlcache.DefaultTTL) - pokemon.Lock() if config.Config.PokemonMemoryOnly { pokemon.snapshotOldValues() @@ -580,6 +581,7 @@ func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId } else { // We loaded pokemon.newRecord = false + pokemonRtreeUpdatePokemonOnGet(pokemon) } } diff --git a/decoder/sharded_cache.go b/decoder/sharded_cache.go index 4ae9ea9f..dced7c27 100644 --- a/decoder/sharded_cache.go +++ b/decoder/sharded_cache.go @@ -121,6 +121,6 @@ func Int64KeyToShard(key int64) uint64 { // StringKeyToShard hashes string keys to uint64 for sharding using FNV-1a func StringKeyToShard(key string) uint64 { h := fnv.New64a() - h.Write([]byte(key)) + _, _ = h.Write([]byte(key)) return h.Sum64() } diff --git a/go.mod b/go.mod index 66ce0646..83952183 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/tidwall/rtree v1.10.0 github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f google.golang.org/grpc v1.74.2 - google.golang.org/protobuf v1.36.7 + google.golang.org/protobuf v1.36.11 gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) diff --git a/go.sum b/go.sum index 9f2f8889..955a7b27 100644 --- a/go.sum +++ b/go.sum @@ -447,6 +447,8 @@ google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9x google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= From ca4f0d6bbe0aced319d0763436b473a304470321 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 15:24:53 +0000 Subject: [PATCH 17/35] optimise locking in getOrCreatePokemonRecord --- decoder/pokemon.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 30789340..7e1651b2 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -560,11 +560,11 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { // Create new Pokemon atomically - function only called if key doesn't exist pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - p := &Pokemon{Id: encounterId, newRecord: true} - p.Lock() - return p + return &Pokemon{Id: encounterId, newRecord: true} }, ttlcache.DefaultTTL) + pokemon.Lock() + if config.Config.PokemonMemoryOnly { pokemon.snapshotOldValues() return pokemon, func() { pokemon.Unlock() }, nil From 08254456e5cfacf46580a1ad27fa317f1d766897 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 16:04:16 +0000 Subject: [PATCH 18/35] Pokestop to new locking model --- decoder/fort.go | 23 +++--- decoder/fortRtree.go | 5 +- decoder/fort_tracker.go | 10 ++- decoder/incident.go | 37 ++++++---- decoder/main.go | 17 ++--- decoder/pokemon.go | 9 ++- decoder/pokestop.go | 157 +++++++++++++++++++++++++++------------- routes.go | 9 ++- 8 files changed, 166 insertions(+), 101 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index 2855d149..9c308a66 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -97,8 +97,6 @@ func InitWebHookFortFromPokestop(stop *Pokestop) *FortWebhook { } func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []string, fortType FortType, change FortChange) { - var gyms []Gym - var stops []Pokestop if fortType == GYM { for _, id := range ids { gym, err := GetGymRecord(ctx, dbDetails, id) @@ -108,29 +106,26 @@ func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []strin if gym == nil { continue } - gyms = append(gyms, *gym) + fort := InitWebHookFortFromGym(gym) + CreateFortWebHooks(fort, &FortWebhook{}, change) } } if fortType == POKESTOP { for _, id := range ids { - stop, err := GetPokestopRecord(ctx, dbDetails, id) + stop, unlock, err := getPokestopRecordReadOnly(ctx, dbDetails, id) if err != nil { continue } if stop == nil { continue } - stops = append(stops, *stop) + + fort := InitWebHookFortFromPokestop(stop) + unlock() + + CreateFortWebHooks(fort, &FortWebhook{}, change) } } - for _, gym := range gyms { - fort := InitWebHookFortFromGym(&gym) - CreateFortWebHooks(fort, &FortWebhook{}, change) - } - for _, stop := range stops { - fort := InitWebHookFortFromPokestop(&stop) - CreateFortWebHooks(fort, &FortWebhook{}, change) - } } func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { @@ -149,7 +144,7 @@ func CreateFortWebHooks(old *FortWebhook, new *FortWebhook, change FortChange) { Old: old, } webhooksSender.AddMessage(webhooks.FortUpdate, hook, areas) - statsCollector.UpdateFortCount(areas, new.Type, "removal") + statsCollector.UpdateFortCount(areas, old.Type, "removal") } else if change == EDIT { areas := MatchStatsGeofence(new.Location.Latitude, new.Location.Longitude) var editTypes []string diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index ce949d00..8a52792b 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -46,7 +46,10 @@ func LoadAllPokestops(details db.DbDetails) { if err != nil { log.Fatalln(err) } - GetPokestopRecord(context.Background(), details, place.Id) + _, unlock, _ := getPokestopRecordReadOnly(context.Background(), details, place.Id) + if unlock != nil { + unlock() + } } log.Infof("Loaded %d pokestops [finished]", count) } diff --git a/decoder/fort_tracker.go b/decoder/fort_tracker.go index d9839e67..36103b5a 100644 --- a/decoder/fort_tracker.go +++ b/decoder/fort_tracker.go @@ -431,11 +431,13 @@ func clearGymWithLock(ctx context.Context, dbDetails db.DbDetails, gymId string, } } -// clearPokestopWithLock marks a pokestop as deleted while holding the striped mutex +// clearPokestopWithLock marks a pokestop as deleted while holding the object-level mutex func clearPokestopWithLock(ctx context.Context, dbDetails db.DbDetails, stopId string, cellId uint64, removeFromTracker bool) { - pokestopMutex, _ := pokestopStripedMutex.GetLock(stopId) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() + // Lock the pokestop if it exists in cache + pokestop, unlock, _ := PeekPokestopRecord(stopId) + if pokestop != nil { + defer unlock() + } pokestopCache.Delete(stopId) if err := db.ClearOldPokestops(ctx, dbDetails, []string{stopId}); err != nil { diff --git a/decoder/incident.go b/decoder/incident.go index 8eb65949..a432f2e5 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -281,12 +281,14 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident createIncidentWebhooks(ctx, db, incident) - stop, _ := GetPokestopRecord(ctx, db, incident.PokestopId) - if stop == nil { - stop = &Pokestop{} + var stopLat, stopLon float64 + stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) + if stop != nil { + stopLat, stopLon = stop.Lat, stop.Lon + unlock() } - areas := MatchStatsGeofence(stop.Lat, stop.Lon) + areas := MatchStatsGeofence(stopLat, stopLon) updateIncidentStats(incident, areas) incident.ClearDirty() @@ -299,14 +301,19 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Inci isNew := incident.IsNewRecord() if isNew || (old.ExpirationTime != incident.ExpirationTime || old.Character != incident.Character || old.Confirmed != incident.Confirmed || old.Slot1PokemonId != incident.Slot1PokemonId) { - stop, _ := GetPokestopRecord(ctx, db, incident.PokestopId) - if stop == nil { - stop = &Pokestop{} + var pokestopName, stopUrl string + var stopLat, stopLon float64 + var stopEnabled bool + stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) + if stop != nil { + pokestopName = stop.Name.ValueOrZero() + stopLat, stopLon = stop.Lat, stop.Lon + stopUrl = stop.Url.ValueOrZero() + stopEnabled = stop.Enabled.ValueOrZero() + unlock() } - - pokestopName := "Unknown" - if stop.Name.Valid { - pokestopName = stop.Name.String + if pokestopName == "" { + pokestopName = "Unknown" } var lineup []webhookLineup @@ -333,11 +340,11 @@ func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Inci incidentHook := IncidentWebhook{ Id: incident.Id, PokestopId: incident.PokestopId, - Latitude: stop.Lat, - Longitude: stop.Lon, + Latitude: stopLat, + Longitude: stopLon, PokestopName: pokestopName, - Url: stop.Url.ValueOrZero(), - Enabled: stop.Enabled.ValueOrZero(), + Url: stopUrl, + Enabled: stopEnabled, Start: incident.StartTime, IncidentExpireTimestamp: incident.ExpirationTime, Expiration: incident.ExpirationTime, diff --git a/decoder/main.go b/decoder/main.go index 897a7361..4d2324a6 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -74,7 +74,6 @@ var diskEncounterCache *ttlcache.Cache[uint64, *pogo.DiskEncounterOutProto] var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto] var gymStripedMutex = stripedmutex.New(1103) -var pokestopStripedMutex = stripedmutex.New(1103) var stationStripedMutex = stripedmutex.New(1103) var tappableStripedMutex = intstripedmutex.New(563) var incidentStripedMutex = stripedmutex.New(157) @@ -271,19 +270,12 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa for _, fort := range p { fortId := fort.Data.FortId if fort.Data.FortType == pogo.FortType_CHECKPOINT && scanParameters.ProcessPokestops { - pokestopMutex, _ := pokestopStripedMutex.GetLock(fortId) - - pokestopMutex.Lock() - pokestop, err := GetPokestopRecord(ctx, db, fortId) // should check error + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fortId) if err != nil { - log.Errorf("getPokestopRecord: %s", err) - pokestopMutex.Unlock() + log.Errorf("getOrCreatePokestopRecord: %s", err) continue } - if pokestop == nil { - pokestop = &Pokestop{newRecord: true} - } pokestop.updatePokestopFromFort(fort.Data, fort.Cell, fort.Timestamp/1000) // If this is a new pokestop, check if it was converted from a gym and copy shared fields @@ -295,6 +287,7 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa } savePokestopRecord(ctx, db, pokestop) + unlock() incidents := fort.Data.PokestopDisplays if incidents == nil && fort.Data.PokestopDisplay != nil { @@ -324,7 +317,6 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa incidentMutex.Unlock() } } - pokestopMutex.Unlock() } if fort.Data.FortType == pogo.FortType_GYM && scanParameters.ProcessGyms { @@ -346,9 +338,10 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa // If this is a new gym, check if it was converted from a pokestop and copy shared fields if gym.IsNewRecord() { - pokestop, _ := GetPokestopRecord(ctx, db, fortId) + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, fortId) if pokestop != nil { gym.copySharedFieldsFrom(pokestop) + unlock() } } diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 7e1651b2..e4c079bb 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -884,10 +884,11 @@ func createPokemonWebhooks(ctx context.Context, db db.DbDetails, pokemon *Pokemo var pokestopName *string if pokemon.PokestopId.Valid { - pokestop, _ := GetPokestopRecord(ctx, db, pokemon.PokestopId.String) + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokemon.PokestopId.String) name := "Unknown" if pokestop != nil { name = pokestop.Name.ValueOrZero() + unlock() } pokestopName = &name } @@ -1086,7 +1087,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP spawnpointId := mapPokemon.SpawnpointId - pokestop, _ := GetPokestopRecord(ctx, db, spawnpointId) + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, spawnpointId) if pokestop == nil { // Unrecognised pokestop return @@ -1095,6 +1096,7 @@ func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapP pokemon.SetLat(pokestop.Lat) pokemon.SetLon(pokestop.Lon) pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) + unlock() if mapPokemon.PokemonDisplay != nil { pokemon.setPokemonDisplay(int16(mapPokemon.PokedexTypeId), mapPokemon.PokemonDisplay) @@ -1149,7 +1151,7 @@ func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, n default: return } - pokestop, _ := GetPokestopRecord(ctx, db, pokestopId) + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokestopId) if pokestop == nil { // Unrecognised pokestop, rollback changes overrideLatLon = pokemon.isNewRecord() @@ -1158,6 +1160,7 @@ func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, n pokemon.SetPokestopId(null.StringFrom(pokestopId)) lat, lon = pokestop.Lat, pokestop.Lon useCellLatLon = false + unlock() } } if useCellLatLon { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index b5139d98..c0fd7141 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" "github.com/jellydator/ttlcache/v3" @@ -24,6 +25,8 @@ import ( // Pokestop struct. type Pokestop struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` Lon float64 `db:"lon" json:"lon"` @@ -159,6 +162,16 @@ func (p *Pokestop) snapshotOldValues() { } } +// Lock acquires the Pokestop's mutex +func (p *Pokestop) Lock() { + p.mu.Lock() +} + +// Unlock releases the Pokestop's mutex +func (p *Pokestop) Unlock() { + p.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (p *Pokestop) SetId(v string) { @@ -622,16 +635,8 @@ type PokestopWebhook struct { ShowcaseRankings json.RawMessage `json:"showcase_rankings"` } -func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, error) { - stop := pokestopCache.Get(fortId) - if stop != nil { - //log.Debugf("GetPokestopRecord %s (from cache)", fortId) - pokestop := stop.Value() - pokestop.snapshotOldValues() // Snapshot for webhook comparison - return pokestop, nil - } - pokestop := Pokestop{} - err := db.GeneralDb.GetContext(ctx, &pokestop, +func loadPokestopFromDatabase(ctx context.Context, db db.DbDetails, fortId string, pokestop *Pokestop) error { + err := db.GeneralDb.GetContext(ctx, pokestop, `SELECT pokestop.id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, pokestop.updated, quest_type, quest_timestamp, quest_target, quest_conditions, quest_rewards, quest_template, quest_title, @@ -643,22 +648,96 @@ func GetPokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Po showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings FROM pokestop WHERE pokestop.id = ? `, fortId) - //log.Debugf("GetPokestopRecord %s (from db)", fortId) - statsCollector.IncDbQuery("select pokestop", err) + return err +} + +// PeekPokestopRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekPokestopRecord(fortId string) (*Pokestop, func(), error) { + if item := pokestopCache.Get(fortId); item != nil { + pokestop := item.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil + } + return nil, nil, nil +} + +// getPokestopRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + // Check cache first + if item := pokestopCache.Get(fortId); item != nil { + pokestop := item.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil + } + + dbPokestop := Pokestop{} + err := loadPokestopFromDatabase(ctx, db, fortId, &dbPokestop) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil, nil } if err != nil { - return nil, err + return nil, nil, err } - pokestop.snapshotOldValues() // Snapshot for webhook comparison - pokestopCache.Set(fortId, &pokestop, ttlcache.DefaultTTL) - if config.Config.TestFortInMemory { - fortRtreeUpdatePokestopOnGet(&pokestop) + // Atomically cache the loaded Pokestop - if another goroutine raced us, + // we'll get their Pokestop and use that instead (ensuring same mutex) + pokestop := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + // Only called if key doesn't exist - our Pokestop wins + if config.Config.TestFortInMemory { + fortRtreeUpdatePokestopOnGet(&dbPokestop) + } + return &dbPokestop + }, ttlcache.DefaultTTL) + + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil +} + +// getPokestopRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Pokestop. +// Caller MUST call returned unlock function if non-nil. +func getPokestopRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + pokestop, unlock, err := getPokestopRecordReadOnly(ctx, db, fortId) + if err != nil || pokestop == nil { + return nil, nil, err } - return &pokestop, nil + pokestop.snapshotOldValues() + return pokestop, unlock, nil +} + +// getOrCreatePokestopRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + // Create new Pokestop atomically - function only called if key doesn't exist + pokestop := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + return &Pokestop{Id: fortId, newRecord: true} + }, ttlcache.DefaultTTL) + + pokestop.Lock() + + if pokestop.newRecord { + // We should attempt to load from database + err := loadPokestopFromDatabase(ctx, db, fortId, pokestop) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + pokestop.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + pokestop.newRecord = false + if config.Config.TestFortInMemory { + fortRtreeUpdatePokestopOnGet(pokestop) + } + } + } + + pokestop.snapshotOldValues() + return pokestop, func() { pokestop.Unlock() }, nil } var LureTime int64 = 1800 @@ -1351,19 +1430,13 @@ func updatePokestopGetMapFortCache(pokestop *Pokestop) { } func UpdatePokestopRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { - pokestopMutex, _ := pokestopStripedMutex.GetLock(fort.Id) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - - pokestop, err := GetPokestopRecord(ctx, db, fort.Id) // should check error + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fort.Id) if err != nil { log.Printf("Update pokestop %s", err) return fmt.Sprintf("Error %s", err) } + defer unlock() - if pokestop == nil { - pokestop = &Pokestop{newRecord: true} - } pokestop.updatePokestopFromFortDetailsProto(fort) updatePokestopGetMapFortCache(pokestop) @@ -1383,19 +1456,14 @@ func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.F } statsCollector.IncDecodeQuest("ok", haveArStr) - pokestopMutex, _ := pokestopStripedMutex.GetLock(quest.FortId) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - pokestop, err := GetPokestopRecord(ctx, db, quest.FortId) + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, quest.FortId) if err != nil { log.Printf("Update quest %s", err) return fmt.Sprintf("error %s", err) } + defer unlock() - if pokestop == nil { - pokestop = &Pokestop{newRecord: true} - } questTitle := pokestop.updatePokestopFromQuestProto(quest, haveAr) updatePokestopGetMapFortCache(pokestop) @@ -1428,11 +1496,7 @@ func GetQuestStatusWithGeofence(dbDetails db.DbDetails, geofence *geojson.Featur } func UpdatePokestopRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { - pokestopMutex, _ := pokestopStripedMutex.GetLock(mapFort.Id) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - - pokestop, err := GetPokestopRecord(ctx, db, mapFort.Id) + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, mapFort.Id) if err != nil { log.Printf("Update pokestop %s", err) return false, fmt.Sprintf("Error %s", err) @@ -1441,6 +1505,7 @@ func UpdatePokestopRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDe if pokestop == nil { return false, "" } + defer unlock() pokestop.updatePokestopFromGetMapFortsOutProto(mapFort) savePokestopRecord(ctx, db, pokestop) @@ -1474,11 +1539,7 @@ func UpdatePokestopWithContestData(ctx context.Context, db db.DbDetails, request contest := contestData.ContestIncident.Contests[0] - pokestopMutex, _ := pokestopStripedMutex.GetLock(fortId) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - - pokestop, err := GetPokestopRecord(ctx, db, fortId) + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) if err != nil { log.Printf("Get pokestop %s", err) return "Error getting pokestop" @@ -1488,6 +1549,7 @@ func UpdatePokestopWithContestData(ctx context.Context, db db.DbDetails, request log.Infof("Contest data for pokestop %s not found", fortId) return fmt.Sprintf("Contest data for pokestop %s not found", fortId) } + defer unlock() pokestop.updatePokestopFromGetContestDataOutProto(contest) savePokestopRecord(ctx, db, pokestop) @@ -1502,11 +1564,7 @@ func getFortIdFromContest(id string) string { func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDetails, request *pogo.GetPokemonSizeLeaderboardEntryProto, contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) string { fortId := getFortIdFromContest(request.GetContestId()) - pokestopMutex, _ := pokestopStripedMutex.GetLock(fortId) - pokestopMutex.Lock() - defer pokestopMutex.Unlock() - - pokestop, err := GetPokestopRecord(ctx, db, fortId) + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) if err != nil { log.Printf("Get pokestop %s", err) return "Error getting pokestop" @@ -1516,6 +1574,7 @@ func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDeta log.Infof("Contest data for pokestop %s not found", fortId) return fmt.Sprintf("Contest data for pokestop %s not found", fortId) } + defer unlock() pokestop.updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData) savePokestopRecord(ctx, db, pokestop) diff --git a/routes.go b/routes.go index ca840480..ff1fda73 100644 --- a/routes.go +++ b/routes.go @@ -489,9 +489,12 @@ func GetPokestopPositions(c *gin.Context) { func GetPokestop(c *gin.Context) { fortId := c.Param("fort_id") - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - pokestop, err := decoder.GetPokestopRecord(ctx, dbDetails, fortId) - cancel() + //ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + pokestop, unlock, err := decoder.PeekPokestopRecord(fortId) + if unlock != nil { + defer unlock() + } + //cancel() if err != nil { log.Warnf("GET /api/pokestop/id/:fort_id/ Error during post retrieve %v", err) c.Status(http.StatusInternalServerError) From 642eb3288d5d842daa9e867202b4a4ca54ff4eef Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 16:39:22 +0000 Subject: [PATCH 19/35] Gym locking model changes --- decoder/fort.go | 5 +- decoder/fortRtree.go | 5 +- decoder/fort_tracker.go | 10 ++- decoder/gym.go | 175 ++++++++++++++++++++++++++++----------- decoder/main.go | 18 ++-- decoder/pokemon.go | 10 ++- decoder/pokestop.go | 10 ++- decoder/sharded_cache.go | 15 ++-- 8 files changed, 165 insertions(+), 83 deletions(-) diff --git a/decoder/fort.go b/decoder/fort.go index 9c308a66..979bc2e1 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -99,14 +99,17 @@ func InitWebHookFortFromPokestop(stop *Pokestop) *FortWebhook { func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []string, fortType FortType, change FortChange) { if fortType == GYM { for _, id := range ids { - gym, err := GetGymRecord(ctx, dbDetails, id) + gym, unlock, err := getGymRecordReadOnly(ctx, dbDetails, id) if err != nil { continue } if gym == nil { continue } + fort := InitWebHookFortFromGym(gym) + unlock() + CreateFortWebHooks(fort, &FortWebhook{}, change) } } diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 8a52792b..14222171 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -71,7 +71,10 @@ func LoadAllGyms(details db.DbDetails) { if err != nil { log.Fatalln(err) } - GetGymRecord(context.Background(), details, place.Id) + _, unlock, _ := getGymRecordReadOnly(context.Background(), details, place.Id) + if unlock != nil { + unlock() + } } log.Infof("Loaded %d gyms [finished]", count) } diff --git a/decoder/fort_tracker.go b/decoder/fort_tracker.go index 36103b5a..b3a2ae0d 100644 --- a/decoder/fort_tracker.go +++ b/decoder/fort_tracker.go @@ -409,11 +409,13 @@ func GetFortTracker() *FortTracker { return fortTracker } -// clearGymWithLock marks a gym as deleted while holding the striped mutex +// clearGymWithLock marks a gym as deleted while holding the object-level mutex func clearGymWithLock(ctx context.Context, dbDetails db.DbDetails, gymId string, cellId uint64, removeFromTracker bool) { - gymMutex, _ := gymStripedMutex.GetLock(gymId) - gymMutex.Lock() - defer gymMutex.Unlock() + // Lock the gym if it exists in cache + gym, unlock, _ := PeekGymRecord(gymId) + if gym != nil { + defer unlock() + } gymCache.Delete(gymId) if err := db.ClearOldGyms(ctx, dbDetails, []string{gymId}); err != nil { diff --git a/decoder/gym.go b/decoder/gym.go index d89ff26a..261cfff6 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -5,9 +5,11 @@ import ( "context" "database/sql" "encoding/json" + "errors" "fmt" "slices" "strings" + "sync" "time" "golbat/geo" @@ -26,6 +28,8 @@ import ( // Gym struct. // REMINDER! Keep hasChangesGym updated after making changes type Gym struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` Lon float64 `db:"lon" json:"lon"` @@ -179,6 +183,16 @@ func (gym *Gym) snapshotOldValues() { } } +// Lock acquires the Gym's mutex +func (gym *Gym) Lock() { + gym.mu.Lock() +} + +// Unlock releases the Gym's mutex +func (gym *Gym) Unlock() { + gym.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (gym *Gym) SetId(v string) { @@ -567,31 +581,116 @@ func (gym *Gym) SetRsvps(v null.String) { } } -func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, error) { - inMemoryGym := gymCache.Get(fortId) - if inMemoryGym != nil { - gym := inMemoryGym.Value() - gym.snapshotOldValues() // Snapshot for webhook comparison - return gym, nil +func loadGymFromDatabase(ctx context.Context, db db.DbDetails, fortId string, gym *Gym) error { + err := db.GeneralDb.GetContext(ctx, gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) + statsCollector.IncDbQuery("select gym", err) + return err +} + +// PeekGymRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekGymRecord(fortId string) (*Gym, func(), error) { + if item := gymCache.Get(fortId); item != nil { + gym := item.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil } - gym := Gym{} - err := db.GeneralDb.GetContext(ctx, &gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) + return nil, nil, nil +} - statsCollector.IncDbQuery("select gym", err) - if err == sql.ErrNoRows { - return nil, nil +// getGymRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getGymRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + // Check cache first + if item := gymCache.Get(fortId); item != nil { + gym := item.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil } + dbGym := Gym{} + err := loadGymFromDatabase(ctx, db, fortId, &dbGym) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return nil, err + return nil, nil, err + } + + // Atomically cache the loaded Gym - if another goroutine raced us, + // we'll get their Gym and use that instead (ensuring same mutex) + existingGym, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { + // Only called if key doesn't exist - our Pokestop wins + if config.Config.TestFortInMemory { + fortRtreeUpdateGymOnGet(&dbGym) + } + return &dbGym + }) + + gym := existingGym.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil +} + +// getGymRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Gym. +// Caller MUST call returned unlock function if non-nil. +func getGymRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + gym, unlock, err := getGymRecordReadOnly(ctx, db, fortId) + if err != nil || gym == nil { + return nil, nil, err + } + gym.snapshotOldValues() + return gym, unlock, nil +} + +// getOrCreateGymRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + // Create new Gym atomically - function only called if key doesn't exist + gymItem, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { + return &Gym{Id: fortId, newRecord: true} + }) + + gym := gymItem.Value() + gym.Lock() + + if gym.newRecord { + // We should attempt to load from database + err := loadGymFromDatabase(ctx, db, fortId, gym) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + gym.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + gym.newRecord = false + if config.Config.TestFortInMemory { + fortRtreeUpdateGymOnGet(gym) + } + } } - gym.snapshotOldValues() // Snapshot for webhook comparison - gymCache.Set(fortId, &gym, ttlcache.DefaultTTL) - if config.Config.TestFortInMemory { - fortRtreeUpdateGymOnGet(&gym) + gym.snapshotOldValues() + return gym, func() { gym.Unlock() }, nil +} + +// GetGymRecord returns a copy of the Gym for external/API use. +// For internal use, prefer getGymRecordReadOnly or getGymRecordForUpdate. +func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, error) { + gym, unlock, err := getGymRecordReadOnly(ctx, db, fortId) + if err != nil { + return nil, err + } + if gym == nil { + return nil, nil } - return &gym, nil + // Make a copy for safe external use + gymCopy := *gym + unlock() + return &gymCopy, nil } func escapeLike(s string) string { @@ -1205,18 +1304,12 @@ func updateGymGetMapFortCache(gym *Gym, skipName bool) { } func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { - gymMutex, _ := gymStripedMutex.GetLock(fort.Id) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, fort.Id) // should check error + gym, unlock, err := getOrCreateGymRecord(ctx, db, fort.Id) if err != nil { return err.Error() } + defer unlock() - if gym == nil { - gym = &Gym{newRecord: true} - } gym.updateGymFromFortProto(fort) updateGymGetMapFortCache(gym, true) @@ -1226,18 +1319,12 @@ func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails } func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymInfo *pogo.GymGetInfoOutProto) string { - gymMutex, _ := gymStripedMutex.GetLock(gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) // should check error + gym, unlock, err := getOrCreateGymRecord(ctx, db, gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) if err != nil { return err.Error() } + defer unlock() - if gym == nil { - gym = &Gym{newRecord: true} - } gym.updateGymFromGymInfoOutProto(gymInfo) updateGymGetMapFortCache(gym, true) @@ -1246,11 +1333,7 @@ func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymIn } func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { - gymMutex, _ := gymStripedMutex.GetLock(mapFort.Id) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, mapFort.Id) + gym, unlock, err := getGymRecordForUpdate(ctx, db, mapFort.Id) if err != nil { return false, err.Error() } @@ -1259,6 +1342,7 @@ func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails if gym == nil { return false, "" } + defer unlock() gym.updateGymFromGetMapFortsOutProto(mapFort, false) saveGymRecord(ctx, db, gym) @@ -1266,11 +1350,7 @@ func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails } func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pogo.RaidDetails, resp *pogo.GetEventRsvpsOutProto) string { - gymMutex, _ := gymStripedMutex.GetLock(req.FortId) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, req.FortId) + gym, unlock, err := getGymRecordForUpdate(ctx, db, req.FortId) if err != nil { return err.Error() } @@ -1279,6 +1359,8 @@ func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pog // Do not add RSVP details to unknown gyms return fmt.Sprintf("%s Gym not present", req.FortId) } + defer unlock() + gym.updateGymFromRsvpProto(resp) saveGymRecord(ctx, db, gym) @@ -1287,11 +1369,7 @@ func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pog } func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { - gymMutex, _ := gymStripedMutex.GetLock(fortId) - gymMutex.Lock() - defer gymMutex.Unlock() - - gym, err := GetGymRecord(ctx, db, fortId) + gym, unlock, err := getGymRecordForUpdate(ctx, db, fortId) if err != nil { return err.Error() } @@ -1300,6 +1378,7 @@ func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { // Do not add RSVP details to unknown gyms return fmt.Sprintf("%s Gym not present", fortId) } + defer unlock() if gym.Rsvps.Valid { gym.SetRsvps(null.NewString("", false)) diff --git a/decoder/main.go b/decoder/main.go index 4d2324a6..c66c8ea7 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -73,7 +73,6 @@ var routeCache *ttlcache.Cache[string, *Route] var diskEncounterCache *ttlcache.Cache[uint64, *pogo.DiskEncounterOutProto] var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto] -var gymStripedMutex = stripedmutex.New(1103) var stationStripedMutex = stripedmutex.New(1103) var tappableStripedMutex = intstripedmutex.New(563) var incidentStripedMutex = stripedmutex.New(157) @@ -280,9 +279,10 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa // If this is a new pokestop, check if it was converted from a gym and copy shared fields if pokestop.IsNewRecord() { - gym, _ := GetGymRecord(ctx, db, fortId) + gym, gymUnlock, _ := getGymRecordReadOnly(ctx, db, fortId) if gym != nil { pokestop.copySharedFieldsFrom(gym) + gymUnlock() } } @@ -320,20 +320,12 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa } if fort.Data.FortType == pogo.FortType_GYM && scanParameters.ProcessGyms { - gymMutex, _ := gymStripedMutex.GetLock(fortId) - - gymMutex.Lock() - gym, err := GetGymRecord(ctx, db, fortId) + gym, gymUnlock, err := getOrCreateGymRecord(ctx, db, fortId) if err != nil { - log.Errorf("GetGymRecord: %s", err) - gymMutex.Unlock() + log.Errorf("getOrCreateGymRecord: %s", err) continue } - if gym == nil { - gym = &Gym{newRecord: true} - } - gym.updateGymFromFort(fort.Data, fort.Cell) // If this is a new gym, check if it was converted from a pokestop and copy shared fields @@ -346,7 +338,7 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa } saveGymRecord(ctx, db, gym) - gymMutex.Unlock() + gymUnlock() } } } diff --git a/decoder/pokemon.go b/decoder/pokemon.go index e4c079bb..2518a237 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -533,12 +533,13 @@ func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId // Atomically cache the loaded Pokemon - if another goroutine raced us, // we'll get their Pokemon and use that instead (ensuring same mutex) - pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + existingPokemon, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { // Only called if key doesn't exist - our Pokemon wins pokemonRtreeUpdatePokemonOnGet(&dbPokemon) return &dbPokemon - }, ttlcache.DefaultTTL) + }) + pokemon := existingPokemon.Value() pokemon.Lock() return pokemon, func() { pokemon.Unlock() }, nil } @@ -559,10 +560,11 @@ func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId // Caller MUST call returned unlock function. func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { // Create new Pokemon atomically - function only called if key doesn't exist - pokemon := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + pokemonItem, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { return &Pokemon{Id: encounterId, newRecord: true} - }, ttlcache.DefaultTTL) + }) + pokemon := pokemonItem.Value() pokemon.Lock() if config.Config.PokemonMemoryOnly { diff --git a/decoder/pokestop.go b/decoder/pokestop.go index c0fd7141..d4afe735 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -685,14 +685,15 @@ func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId stri // Atomically cache the loaded Pokestop - if another goroutine raced us, // we'll get their Pokestop and use that instead (ensuring same mutex) - pokestop := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + existingPokestop, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { // Only called if key doesn't exist - our Pokestop wins if config.Config.TestFortInMemory { fortRtreeUpdatePokestopOnGet(&dbPokestop) } return &dbPokestop - }, ttlcache.DefaultTTL) + }) + pokestop := existingPokestop.Value() pokestop.Lock() return pokestop, func() { pokestop.Unlock() }, nil } @@ -713,10 +714,11 @@ func getPokestopRecordForUpdate(ctx context.Context, db db.DbDetails, fortId str // Caller MUST call returned unlock function. func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { // Create new Pokestop atomically - function only called if key doesn't exist - pokestop := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + pokestopItem, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { return &Pokestop{Id: fortId, newRecord: true} - }, ttlcache.DefaultTTL) + }) + pokestop := pokestopItem.Value() pokestop.Lock() if pokestop.newRecord { diff --git a/decoder/sharded_cache.go b/decoder/sharded_cache.go index dced7c27..84777838 100644 --- a/decoder/sharded_cache.go +++ b/decoder/sharded_cache.go @@ -96,14 +96,13 @@ func (sc *ShardedCache[K, V]) DeleteAll() { } } -// GetOrSetFunc atomically gets an existing item or creates and sets a new one. -// If key exists, returns existing value (createFunc NOT called). -// If key doesn't exist, calls createFunc to create value, sets it, and returns it. -// This prevents race conditions and avoids creating objects unnecessarily. -func (sc *ShardedCache[K, V]) GetOrSetFunc(key K, createFunc func() V, ttl time.Duration) V { - shard := sc.getShard(key) - item, _ := shard.GetOrSetFunc(key, createFunc, ttlcache.WithTTL[K, V](ttl)) - return item.Value() +// GetOrSetFunc retrieves an item from the cache by the provided key. +// If the element is not found, it is created by executing the fn function +// with the provided options and then returned. +// The bool return value is true if the item was found, false if created +// during the execution of the method. +func (sc *ShardedCache[K, V]) GetOrSetFunc(key K, createFunc func() V, opts ...ttlcache.Option[K, V]) (*ttlcache.Item[K, V], bool) { + return sc.getShard(key).GetOrSetFunc(key, createFunc, opts...) } // --- Key conversion helpers --- From c56b3baafb314f093c2b0954f4c4c23826c42282 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 17:26:53 +0000 Subject: [PATCH 20/35] copilot review suggestion fixes --- decoder/api_pokemon.go | 1 - decoder/incident.go | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/decoder/api_pokemon.go b/decoder/api_pokemon.go index 7e017fdd..6a414eb6 100644 --- a/decoder/api_pokemon.go +++ b/decoder/api_pokemon.go @@ -154,7 +154,6 @@ func SearchPokemon(request ApiPokemonSearch) ([]*ApiPokemonResult, error) { unlock() } } - pokemonMatched++ return apiResults, nil } diff --git a/decoder/incident.go b/decoder/incident.go index a432f2e5..c94dd598 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -292,8 +292,10 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident updateIncidentStats(incident, areas) incident.ClearDirty() - incident.newRecord = false - //incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) + if incident.IsNewRecord() { + incident.newRecord = false + incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) + } } func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Incident) { From a28a03979798f51cab71ae8e85cae99915367559 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 18:10:17 +0000 Subject: [PATCH 21/35] update modules --- go.mod | 78 +++++++------ go.sum | 362 ++++++++++++++++++++------------------------------------- 2 files changed, 163 insertions(+), 277 deletions(-) diff --git a/go.mod b/go.mod index 83952183..fc946286 100644 --- a/go.mod +++ b/go.mod @@ -5,32 +5,32 @@ go 1.25 toolchain go1.25.0 require ( - github.com/Depado/ginprom v1.8.1 + github.com/Depado/ginprom v1.8.2 github.com/UnownHash/gohbem v0.12.0 - github.com/getsentry/sentry-go v0.35.1 - github.com/gin-gonic/gin v1.10.1 + github.com/getsentry/sentry-go v0.42.0 + github.com/gin-gonic/gin v1.11.0 github.com/go-sql-driver/mysql v1.9.3 github.com/goccy/go-json v0.10.5 - github.com/golang-migrate/migrate/v4 v4.18.3 - github.com/golang/geo v0.0.0-20250813021530-247f39904721 - github.com/grafana/pyroscope-go v1.2.4 + github.com/golang-migrate/migrate/v4 v4.19.1 + github.com/golang/geo v0.0.0-20260129164528-943061e2742c + github.com/grafana/pyroscope-go v1.2.7 github.com/jellydator/ttlcache/v3 v3.4.0 github.com/jmoiron/sqlx v1.4.0 github.com/knadh/koanf/maps v0.1.2 github.com/knadh/koanf/parsers/toml v0.1.0 - github.com/knadh/koanf/providers/file v1.2.0 + github.com/knadh/koanf/providers/file v1.2.1 github.com/knadh/koanf/providers/structs v1.0.0 - github.com/knadh/koanf/v2 v2.2.2 + github.com/knadh/koanf/v2 v2.3.2 github.com/nmvalera/striped-mutex v0.1.0 - github.com/paulmach/orb v0.11.1 - github.com/prometheus/client_golang v1.23.0 + github.com/paulmach/orb v0.12.0 + github.com/prometheus/client_golang v1.23.2 github.com/puzpuzpuz/xsync/v3 v3.5.1 - github.com/ringsaturn/tzf v1.0.0 - github.com/ringsaturn/tzf-rel v0.0.2025-b - github.com/sirupsen/logrus v1.9.3 + github.com/ringsaturn/tzf v1.0.3 + github.com/ringsaturn/tzf-rel v0.0.2025-c + github.com/sirupsen/logrus v1.9.4 github.com/tidwall/rtree v1.10.0 github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f - google.golang.org/grpc v1.74.2 + google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -39,23 +39,23 @@ require ( require ( filippo.io/edwards25519 v1.1.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bytedance/sonic v1.14.0 // indirect - github.com/bytedance/sonic/loader v0.3.0 // indirect + github.com/bytedance/gopkg v0.1.3 // indirect + github.com/bytedance/sonic v1.15.0 // indirect + github.com/bytedance/sonic/loader v0.5.0 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cloudwego/base64x v0.1.6 // indirect github.com/fatih/structs v1.1.0 // indirect github.com/fsnotify/fsnotify v1.9.0 // indirect - github.com/gabriel-vasile/mimetype v1.4.9 // indirect + github.com/gabriel-vasile/mimetype v1.4.12 // indirect github.com/gin-contrib/sse v1.1.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.27.0 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect - github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect - github.com/hashicorp/errwrap v1.1.0 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/go-playground/validator/v10 v10.30.1 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/goccy/go-yaml v1.19.2 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.18.0 // indirect + github.com/klauspost/compress v1.18.3 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect @@ -68,22 +68,24 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.4 // indirect github.com/prometheus/client_model v0.6.2 // indirect - github.com/prometheus/common v0.65.0 // indirect - github.com/prometheus/procfs v0.17.0 // indirect - github.com/ringsaturn/tzf-rel-lite v0.0.2025-b // indirect + github.com/prometheus/common v0.67.5 // indirect + github.com/prometheus/procfs v0.19.2 // indirect + github.com/quic-go/qpack v0.6.0 // indirect + github.com/quic-go/quic-go v0.59.0 // indirect + github.com/ringsaturn/tzf-rel-lite v0.0.2025-c // indirect github.com/tidwall/geoindex v1.7.0 // indirect - github.com/tidwall/geojson v1.4.5 // indirect + github.com/tidwall/geojson v1.4.6 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/twpayne/go-polyline v1.1.1 // indirect - github.com/ugorji/go/codec v1.3.0 // indirect - go.mongodb.org/mongo-driver v1.17.4 // indirect - go.uber.org/atomic v1.11.0 // indirect - golang.org/x/arch v0.20.0 // indirect - golang.org/x/crypto v0.41.0 // indirect - golang.org/x/net v0.43.0 // indirect - golang.org/x/sync v0.16.0 // indirect - golang.org/x/sys v0.35.0 // indirect - golang.org/x/text v0.28.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/ugorji/go/codec v1.3.1 // indirect + go.mongodb.org/mongo-driver v1.17.8 // indirect + go.uber.org/mock v0.6.0 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect + golang.org/x/arch v0.23.0 // indirect + golang.org/x/crypto v0.47.0 // indirect + golang.org/x/net v0.49.0 // indirect + golang.org/x/sync v0.19.0 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/text v0.33.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 // indirect ) diff --git a/go.sum b/go.sum index 955a7b27..a16aadf3 100644 --- a/go.sum +++ b/go.sum @@ -2,46 +2,40 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= -github.com/Depado/ginprom v1.8.1 h1:lrQTddbRqlHq1j6SpJDySDumJlR7FEybzdX0PS3HXPc= -github.com/Depado/ginprom v1.8.1/go.mod h1:9Z+ahPJLSeMndDfnDTfiuBn2SKVAuL2yvihApWzof9A= +github.com/Depado/ginprom v1.8.2 h1:H3sXqXlHfXpoUHciuWSbod1jzc9OyaZ4edM5oYL/nUI= +github.com/Depado/ginprom v1.8.2/go.mod h1:uq9dl4TqwBr0OpkvswJURh5fmjZcbrrMoDiDFHN8dMw= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= github.com/UnownHash/gohbem v0.12.0 h1:eSyioEWJSU/81i6wf5x4XaiRZBXm6dW/KuYiHKjcELI= github.com/UnownHash/gohbem v0.12.0/go.mod h1:PUeicvRH6HyjTgkuaivjYHzDUzErf2QlsXZ24m0DaNU= -github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= -github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= +github.com/appleboy/gofight/v2 v2.2.0 h1:uqQ3wzTlF1ma+r4jRCQ4cygCjrGZyZEBMBCjT/t9zRw= +github.com/appleboy/gofight/v2 v2.2.0/go.mod h1:USTV3UbA5kHBs4I91EsPi+6PIVZAx3KLorYjvtON91A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/bytedance/sonic v1.12.9 h1:Od1BvK55NnewtGaJsTDeAOSnLVO2BTSLOe0+ooKokmQ= -github.com/bytedance/sonic v1.12.9/go.mod h1:uVvFidNmlt9+wa31S1urfwwthTWteBgG0hWuoKAXTx8= -github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= -github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= -github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQQ= -github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA= -github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= -github.com/bytedance/sonic/loader v0.2.3 h1:yctD0Q3v2NOGfSWPLPvG2ggA2kV6TS6s4wioyEqssH0= -github.com/bytedance/sonic/loader v0.2.3/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= -github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= -github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA= -github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= +github.com/bytedance/gopkg v0.1.3 h1:TPBSwH8RsouGCBcMBktLt1AymVo2TVsBVCY4b6TnZ/M= +github.com/bytedance/gopkg v0.1.3/go.mod h1:576VvJ+eJgyCzdjS+c4+77QF3p7ubbtiKARP3TxducM= +github.com/bytedance/sonic v1.15.0 h1:/PXeWFaR5ElNcVE84U0dOHjiMHQOwNIx3K4ymzh/uSE= +github.com/bytedance/sonic v1.15.0/go.mod h1:tFkWrPz0/CUCLEF4ri4UkHekCIcdnkqXw9VduqpJh0k= +github.com/bytedance/sonic/loader v0.5.0 h1:gXH3KVnatgY7loH5/TkeVyXPfESoqSBSBEiDd5VjlgE= +github.com/bytedance/sonic/loader v0.5.0/go.mod h1:AR4NYCk5DdzZizZ5djGqQ92eEhCCcdf5x77udYiSJRo= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= -github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M= github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU= -github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dhui/dktest v0.4.4 h1:+I4s6JRE1yGuqflzwqG+aIaMdgXIorCf5P98JnaAWa8= -github.com/dhui/dktest v0.4.4/go.mod h1:4+22R4lgsdAXrDyaH4Nqx2JEz2hLp49MqQmm9HLCQhM= -github.com/dhui/dktest v0.4.5 h1:uUfYBIVREmj/Rw6MvgmqNAYzTiKOHJak+enB5Di73MM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4= +github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/docker/docker v27.2.0+incompatible h1:Rk9nIVdfH3+Vz4cyI/uhbINhEZ/oLmc+CBXmH6fbNk4= -github.com/docker/docker v27.2.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI= +github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c= github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= @@ -51,30 +45,20 @@ github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M= -github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k= github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0= -github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= -github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= -github.com/gabriel-vasile/mimetype v1.4.9 h1:5k+WDwEsD9eTLL8Tz3L0VnmVh9QxGjRmjBvAG7U/oYY= -github.com/gabriel-vasile/mimetype v1.4.9/go.mod h1:WnSQhFKJuBlRyLiKohA/2DtIlPFAbguNaG7QCHcyGok= -github.com/getsentry/sentry-go v0.31.1 h1:ELVc0h7gwyhnXHDouXkhqTFSO5oslsRDk0++eyE0KJ4= -github.com/getsentry/sentry-go v0.31.1/go.mod h1:CYNcMMz73YigoHljQRG+qPF+eMq8gG72XcGN/p71BAY= -github.com/getsentry/sentry-go v0.35.1 h1:iopow6UVLE2aXu46xKVIs8Z9D/YZkJrHkgozrxa+tOQ= -github.com/getsentry/sentry-go v0.35.1/go.mod h1:C55omcY9ChRQIUcVcGcs+Zdy4ZpQGvNJ7JYHIoSWOtE= -github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= -github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= +github.com/gabriel-vasile/mimetype v1.4.12 h1:e9hWvmLYvtp846tLHam2o++qitpguFiYCKbn0w9jyqw= +github.com/gabriel-vasile/mimetype v1.4.12/go.mod h1:d+9Oxyo1wTzWdyVUPMmXFvp4F9tea18J8ufA774AB3s= +github.com/getsentry/sentry-go v0.42.0 h1:eeFMACuZTbUQf90RE8dE4tXeSe4CZyfvR1MBL7RLEt8= +github.com/getsentry/sentry-go v0.42.0/go.mod h1:eRXCoh3uvmjQLY6qu63BjUZnaBu5L5WhMV1RwYO8W5s= github.com/gin-contrib/sse v1.1.0 h1:n0w2GMuUpWDVp7qSpvze6fAu9iRxJY4Hmj6AmBOU05w= github.com/gin-contrib/sse v1.1.0/go.mod h1:hxRZ5gVpWMT7Z0B0gSNYqqsSCNIJMjzvm6fqCz9vjwM= -github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= -github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= -github.com/gin-gonic/gin v1.10.1 h1:T0ujvqyCSqRopADpgPgiTT63DUQVSfojyME59Ei63pQ= -github.com/gin-gonic/gin v1.10.1/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= +github.com/gin-gonic/gin v1.11.0 h1:OW/6PLjyusp2PPXtyxKHU0RbX6I/l28FTdDlae5ueWk= +github.com/gin-gonic/gin v1.11.0/go.mod h1:+iq/FyxlGzII0KHiBGjuNn4UNENUlKbGlNmc+W50Dls= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= @@ -83,64 +67,38 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.25.0 h1:5Dh7cjvzR7BRZadnsVOzPhWsrwUr0nmsZJxEAnFLNO8= -github.com/go-playground/validator/v10 v10.25.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus= -github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= -github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= -github.com/go-playground/validator/v10 v10.27.0 h1:w8+XrWVMhGkxOaaowyKH35gFydVHOvC0/uWoy2Fzwn4= -github.com/go-playground/validator/v10 v10.27.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= +github.com/go-playground/validator/v10 v10.30.1 h1:f3zDSN/zOma+w6+1Wswgd9fLkdwy06ntQJp0BBvFG0w= +github.com/go-playground/validator/v10 v10.30.1/go.mod h1:oSuBIQzuJxL//3MelwSLD5hc2Tu889bF0Idm9Dg26cM= github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg= -github.com/go-sql-driver/mysql v1.9.0 h1:Y0zIbQXhQKmQgTp44Y1dp3wTXcn804QoTptLZT1vtvo= -github.com/go-sql-driver/mysql v1.9.0/go.mod h1:pDetrLJeA3oMujJuvXc8RJoasr589B6A9fwzD3QMrqw= -github.com/go-sql-driver/mysql v1.9.1 h1:FrjNGn/BsJQjVRuSa8CBrM5BWA9BWoXXat3KrtSb/iI= -github.com/go-sql-driver/mysql v1.9.1/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= -github.com/go-viper/mapstructure/v2 v2.2.1 h1:ZAaOCxANMuZx5RCeg0mBdEZk7DZasvvZIxtHqx8aGss= -github.com/go-viper/mapstructure/v2 v2.2.1/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= -github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= -github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-yaml v1.19.2 h1:PmFC1S6h8ljIz6gMRBopkjP1TVT7xuwrButHID66PoM= +github.com/goccy/go-yaml v1.19.2/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-migrate/migrate/v4 v4.18.2 h1:2VSCMz7x7mjyTXx3m2zPokOY82LTRgxK1yQYKo6wWQ8= -github.com/golang-migrate/migrate/v4 v4.18.2/go.mod h1:2CM6tJvn2kqPXwnXO/d3rAQYiyoIm180VsO8PRX6Rpk= -github.com/golang-migrate/migrate/v4 v4.18.3 h1:EYGkoOsvgHHfm5U/naS1RP/6PL/Xv3S4B/swMiAmDLs= -github.com/golang-migrate/migrate/v4 v4.18.3/go.mod h1:99BKpIi6ruaaXRM1A77eqZ+FWPQ3cfRa+ZVy5bmWMaY= -github.com/golang/geo v0.0.0-20230421003525-6adc56603217 h1:HKlyj6in2JV6wVkmQ4XmG/EIm+SCYlPZ+V4GWit7Z+I= -github.com/golang/geo v0.0.0-20230421003525-6adc56603217/go.mod h1:8wI0hitZ3a1IxZfeH3/5I97CI8i5cLGsYe7xNhQGs9U= -github.com/golang/geo v0.0.0-20250328065203-0b6e08c212fb h1:eqdj1jSZjgmPdSl2lr3rAwJykSe9jHxPN1zLuasKVh0= -github.com/golang/geo v0.0.0-20250328065203-0b6e08c212fb/go.mod h1:J+F9/3Ofc8ysEOY2/cNjxTMl2eB1gvPIywEHUplPgDA= -github.com/golang/geo v0.0.0-20250813021530-247f39904721 h1:Hlto+T7Ba4CJM4SN8WiA9mw3MdMUboxWsWBaUzRuJuA= -github.com/golang/geo v0.0.0-20250813021530-247f39904721/go.mod h1:AN0OjM34c3PbjAsX+QNma1nYtJtRxl+s9MZNV7S+efw= +github.com/golang-migrate/migrate/v4 v4.19.1 h1:OCyb44lFuQfYXYLx1SCxPZQGU7mcaZ7gH9yH4jSFbBA= +github.com/golang-migrate/migrate/v4 v4.19.1/go.mod h1:CTcgfjxhaUtsLipnLoQRWCrjYXycRz/g5+RWDuYgPrE= +github.com/golang/geo v0.0.0-20260129164528-943061e2742c h1:ysO2h2Odnl1AJM1I2Lm/fa6JvO0pECMSt2CwBaa+ITo= +github.com/golang/geo v0.0.0-20260129164528-943061e2742c/go.mod h1:Mymr9kRGDc64JPr03TSZmuIBODZ3KyswLzm1xL0HFA8= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= -github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= -github.com/grafana/pyroscope-go v1.2.1 h1:ewi38pE6XMnoHlZYhGxS3uH5TGKA7vDhkT1T3RVkjq0= -github.com/grafana/pyroscope-go v1.2.1/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= -github.com/grafana/pyroscope-go v1.2.4 h1:B22GMXz+O0nWLatxLuaP7o7L9dvP0clLvIpmeEQQM0Q= -github.com/grafana/pyroscope-go v1.2.4/go.mod h1:zzT9QXQAp2Iz2ZdS216UiV8y9uXJYQiGE1q8v1FyhqU= -github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= -github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= -github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= -github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= -github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= -github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= -github.com/jellydator/ttlcache/v3 v3.3.0 h1:BdoC9cE81qXfrxeb9eoJi9dWrdhSuwXMAnHTbnBm4Wc= -github.com/jellydator/ttlcache/v3 v3.3.0/go.mod h1:bj2/e0l4jRnQdrnSTaGTsh4GSXvMjQcy41i7th0GVGw= +github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac= +github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= @@ -150,34 +108,20 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= -github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= -github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= -github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= -github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= -github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y= github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= -github.com/knadh/koanf/maps v0.1.1 h1:G5TjmUh2D7G2YWf5SQQqSiHRJEjaicvU0KpypqB3NIs= -github.com/knadh/koanf/maps v0.1.1/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpbo= github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI= github.com/knadh/koanf/parsers/toml v0.1.0 h1:S2hLqS4TgWZYj4/7mI5m1CQQcWurxUz6ODgOub/6LCI= github.com/knadh/koanf/parsers/toml v0.1.0/go.mod h1:yUprhq6eo3GbyVXFFMdbfZSo928ksS+uo0FFqNMnO18= -github.com/knadh/koanf/providers/file v1.1.2 h1:aCC36YGOgV5lTtAFz2qkgtWdeQsgfxUkxDOe+2nQY3w= -github.com/knadh/koanf/providers/file v1.1.2/go.mod h1:/faSBcv2mxPVjFrXck95qeoyoZ5myJ6uxN8OOVNJJCI= -github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U= -github.com/knadh/koanf/providers/file v1.2.0/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= -github.com/knadh/koanf/providers/structs v0.1.0 h1:wJRteCNn1qvLtE5h8KQBvLJovidSdntfdyIbbCzEyE0= -github.com/knadh/koanf/providers/structs v0.1.0/go.mod h1:sw2YZ3txUcqA3Z27gPlmmBzWn1h8Nt9O6EP/91MkcWE= +github.com/knadh/koanf/providers/file v1.2.1 h1:bEWbtQwYrA+W2DtdBrQWyXqJaJSG3KrP3AESOJYp9wM= +github.com/knadh/koanf/providers/file v1.2.1/go.mod h1:bp1PM5f83Q+TOUu10J/0ApLBd9uIzg+n9UgthfY+nRA= github.com/knadh/koanf/providers/structs v1.0.0 h1:DznjB7NQykhqCar2LvNug3MuxEQsZ5KvfgMbio+23u4= github.com/knadh/koanf/providers/structs v1.0.0/go.mod h1:kjo5TFtgpaZORlpoJqcbeLowM2cINodv8kX+oFAeQ1w= -github.com/knadh/koanf/v2 v2.1.2 h1:I2rtLRqXRy1p01m/utEtpZSSA6dcJbgGVuE27kW2PzQ= -github.com/knadh/koanf/v2 v2.1.2/go.mod h1:Gphfaen0q1Fc1HTgJgSTC4oRX9R2R5ErYMZJy8fLJBo= -github.com/knadh/koanf/v2 v2.2.2 h1:ghbduIkpFui3L587wavneC9e3WIliCgiCgdxYO/wd7A= -github.com/knadh/koanf/v2 v2.2.2/go.mod h1:abWQc0cBXLSF/PSOMCB/SK+T13NXDsPvOksbpi5e/9Q= -github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= +github.com/knadh/koanf/v2 v2.3.2 h1:Ee6tuzQYFwcZXQpc2MiVeC6qHMandf5SMUJJNoFp/c4= +github.com/knadh/koanf/v2 v2.3.2/go.mod h1:gRb40VRAbd4iJMYYD5IxZ6hfuopFcXBpc9bbQpZwo28= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -191,8 +135,8 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/loov/hrtime v1.0.3 h1:LiWKU3B9skJwRPUf0Urs9+0+OE3TxdMuiRPOTwR0gcU= -github.com/loov/hrtime v1.0.3/go.mod h1:yDY3Pwv2izeY4sq7YcPX/dtLwzg5NU1AxWuWxKwd0p0= +github.com/loov/hrtime v1.0.4 h1:K0wPQBsd9mWer2Sx8zIfpyAlF4ckZovtkEMUR/l9wpU= +github.com/loov/hrtime v1.0.4/go.mod h1:VbIwDNS2gYTRoo0RjQFdqdDlBjJLXrkDIOgoA7Jvupk= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= @@ -223,64 +167,46 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= -github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= -github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/orb v0.12.0 h1:z+zOwjmG3MyEEqzv92UN49Lg1JFYx0L9GpGKNVDKk1s= +github.com/paulmach/orb v0.12.0/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= -github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4= github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.21.0 h1:DIsaGmiaBkSangBgMtWdNfxbMNdku5IK6iNhrEqWvdA= -github.com/prometheus/client_golang v1.21.0/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= -github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk= -github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg= -github.com/prometheus/client_golang v1.23.0 h1:ust4zpdl9r4trLY/gSjlm07PuiBq2ynaXXlptpfy8Uc= -github.com/prometheus/client_golang v1.23.0/go.mod h1:i/o0R9ByOnHX0McrTMTyhYvKE4haaf2mW08I+jGAjEE= -github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= -github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o= +github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg= github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= -github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= -github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= -github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= -github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= -github.com/prometheus/common v0.65.0 h1:QDwzd+G1twt//Kwj/Ww6E9FQq1iVMmODnILtW1t2VzE= -github.com/prometheus/common v0.65.0/go.mod h1:0gZns+BLRQ3V6NdaerOhMbwwRbNh9hkGINtQAsP5GS8= -github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= -github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/prometheus/procfs v0.16.0 h1:xh6oHhKwnOJKMYiYBDWmkHqQPyiY40sny36Cmx2bbsM= -github.com/prometheus/procfs v0.16.0/go.mod h1:8veyXUu3nGP7oaCxhX6yeaM5u4stL2FeMXnCqhDthZg= -github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0= -github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw= +github.com/prometheus/common v0.67.5 h1:pIgK94WWlQt1WLwAC5j2ynLaBRDiinoAb86HZHTUGI4= +github.com/prometheus/common v0.67.5/go.mod h1:SjE/0MzDEEAyrdr5Gqc6G+sXI67maCxzaT3A2+HqjUw= +github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws= +github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw= github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg= github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= -github.com/ringsaturn/go-cities.json v0.6.6 h1:zXEsYeQBjgJwKdGHMrq1j8d9uVNxmnbVvi2fAQAyfNM= -github.com/ringsaturn/go-cities.json v0.6.6/go.mod h1:RWApnQPG6nU558XXbY1try5mi9u9Hd667J6vr948VBo= -github.com/ringsaturn/go-cities.json v0.6.8 h1:PsbjBrEANywxKSMMbagv4qnLSg4Rz+xWeoVKY3pkWnA= -github.com/ringsaturn/tzf v0.17.0 h1:hGHBPBzJfSLmCAKg0khA7yxx9kqPPNYkggdodpKNaKI= -github.com/ringsaturn/tzf v0.17.0/go.mod h1:4Z139lkC4Btg5tse9qdQ78gWmj7X0VwJjDxxO4ydAiU= -github.com/ringsaturn/tzf v1.0.0 h1:z0M2wKJNkyCsNFv/D1cveh6F5jhGwm8aysl0GLh2rnY= -github.com/ringsaturn/tzf v1.0.0/go.mod h1:H/Fl+lPWq+5oD72UZQzFXQnYXcWs3nnyGq6PIYEN8YY= -github.com/ringsaturn/tzf-rel v0.0.2025-a h1:OLJe11vif6JMDtECkS71hAOdIzDHV+uBVgWUW3V/dZQ= -github.com/ringsaturn/tzf-rel v0.0.2025-a/go.mod h1:R6HljjIcUQclZ6bOzWDa8llkEb/x+dmMWU60jXx2BEI= -github.com/ringsaturn/tzf-rel v0.0.2025-b h1:R/YuE6HKj/IwSbb6LilwhwsHoXEkUtnHrtz29Mnkdfw= -github.com/ringsaturn/tzf-rel v0.0.2025-b/go.mod h1:p5HyM9AThIOOr5ZCyheMFAdg236o/HMMnz/DObWdnys= -github.com/ringsaturn/tzf-rel-lite v0.0.2025-a h1:V+B0brHLxYqSyAeEWim/0BhM7Hnz587ZLGe3+8YgJ6E= -github.com/ringsaturn/tzf-rel-lite v0.0.2025-a/go.mod h1:nVMtMUFC40aMfjXzc+fj3CCXrHxD8sRUXLEVfxhFVAs= -github.com/ringsaturn/tzf-rel-lite v0.0.2025-b h1:YYuKav8cpkRtDZ9yFF0kBTO3bU/TjtcawjKI91GWCa4= -github.com/ringsaturn/tzf-rel-lite v0.0.2025-b/go.mod h1:SyVF6OU+Le0vKajtTA7PvYabdYCJsDlmplHuXeCZDrw= -github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= -github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= -github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= -github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/quic-go/qpack v0.6.0 h1:g7W+BMYynC1LbYLSqRt8PBg5Tgwxn214ZZR34VIOjz8= +github.com/quic-go/qpack v0.6.0/go.mod h1:lUpLKChi8njB4ty2bFLX2x4gzDqXwUpaO1DP9qMDZII= +github.com/quic-go/quic-go v0.59.0 h1:OLJkp1Mlm/aS7dpKgTc6cnpynnD2Xg7C1pwL6vy/SAw= +github.com/quic-go/quic-go v0.59.0/go.mod h1:upnsH4Ju1YkqpLXC305eW3yDZ4NfnNbmQRCMWS58IKU= +github.com/ringsaturn/go-cities.json v0.6.13 h1:p5afPcJ/tEE6uzFCOzLSHJYXgWnGdPmwZB9KBrEASxc= +github.com/ringsaturn/go-cities.json v0.6.13/go.mod h1:VtklT4Sod9i6kvXXNZV63sfjeCX9l11OQfaAvPu+p4M= +github.com/ringsaturn/tzf v1.0.3 h1:DdGcCiHpS6kg0Fo0XK+YlwNGIRXK3rs+KvFAd6b5XfQ= +github.com/ringsaturn/tzf v1.0.3/go.mod h1:8wWHQjIYklMR3uG9cIkzc/otIBhit3vtAX/D78cXNpY= +github.com/ringsaturn/tzf-rel v0.0.2025-c h1:hx2KHcZzMnO2VLg/GXKKJ6vMubmPwimcen9Gf/t1KzY= +github.com/ringsaturn/tzf-rel v0.0.2025-c/go.mod h1:p5HyM9AThIOOr5ZCyheMFAdg236o/HMMnz/DObWdnys= +github.com/ringsaturn/tzf-rel-lite v0.0.2025-c h1:CUs4l73ApN87MhlAhp1UtcRe3E5UFMQnl9d9XtJiHvg= +github.com/ringsaturn/tzf-rel-lite v0.0.2025-c/go.mod h1:SyVF6OU+Le0vKajtTA7PvYabdYCJsDlmplHuXeCZDrw= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.3.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -292,21 +218,21 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/cities v0.1.0 h1:CVNkmMf7NEC9Bvokf5GoSsArHCKRMTgLuubRTHnH0mE= github.com/tidwall/cities v0.1.0/go.mod h1:lV/HDp2gCcRcHJWqgt6Di54GiDrTZwh1aG2ZUPNbqa4= github.com/tidwall/geoindex v1.4.4/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= github.com/tidwall/geoindex v1.7.0 h1:jtk41sfgwIt8MEDyC3xyKSj75iXXf6rjReJGDNPtR5o= github.com/tidwall/geoindex v1.7.0/go.mod h1:rvVVNEFfkJVWGUdEfU8QaoOg/9zFX0h9ofWzA60mz1I= -github.com/tidwall/geojson v1.4.5 h1:BFVb5Pr7WZJMqFXy1LVudt5hPEWR3g4uhjk5Ezc3GzA= -github.com/tidwall/geojson v1.4.5/go.mod h1:1cn3UWfSYCJOq53NZoQ9rirdw89+DM0vw+ZOAVvuReg= +github.com/tidwall/geojson v1.4.6 h1:HpEGer4tc5ieFn8Ts8aTG9fo+hgFJkqfql4O9cgphmg= +github.com/tidwall/geojson v1.4.6/go.mod h1:1cn3UWfSYCJOq53NZoQ9rirdw89+DM0vw+ZOAVvuReg= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8= -github.com/tidwall/lotsa v1.0.3 h1:lFAp3PIsS58FPmz+LzhE1mcZ67tBBCRPv5j66g6y7sg= -github.com/tidwall/lotsa v1.0.3/go.mod h1:cPF+z88hamDNDjvE+u3suxCtRMVw24Gvze9eeWGYook= +github.com/tidwall/lotsa v1.0.4 h1:7jF9n2JVRuI42E4AqBlbAcjF6ACyI+8v46/CYQY47ZI= +github.com/tidwall/lotsa v1.0.4/go.mod h1:cPF+z88hamDNDjvE+u3suxCtRMVw24Gvze9eeWGYook= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -320,10 +246,8 @@ github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/twpayne/go-polyline v1.1.1 h1:/tSF1BR7rN4HWj4XKqvRUNrCiYVMCvywxTFVofvDV0w= github.com/twpayne/go-polyline v1.1.1/go.mod h1:ybd9IWWivW/rlXPXuuckeKUyF3yrIim+iqA7kSl4NFY= -github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= -github.com/ugorji/go/codec v1.2.12/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= -github.com/ugorji/go/codec v1.3.0 h1:Qd2W2sQawAfG8XSvzwhBeoGq71zXOC/Q1E9y/wUcsUA= -github.com/ugorji/go/codec v1.3.0/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= +github.com/ugorji/go/codec v1.3.1 h1:waO7eEiFDwidsBN6agj1vJQ4AG7lh2yqXyOXqhgQuyY= +github.com/ugorji/go/codec v1.3.1/go.mod h1:pRBVtBSKl77K30Bv8R2P+cLSGaTtex6fsA2Wjqmfxj4= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= @@ -331,46 +255,36 @@ github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7Jul github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= -go.mongodb.org/mongo-driver v1.17.2 h1:gvZyk8352qSfzyZ2UMWcpDpMSGEr1eqE4T793SqyhzM= -go.mongodb.org/mongo-driver v1.17.2/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= -go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.mongodb.org/mongo-driver v1.17.4 h1:jUorfmVzljjr0FLzYQsGP8cgN/qzzxlY9Vh0C9KFXVw= -go.mongodb.org/mongo-driver v1.17.4/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U= -go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg= -go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M= -go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8= -go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4= -go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU= -go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= -go.opentelemetry.io/otel/sdk/metric v1.32.0 h1:rZvFnvmvawYb0alrYkjraqJq0Z4ZUJAiyYCU9snn1CU= -go.opentelemetry.io/otel/sdk/metric v1.32.0/go.mod h1:PWeZlq0zt9YkYAp3gjKZ0eicRYvOh1Gd+X99x6GHpCQ= -go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= -go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM= -go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8= -go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= -go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.mongodb.org/mongo-driver v1.17.8 h1:BDP3+U3Y8K0vTrpqDJIRaXNhb/bKyoVeg6tIJsW5EhM= +go.mongodb.org/mongo-driver v1.17.8/go.mod h1:LlOhpH5NUEfhxcAwG0UEkMqwYcc4JU18gtCdGudk/tQ= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8= +go.opentelemetry.io/otel v1.38.0/go.mod h1:zcmtmQ1+YmQM9wrNsTGV/q/uyusom3P8RxwExxkZhjM= +go.opentelemetry.io/otel/metric v1.38.0 h1:Kl6lzIYGAh5M159u9NgiRkmoMKjvbsKtYRwgfrA6WpA= +go.opentelemetry.io/otel/metric v1.38.0/go.mod h1:kB5n/QoRM8YwmUahxvI3bO34eVtQf2i4utNVLr9gEmI= +go.opentelemetry.io/otel/sdk v1.38.0 h1:l48sr5YbNf2hpCUj/FoGhW9yDkl+Ma+LrVl8qaM5b+E= +go.opentelemetry.io/otel/sdk v1.38.0/go.mod h1:ghmNdGlVemJI3+ZB5iDEuk4bWA3GkTpW+DOoZMYBVVg= +go.opentelemetry.io/otel/sdk/metric v1.38.0 h1:aSH66iL0aZqo//xXzQLYozmWrXxyFkBJ6qT5wthqPoM= +go.opentelemetry.io/otel/sdk/metric v1.38.0/go.mod h1:dg9PBnW9XdQ1Hd6ZnRz689CbtrUp0wMMs9iPcgT9EZA= +go.opentelemetry.io/otel/trace v1.38.0 h1:Fxk5bKrDZJUH+AMyyIXGcFAPah0oRcT+LuNtJrmcNLE= +go.opentelemetry.io/otel/trace v1.38.0/go.mod h1:j1P9ivuFsTceSWe1oY+EeW3sc+Pp42sO++GHkg4wwhs= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/arch v0.14.0 h1:z9JUEZWr8x4rR0OU6c4/4t6E6jOZ8/QBS2bBYBm4tx4= -golang.org/x/arch v0.14.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= -golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= -golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= -golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c= -golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk= +go.uber.org/mock v0.6.0 h1:hyF9dfmbgIX5EfOdasqLsWD6xqpNZlXblLB/Dbnwv3Y= +go.uber.org/mock v0.6.0/go.mod h1:KiVJ4BqZJaMj4svdfmHM0AUx4NJYO8ZNpPnZn1Z+BBU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= +golang.org/x/arch v0.23.0 h1:lKF64A2jF6Zd8L0knGltUnegD62JMFBiCPBmQpToHhg= +golang.org/x/arch v0.23.0/go.mod h1:dNHoOeKiyja7GTvF9NJS1l3Z2yntpQNzgrjh1cU103A= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus= -golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= -golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc= +golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8= +golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -378,47 +292,30 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8= -golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/net v0.43.0 h1:lat02VYK2j4aLzMzecihNvTlJNQUq316m2Mr9rnM6YE= -golang.org/x/net v0.43.0/go.mod h1:vhO1fvI4dGsIjh73sWfUVjj3N7CA9WkKJNQm2svM6Jg= +golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o= +golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= -golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw= -golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= +golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc= -golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI= -golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM= -golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng= -golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU= +golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE= +golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -427,26 +324,14 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250219182151-9fdb1cabc7b2 h1:DMTIbak9GhdaSxEjvVzAeNZvyc03I61duqNbnm3SU0M= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250219182151-9fdb1cabc7b2/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463 h1:e0AIkUUhxyBKh6ssZNrAMeqhA7RKUj42346d1y02i2g= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250324211829-b45e905df463/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a h1:tPE/Kp+x9dMSwUm/uM0JKK0IfdiJkwAbSMSeZBXXJXc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20250811230008-5f3141c8851a/go.mod h1:gw1tLEfykwDz2ET4a12jcXt4couGAm7IwsVaTy0Sflo= -google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ= -google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw= -google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= -google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= -google.golang.org/grpc v1.74.2 h1:WoosgB65DlWVC9FqI82dGsZhWFNBSLjQ84bjROOpMu4= -google.golang.org/grpc v1.74.2/go.mod h1:CtQ+BGjaAIXHs/5YS3i473GqwBBa1zGQNevxdeBEXrM= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409 h1:H86B94AW+VfJWDqFeEbBPhEtHzJwJfTbgE2lZa54ZAQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260128011058-8636f8732409/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= +google.golang.org/grpc v1.78.0 h1:K1XZG/yGDJnzMdd/uZHAkVqJE+xIDOcmdSFZkBUicNc= +google.golang.org/grpc v1.78.0/go.mod h1:I47qjTo4OKbMkjA/aOOwxDIiPSBofUtQUI5EfpWvW7U= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= -google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= -google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -461,4 +346,3 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -nullprogram.com/x/optparse v1.0.0/go.mod h1:KdyPE+Igbe0jQUrVfMqDMeJQIJZEuyV7pjYmp6pbG50= From 03c897ad5b4cf0cdaa42f4a7790ca62836e0102a Mon Sep 17 00:00:00 2001 From: Fabio1988 Date: Thu, 29 Jan 2026 18:30:54 +0100 Subject: [PATCH 22/35] fix: increase spawnpoint updates --- decoder/spawnpoint.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 9d411a2f..09632856 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -213,7 +213,8 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { spawnpoint := inMemorySpawnpoint.Value() now := time.Now().Unix() - if now-spawnpoint.LastSeen > 3600 { + // update at least every 6 hours + if now-spawnpoint.LastSeen > 21600 { spawnpoint.LastSeen = now _, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ From d2ba3b7d771e15e1b49f9d2fe2351965506f0485 Mon Sep 17 00:00:00 2001 From: Fabio1988 Date: Fri, 30 Jan 2026 16:55:18 +0100 Subject: [PATCH 23/35] feat: reduce updates config --- config.toml.example | 6 ++++++ config/config.go | 1 + config/reader.go | 1 + decoder/gym.go | 5 +++-- decoder/main.go | 10 ++++++++++ decoder/pokestop.go | 9 +++++---- decoder/routes.go | 2 +- decoder/s2cell.go | 2 +- decoder/spawnpoint.go | 4 ++-- decoder/station.go | 2 +- 10 files changed, 31 insertions(+), 11 deletions(-) diff --git a/config.toml.example b/config.toml.example index 561410c4..2c018fcb 100644 --- a/config.toml.example +++ b/config.toml.example @@ -3,6 +3,12 @@ port = 9001 # Listening port for golbat raw_bearer = "" # Raw bearer (password) required api_secret = "golbat" # Golbat secret required on api calls (blank for none) +# When enabled, reduce_updates will make fort update debounce windows much longer +# to reduce database churn. Specifically, gym/pokestop/station debounce will be +# extended from 15 minutes (900s) to 12 hours (43200s) and spawnpoint last_seen +# will be updated every 12 hours instead of the default 6 hours. +reduce_updates = false + pokemon_memory_only = false # Use in-memory storage for pokemon only [koji] diff --git a/config/config.go b/config/config.go index 09463131..fb66aa37 100644 --- a/config/config.go +++ b/config/config.go @@ -27,6 +27,7 @@ type configDefinition struct { Weather weather `koanf:"weather"` ScanRules []scanRule `koanf:"scan_rules"` MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` + ReduceUpdates bool `koanf:"reduce_updates"` } func (configDefinition configDefinition) GetWebhookInterval() time.Duration { diff --git a/config/reader.go b/config/reader.go index 270edc93..e0daa36c 100644 --- a/config/reader.go +++ b/config/reader.go @@ -62,6 +62,7 @@ func ReadConfig() (configDefinition, error) { LevelCaps: []int{50, 51}, }, MaxConcurrentProactiveIVSwitch: 6, + ReduceUpdates: false, }, "koanf"), nil) if defaultErr != nil { fmt.Println(fmt.Errorf("failed to load default config: %w", defaultErr)) diff --git a/decoder/gym.go b/decoder/gym.go index 261cfff6..f43c48f5 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -1202,8 +1202,9 @@ func createGymWebhooks(gym *Gym, areas []geo.AreaName) { func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { now := time.Now().Unix() if !gym.IsNewRecord() && !gym.IsDirty() && !gym.IsInternalDirty() { - if gym.Updated > now-900 { - // if a gym is unchanged, but we did see it again after 15 minutes, then save again + // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. + if gym.Updated > now-GetUpdateThreshold(900) { + // if a gym is unchanged and was seen recently, skip saving return } } diff --git a/decoder/main.go b/decoder/main.go index c66c8ea7..a9a99a89 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -551,3 +551,13 @@ func SetWebhooksSender(whSender webhooksSenderInterface) { func SetStatsCollector(collector stats_collector.StatsCollector) { statsCollector = collector } + +// GetUpdateThreshold returns the number of seconds that should be used as a +// debounce/last-seen threshold. Pass the default seconds for normal operation +// If ReduceUpdates is enabled in the loaded config.Config, this returns 43200 (12 hours). +func GetUpdateThreshold(defaultSeconds int64) int64 { + if config.Config.ReduceUpdates { + return 43200 // 12 hours + } + return defaultSeconds +} diff --git a/decoder/pokestop.go b/decoder/pokestop.go index d4afe735..024ef7a6 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1306,7 +1306,8 @@ func createPokestopWebhooks(stop *Pokestop) { func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop) { now := time.Now().Unix() if !pokestop.IsNewRecord() && !pokestop.IsDirty() { - if pokestop.Updated > now-900 { + // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. + if pokestop.Updated > now-GetUpdateThreshold(900) { // if a pokestop is unchanged, but we did see it again after 15 minutes, then save again return } @@ -1343,12 +1344,12 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop statsCollector.IncDbQuery("insert pokestop", err) //log.Debugf("Insert pokestop %s %+v", pokestop.Id, pokestop) if err != nil { - log.Errorf("insert pokestop %s: %s", pokestop.Id, err) + log.Errorf("insert pokestop: %s", err) return } - _ = res + + _, _ = res, err } else { - // Existing record - UPDATE if dbDebugEnabled { dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) } diff --git a/decoder/routes.go b/decoder/routes.go index 8407e963..3ac86969 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -293,7 +293,7 @@ func getRouteRecord(db db.DbDetails, id string) (*Route, error) { func saveRouteRecord(db db.DbDetails, route *Route) error { // Skip save if not dirty and not new, unless 15-minute debounce expired if !route.IsDirty() && !route.IsNewRecord() { - if route.Updated > time.Now().Unix()-900 { + if route.Updated > time.Now().Unix()-GetUpdateThreshold(900) { // if a route is unchanged, but we did see it again after 15 minutes, then save again return nil } diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 5dca4187..4f59ebaf 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -41,7 +41,7 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { if c := s2CellCache.Get(cellId); c != nil { cachedCell := c.Value() - if cachedCell.Updated > now-900 { + if cachedCell.Updated > now-GetUpdateThreshold(900) { continue } s2Cell = cachedCell diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 09632856..176eac2b 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -213,8 +213,8 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { spawnpoint := inMemorySpawnpoint.Value() now := time.Now().Unix() - // update at least every 6 hours - if now-spawnpoint.LastSeen > 21600 { + // update at least every 6 hours (21600s). If reduce_updates is enabled, use 12 hours. + if now-spawnpoint.LastSeen > GetUpdateThreshold(21600) { spawnpoint.LastSeen = now _, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ diff --git a/decoder/station.go b/decoder/station.go index 029fe168..b2517bfe 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -416,7 +416,7 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { // Skip save if not dirty and was updated recently (15-min debounce) if !station.IsDirty() && !station.IsNewRecord() { - if station.Updated > now-900 { + if station.Updated > now-GetUpdateThreshold(900) { return } } From f4976ac238204fd40301471e85b5318b86a68913 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 21:04:12 +0000 Subject: [PATCH 24/35] more locking update --- decoder/gym.go | 2 + decoder/incident.go | 113 ++++++++++++++++++++++++++----- decoder/main.go | 96 +++++++------------------- decoder/pokemon.go | 14 +++- decoder/pokestop.go | 3 + decoder/routes.go | 160 +++++++++++++++++++++++++++++++++----------- decoder/station.go | 126 +++++++++++++++++++++++++++------- decoder/tappable.go | 123 +++++++++++++++++++++++++++------- decoder/weather.go | 112 ++++++++++++++++++++++++++----- main.go | 6 +- 10 files changed, 557 insertions(+), 198 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index f43c48f5..5fc58626 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -617,6 +617,7 @@ func getGymRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) ( if err != nil { return nil, nil, err } + dbGym.ClearDirty() // Atomically cache the loaded Gym - if another goroutine raced us, // we'll get their Gym and use that instead (ensuring same mutex) @@ -667,6 +668,7 @@ func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) ( } else { // We loaded from DB gym.newRecord = false + gym.ClearDirty() if config.Config.TestFortInMemory { fortRtreeUpdateGymOnGet(gym) } diff --git a/decoder/incident.go b/decoder/incident.go index c94dd598..38b5a992 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -3,6 +3,8 @@ package decoder import ( "context" "database/sql" + "errors" + "sync" "time" "github.com/jellydator/ttlcache/v3" @@ -17,6 +19,8 @@ import ( // Incident struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Incident struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id"` PokestopId string `db:"pokestop_id"` StartTime int64 `db:"start"` @@ -98,6 +102,16 @@ func (incident *Incident) IsNewRecord() bool { return incident.newRecord } +// Lock acquires the Incident's mutex +func (incident *Incident) Lock() { + incident.mu.Lock() +} + +// Unlock releases the Incident's mutex +func (incident *Incident) Unlock() { + incident.mu.Unlock() +} + // snapshotOldValues saves current values for webhook comparison // Call this after loading from cache/DB but before modifications func (incident *Incident) snapshotOldValues() { @@ -210,31 +224,96 @@ func (incident *Incident) SetSlot3Form(v null.Int) { } } -func getIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, error) { - inMemoryIncident := incidentCache.Get(incidentId) - if inMemoryIncident != nil { - incident := inMemoryIncident.Value() - incident.snapshotOldValues() - return incident, nil - } - - incident := Incident{} - err := db.GeneralDb.GetContext(ctx, &incident, +func loadIncidentFromDatabase(ctx context.Context, db db.DbDetails, incidentId string, incident *Incident) error { + err := db.GeneralDb.GetContext(ctx, incident, "SELECT id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form "+ - "FROM incident "+ - "WHERE incident.id = ? ", incidentId) + "FROM incident WHERE incident.id = ?", incidentId) statsCollector.IncDbQuery("select incident", err) - if err == sql.ErrNoRows { - return nil, nil + return err +} + +// peekIncidentRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekIncidentRecord(incidentId string) (*Incident, func(), error) { + if item := incidentCache.Get(incidentId); item != nil { + incident := item.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil } + return nil, nil, nil +} +// getIncidentRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getIncidentRecordReadOnly(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { + // Check cache first + if item := incidentCache.Get(incidentId); item != nil { + incident := item.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil + } + + dbIncident := Incident{} + err := loadIncidentFromDatabase(ctx, db, incidentId, &dbIncident) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return nil, err + return nil, nil, err + } + dbIncident.ClearDirty() + + // Atomically cache the loaded Incident - if another goroutine raced us, + // we'll get their Incident and use that instead (ensuring same mutex) + existingIncident, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { + return &dbIncident + }) + + incident := existingIncident.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil +} + +// getIncidentRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getIncidentRecordForUpdate(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { + incident, unlock, err := getIncidentRecordReadOnly(ctx, db, incidentId) + if err != nil || incident == nil { + return nil, nil, err + } + incident.snapshotOldValues() + return incident, unlock, nil +} + +// getOrCreateIncidentRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string, pokestopId string) (*Incident, func(), error) { + // Create new Incident atomically - function only called if key doesn't exist + incidentItem, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { + return &Incident{Id: incidentId, PokestopId: pokestopId, newRecord: true} + }) + + incident := incidentItem.Value() + incident.Lock() + + if incident.newRecord { + // We should attempt to load from database + err := loadIncidentFromDatabase(ctx, db, incidentId, incident) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + incident.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + incident.newRecord = false + incident.ClearDirty() + } } - incidentCache.Set(incidentId, &incident, ttlcache.DefaultTTL) incident.snapshotOldValues() - return &incident, nil + return incident, func() { incident.Unlock() }, nil } func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident) { diff --git a/decoder/main.go b/decoder/main.go index a9a99a89..21467b7c 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -7,11 +7,8 @@ import ( "runtime" "time" - "golbat/intstripedmutex" - "github.com/UnownHash/gohbem" "github.com/jellydator/ttlcache/v3" - stripedmutex "github.com/nmvalera/striped-mutex" log "github.com/sirupsen/logrus" "gopkg.in/guregu/null.v4" @@ -73,12 +70,6 @@ var routeCache *ttlcache.Cache[string, *Route] var diskEncounterCache *ttlcache.Cache[uint64, *pogo.DiskEncounterOutProto] var getMapFortsCache *ttlcache.Cache[string, *pogo.GetMapFortsOutProto_FortProto] -var stationStripedMutex = stripedmutex.New(1103) -var tappableStripedMutex = intstripedmutex.New(563) -var incidentStripedMutex = stripedmutex.New(157) -var weatherStripedMutex = intstripedmutex.New(157) -var routeStripedMutex = stripedmutex.New(157) - var ProactiveIVSwitchSem chan bool var ohbem *gohbem.Ohbem @@ -296,25 +287,14 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa if incidents != nil { for _, incidentProto := range incidents { - incidentMutex, _ := incidentStripedMutex.GetLock(incidentProto.IncidentId) - - incidentMutex.Lock() - incident, err := getIncidentRecord(ctx, db, incidentProto.IncidentId) + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, incidentProto.IncidentId, fortId) if err != nil { - log.Errorf("getIncident: %s", err) - incidentMutex.Unlock() + log.Errorf("getOrCreateIncidentRecord: %s", err) continue } - if incident == nil { - incident = &Incident{ - PokestopId: fortId, - newRecord: true, - } - } incident.updateFromPokestopIncidentDisplay(incidentProto) saveIncidentRecord(ctx, db, incident) - - incidentMutex.Unlock() + unlock() } } } @@ -346,20 +326,14 @@ func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanPa func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawStationData) { for _, stationProto := range p { stationId := stationProto.Data.Id - stationMutex, _ := stationStripedMutex.GetLock(stationId) - stationMutex.Lock() - station, err := getStationRecord(ctx, db, stationId) + station, unlock, err := getOrCreateStationRecord(ctx, db, stationId) if err != nil { - log.Errorf("getStationRecord: %s", err) - stationMutex.Unlock() + log.Errorf("getOrCreateStationRecord: %s", err) continue } - if station == nil { - station = &Station{newRecord: true} - } station.updateFromStationProto(stationProto.Data, stationProto.Cell) saveStationRecord(ctx, db, station) - stationMutex.Unlock() + unlock() } } @@ -452,13 +426,13 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.ClientWeatherProto, timestampMs int64, account string) (updates []WeatherUpdate) { hourKey := timestampMs / time.Hour.Milliseconds() for _, weatherProto := range p { - weatherMutex, _ := weatherStripedMutex.GetLock(uint64(weatherProto.S2CellId)) - weatherMutex.Lock() - - weather, err := getWeatherRecord(ctx, db, weatherProto.S2CellId) + weather, unlock, err := getOrCreateWeatherRecord(ctx, db, weatherProto.S2CellId) if err != nil { - log.Printf("getWeatherRecord: %s", err) - } else if weather == nil || timestampMs >= weather.UpdatedMs { + log.Printf("getOrCreateWeatherRecord: %s", err) + continue + } + + if weather.newRecord || timestampMs >= weather.UpdatedMs { state := getWeatherConsensusState(weatherProto.S2CellId, hourKey) if state != nil { publish, publishProto := state.applyObservation(hourKey, account, weatherProto) @@ -466,9 +440,6 @@ func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.Cl if publishProto == nil { publishProto = weatherProto } - if weather == nil { - weather = &Weather{newRecord: true} - } weather.UpdatedMs = timestampMs weather.updateWeatherFromClientWeatherProto(publishProto) saveWeatherRecord(ctx, db, weather) @@ -482,7 +453,7 @@ func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.Cl } } - weatherMutex.Unlock() + unlock() } return updates } @@ -492,55 +463,34 @@ func UpdateClientMapS2CellBatch(ctx context.Context, db db.DbDetails, cellIds [] } func UpdateIncidentLineup(ctx context.Context, db db.DbDetails, protoReq *pogo.OpenInvasionCombatSessionProto, protoRes *pogo.OpenInvasionCombatSessionOutProto) string { - incidentMutex, _ := incidentStripedMutex.GetLock(protoReq.IncidentLookup.IncidentId) - - incidentMutex.Lock() - incident, err := getIncidentRecord(ctx, db, protoReq.IncidentLookup.IncidentId) + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, protoReq.IncidentLookup.IncidentId, protoReq.IncidentLookup.FortId) if err != nil { - incidentMutex.Unlock() - return fmt.Sprintf("getIncident: %s", err) + return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) } - if incident == nil { + defer unlock() + + if incident.newRecord { log.Debugf("Updating lineup before it was saved: %s", protoReq.IncidentLookup.IncidentId) - incident = &Incident{ - Id: protoReq.IncidentLookup.IncidentId, - PokestopId: protoReq.IncidentLookup.FortId, - newRecord: true, - } } incident.updateFromOpenInvasionCombatSessionOut(protoRes) saveIncidentRecord(ctx, db, incident) - incidentMutex.Unlock() return "" } func ConfirmIncident(ctx context.Context, db db.DbDetails, proto *pogo.StartIncidentOutProto) string { - incidentMutex, _ := incidentStripedMutex.GetLock(proto.Incident.IncidentId) - - incidentMutex.Lock() - incident, err := getIncidentRecord(ctx, db, proto.Incident.IncidentId) + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, proto.Incident.IncidentId, proto.Incident.FortId) if err != nil { - incidentMutex.Unlock() - return fmt.Sprintf("getIncident: %s", err) + return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) } - if incident == nil { + defer unlock() + + if incident.newRecord { log.Debugf("Confirming incident before it was saved: %s", proto.Incident.IncidentId) - incident = &Incident{ - Id: proto.Incident.IncidentId, - PokestopId: proto.Incident.FortId, - newRecord: true, - } } incident.updateFromStartIncidentOut(proto) - if incident == nil { - incidentMutex.Unlock() - // I only saw this once during testing but I couldn't reproduce it so just in case - return "Unable to process incident" - } saveIncidentRecord(ctx, db, incident) - incidentMutex.Unlock() return "" } diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 2518a237..1f5494e2 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -530,6 +530,7 @@ func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId if err != nil { return nil, nil, err } + dbPokemon.ClearDirty() // Atomically cache the loaded Pokemon - if another goroutine raced us, // we'll get their Pokemon and use that instead (ensuring same mutex) @@ -583,6 +584,7 @@ func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId } else { // We loaded pokemon.newRecord = false + pokemon.ClearDirty() pokemonRtreeUpdatePokemonOnGet(pokemon) } } @@ -1613,12 +1615,15 @@ func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails Form: int32(proto.PokemonDisplay.Form), } if scan.CellWeather == int32(pogo.GameplayWeatherProto_NONE) { - weather, err := getWeatherRecord(ctx, db, weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon)) - if err != nil || weather == nil || !weather.GameplayCondition.Valid { + weather, unlock, err := peekWeatherRecord(weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon)) + if weather == nil || !weather.GameplayCondition.Valid { log.Warnf("Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) } else { scan.CellWeather = int32(weather.GameplayCondition.Int64) } + if unlock != nil { + unlock() + } } if proto.CpMultiplier < 0.734 { scan.Level = int32((58.215688455154954*proto.CpMultiplier-2.7012478057856497)*proto.CpMultiplier + 1.3220677708486794) @@ -1838,7 +1843,7 @@ func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails cellId := weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon) cellWeather, found := weather[cellId] if !found { - record, err := getWeatherRecord(ctx, db, cellId) + record, unlock, err := getWeatherRecordReadOnly(ctx, db, cellId) if err != nil || record == nil || !record.GameplayCondition.Valid { log.Warnf("[POKEMON] Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) } else { @@ -1846,6 +1851,9 @@ func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails cellWeather = pogo.GameplayWeatherProto_WeatherCondition(record.GameplayCondition.Int64) found = true } + if unlock != nil { + unlock() + } } if found && cellWeather == pogo.GameplayWeatherProto_PARTLY_CLOUDY { shouldOverrideIv = true diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 024ef7a6..8e25200f 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -683,6 +683,8 @@ func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId stri return nil, nil, err } + dbPokestop.ClearDirty() + // Atomically cache the loaded Pokestop - if another goroutine raced us, // we'll get their Pokestop and use that instead (ensuring same mutex) existingPokestop, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { @@ -732,6 +734,7 @@ func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId stri } else { // We loaded from DB pokestop.newRecord = false + pokestop.ClearDirty() if config.Config.TestFortInMemory { fortRtreeUpdatePokestopOnGet(pokestop) } diff --git a/decoder/routes.go b/decoder/routes.go index 3ac86969..e707b806 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -1,9 +1,12 @@ package decoder import ( + "context" "database/sql" "encoding/json" + "errors" "fmt" + "sync" "time" "golbat/db" @@ -18,6 +21,8 @@ import ( // Route struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Route struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id"` Name string `db:"name"` Shortcode string `db:"shortcode"` @@ -44,6 +49,13 @@ type Route struct { dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) + + oldValues RouteOldValues `db:"-" json:"-"` // Old values for webhook comparison +} + +// RouteOldValues holds old field values for webhook comparison +type RouteOldValues struct { + Version int64 } // IsDirty returns true if any field has been modified @@ -61,6 +73,24 @@ func (r *Route) IsNewRecord() bool { return r.newRecord } +// Lock acquires the Route's mutex +func (r *Route) Lock() { + r.mu.Lock() +} + +// Unlock releases the Route's mutex +func (r *Route) Unlock() { + r.mu.Unlock() +} + +// snapshotOldValues saves current values for webhook comparison +// Call this after loading from cache/DB but before modifications +func (r *Route) snapshotOldValues() { + r.oldValues = RouteOldValues{ + Version: r.Version, + } +} + // --- Set methods with dirty tracking --- func (r *Route) SetName(v string) { @@ -263,34 +293,98 @@ func (r *Route) SetWaypoints(v string) { } } -func getRouteRecord(db db.DbDetails, id string) (*Route, error) { - inMemoryRoute := routeCache.Get(id) - if inMemoryRoute != nil { - route := inMemoryRoute.Value() - return route, nil +func loadRouteFromDatabase(ctx context.Context, db db.DbDetails, routeId string, route *Route) error { + err := db.GeneralDb.GetContext(ctx, route, + `SELECT * FROM route WHERE route.id = ?`, routeId) + statsCollector.IncDbQuery("select route", err) + return err +} + +// peekRouteRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekRouteRecord(routeId string) (*Route, func(), error) { + if item := routeCache.Get(routeId); item != nil { + route := item.Value() + route.Lock() + return route, func() { route.Unlock() }, nil } + return nil, nil, nil +} - route := Route{} - err := db.GeneralDb.Get(&route, - ` - SELECT * - FROM route - WHERE route.id = ? - `, - id, - ) - statsCollector.IncDbQuery("select route", err) - if err == sql.ErrNoRows { - return nil, nil - } else if err != nil { - return nil, err +// getRouteRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getRouteRecordReadOnly(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + // Check cache first + if item := routeCache.Get(routeId); item != nil { + route := item.Value() + route.Lock() + return route, func() { route.Unlock() }, nil + } + + dbRoute := Route{} + err := loadRouteFromDatabase(ctx, db, routeId, &dbRoute) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil } + if err != nil { + return nil, nil, err + } + dbRoute.ClearDirty() + + // Atomically cache the loaded Route - if another goroutine raced us, + // we'll get their Route and use that instead (ensuring same mutex) + existingRoute, _ := routeCache.GetOrSetFunc(routeId, func() *Route { + return &dbRoute + }) + + route := existingRoute.Value() + route.Lock() + return route, func() { route.Unlock() }, nil +} + +// getRouteRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getRouteRecordForUpdate(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + route, unlock, err := getRouteRecordReadOnly(ctx, db, routeId) + if err != nil || route == nil { + return nil, nil, err + } + route.snapshotOldValues() + return route, unlock, nil +} + +// getOrCreateRouteRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateRouteRecord(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + // Create new Route atomically - function only called if key doesn't exist + routeItem, _ := routeCache.GetOrSetFunc(routeId, func() *Route { + return &Route{Id: routeId, newRecord: true} + }) - routeCache.Set(id, &route, ttlcache.DefaultTTL) - return &route, nil + route := routeItem.Value() + route.Lock() + + if route.newRecord { + // We should attempt to load from database + err := loadRouteFromDatabase(ctx, db, routeId, route) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + route.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + route.newRecord = false + route.ClearDirty() + } + } + + route.snapshotOldValues() + return route, func() { route.Unlock() }, nil } -func saveRouteRecord(db db.DbDetails, route *Route) error { +func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { // Skip save if not dirty and not new, unless 15-minute debounce expired if !route.IsDirty() && !route.IsNewRecord() { if route.Updated > time.Now().Unix()-GetUpdateThreshold(900) { @@ -305,7 +399,7 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { if dbDebugEnabled { dbDebugLog("INSERT", "Route", route.Id, route.changedFields) } - _, err := db.GeneralDb.NamedExec( + _, err := db.GeneralDb.NamedExecContext(ctx, ` INSERT INTO route ( id, name, shortcode, description, distance_meters, @@ -337,7 +431,7 @@ func saveRouteRecord(db db.DbDetails, route *Route) error { if dbDebugEnabled { dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) } - _, err := db.GeneralDb.NamedExec( + _, err := db.GeneralDb.NamedExecContext(ctx, ` UPDATE route SET name = :name, @@ -422,24 +516,14 @@ func (route *Route) updateFromSharedRouteProto(sharedRouteProto *pogo.SharedRout } } -func UpdateRouteRecordWithSharedRouteProto(db db.DbDetails, sharedRouteProto *pogo.SharedRouteProto) error { - routeMutex, _ := routeStripedMutex.GetLock(sharedRouteProto.GetId()) - routeMutex.Lock() - defer routeMutex.Unlock() - - route, err := getRouteRecord(db, sharedRouteProto.GetId()) +func UpdateRouteRecordWithSharedRouteProto(ctx context.Context, db db.DbDetails, sharedRouteProto *pogo.SharedRouteProto) error { + route, unlock, err := getOrCreateRouteRecord(ctx, db, sharedRouteProto.GetId()) if err != nil { return err } - - if route == nil { - route = &Route{ - Id: sharedRouteProto.GetId(), - newRecord: true, - } - } + defer unlock() route.updateFromSharedRouteProto(sharedRouteProto) - saveError := saveRouteRecord(db, route) + saveError := saveRouteRecord(ctx, db, route) return saveError } diff --git a/decoder/station.go b/decoder/station.go index b2517bfe..bbf5c79a 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "fmt" + "sync" "time" "golbat/db" @@ -21,6 +22,8 @@ import ( // Station struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Station struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id string `db:"id"` Lat float64 `db:"lat"` Lon float64 `db:"lon"` @@ -84,6 +87,16 @@ func (station *Station) IsNewRecord() bool { return station.newRecord } +// Lock acquires the Station's mutex +func (station *Station) Lock() { + station.mu.Lock() +} + +// Unlock releases the Station's mutex +func (station *Station) Unlock() { + station.mu.Unlock() +} + // snapshotOldValues saves current values for webhook comparison // Call this after loading from cache/DB but before modifications func (station *Station) snapshotOldValues() { @@ -384,31 +397,96 @@ type StationWebhook struct { Updated int64 `json:"updated"` } -func getStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, error) { - inMemoryStation := stationCache.Get(stationId) - if inMemoryStation != nil { - station := inMemoryStation.Value() - station.snapshotOldValues() - return station, nil - } - station := Station{} - err := db.GeneralDb.GetContext(ctx, &station, - ` - SELECT id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon - FROM station WHERE id = ? - `, stationId) +func loadStationFromDatabase(ctx context.Context, db db.DbDetails, stationId string, station *Station) error { + err := db.GeneralDb.GetContext(ctx, station, + `SELECT id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon + FROM station WHERE id = ?`, stationId) statsCollector.IncDbQuery("select station", err) + return err +} - if errors.Is(err, sql.ErrNoRows) { - return nil, nil +// peekStationRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekStationRecord(stationId string) (*Station, func(), error) { + if item := stationCache.Get(stationId); item != nil { + station := item.Value() + station.Lock() + return station, func() { station.Unlock() }, nil + } + return nil, nil, nil +} + +// getStationRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + // Check cache first + if item := stationCache.Get(stationId); item != nil { + station := item.Value() + station.Lock() + return station, func() { station.Unlock() }, nil } + dbStation := Station{} + err := loadStationFromDatabase(ctx, db, stationId, &dbStation) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return nil, err + return nil, nil, err + } + dbStation.ClearDirty() + + // Atomically cache the loaded Station - if another goroutine raced us, + // we'll get their Station and use that instead (ensuring same mutex) + existingStation, _ := stationCache.GetOrSetFunc(stationId, func() *Station { + return &dbStation + }) + + station := existingStation.Value() + station.Lock() + return station, func() { station.Unlock() }, nil +} + +// getStationRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getStationRecordForUpdate(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + station, unlock, err := getStationRecordReadOnly(ctx, db, stationId) + if err != nil || station == nil { + return nil, nil, err + } + station.snapshotOldValues() + return station, unlock, nil +} + +// getOrCreateStationRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + // Create new Station atomically - function only called if key doesn't exist + stationItem, _ := stationCache.GetOrSetFunc(stationId, func() *Station { + return &Station{Id: stationId, newRecord: true} + }) + + station := stationItem.Value() + station.Lock() + + if station.newRecord { + // We should attempt to load from database + err := loadStationFromDatabase(ctx, db, stationId, station) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + station.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + station.newRecord = false + station.ClearDirty() + } } - stationCache.Set(stationId, &station, ttlcache.DefaultTTL) + station.snapshotOldValues() - return &station, nil + return station, func() { station.Unlock() }, nil } func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { @@ -590,11 +668,8 @@ func (station *Station) resetStationedPokemonFromStationDetailsNotFound() *Stati func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto) string { stationId := request.StationId - stationMutex, _ := stationStripedMutex.GetLock(stationId) - stationMutex.Lock() - defer stationMutex.Unlock() - station, err := getStationRecord(ctx, db, stationId) + station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) if err != nil { log.Printf("Get station %s", err) return "Error getting station" @@ -604,6 +679,7 @@ func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db. log.Infof("Stationed pokemon details for station %s not found", stationId) return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) } + defer unlock() station.resetStationedPokemonFromStationDetailsNotFound() saveStationRecord(ctx, db, station) @@ -612,11 +688,8 @@ func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db. func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto, stationDetails *pogo.GetStationedPokemonDetailsOutProto) string { stationId := request.StationId - stationMutex, _ := stationStripedMutex.GetLock(stationId) - stationMutex.Lock() - defer stationMutex.Unlock() - station, err := getStationRecord(ctx, db, stationId) + station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) if err != nil { log.Printf("Get station %s", err) return "Error getting station" @@ -626,6 +699,7 @@ func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, reque log.Infof("Stationed pokemon details for station %s not found", stationId) return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) } + defer unlock() station.updateFromGetStationedPokemonDetailsOutProto(stationDetails) saveStationRecord(ctx, db, station) diff --git a/decoder/tappable.go b/decoder/tappable.go index cf089a71..5454039e 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "strconv" + "sync" "time" "golbat/db" @@ -19,6 +20,8 @@ import ( // Tappable struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Tappable struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id uint64 `db:"id" json:"id"` Lat float64 `db:"lat" json:"lat"` Lon float64 `db:"lon" json:"lon"` @@ -52,6 +55,16 @@ func (ta *Tappable) IsNewRecord() bool { return ta.newRecord } +// Lock acquires the Tappable's mutex +func (ta *Tappable) Lock() { + ta.mu.Lock() +} + +// Unlock releases the Tappable's mutex +func (ta *Tappable) Unlock() { + ta.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (ta *Tappable) SetLat(v float64) { @@ -254,28 +267,98 @@ func (ta *Tappable) setUnknownTimestamp(now int64) { } } -func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, error) { - inMemoryTappable := tappableCache.Get(id) - if inMemoryTappable != nil { - tappable := inMemoryTappable.Value() - return tappable, nil - } - tappable := Tappable{} - err := db.GeneralDb.GetContext(ctx, &tappable, +func loadTappableFromDatabase(ctx context.Context, db db.DbDetails, id uint64, tappable *Tappable) error { + err := db.GeneralDb.GetContext(ctx, tappable, `SELECT id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated - FROM tappable - WHERE id = ?`, strconv.FormatUint(id, 10)) + FROM tappable WHERE id = ?`, strconv.FormatUint(id, 10)) statsCollector.IncDbQuery("select tappable", err) + return err +} + +// peekTappableRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekTappableRecord(id uint64) (*Tappable, func(), error) { + if item := tappableCache.Get(id); item != nil { + tappable := item.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil + } + return nil, nil, nil +} + +// getTappableRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getTappableRecordReadOnly(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { + // Check cache first + if item := tappableCache.Get(id); item != nil { + tappable := item.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil + } + + dbTappable := Tappable{} + err := loadTappableFromDatabase(ctx, db, id, &dbTappable) if errors.Is(err, sql.ErrNoRows) { - return nil, nil + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbTappable.ClearDirty() + + // Atomically cache the loaded Tappable - if another goroutine raced us, + // we'll get their Tappable and use that instead (ensuring same mutex) + existingTappable, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { + return &dbTappable + }) + + tappable := existingTappable.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil +} + +// getOrCreateTappableRecord gets existing or creates new, locked. +// Caller MUST call returned unlock function. +func getOrCreateTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { + // Create new Tappable atomically - function only called if key doesn't exist + tappableItem, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { + return &Tappable{Id: id, newRecord: true} + }) + + tappable := tappableItem.Value() + tappable.Lock() + + if tappable.newRecord { + // We should attempt to load from database + err := loadTappableFromDatabase(ctx, db, id, tappable) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + tappable.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + tappable.newRecord = false + tappable.ClearDirty() + } } + return tappable, func() { tappable.Unlock() }, nil +} + +// GetTappableRecord is an exported function for API use. +// Returns a tappable if found, nil if not found. +func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, error) { + tappable, unlock, err := getTappableRecordReadOnly(ctx, db, id) if err != nil { return nil, err } - - tappableCache.Set(id, &tappable, ttlcache.DefaultTTL) - return &tappable, nil + if tappable == nil { + return nil, nil + } + defer unlock() + return tappable, nil } func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tappable) { @@ -342,19 +425,13 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { id := request.GetEncounterId() - tappableMutex, _ := tappableStripedMutex.GetLock(id) - tappableMutex.Lock() - defer tappableMutex.Unlock() - tappable, err := GetTappableRecord(ctx, db, id) + tappable, unlock, err := getOrCreateTappableRecord(ctx, db, id) if err != nil { - log.Printf("Get tappable %s", err) + log.Printf("getOrCreateTappableRecord: %s", err) return "Error getting tappable" } - - if tappable == nil { - tappable = &Tappable{newRecord: true} - } + defer unlock() tappable.updateFromProcessTappableProto(ctx, db, tappableDetails, request, timestampMs) saveTappableRecord(ctx, db, tappable) diff --git a/decoder/weather.go b/decoder/weather.go index 7aab48a5..7f346761 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -3,6 +3,8 @@ package decoder import ( "context" "database/sql" + "errors" + "sync" "golbat/db" "golbat/pogo" @@ -17,6 +19,8 @@ import ( // Weather struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Weather struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id int64 `db:"id"` Latitude float64 `db:"latitude"` Longitude float64 `db:"longitude"` @@ -79,6 +83,16 @@ func (weather *Weather) IsNewRecord() bool { return weather.newRecord } +// Lock acquires the Weather's mutex +func (weather *Weather) Lock() { + weather.mu.Lock() +} + +// Unlock releases the Weather's mutex +func (weather *Weather) Unlock() { + weather.mu.Unlock() +} + // snapshotOldValues saves current values for webhook comparison // Call this after loading from cache/DB but before modifications func (weather *Weather) snapshotOldValues() { @@ -188,30 +202,98 @@ func (weather *Weather) SetWarnWeather(v null.Bool) { } } -func getWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, error) { - inMemoryWeather := weatherCache.Get(weatherId) - if inMemoryWeather != nil { - weather := inMemoryWeather.Value() - weather.snapshotOldValues() - return weather, nil +func loadWeatherFromDatabase(ctx context.Context, db db.DbDetails, weatherId int64, weather *Weather) error { + err := db.GeneralDb.GetContext(ctx, weather, + "SELECT id, latitude, longitude, level, gameplay_condition, wind_direction, cloud_level, rain_level, wind_level, snow_level, fog_level, special_effect_level, severity, warn_weather, updated FROM weather WHERE id = ?", weatherId) + statsCollector.IncDbQuery("select weather", err) + if err == nil { + weather.UpdatedMs *= 1000 } - weather := Weather{} + return err +} - err := db.GeneralDb.GetContext(ctx, &weather, "SELECT id, latitude, longitude, level, gameplay_condition, wind_direction, cloud_level, rain_level, wind_level, snow_level, fog_level, special_effect_level, severity, warn_weather, updated FROM weather WHERE id = ?", weatherId) +// peekWeatherRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekWeatherRecord(weatherId int64) (*Weather, func(), error) { + if item := weatherCache.Get(weatherId); item != nil { + weather := item.Value() + weather.Lock() + return weather, func() { weather.Unlock() }, nil + } + return nil, nil, nil +} - statsCollector.IncDbQuery("select weather", err) - if err == sql.ErrNoRows { - return nil, nil +// getWeatherRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getWeatherRecordReadOnly(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, func(), error) { + // Check cache first + if item := weatherCache.Get(weatherId); item != nil { + weather := item.Value() + weather.Lock() + return weather, func() { weather.Unlock() }, nil } + dbWeather := Weather{} + err := loadWeatherFromDatabase(ctx, db, weatherId, &dbWeather) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return nil, err + return nil, nil, err + } + dbWeather.ClearDirty() + + // Atomically cache the loaded Weather - if another goroutine raced us, + // we'll get their Weather and use that instead (ensuring same mutex) + existingWeather, _ := weatherCache.GetOrSetFunc(weatherId, func() *Weather { + return &dbWeather + }) + + weather := existingWeather.Value() + weather.Lock() + return weather, func() { weather.Unlock() }, nil +} + +// getWeatherRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getWeatherRecordForUpdate(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, func(), error) { + weather, unlock, err := getWeatherRecordReadOnly(ctx, db, weatherId) + if err != nil || weather == nil { + return nil, nil, err + } + weather.snapshotOldValues() + return weather, unlock, nil +} + +// getOrCreateWeatherRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateWeatherRecord(ctx context.Context, db db.DbDetails, weatherId int64) (*Weather, func(), error) { + // Create new Weather atomically - function only called if key doesn't exist + weatherItem, _ := weatherCache.GetOrSetFunc(weatherId, func() *Weather { + return &Weather{Id: weatherId, newRecord: true} + }) + + weather := weatherItem.Value() + weather.Lock() + + if weather.newRecord { + // We should attempt to load from database + err := loadWeatherFromDatabase(ctx, db, weatherId, weather) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + weather.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + weather.newRecord = false + weather.ClearDirty() + } } - weather.UpdatedMs *= 1000 weather.snapshotOldValues() - weatherCache.Set(weatherId, &weather, ttlcache.DefaultTTL) - return &weather, nil + return weather, func() { weather.Unlock() }, nil } func weatherCellIdFromLatLon(lat, lon float64) int64 { diff --git a/main.go b/main.go index 209599b8..bf486171 100644 --- a/main.go +++ b/main.go @@ -454,7 +454,7 @@ func decode(ctx context.Context, method int, protoData *ProtoData) { result = decodeGetMapForts(ctx, protoData.Data) processed = true case pogo.Method_METHOD_GET_ROUTES: - result = decodeGetRoutes(protoData.Data) + result = decodeGetRoutes(ctx, protoData.Data) processed = true case pogo.Method_METHOD_GET_CONTEST_DATA: if getScanParameters(protoData).ProcessPokestops { @@ -690,7 +690,7 @@ func decodeGetMapForts(ctx context.Context, sDec []byte) string { return "No forts updated" } -func decodeGetRoutes(payload []byte) string { +func decodeGetRoutes(ctx context.Context, payload []byte) string { getRoutesOutProto := &pogo.GetRoutesOutProto{} if err := proto.Unmarshal(payload, getRoutesOutProto); err != nil { return fmt.Sprintf("failed to decode GetRoutesOutProto %s", err) @@ -711,7 +711,7 @@ func decodeGetRoutes(payload []byte) string { log.Warnf("Non published Route found in GetRoutesOutProto, status: %s", routeSubmissionStatus.String()) continue } - decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(dbDetails, route) + decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(ctx, dbDetails, route) if decodeError != nil { if decodeErrors[route.Id] != true { decodeErrors[route.Id] = true From 0f2bd79065a48bb400ee661181f46c324ce84538 Mon Sep 17 00:00:00 2001 From: James Berry Date: Fri, 30 Jan 2026 21:22:32 +0000 Subject: [PATCH 25/35] spawnpoint locking --- decoder/pokemon.go | 6 +- decoder/spawnpoint.go | 136 +++++++++++++++++++++++++++++++----------- decoder/tappable.go | 6 +- 3 files changed, 111 insertions(+), 37 deletions(-) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 1f5494e2..47096a4d 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -1221,9 +1221,10 @@ func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db } pokemon.ExpireTimestampVerified = false - spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) + spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) if spawnPoint != nil && spawnPoint.DespawnSec.Valid { despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) + unlock() date := time.Unix(timestampMs/1000, 0) secondOfHour := date.Second() + date.Minute()*60 @@ -1235,6 +1236,9 @@ func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) pokemon.SetExpireTimestampVerified(true) } else { + if unlock != nil { + unlock() + } pokemon.setUnknownTimestamp(timestampMs / 1000) } } diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 176eac2b..7e1d8811 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -3,7 +3,9 @@ package decoder import ( "context" "database/sql" + "errors" "strconv" + "sync" "time" "golbat/db" @@ -17,6 +19,8 @@ import ( // Spawnpoint struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Spawnpoint struct { + mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + Id int64 `db:"id"` Lat float64 `db:"lat"` Lon float64 `db:"lon"` @@ -56,6 +60,16 @@ func (s *Spawnpoint) IsNewRecord() bool { return s.newRecord } +// Lock acquires the Spawnpoint's mutex +func (s *Spawnpoint) Lock() { + s.mu.Lock() +} + +// Unlock releases the Spawnpoint's mutex +func (s *Spawnpoint) Unlock() { + s.mu.Unlock() +} + // --- Set methods with dirty tracking --- func (s *Spawnpoint) SetLat(v float64) { @@ -105,28 +119,82 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { } } -func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int64) (*Spawnpoint, error) { - inMemorySpawnpoint := spawnpointCache.Get(spawnpointId) - if inMemorySpawnpoint != nil { - spawnpoint := inMemorySpawnpoint.Value() - return spawnpoint, nil - } - spawnpoint := Spawnpoint{} +func loadSpawnpointFromDatabase(ctx context.Context, db db.DbDetails, spawnpointId int64, spawnpoint *Spawnpoint) error { + err := db.GeneralDb.GetContext(ctx, spawnpoint, + "SELECT id, lat, lon, updated, last_seen, despawn_sec FROM spawnpoint WHERE id = ?", spawnpointId) + statsCollector.IncDbQuery("select spawnpoint", err) + return err +} - err := db.GeneralDb.GetContext(ctx, &spawnpoint, "SELECT id, lat, lon, updated, last_seen, despawn_sec FROM spawnpoint WHERE id = ?", spawnpointId) +// peekSpawnpointRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekSpawnpointRecord(spawnpointId int64) (*Spawnpoint, func(), error) { + if item := spawnpointCache.Get(spawnpointId); item != nil { + spawnpoint := item.Value() + spawnpoint.Lock() + return spawnpoint, func() { spawnpoint.Unlock() }, nil + } + return nil, nil, nil +} - statsCollector.IncDbQuery("select spawnpoint", err) - if err == sql.ErrNoRows { - return nil, nil +// getSpawnpointRecord acquires lock. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int64) (*Spawnpoint, func(), error) { + // Check cache first + if item := spawnpointCache.Get(spawnpointId); item != nil { + spawnpoint := item.Value() + spawnpoint.Lock() + return spawnpoint, func() { spawnpoint.Unlock() }, nil } + dbSpawnpoint := Spawnpoint{} + err := loadSpawnpointFromDatabase(ctx, db, spawnpointId, &dbSpawnpoint) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } if err != nil { - return &Spawnpoint{Id: spawnpointId}, err + return nil, nil, err } + dbSpawnpoint.ClearDirty() - spawnpointCache.Set(spawnpointId, &spawnpoint, ttlcache.DefaultTTL) + // Atomically cache the loaded Spawnpoint - if another goroutine raced us, + // we'll get their Spawnpoint and use that instead (ensuring same mutex) + existingSpawnpoint, _ := spawnpointCache.GetOrSetFunc(spawnpointId, func() *Spawnpoint { + return &dbSpawnpoint + }) - return &spawnpoint, nil + spawnpoint := existingSpawnpoint.Value() + spawnpoint.Lock() + return spawnpoint, func() { spawnpoint.Unlock() }, nil +} + +// getOrCreateSpawnpointRecord gets existing or creates new, locked. +// Caller MUST call returned unlock function. +func getOrCreateSpawnpointRecord(ctx context.Context, db db.DbDetails, spawnpointId int64) (*Spawnpoint, func(), error) { + // Create new Spawnpoint atomically - function only called if key doesn't exist + spawnpointItem, _ := spawnpointCache.GetOrSetFunc(spawnpointId, func() *Spawnpoint { + return &Spawnpoint{Id: spawnpointId, newRecord: true} + }) + + spawnpoint := spawnpointItem.Value() + spawnpoint.Lock() + + if spawnpoint.newRecord { + // We should attempt to load from database + err := loadSpawnpointFromDatabase(ctx, db, spawnpointId, spawnpoint) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + spawnpoint.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + spawnpoint.newRecord = false + spawnpoint.ClearDirty() + } + } + + return spawnpoint, func() { spawnpoint.Unlock() }, nil } func Abs(x int64) int64 { @@ -148,27 +216,30 @@ func spawnpointUpdateFromWild(ctx context.Context, db db.DbDetails, wildPokemon date := time.Unix(expireTimeStamp, 0) secondOfHour := date.Second() + date.Minute()*60 - spawnpoint, _ := getSpawnpointRecord(ctx, db, spawnId) - if spawnpoint == nil { - spawnpoint = &Spawnpoint{Id: spawnId, newRecord: true} + spawnpoint, unlock, err := getOrCreateSpawnpointRecord(ctx, db, spawnId) + if err != nil { + log.Errorf("getOrCreateSpawnpointRecord: %s", err) + return } spawnpoint.SetLat(wildPokemon.Latitude) spawnpoint.SetLon(wildPokemon.Longitude) spawnpoint.SetDespawnSec(null.IntFrom(int64(secondOfHour))) spawnpointUpdate(ctx, db, spawnpoint) + unlock() } else { - spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) - if spawnPoint == nil { - spawnpoint := &Spawnpoint{ - Id: spawnId, - Lat: wildPokemon.Latitude, - Lon: wildPokemon.Longitude, - newRecord: true, - } + spawnpoint, unlock, err := getOrCreateSpawnpointRecord(ctx, db, spawnId) + if err != nil { + log.Errorf("getOrCreateSpawnpointRecord: %s", err) + return + } + if spawnpoint.newRecord { + spawnpoint.SetLat(wildPokemon.Latitude) + spawnpoint.SetLon(wildPokemon.Longitude) spawnpointUpdate(ctx, db, spawnpoint) } else { - spawnpointSeen(ctx, db, spawnId) + spawnpointSeen(ctx, db, spawnpoint) } + unlock() } } @@ -203,14 +274,9 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi } } -func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { - inMemorySpawnpoint := spawnpointCache.Get(spawnpointId) - if inMemorySpawnpoint == nil { - // This should never happen, since all routes here have previously created a spawnpoint in the cache - return - } - - spawnpoint := inMemorySpawnpoint.Value() +// spawnpointSeen updates the last_seen timestamp for a spawnpoint. +// The spawnpoint must already be locked by the caller. +func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoint) { now := time.Now().Unix() // update at least every 6 hours (21600s). If reduce_updates is enabled, use 12 hours. @@ -219,7 +285,7 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpointId int64) { _, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ "SET last_seen=? "+ - "WHERE id = ? ", now, spawnpointId) + "WHERE id = ? ", now, spawnpoint.Id) statsCollector.IncDbQuery("update spawnpoint", err) if err != nil { log.Printf("Error updating spawnpoint last seen %s", err) diff --git a/decoder/tappable.go b/decoder/tappable.go index 5454039e..3a498ef2 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -235,9 +235,10 @@ func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.Db func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, timestampMs int64) { ta.SetExpireTimestampVerified(false) if spawnId := ta.SpawnId.ValueOrZero(); spawnId != 0 { - spawnPoint, _ := getSpawnpointRecord(ctx, db, spawnId) + spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) if spawnPoint != nil && spawnPoint.DespawnSec.Valid { despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) + unlock() date := time.Unix(timestampMs/1000, 0) secondOfHour := date.Second() + date.Minute()*60 @@ -249,6 +250,9 @@ func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, tim ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) ta.SetExpireTimestampVerified(true) } else { + if unlock != nil { + unlock() + } ta.setUnknownTimestamp(timestampMs / 1000) } } else if fortId := ta.FortId.ValueOrZero(); fortId != "" { From 07efbe6858e7435e2ab190a5fe3aec015e1ff5e7 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 31 Jan 2026 14:22:56 +0000 Subject: [PATCH 26/35] Tidy source --- decode.go | 738 ++++++++++++++ decoder/api_gym.go | 96 ++ decoder/api_pokemon_common.go | 2 +- decoder/api_pokestop.go | 101 ++ decoder/api_tappable.go | 39 + decoder/fort.go | 18 +- decoder/fortRtree.go | 2 +- decoder/gmo_decode.go | 224 +++++ decoder/gym.go | 832 +-------------- decoder/gym_decode.go | 311 ++++++ decoder/gym_process.go | 97 ++ decoder/gym_state.go | 430 ++++++++ decoder/incident.go | 278 +---- decoder/incident_decode.go | 54 + decoder/incident_process.go | 43 + decoder/incident_state.go | 234 +++++ decoder/main.go | 250 +---- decoder/player.go | 2 +- decoder/pokemon.go | 1512 +--------------------------- decoder/pokemonRtree.go | 2 +- decoder/pokemon_decode.go | 968 ++++++++++++++++++ decoder/pokemon_process.go | 92 ++ decoder/pokemon_state.go | 486 +++++++++ decoder/pokestop.go | 1013 +------------------ decoder/pokestop_decode.go | 475 +++++++++ decoder/pokestop_process.go | 167 +++ decoder/pokestop_showcase.go | 2 +- decoder/pokestop_state.go | 397 ++++++++ decoder/routes.go | 249 +---- decoder/routes_decode.go | 51 + decoder/routes_process.go | 20 + decoder/routes_state.go | 196 ++++ decoder/s2cell.go | 2 +- decoder/spawnpoint.go | 2 +- decoder/station.go | 388 +------ decoder/station_decode.go | 106 ++ decoder/station_process.go | 51 + decoder/station_state.go | 253 +++++ decoder/tappable.go | 288 +----- decoder/tappable_decode.go | 117 +++ decoder/tappable_process.go | 26 + decoder/tappable_state.go | 157 +++ decoder/weather.go | 2 +- decoder/weather_iv.go | 2 +- go.mod | 3 +- go.sum | 6 +- main.go | 743 +------------- routes.go | 60 +- stats_collector/noop.go | 2 +- stats_collector/prometheus.go | 5 +- stats_collector/stats_collector.go | 2 +- 51 files changed, 6016 insertions(+), 5580 deletions(-) create mode 100644 decode.go create mode 100644 decoder/api_pokestop.go create mode 100644 decoder/api_tappable.go create mode 100644 decoder/gmo_decode.go create mode 100644 decoder/gym_decode.go create mode 100644 decoder/gym_process.go create mode 100644 decoder/gym_state.go create mode 100644 decoder/incident_decode.go create mode 100644 decoder/incident_process.go create mode 100644 decoder/incident_state.go create mode 100644 decoder/pokemon_decode.go create mode 100644 decoder/pokemon_process.go create mode 100644 decoder/pokemon_state.go create mode 100644 decoder/pokestop_decode.go create mode 100644 decoder/pokestop_process.go create mode 100644 decoder/pokestop_state.go create mode 100644 decoder/routes_decode.go create mode 100644 decoder/routes_process.go create mode 100644 decoder/routes_state.go create mode 100644 decoder/station_decode.go create mode 100644 decoder/station_process.go create mode 100644 decoder/station_state.go create mode 100644 decoder/tappable_decode.go create mode 100644 decoder/tappable_process.go create mode 100644 decoder/tappable_state.go diff --git a/decode.go b/decode.go new file mode 100644 index 00000000..daab1cb3 --- /dev/null +++ b/decode.go @@ -0,0 +1,738 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" + + "golbat/decoder" + "golbat/pogo" + + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" +) + +func decode(ctx context.Context, method int, protoData *ProtoData) { + getMethodName := func(method int, trimString bool) string { + if val, ok := pogo.Method_name[int32(method)]; ok { + if trimString && strings.HasPrefix(val, "METHOD_") { + return strings.TrimPrefix(val, "METHOD_") + } + return val + } + return fmt.Sprintf("#%d", method) + } + + if method != int(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION) && protoData.Level < 30 { + statsCollector.IncDecodeMethods("error", "low_level", getMethodName(method, true)) + log.Debugf("Insufficient Level %d Did not process hook type %s", protoData.Level, pogo.Method(method)) + return + } + + processed := false + ignore := false + start := time.Now() + result := "" + + switch pogo.Method(method) { + case pogo.Method_METHOD_START_INCIDENT: + result = decodeStartIncident(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_INVASION_OPEN_COMBAT_SESSION: + if protoData.Request != nil { + result = decodeOpenInvasion(ctx, protoData.Request, protoData.Data) + processed = true + } + case pogo.Method_METHOD_FORT_DETAILS: + result = decodeFortDetails(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_GET_MAP_OBJECTS: + result = decodeGMO(ctx, protoData, getScanParameters(protoData)) + processed = true + case pogo.Method_METHOD_GYM_GET_INFO: + result = decodeGetGymInfo(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_ENCOUNTER: + if getScanParameters(protoData).ProcessPokemon { + result = decodeEncounter(ctx, protoData.Data, protoData.Account, protoData.TimestampMs) + } + processed = true + case pogo.Method_METHOD_DISK_ENCOUNTER: + result = decodeDiskEncounter(ctx, protoData.Data, protoData.Account) + processed = true + case pogo.Method_METHOD_FORT_SEARCH: + result = decodeQuest(ctx, protoData.Data, protoData.HaveAr) + processed = true + case pogo.Method_METHOD_GET_PLAYER: + ignore = true + case pogo.Method_METHOD_GET_HOLOHOLO_INVENTORY: + ignore = true + case pogo.Method_METHOD_CREATE_COMBAT_CHALLENGE: + ignore = true + case pogo.Method(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION): + if protoData.Request != nil { + result = decodeSocialActionWithRequest(protoData.Request, protoData.Data) + processed = true + } + case pogo.Method_METHOD_GET_MAP_FORTS: + result = decodeGetMapForts(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_GET_ROUTES: + result = decodeGetRoutes(ctx, protoData.Data) + processed = true + case pogo.Method_METHOD_GET_CONTEST_DATA: + if getScanParameters(protoData).ProcessPokestops { + // Request helps, but can be decoded without it + result = decodeGetContestData(ctx, protoData.Request, protoData.Data) + } + processed = true + case pogo.Method_METHOD_GET_POKEMON_SIZE_CONTEST_ENTRY: + // Request is essential to decode this + if protoData.Request != nil { + if getScanParameters(protoData).ProcessPokestops { + result = decodeGetPokemonSizeContestEntry(ctx, protoData.Request, protoData.Data) + } + processed = true + } + case pogo.Method_METHOD_GET_STATION_DETAILS: + if getScanParameters(protoData).ProcessStations { + // Request is essential to decode this + result = decodeGetStationDetails(ctx, protoData.Request, protoData.Data) + } + processed = true + case pogo.Method_METHOD_PROCESS_TAPPABLE: + if getScanParameters(protoData).ProcessTappables { + // Request is essential to decode this + result = decodeTappable(ctx, protoData.Request, protoData.Data, protoData.Account, protoData.TimestampMs) + } + processed = true + case pogo.Method_METHOD_GET_EVENT_RSVPS: + if getScanParameters(protoData).ProcessGyms { + result = decodeGetEventRsvp(ctx, protoData.Request, protoData.Data) + } + processed = true + case pogo.Method_METHOD_GET_EVENT_RSVP_COUNT: + if getScanParameters(protoData).ProcessGyms { + result = decodeGetEventRsvpCount(ctx, protoData.Data) + } + processed = true + default: + log.Debugf("Did not know hook type %s", pogo.Method(method)) + } + if !ignore { + elapsed := time.Since(start) + if processed == true { + statsCollector.IncDecodeMethods("ok", "", getMethodName(method, true)) + log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, result) + } else { + log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, "**Did not process**") + statsCollector.IncDecodeMethods("unprocessed", "", getMethodName(method, true)) + } + } +} + +func getScanParameters(protoData *ProtoData) decoder.ScanParameters { + return decoder.FindScanConfiguration(protoData.ScanContext, protoData.Lat, protoData.Lon) +} + +func decodeQuest(ctx context.Context, sDec []byte, haveAr *bool) string { + if haveAr == nil { + statsCollector.IncDecodeQuest("error", "missing_ar_info") + log.Infoln("Cannot determine AR quest - ignoring") + // We should either assume AR quest, or trace inventory like RDM probably + return "No AR quest info" + } + decodedQuest := &pogo.FortSearchOutProto{} + if err := proto.Unmarshal(sDec, decodedQuest); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeQuest("error", "parse") + return "Parse failure" + } + + if decodedQuest.Result != pogo.FortSearchOutProto_SUCCESS { + statsCollector.IncDecodeQuest("error", "non_success") + res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedQuest.Result, + pogo.FortSearchOutProto_Result_name[int32(decodedQuest.Result)]) + return res + } + + return decoder.UpdatePokestopWithQuest(ctx, dbDetails, decodedQuest, *haveAr) + +} + +func decodeSocialActionWithRequest(request []byte, payload []byte) string { + var proxyRequestProto pogo.ProxyRequestProto + + if err := proto.Unmarshal(request, &proxyRequestProto); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeSocialActionWithRequest("error", "request_parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + var proxyResponseProto pogo.ProxyResponseProto + + if err := proto.Unmarshal(payload, &proxyResponseProto); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeSocialActionWithRequest("error", "response_parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED && proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED_AND_REASSIGNED { + statsCollector.IncDecodeSocialActionWithRequest("error", "non_success") + return fmt.Sprintf("unsuccessful proxyResponseProto response %d %s", int(proxyResponseProto.Status), proxyResponseProto.Status) + } + + switch pogo.InternalSocialAction(proxyRequestProto.GetAction()) { + case pogo.InternalSocialAction_SOCIAL_ACTION_LIST_FRIEND_STATUS: + statsCollector.IncDecodeSocialActionWithRequest("ok", "list_friend_status") + return decodeGetFriendDetails(proxyResponseProto.Payload) + case pogo.InternalSocialAction_SOCIAL_ACTION_SEARCH_PLAYER: + statsCollector.IncDecodeSocialActionWithRequest("ok", "search_player") + return decodeSearchPlayer(&proxyRequestProto, proxyResponseProto.Payload) + + } + + statsCollector.IncDecodeSocialActionWithRequest("ok", "unknown") + return fmt.Sprintf("Did not process %s", pogo.InternalSocialAction(proxyRequestProto.GetAction()).String()) +} + +func decodeGetFriendDetails(payload []byte) string { + var getFriendDetailsOutProto pogo.InternalGetFriendDetailsOutProto + getFriendDetailsError := proto.Unmarshal(payload, &getFriendDetailsOutProto) + + if getFriendDetailsError != nil { + statsCollector.IncDecodeGetFriendDetails("error", "parse") + log.Errorf("Failed to parse %s", getFriendDetailsError) + return fmt.Sprintf("Failed to parse %s", getFriendDetailsError) + } + + if getFriendDetailsOutProto.GetResult() != pogo.InternalGetFriendDetailsOutProto_SUCCESS || getFriendDetailsOutProto.GetFriend() == nil { + statsCollector.IncDecodeGetFriendDetails("error", "non_success") + return fmt.Sprintf("unsuccessful get friends details") + } + + failures := 0 + + for _, friend := range getFriendDetailsOutProto.GetFriend() { + player := friend.GetPlayer() + + updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, "", player.GetPlayerId()) + if updatePlayerError != nil { + failures++ + } + } + + statsCollector.IncDecodeGetFriendDetails("ok", "") + return fmt.Sprintf("%d players decoded on %d", len(getFriendDetailsOutProto.GetFriend())-failures, len(getFriendDetailsOutProto.GetFriend())) +} + +func decodeSearchPlayer(proxyRequestProto *pogo.ProxyRequestProto, payload []byte) string { + var searchPlayerOutProto pogo.InternalSearchPlayerOutProto + searchPlayerOutError := proto.Unmarshal(payload, &searchPlayerOutProto) + + if searchPlayerOutError != nil { + log.Errorf("Failed to parse %s", searchPlayerOutError) + statsCollector.IncDecodeSearchPlayer("error", "parse") + return fmt.Sprintf("Failed to parse %s", searchPlayerOutError) + } + + if searchPlayerOutProto.GetResult() != pogo.InternalSearchPlayerOutProto_SUCCESS || searchPlayerOutProto.GetPlayer() == nil { + statsCollector.IncDecodeSearchPlayer("error", "non_success") + return fmt.Sprintf("unsuccessful search player response") + } + + var searchPlayerProto pogo.InternalSearchPlayerProto + searchPlayerError := proto.Unmarshal(proxyRequestProto.GetPayload(), &searchPlayerProto) + + if searchPlayerError != nil || searchPlayerProto.GetFriendCode() == "" { + statsCollector.IncDecodeSearchPlayer("error", "parse") + return fmt.Sprintf("Failed to parse %s", searchPlayerError) + } + + player := searchPlayerOutProto.GetPlayer() + updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, searchPlayerProto.GetFriendCode(), "") + if updatePlayerError != nil { + statsCollector.IncDecodeSearchPlayer("error", "update") + return fmt.Sprintf("Failed update player %s", updatePlayerError) + } + + statsCollector.IncDecodeSearchPlayer("ok", "") + return fmt.Sprintf("1 player decoded from SearchPlayerProto") +} + +func decodeFortDetails(ctx context.Context, sDec []byte) string { + decodedFort := &pogo.FortDetailsOutProto{} + if err := proto.Unmarshal(sDec, decodedFort); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeFortDetails("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + switch decodedFort.FortType { + case pogo.FortType_CHECKPOINT: + statsCollector.IncDecodeFortDetails("ok", "pokestop") + return decoder.UpdatePokestopRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) + case pogo.FortType_GYM: + statsCollector.IncDecodeFortDetails("ok", "gym") + return decoder.UpdateGymRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) + } + + statsCollector.IncDecodeFortDetails("ok", "unknown") + return "Unknown fort type" +} + +func decodeGetMapForts(ctx context.Context, sDec []byte) string { + decodedMapForts := &pogo.GetMapFortsOutProto{} + if err := proto.Unmarshal(sDec, decodedMapForts); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeGetMapForts("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedMapForts.Status != pogo.GetMapFortsOutProto_SUCCESS { + statsCollector.IncDecodeGetMapForts("error", "non_success") + res := fmt.Sprintf(`GetMapFortsOutProto: Ignored non-success value %d:%s`, decodedMapForts.Status, + pogo.GetMapFortsOutProto_Status_name[int32(decodedMapForts.Status)]) + return res + } + + statsCollector.IncDecodeGetMapForts("ok", "") + var outputString string + processedForts := 0 + + for _, fort := range decodedMapForts.Fort { + status, output := decoder.UpdateFortRecordWithGetMapFortsOutProto(ctx, dbDetails, fort) + if status { + processedForts += 1 + outputString += output + ", " + } + } + + if processedForts > 0 { + return fmt.Sprintf("Updated %d forts: %s", processedForts, outputString) + } + return "No forts updated" +} + +func decodeGetRoutes(ctx context.Context, payload []byte) string { + getRoutesOutProto := &pogo.GetRoutesOutProto{} + if err := proto.Unmarshal(payload, getRoutesOutProto); err != nil { + return fmt.Sprintf("failed to decode GetRoutesOutProto %s", err) + } + + if getRoutesOutProto.Status != pogo.GetRoutesOutProto_SUCCESS { + return fmt.Sprintf("GetRoutesOutProto: Ignored non-success value %d:%s", getRoutesOutProto.Status, getRoutesOutProto.Status.String()) + } + + decodeSuccesses := map[string]bool{} + decodeErrors := map[string]bool{} + + for _, routeMapCell := range getRoutesOutProto.GetRouteMapCell() { + for _, route := range routeMapCell.GetRoute() { + //TODO we need to check the repeated field, for now access last element + routeSubmissionStatus := route.RouteSubmissionStatus[len(route.RouteSubmissionStatus)-1] + if routeSubmissionStatus != nil && routeSubmissionStatus.Status != pogo.RouteSubmissionStatus_PUBLISHED { + log.Warnf("Non published Route found in GetRoutesOutProto, status: %s", routeSubmissionStatus.String()) + continue + } + decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(ctx, dbDetails, route) + if decodeError != nil { + if decodeErrors[route.Id] != true { + decodeErrors[route.Id] = true + } + log.Errorf("Failed to decode route %s", decodeError) + } else if decodeSuccesses[route.Id] != true { + decodeSuccesses[route.Id] = true + } + } + } + + return fmt.Sprintf( + "Decoded %d routes, failed to decode %d routes, from %d cells", + len(decodeSuccesses), + len(decodeErrors), + len(getRoutesOutProto.GetRouteMapCell()), + ) +} + +func decodeGetGymInfo(ctx context.Context, sDec []byte) string { + decodedGymInfo := &pogo.GymGetInfoOutProto{} + if err := proto.Unmarshal(sDec, decodedGymInfo); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeGetGymInfo("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedGymInfo.Result != pogo.GymGetInfoOutProto_SUCCESS { + statsCollector.IncDecodeGetGymInfo("error", "non_success") + res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedGymInfo.Result, + pogo.GymGetInfoOutProto_Result_name[int32(decodedGymInfo.Result)]) + return res + } + + statsCollector.IncDecodeGetGymInfo("ok", "") + return decoder.UpdateGymRecordWithGymInfoProto(ctx, dbDetails, decodedGymInfo) +} + +func decodeEncounter(ctx context.Context, sDec []byte, username string, timestampMs int64) string { + decodedEncounterInfo := &pogo.EncounterOutProto{} + if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeEncounter("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedEncounterInfo.Status != pogo.EncounterOutProto_ENCOUNTER_SUCCESS { + statsCollector.IncDecodeEncounter("error", "non_success") + res := fmt.Sprintf(`EncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Status, + pogo.EncounterOutProto_Status_name[int32(decodedEncounterInfo.Status)]) + return res + } + + statsCollector.IncDecodeEncounter("ok", "") + return decoder.UpdatePokemonRecordWithEncounterProto(ctx, dbDetails, decodedEncounterInfo, username, timestampMs) +} + +func decodeDiskEncounter(ctx context.Context, sDec []byte, username string) string { + decodedEncounterInfo := &pogo.DiskEncounterOutProto{} + if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeDiskEncounter("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedEncounterInfo.Result != pogo.DiskEncounterOutProto_SUCCESS { + statsCollector.IncDecodeDiskEncounter("error", "non_success") + res := fmt.Sprintf(`DiskEncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Result, + pogo.DiskEncounterOutProto_Result_name[int32(decodedEncounterInfo.Result)]) + return res + } + + statsCollector.IncDecodeDiskEncounter("ok", "") + return decoder.UpdatePokemonRecordWithDiskEncounterProto(ctx, dbDetails, decodedEncounterInfo, username) +} + +func decodeStartIncident(ctx context.Context, sDec []byte) string { + decodedIncident := &pogo.StartIncidentOutProto{} + if err := proto.Unmarshal(sDec, decodedIncident); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeStartIncident("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedIncident.Status != pogo.StartIncidentOutProto_SUCCESS { + statsCollector.IncDecodeStartIncident("error", "non_success") + res := fmt.Sprintf(`GiovanniOutProto: Ignored non-success value %d:%s`, decodedIncident.Status, + pogo.StartIncidentOutProto_Status_name[int32(decodedIncident.Status)]) + return res + } + + statsCollector.IncDecodeStartIncident("ok", "") + return decoder.ConfirmIncident(ctx, dbDetails, decodedIncident) +} + +func decodeOpenInvasion(ctx context.Context, request []byte, payload []byte) string { + decodeOpenInvasionRequest := &pogo.OpenInvasionCombatSessionProto{} + + if err := proto.Unmarshal(request, decodeOpenInvasionRequest); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeOpenInvasion("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + if decodeOpenInvasionRequest.IncidentLookup == nil { + return "Invalid OpenInvasionCombatSessionProto received" + } + + decodedOpenInvasionResponse := &pogo.OpenInvasionCombatSessionOutProto{} + if err := proto.Unmarshal(payload, decodedOpenInvasionResponse); err != nil { + log.Errorf("Failed to parse %s", err) + statsCollector.IncDecodeOpenInvasion("error", "parse") + return fmt.Sprintf("Failed to parse %s", err) + } + + if decodedOpenInvasionResponse.Status != pogo.InvasionStatus_SUCCESS { + statsCollector.IncDecodeOpenInvasion("error", "non_success") + res := fmt.Sprintf(`InvasionLineupOutProto: Ignored non-success value %d:%s`, decodedOpenInvasionResponse.Status, + pogo.InvasionStatus_Status_name[int32(decodedOpenInvasionResponse.Status)]) + return res + } + + statsCollector.IncDecodeOpenInvasion("ok", "") + return decoder.UpdateIncidentLineup(ctx, dbDetails, decodeOpenInvasionRequest, decodedOpenInvasionResponse) +} + +func decodeGMO(ctx context.Context, protoData *ProtoData, scanParameters decoder.ScanParameters) string { + decodedGmo := &pogo.GetMapObjectsOutProto{} + + if err := proto.Unmarshal(protoData.Data, decodedGmo); err != nil { + statsCollector.IncDecodeGMO("error", "parse") + log.Errorf("Failed to parse %s", err) + } + + if decodedGmo.Status != pogo.GetMapObjectsOutProto_SUCCESS { + statsCollector.IncDecodeGMO("error", "non_success") + res := fmt.Sprintf(`GetMapObjectsOutProto: Ignored non-success value %d:%s`, decodedGmo.Status, + pogo.GetMapObjectsOutProto_Status_name[int32(decodedGmo.Status)]) + return res + } + + var newForts []decoder.RawFortData + var newStations []decoder.RawStationData + var newWildPokemon []decoder.RawWildPokemonData + var newNearbyPokemon []decoder.RawNearbyPokemonData + var newMapPokemon []decoder.RawMapPokemonData + var newMapCells []uint64 + var cellsToBeCleaned []uint64 + + // track forts per cell for memory-based cleanup (only if tracker enabled) + cellForts := make(map[uint64]*decoder.FortTrackerGMOContents) + + if len(decodedGmo.MapCell) == 0 { + return "Skipping GetMapObjectsOutProto: No map cells found" + } + for _, mapCell := range decodedGmo.MapCell { + // initialize cell forts tracking for every map cell (so empty fort lists are seen as "no forts") + cellForts[mapCell.S2CellId] = &decoder.FortTrackerGMOContents{ + Pokestops: make([]string, 0), + Gyms: make([]string, 0), + Timestamp: mapCell.AsOfTimeMs, + } + // always mark this mapCell to be checked for removed forts. Previously only cells with forts were + // added which meant an empty fort list (all forts removed) was never passed to the tracker. + cellsToBeCleaned = append(cellsToBeCleaned, mapCell.S2CellId) + + if isCellNotEmpty(mapCell) { + newMapCells = append(newMapCells, mapCell.S2CellId) + } + + for _, fort := range mapCell.Fort { + newForts = append(newForts, decoder.RawFortData{Cell: mapCell.S2CellId, Data: fort, Timestamp: mapCell.AsOfTimeMs}) + + // track fort by type for memory-based cleanup (only if tracker enabled) + if cf, ok := cellForts[mapCell.S2CellId]; ok { + switch fort.FortType { + case pogo.FortType_GYM: + cf.Gyms = append(cf.Gyms, fort.FortId) + case pogo.FortType_CHECKPOINT: + cf.Pokestops = append(cf.Pokestops, fort.FortId) + } + } + + if fort.ActivePokemon != nil { + newMapPokemon = append(newMapPokemon, decoder.RawMapPokemonData{Cell: mapCell.S2CellId, Data: fort.ActivePokemon, Timestamp: mapCell.AsOfTimeMs}) + } + } + for _, mon := range mapCell.WildPokemon { + newWildPokemon = append(newWildPokemon, decoder.RawWildPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) + } + for _, mon := range mapCell.NearbyPokemon { + newNearbyPokemon = append(newNearbyPokemon, decoder.RawNearbyPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) + } + for _, station := range mapCell.Stations { + newStations = append(newStations, decoder.RawStationData{Cell: mapCell.S2CellId, Data: station}) + } + } + + if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { + decoder.UpdateFortBatch(ctx, dbDetails, scanParameters, newForts) + } + var weatherUpdates []decoder.WeatherUpdate + if scanParameters.ProcessWeather { + weatherUpdates = decoder.UpdateClientWeatherBatch(ctx, dbDetails, decodedGmo.ClientWeather, decodedGmo.MapCell[0].AsOfTimeMs, protoData.Account) + } + if scanParameters.ProcessPokemon { + decoder.UpdatePokemonBatch(ctx, dbDetails, scanParameters, newWildPokemon, newNearbyPokemon, newMapPokemon, decodedGmo.ClientWeather, protoData.Account) + if scanParameters.ProcessWeather && scanParameters.ProactiveIVSwitching { + for _, weatherUpdate := range weatherUpdates { + go func(weatherUpdate decoder.WeatherUpdate) { + decoder.ProactiveIVSwitchSem <- true + defer func() { <-decoder.ProactiveIVSwitchSem }() + decoder.ProactiveIVSwitch(ctx, dbDetails, weatherUpdate, scanParameters.ProactiveIVSwitchingToDB, decodedGmo.MapCell[0].AsOfTimeMs/1000) + }(weatherUpdate) + } + } + } + if scanParameters.ProcessStations { + decoder.UpdateStationBatch(ctx, dbDetails, scanParameters, newStations) + } + + if scanParameters.ProcessCells { + decoder.UpdateClientMapS2CellBatch(ctx, dbDetails, newMapCells) + } + + if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + decoder.CheckRemovedForts(ctx, dbDetails, cellsToBeCleaned, cellForts) + }() + } + + newFortsLen := len(newForts) + newStationsLen := len(newStations) + newWildPokemonLen := len(newWildPokemon) + newNearbyPokemonLen := len(newNearbyPokemon) + newMapPokemonLen := len(newMapPokemon) + newClientWeatherLen := len(decodedGmo.ClientWeather) + newMapCellsLen := len(newMapCells) + + statsCollector.IncDecodeGMO("ok", "") + statsCollector.AddDecodeGMOType("fort", float64(newFortsLen)) + statsCollector.AddDecodeGMOType("station", float64(newStationsLen)) + statsCollector.AddDecodeGMOType("wild_pokemon", float64(newWildPokemonLen)) + statsCollector.AddDecodeGMOType("nearby_pokemon", float64(newNearbyPokemonLen)) + statsCollector.AddDecodeGMOType("map_pokemon", float64(newMapPokemonLen)) + statsCollector.AddDecodeGMOType("weather", float64(newClientWeatherLen)) + statsCollector.AddDecodeGMOType("cell", float64(newMapCellsLen)) + + return fmt.Sprintf("%d cells containing %d forts %d stations %d mon %d nearby", newMapCellsLen, newFortsLen, newStationsLen, newWildPokemonLen, newNearbyPokemonLen) +} + +func isCellNotEmpty(mapCell *pogo.ClientMapCellProto) bool { + return len(mapCell.Stations) > 0 || len(mapCell.Fort) > 0 || len(mapCell.WildPokemon) > 0 || len(mapCell.NearbyPokemon) > 0 || len(mapCell.CatchablePokemon) > 0 +} + +func cellContainsForts(mapCell *pogo.ClientMapCellProto) bool { + return len(mapCell.Fort) > 0 +} + +func decodeGetContestData(ctx context.Context, request []byte, data []byte) string { + var decodedContestData pogo.GetContestDataOutProto + if err := proto.Unmarshal(data, &decodedContestData); err != nil { + log.Errorf("Failed to parse GetContestDataOutProto %s", err) + return fmt.Sprintf("Failed to parse GetContestDataOutProto %s", err) + } + + var decodedContestDataRequest pogo.GetContestDataProto + if request != nil { + if err := proto.Unmarshal(request, &decodedContestDataRequest); err != nil { + log.Errorf("Failed to parse GetContestDataProto %s", err) + return fmt.Sprintf("Failed to parse GetContestDataProto %s", err) + } + } + return decoder.UpdatePokestopWithContestData(ctx, dbDetails, &decodedContestDataRequest, &decodedContestData) +} + +func decodeGetPokemonSizeContestEntry(ctx context.Context, request []byte, data []byte) string { + var decodedPokemonSizeContestEntry pogo.GetPokemonSizeLeaderboardEntryOutProto + if err := proto.Unmarshal(data, &decodedPokemonSizeContestEntry); err != nil { + log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + } + + if decodedPokemonSizeContestEntry.Status != pogo.GetPokemonSizeLeaderboardEntryOutProto_SUCCESS { + return fmt.Sprintf("Ignored GetPokemonSizeLeaderboardEntryOutProto non-success status %s", decodedPokemonSizeContestEntry.Status) + } + + var decodedPokemonSizeContestEntryRequest pogo.GetPokemonSizeLeaderboardEntryProto + if request != nil { + if err := proto.Unmarshal(request, &decodedPokemonSizeContestEntryRequest); err != nil { + log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) + } + } + + return decoder.UpdatePokestopWithPokemonSizeContestEntry(ctx, dbDetails, &decodedPokemonSizeContestEntryRequest, &decodedPokemonSizeContestEntry) +} + +func decodeGetStationDetails(ctx context.Context, request []byte, data []byte) string { + var decodedGetStationDetails pogo.GetStationedPokemonDetailsOutProto + if err := proto.Unmarshal(data, &decodedGetStationDetails); err != nil { + log.Errorf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) + return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) + } + + var decodedGetStationDetailsRequest pogo.GetStationedPokemonDetailsProto + if request != nil { + if err := proto.Unmarshal(request, &decodedGetStationDetailsRequest); err != nil { + log.Errorf("Failed to parse GetStationedPokemonDetailsProto %s", err) + return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsProto %s", err) + } + } + + if decodedGetStationDetails.Result == pogo.GetStationedPokemonDetailsOutProto_STATION_NOT_FOUND { + // station without stationed pokemon found, therefore we need to reset the columns + return decoder.ResetStationedPokemonWithStationDetailsNotFound(ctx, dbDetails, &decodedGetStationDetailsRequest) + } else if decodedGetStationDetails.Result != pogo.GetStationedPokemonDetailsOutProto_SUCCESS { + return fmt.Sprintf("Ignored GetStationedPokemonDetailsOutProto non-success status %s", decodedGetStationDetails.Result) + } + + return decoder.UpdateStationWithStationDetails(ctx, dbDetails, &decodedGetStationDetailsRequest, &decodedGetStationDetails) +} + +func decodeTappable(ctx context.Context, request, data []byte, username string, timestampMs int64) string { + var tappable pogo.ProcessTappableOutProto + if err := proto.Unmarshal(data, &tappable); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse ProcessTappableOutProto %s", err) + } + + var tappableRequest pogo.ProcessTappableProto + if request != nil { + if err := proto.Unmarshal(request, &tappableRequest); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse ProcessTappableProto %s", err) + } + } + + if tappable.Status != pogo.ProcessTappableOutProto_SUCCESS { + return fmt.Sprintf("Ignored ProcessTappableOutProto non-success status %s", tappable.Status) + } + var result string + if encounter := tappable.GetEncounter(); encounter != nil { + result = decoder.UpdatePokemonRecordWithTappableEncounter(ctx, dbDetails, &tappableRequest, encounter, username, timestampMs) + } + return result + " " + decoder.UpdateTappable(ctx, dbDetails, &tappableRequest, &tappable, timestampMs) +} + +func decodeGetEventRsvp(ctx context.Context, request []byte, data []byte) string { + var rsvp pogo.GetEventRsvpsOutProto + if err := proto.Unmarshal(data, &rsvp); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse GetEventRsvpsOutProto %s", err) + } + + var rsvpRequest pogo.GetEventRsvpsProto + if request != nil { + if err := proto.Unmarshal(request, &rsvpRequest); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse GetEventRsvpsProto %s", err) + } + } + + if rsvp.Status != pogo.GetEventRsvpsOutProto_SUCCESS { + return fmt.Sprintf("Ignored GetEventRsvpsOutProto non-success status %s", rsvp.Status) + } + + switch op := rsvpRequest.EventDetails.(type) { + case *pogo.GetEventRsvpsProto_Raid: + return decoder.UpdateGymRecordWithRsvpProto(ctx, dbDetails, op.Raid, &rsvp) + case *pogo.GetEventRsvpsProto_GmaxBattle: + return "Unsupported GmaxBattle Rsvp received" + } + + return "Failed to parse GetEventRsvpsProto - unknown event type" +} + +func decodeGetEventRsvpCount(ctx context.Context, data []byte) string { + var rsvp pogo.GetEventRsvpCountOutProto + if err := proto.Unmarshal(data, &rsvp); err != nil { + log.Errorf("Failed to parse %s", err) + return fmt.Sprintf("Failed to parse GetEventRsvpCountOutProto %s", err) + } + + if rsvp.Status != pogo.GetEventRsvpCountOutProto_SUCCESS { + return fmt.Sprintf("Ignored GetEventRsvpCountOutProto non-success status %s", rsvp.Status) + } + + var clearLocations []string + for _, rsvpDetails := range rsvp.RsvpDetails { + if rsvpDetails.MaybeCount == 0 && rsvpDetails.GoingCount == 0 { + clearLocations = append(clearLocations, rsvpDetails.LocationId) + decoder.ClearGymRsvp(ctx, dbDetails, rsvpDetails.LocationId) + } + } + + return "Cleared RSVP @ " + strings.Join(clearLocations, ", ") +} diff --git a/decoder/api_gym.go b/decoder/api_gym.go index 97bd60b9..8003cde3 100644 --- a/decoder/api_gym.go +++ b/decoder/api_gym.go @@ -8,8 +8,104 @@ import ( "golbat/db" "golbat/geo" + + "github.com/guregu/null/v6" ) +type ApiGymResult struct { + Id string `json:"id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + Name null.String `json:"name"` + Url null.String `json:"url"` + LastModifiedTimestamp null.Int `json:"last_modified_timestamp"` + RaidEndTimestamp null.Int `json:"raid_end_timestamp"` + RaidSpawnTimestamp null.Int `json:"raid_spawn_timestamp"` + RaidBattleTimestamp null.Int `json:"raid_battle_timestamp"` + Updated int64 `json:"updated"` + RaidPokemonId null.Int `json:"raid_pokemon_id"` + GuardingPokemonId null.Int `json:"guarding_pokemon_id"` + GuardingPokemonDisplay null.String `json:"guarding_pokemon_display"` + AvailableSlots null.Int `json:"available_slots"` + TeamId null.Int `json:"team_id"` + RaidLevel null.Int `json:"raid_level"` + Enabled null.Int `json:"enabled"` + ExRaidEligible null.Int `json:"ex_raid_eligible"` + InBattle null.Int `json:"in_battle"` + RaidPokemonMove1 null.Int `json:"raid_pokemon_move_1"` + RaidPokemonMove2 null.Int `json:"raid_pokemon_move_2"` + RaidPokemonForm null.Int `json:"raid_pokemon_form"` + RaidPokemonAlignment null.Int `json:"raid_pokemon_alignment"` + RaidPokemonCp null.Int `json:"raid_pokemon_cp"` + RaidIsExclusive null.Int `json:"raid_is_exclusive"` + CellId null.Int `json:"cell_id"` + Deleted bool `json:"deleted"` + TotalCp null.Int `json:"total_cp"` + FirstSeenTimestamp int64 `json:"first_seen_timestamp"` + RaidPokemonGender null.Int `json:"raid_pokemon_gender"` + SponsorId null.Int `json:"sponsor_id"` + PartnerId null.String `json:"partner_id"` + RaidPokemonCostume null.Int `json:"raid_pokemon_costume"` + RaidPokemonEvolution null.Int `json:"raid_pokemon_evolution"` + ArScanEligible null.Int `json:"ar_scan_eligible"` + PowerUpLevel null.Int `json:"power_up_level"` + PowerUpPoints null.Int `json:"power_up_points"` + PowerUpEndTimestamp null.Int `json:"power_up_end_timestamp"` + Description null.String `json:"description"` + Defenders null.String `json:"defenders"` + Rsvps null.String `json:"rsvps"` +} + +func buildGymResult(gym *Gym) ApiGymResult { + return ApiGymResult{ + Id: gym.Id, + Lat: gym.Lat, + Lon: gym.Lon, + Name: gym.Name, + Url: gym.Url, + LastModifiedTimestamp: gym.LastModifiedTimestamp, + RaidEndTimestamp: gym.RaidEndTimestamp, + RaidSpawnTimestamp: gym.RaidSpawnTimestamp, + RaidBattleTimestamp: gym.RaidBattleTimestamp, + Updated: gym.Updated, + RaidPokemonId: gym.RaidPokemonId, + GuardingPokemonId: gym.GuardingPokemonId, + GuardingPokemonDisplay: gym.GuardingPokemonDisplay, + AvailableSlots: gym.AvailableSlots, + TeamId: gym.TeamId, + RaidLevel: gym.RaidLevel, + Enabled: gym.Enabled, + ExRaidEligible: gym.ExRaidEligible, + InBattle: gym.InBattle, + RaidPokemonMove1: gym.RaidPokemonMove1, + RaidPokemonMove2: gym.RaidPokemonMove2, + RaidPokemonForm: gym.RaidPokemonForm, + RaidPokemonAlignment: gym.RaidPokemonAlignment, + RaidPokemonCp: gym.RaidPokemonCp, + RaidIsExclusive: gym.RaidIsExclusive, + CellId: gym.CellId, + Deleted: gym.Deleted, + TotalCp: gym.TotalCp, + FirstSeenTimestamp: gym.FirstSeenTimestamp, + RaidPokemonGender: gym.RaidPokemonGender, + SponsorId: gym.SponsorId, + PartnerId: gym.PartnerId, + RaidPokemonCostume: gym.RaidPokemonCostume, + RaidPokemonEvolution: gym.RaidPokemonEvolution, + ArScanEligible: gym.ArScanEligible, + PowerUpLevel: gym.PowerUpLevel, + PowerUpPoints: gym.PowerUpPoints, + PowerUpEndTimestamp: gym.PowerUpEndTimestamp, + Description: gym.Description, + Defenders: gym.Defenders, + Rsvps: gym.Rsvps, + } +} + +func BuildGymResult(gym *Gym) ApiGymResult { + return buildGymResult(gym) +} + type ApiGymSearch struct { Limit int `json:"limit"` Filters []ApiGymSearchFilter `json:"filters"` diff --git a/decoder/api_pokemon_common.go b/decoder/api_pokemon_common.go index 9eb74919..c9eeca16 100644 --- a/decoder/api_pokemon_common.go +++ b/decoder/api_pokemon_common.go @@ -10,8 +10,8 @@ import ( pb "golbat/grpc" "github.com/UnownHash/gohbem" + "github.com/guregu/null/v6" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) type ApiPokemonDnfId struct { diff --git a/decoder/api_pokestop.go b/decoder/api_pokestop.go new file mode 100644 index 00000000..618c81d4 --- /dev/null +++ b/decoder/api_pokestop.go @@ -0,0 +1,101 @@ +package decoder + +import "github.com/guregu/null/v6" + +type ApiPokestopResult struct { + Id string `json:"id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + Name null.String `json:"name"` + Url null.String `json:"url"` + LureExpireTimestamp null.Int `json:"lure_expire_timestamp"` + LastModifiedTimestamp null.Int `json:"last_modified_timestamp"` + Updated int64 `json:"updated"` + Enabled null.Bool `json:"enabled"` + QuestType null.Int `json:"quest_type"` + QuestTimestamp null.Int `json:"quest_timestamp"` + QuestTarget null.Int `json:"quest_target"` + QuestConditions null.String `json:"quest_conditions"` + QuestRewards null.String `json:"quest_rewards"` + QuestTemplate null.String `json:"quest_template"` + QuestTitle null.String `json:"quest_title"` + QuestExpiry null.Int `json:"quest_expiry"` + CellId null.Int `json:"cell_id"` + Deleted bool `json:"deleted"` + LureId int16 `json:"lure_id"` + FirstSeenTimestamp int16 `json:"first_seen_timestamp"` + SponsorId null.Int `json:"sponsor_id"` + PartnerId null.String `json:"partner_id"` + ArScanEligible null.Int `json:"ar_scan_eligible"` + PowerUpLevel null.Int `json:"power_up_level"` + PowerUpPoints null.Int `json:"power_up_points"` + PowerUpEndTimestamp null.Int `json:"power_up_end_timestamp"` + AlternativeQuestType null.Int `json:"alternative_quest_type"` + AlternativeQuestTimestamp null.Int `json:"alternative_quest_timestamp"` + AlternativeQuestTarget null.Int `json:"alternative_quest_target"` + AlternativeQuestConditions null.String `json:"alternative_quest_conditions"` + AlternativeQuestRewards null.String `json:"alternative_quest_rewards"` + AlternativeQuestTemplate null.String `json:"alternative_quest_template"` + AlternativeQuestTitle null.String `json:"alternative_quest_title"` + AlternativeQuestExpiry null.Int `json:"alternative_quest_expiry"` + Description null.String `json:"description"` + ShowcaseFocus null.String `json:"showcase_focus"` + ShowcasePokemon null.Int `json:"showcase_pokemon_id"` + ShowcasePokemonForm null.Int `json:"showcase_pokemon_form_id"` + ShowcasePokemonType null.Int `json:"showcase_pokemon_type_id"` + ShowcaseRankingStandard null.Int `json:"showcase_ranking_standard"` + ShowcaseExpiry null.Int `json:"showcase_expiry"` + ShowcaseRankings null.String `json:"showcase_rankings"` +} + +func buildPokestopResult(stop *Pokestop) ApiPokestopResult { + return ApiPokestopResult{ + Id: stop.Id, + Lat: stop.Lat, + Lon: stop.Lon, + Name: stop.Name, + Url: stop.Url, + LureExpireTimestamp: stop.LureExpireTimestamp, + LastModifiedTimestamp: stop.LastModifiedTimestamp, + Updated: stop.Updated, + Enabled: stop.Enabled, + QuestType: stop.QuestType, + QuestTimestamp: stop.QuestTimestamp, + QuestTarget: stop.QuestTarget, + QuestConditions: stop.QuestConditions, + QuestRewards: stop.QuestRewards, + QuestTemplate: stop.QuestTemplate, + QuestTitle: stop.QuestTitle, + QuestExpiry: stop.QuestExpiry, + CellId: stop.CellId, + Deleted: stop.Deleted, + LureId: stop.LureId, + FirstSeenTimestamp: stop.FirstSeenTimestamp, + SponsorId: stop.SponsorId, + PartnerId: stop.PartnerId, + ArScanEligible: stop.ArScanEligible, + PowerUpLevel: stop.PowerUpLevel, + PowerUpPoints: stop.PowerUpPoints, + PowerUpEndTimestamp: stop.PowerUpEndTimestamp, + AlternativeQuestType: stop.AlternativeQuestType, + AlternativeQuestTimestamp: stop.AlternativeQuestTimestamp, + AlternativeQuestTarget: stop.AlternativeQuestTarget, + AlternativeQuestConditions: stop.AlternativeQuestConditions, + AlternativeQuestRewards: stop.AlternativeQuestRewards, + AlternativeQuestTemplate: stop.AlternativeQuestTemplate, + AlternativeQuestTitle: stop.AlternativeQuestTitle, + AlternativeQuestExpiry: stop.AlternativeQuestExpiry, + Description: stop.Description, + ShowcaseFocus: stop.ShowcaseFocus, + ShowcasePokemon: stop.ShowcasePokemon, + ShowcasePokemonForm: stop.ShowcasePokemonForm, + ShowcasePokemonType: stop.ShowcasePokemonType, + ShowcaseRankingStandard: stop.ShowcaseRankingStandard, + ShowcaseExpiry: stop.ShowcaseExpiry, + ShowcaseRankings: stop.ShowcaseRankings, + } +} + +func BuildPokestopResult(stop *Pokestop) ApiPokestopResult { + return buildPokestopResult(stop) +} diff --git a/decoder/api_tappable.go b/decoder/api_tappable.go new file mode 100644 index 00000000..96874b65 --- /dev/null +++ b/decoder/api_tappable.go @@ -0,0 +1,39 @@ +package decoder + +import "github.com/guregu/null/v6" + +type ApiTappableResult struct { + Id uint64 `json:"id"` + Lat float64 `json:"lat"` + Lon float64 `json:"lon"` + FortId null.String `json:"fort_id"` + SpawnId null.Int `json:"spawn_id"` + Type string `json:"type"` + Encounter null.Int `json:"pokemon_id"` + ItemId null.Int `json:"item_id"` + Count null.Int `json:"count"` + ExpireTimestamp null.Int `json:"expire_timestamp"` + ExpireTimestampVerified bool `json:"expire_timestamp_verified"` + Updated int64 `json:"updated"` +} + +func buildTappableResult(tappable *Tappable) ApiTappableResult { + return ApiTappableResult{ + Id: tappable.Id, + Lat: tappable.Lat, + Lon: tappable.Lon, + FortId: tappable.FortId, + SpawnId: tappable.SpawnId, + Type: tappable.Type, + Encounter: tappable.Encounter, + ItemId: tappable.ItemId, + Count: tappable.Count, + ExpireTimestamp: tappable.ExpireTimestamp, + ExpireTimestampVerified: tappable.ExpireTimestampVerified, + Updated: tappable.Updated, + } +} + +func BuildTappableResult(tappable *Tappable) ApiTappableResult { + return buildTappableResult(tappable) +} diff --git a/decoder/fort.go b/decoder/fort.go index 979bc2e1..4cc34417 100644 --- a/decoder/fort.go +++ b/decoder/fort.go @@ -99,11 +99,11 @@ func InitWebHookFortFromPokestop(stop *Pokestop) *FortWebhook { func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []string, fortType FortType, change FortChange) { if fortType == GYM { for _, id := range ids { - gym, unlock, err := getGymRecordReadOnly(ctx, dbDetails, id) - if err != nil { - continue - } - if gym == nil { + gym, unlock, err := GetGymRecordReadOnly(ctx, dbDetails, id) + if err != nil || gym == nil { + if unlock != nil { + unlock() + } continue } @@ -116,10 +116,10 @@ func CreateFortWebhooks(ctx context.Context, dbDetails db.DbDetails, ids []strin if fortType == POKESTOP { for _, id := range ids { stop, unlock, err := getPokestopRecordReadOnly(ctx, dbDetails, id) - if err != nil { - continue - } - if stop == nil { + if err != nil || stop == nil { + if unlock != nil { + unlock() + } continue } diff --git a/decoder/fortRtree.go b/decoder/fortRtree.go index 14222171..f914d73c 100644 --- a/decoder/fortRtree.go +++ b/decoder/fortRtree.go @@ -71,7 +71,7 @@ func LoadAllGyms(details db.DbDetails) { if err != nil { log.Fatalln(err) } - _, unlock, _ := getGymRecordReadOnly(context.Background(), details, place.Id) + _, unlock, _ := GetGymRecordReadOnly(context.Background(), details, place.Id) if unlock != nil { unlock() } diff --git a/decoder/gmo_decode.go b/decoder/gmo_decode.go new file mode 100644 index 00000000..95784faf --- /dev/null +++ b/decoder/gmo_decode.go @@ -0,0 +1,224 @@ +package decoder + +import ( + "context" + "time" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawFortData) { + // Logic is: + // 1. Filter out pokestops that are unchanged (last modified time) + // 2. Fetch current stops from database + // 3. Generate batch of inserts as needed (with on duplicate saveGymRecord) + + //var stopsToModify []string + + for _, fort := range p { + fortId := fort.Data.FortId + if fort.Data.FortType == pogo.FortType_CHECKPOINT && scanParameters.ProcessPokestops { + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fortId) + if err != nil { + log.Errorf("getOrCreatePokestopRecord: %s", err) + continue + } + + pokestop.updatePokestopFromFort(fort.Data, fort.Cell, fort.Timestamp/1000) + + // If this is a new pokestop, check if it was converted from a gym and copy shared fields + if pokestop.IsNewRecord() { + gym, gymUnlock, _ := GetGymRecordReadOnly(ctx, db, fortId) + if gym != nil { + pokestop.copySharedFieldsFrom(gym) + gymUnlock() + } + } + + savePokestopRecord(ctx, db, pokestop) + unlock() + + incidents := fort.Data.PokestopDisplays + if incidents == nil && fort.Data.PokestopDisplay != nil { + incidents = []*pogo.PokestopIncidentDisplayProto{fort.Data.PokestopDisplay} + } + + if incidents != nil { + for _, incidentProto := range incidents { + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, incidentProto.IncidentId, fortId) + if err != nil { + log.Errorf("getOrCreateIncidentRecord: %s", err) + continue + } + incident.updateFromPokestopIncidentDisplay(incidentProto) + saveIncidentRecord(ctx, db, incident) + unlock() + } + } + } + + if fort.Data.FortType == pogo.FortType_GYM && scanParameters.ProcessGyms { + gym, gymUnlock, err := getOrCreateGymRecord(ctx, db, fortId) + if err != nil { + log.Errorf("getOrCreateGymRecord: %s", err) + continue + } + + gym.updateGymFromFort(fort.Data, fort.Cell) + + // If this is a new gym, check if it was converted from a pokestop and copy shared fields + if gym.IsNewRecord() { + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, fortId) + if pokestop != nil { + gym.copySharedFieldsFrom(pokestop) + unlock() + } + } + + saveGymRecord(ctx, db, gym) + gymUnlock() + } + } +} + +func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawStationData) { + for _, stationProto := range p { + stationId := stationProto.Data.Id + station, unlock, err := getOrCreateStationRecord(ctx, db, stationId) + if err != nil { + log.Errorf("getOrCreateStationRecord: %s", err) + continue + } + station.updateFromStationProto(stationProto.Data, stationProto.Cell) + saveStationRecord(ctx, db, station) + unlock() + } +} + +func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, wildPokemonList []RawWildPokemonData, nearbyPokemonList []RawNearbyPokemonData, mapPokemonList []RawMapPokemonData, weather []*pogo.ClientWeatherProto, username string) { + weatherLookup := make(map[int64]pogo.GameplayWeatherProto_WeatherCondition) + for _, weatherProto := range weather { + weatherLookup[weatherProto.S2CellId] = weatherProto.GameplayWeather.GameplayCondition + } + + for _, wild := range wildPokemonList { + encounterId := wild.Data.EncounterId + + // spawnpointUpdateFromWild doesn't need Pokemon lock + spawnpointUpdateFromWild(ctx, db, wild.Data, wild.Timestamp) + + if scanParameters.ProcessWild { + // Use read-only getter - we're only checking if update is needed, then queuing + pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) + if err != nil { + log.Errorf("getPokemonRecordReadOnly: %s", err) + continue + } + + updateTime := wild.Timestamp / 1000 + shouldQueue := pokemon == nil || pokemon.wildSignificantUpdate(wild.Data, updateTime) + + if unlock != nil { + unlock() + } + + if shouldQueue { + // The sweeper will process it after timeout if no encounter arrives + pending := &PendingPokemon{ + EncounterId: encounterId, + WildPokemon: wild.Data, + CellId: int64(wild.Cell), + TimestampMs: wild.Timestamp, + UpdateTime: updateTime, + WeatherLookup: weatherLookup, + Username: username, + } + pokemonPendingQueue.AddPending(pending) + } + } + } + + if scanParameters.ProcessNearby { + for _, nearby := range nearbyPokemonList { + encounterId := nearby.Data.EncounterId + + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Printf("getOrCreatePokemonRecord: %s", err) + continue + } + + updateTime := nearby.Timestamp / 1000 + if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { + pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) + } + + unlock() + } + } + + for _, mapPokemon := range mapPokemonList { + encounterId := mapPokemon.Data.EncounterId + + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Printf("getOrCreatePokemonRecord: %s", err) + continue + } + + pokemon.updateFromMap(ctx, db, mapPokemon.Data, int64(mapPokemon.Cell), weatherLookup, mapPokemon.Timestamp, username) + storedDiskEncounter := diskEncounterCache.Get(encounterId) + if storedDiskEncounter != nil { + diskEncounter := storedDiskEncounter.Value() + diskEncounterCache.Delete(encounterId) + pokemon.updatePokemonFromDiskEncounterProto(ctx, db, diskEncounter, username) + //log.Infof("Processed stored disk encounter") + } + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, mapPokemon.Timestamp/1000) + + unlock() + } +} + +func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.ClientWeatherProto, timestampMs int64, account string) (updates []WeatherUpdate) { + hourKey := timestampMs / time.Hour.Milliseconds() + for _, weatherProto := range p { + weather, unlock, err := getOrCreateWeatherRecord(ctx, db, weatherProto.S2CellId) + if err != nil { + log.Printf("getOrCreateWeatherRecord: %s", err) + continue + } + + if weather.newRecord || timestampMs >= weather.UpdatedMs { + state := getWeatherConsensusState(weatherProto.S2CellId, hourKey) + if state != nil { + publish, publishProto := state.applyObservation(hourKey, account, weatherProto) + if publish { + if publishProto == nil { + publishProto = weatherProto + } + weather.UpdatedMs = timestampMs + weather.updateWeatherFromClientWeatherProto(publishProto) + saveWeatherRecord(ctx, db, weather) + if weather.oldValues.GameplayCondition != weather.GameplayCondition { + updates = append(updates, WeatherUpdate{ + S2CellId: publishProto.S2CellId, + NewWeather: int32(publishProto.GetGameplayWeather().GetGameplayCondition()), + }) + } + } + } + } + + unlock() + } + return updates +} + +func UpdateClientMapS2CellBatch(ctx context.Context, db db.DbDetails, cellIds []uint64) { + saveS2CellRecords(ctx, db, cellIds) +} diff --git a/decoder/gym.go b/decoder/gym.go index 5fc58626..c0004247 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -1,28 +1,9 @@ package decoder import ( - "cmp" - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "slices" - "strings" "sync" - "time" - "golbat/geo" - - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" - - "golbat/config" - "golbat/db" - "golbat/pogo" - "golbat/util" - "golbat/webhooks" + "github.com/guregu/null/v6" ) // Gym struct. @@ -580,814 +561,3 @@ func (gym *Gym) SetRsvps(v null.String) { } } } - -func loadGymFromDatabase(ctx context.Context, db db.DbDetails, fortId string, gym *Gym) error { - err := db.GeneralDb.GetContext(ctx, gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) - statsCollector.IncDbQuery("select gym", err) - return err -} - -// PeekGymRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func PeekGymRecord(fortId string) (*Gym, func(), error) { - if item := gymCache.Get(fortId); item != nil { - gym := item.Value() - gym.Lock() - return gym, func() { gym.Unlock() }, nil - } - return nil, nil, nil -} - -// getGymRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getGymRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { - // Check cache first - if item := gymCache.Get(fortId); item != nil { - gym := item.Value() - gym.Lock() - return gym, func() { gym.Unlock() }, nil - } - - dbGym := Gym{} - err := loadGymFromDatabase(ctx, db, fortId, &dbGym) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbGym.ClearDirty() - - // Atomically cache the loaded Gym - if another goroutine raced us, - // we'll get their Gym and use that instead (ensuring same mutex) - existingGym, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { - // Only called if key doesn't exist - our Pokestop wins - if config.Config.TestFortInMemory { - fortRtreeUpdateGymOnGet(&dbGym) - } - return &dbGym - }) - - gym := existingGym.Value() - gym.Lock() - return gym, func() { gym.Unlock() }, nil -} - -// getGymRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Use when modifying the Gym. -// Caller MUST call returned unlock function if non-nil. -func getGymRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { - gym, unlock, err := getGymRecordReadOnly(ctx, db, fortId) - if err != nil || gym == nil { - return nil, nil, err - } - gym.snapshotOldValues() - return gym, unlock, nil -} - -// getOrCreateGymRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { - // Create new Gym atomically - function only called if key doesn't exist - gymItem, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { - return &Gym{Id: fortId, newRecord: true} - }) - - gym := gymItem.Value() - gym.Lock() - - if gym.newRecord { - // We should attempt to load from database - err := loadGymFromDatabase(ctx, db, fortId, gym) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - gym.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - gym.newRecord = false - gym.ClearDirty() - if config.Config.TestFortInMemory { - fortRtreeUpdateGymOnGet(gym) - } - } - } - - gym.snapshotOldValues() - return gym, func() { gym.Unlock() }, nil -} - -// GetGymRecord returns a copy of the Gym for external/API use. -// For internal use, prefer getGymRecordReadOnly or getGymRecordForUpdate. -func GetGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, error) { - gym, unlock, err := getGymRecordReadOnly(ctx, db, fortId) - if err != nil { - return nil, err - } - if gym == nil { - return nil, nil - } - // Make a copy for safe external use - gymCopy := *gym - unlock() - return &gymCopy, nil -} - -func escapeLike(s string) string { - s = strings.ReplaceAll(s, `\`, `\\`) - s = strings.ReplaceAll(s, `%`, `\%`) - s = strings.ReplaceAll(s, `_`, `\_`) - return s -} - -func calculatePowerUpPoints(fortData *pogo.PokemonFortProto) (null.Int, null.Int) { - now := time.Now().Unix() - powerUpLevelExpirationMs := int64(fortData.PowerUpLevelExpirationMs) / 1000 - powerUpPoints := int64(fortData.PowerUpProgressPoints) - powerUpLevel := null.IntFrom(0) - powerUpEndTimestamp := null.NewInt(0, false) - if powerUpPoints < 50 { - powerUpLevel = null.IntFrom(0) - } else if powerUpPoints < 100 && powerUpLevelExpirationMs > now { - powerUpLevel = null.IntFrom(1) - powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) - } else if powerUpPoints < 150 && powerUpLevelExpirationMs > now { - powerUpLevel = null.IntFrom(2) - powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) - } else if powerUpLevelExpirationMs > now { - powerUpLevel = null.IntFrom(3) - powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) - } else { - powerUpLevel = null.IntFrom(0) - } - - return powerUpLevel, powerUpEndTimestamp -} - -func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64) *Gym { - type pokemonDisplay struct { - Form int `json:"form,omitempty"` - Costume int `json:"costume,omitempty"` - Gender int `json:"gender"` - Shiny bool `json:"shiny,omitempty"` - TempEvolution int `json:"temp_evolution,omitempty"` - TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` - Alignment int `json:"alignment,omitempty"` - Badge int `json:"badge,omitempty"` - Background *int64 `json:"background,omitempty"` - } - gym.SetId(fortData.FortId) - gym.SetLat(fortData.Latitude) - gym.SetLon(fortData.Longitude) - gym.SetEnabled(null.IntFrom(util.BoolToInt[int64](fortData.Enabled))) - gym.SetGuardingPokemonId(null.IntFrom(int64(fortData.GuardPokemonId))) - if fortData.GuardPokemonDisplay == nil { - gym.SetGuardingPokemonDisplay(null.NewString("", false)) - } else { - display, _ := json.Marshal(pokemonDisplay{ - Form: int(fortData.GuardPokemonDisplay.Form), - Costume: int(fortData.GuardPokemonDisplay.Costume), - Gender: int(fortData.GuardPokemonDisplay.Gender), - Shiny: fortData.GuardPokemonDisplay.Shiny, - TempEvolution: int(fortData.GuardPokemonDisplay.CurrentTempEvolution), - TempEvolutionFinishMs: fortData.GuardPokemonDisplay.TemporaryEvolutionFinishMs, - Alignment: int(fortData.GuardPokemonDisplay.Alignment), - Badge: int(fortData.GuardPokemonDisplay.PokemonBadge), - Background: util.ExtractBackgroundFromDisplay(fortData.GuardPokemonDisplay), - }) - gym.SetGuardingPokemonDisplay(null.StringFrom(string(display))) - } - gym.SetTeamId(null.IntFrom(int64(fortData.Team))) - if fortData.GymDisplay != nil { - gym.SetAvailableSlots(null.IntFrom(int64(fortData.GymDisplay.SlotsAvailable))) - } else { - gym.SetAvailableSlots(null.IntFrom(6)) // this may be an incorrect assumption - } - gym.SetLastModifiedTimestamp(null.IntFrom(fortData.LastModifiedMs / 1000)) - gym.SetExRaidEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsExRaidEligible))) - - if fortData.ImageUrl != "" { - gym.SetUrl(null.StringFrom(fortData.ImageUrl)) - } - gym.SetInBattle(null.IntFrom(util.BoolToInt[int64](fortData.IsInBattle))) - gym.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) - gym.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) - - powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) - gym.SetPowerUpLevel(powerUpLevel) - gym.SetPowerUpEndTimestamp(powerUpEndTimestamp) - - if fortData.PartnerId == "" { - gym.SetPartnerId(null.NewString("", false)) - } else { - gym.SetPartnerId(null.StringFrom(fortData.PartnerId)) - } - - if fortData.ImageUrl != "" { - gym.SetUrl(null.StringFrom(fortData.ImageUrl)) - } - if fortData.Team == 0 { // check!! - gym.SetTotalCp(null.IntFrom(0)) - } else { - if fortData.GymDisplay != nil { - totalCp := int64(fortData.GymDisplay.TotalGymCp) - if gym.TotalCp.Int64-totalCp > 100 || totalCp-gym.TotalCp.Int64 > 100 { - gym.SetTotalCp(null.IntFrom(totalCp)) - } - } else { - gym.SetTotalCp(null.IntFrom(0)) - } - } - - if fortData.RaidInfo != nil { - gym.SetRaidEndTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000)) - gym.SetRaidSpawnTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000)) - raidBattleTimestamp := int64(fortData.RaidInfo.RaidBattleMs) / 1000 - - if gym.RaidBattleTimestamp.ValueOrZero() != raidBattleTimestamp { - // We are reporting a new raid, clear rsvp data - gym.SetRsvps(null.NewString("", false)) - } - gym.SetRaidBattleTimestamp(null.IntFrom(raidBattleTimestamp)) - - gym.SetRaidLevel(null.IntFrom(int64(fortData.RaidInfo.RaidLevel))) - if fortData.RaidInfo.RaidPokemon != nil { - gym.SetRaidPokemonId(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonId))) - gym.SetRaidPokemonMove1(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move1))) - gym.SetRaidPokemonMove2(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move2))) - gym.SetRaidPokemonForm(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Form))) - gym.SetRaidPokemonAlignment(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Alignment))) - gym.SetRaidPokemonCp(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Cp))) - gym.SetRaidPokemonGender(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Gender))) - gym.SetRaidPokemonCostume(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Costume))) - gym.SetRaidPokemonEvolution(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.CurrentTempEvolution))) - } else { - gym.SetRaidPokemonId(null.IntFrom(0)) - gym.SetRaidPokemonMove1(null.IntFrom(0)) - gym.SetRaidPokemonMove2(null.IntFrom(0)) - gym.SetRaidPokemonForm(null.IntFrom(0)) - gym.SetRaidPokemonAlignment(null.IntFrom(0)) - gym.SetRaidPokemonCp(null.IntFrom(0)) - gym.SetRaidPokemonGender(null.IntFrom(0)) - gym.SetRaidPokemonCostume(null.IntFrom(0)) - gym.SetRaidPokemonEvolution(null.IntFrom(0)) - } - - gym.SetRaidIsExclusive(null.IntFrom(0)) //null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive)) - } - - gym.SetCellId(null.IntFrom(int64(cellId))) - - if gym.Deleted { - gym.SetDeleted(false) - log.Warnf("Cleared Gym with id '%s' is found again in GMO, therefore un-deleted", gym.Id) - // Restore in fort tracker if enabled - if fortTracker != nil { - fortTracker.RestoreFort(gym.Id, cellId, true, time.Now().Unix()) - } - } - - return gym -} - -func (gym *Gym) updateGymFromFortProto(fortData *pogo.FortDetailsOutProto) *Gym { - gym.SetId(fortData.Id) - gym.SetLat(fortData.Latitude) - gym.SetLon(fortData.Longitude) - if len(fortData.ImageUrl) > 0 { - gym.SetUrl(null.StringFrom(fortData.ImageUrl[0])) - } - gym.SetName(null.StringFrom(fortData.Name)) - - return gym -} - -func (gym *Gym) updateGymFromGymInfoOutProto(gymData *pogo.GymGetInfoOutProto) *Gym { - gym.SetId(gymData.GymStatusAndDefenders.PokemonFortProto.FortId) - gym.SetLat(gymData.GymStatusAndDefenders.PokemonFortProto.Latitude) - gym.SetLon(gymData.GymStatusAndDefenders.PokemonFortProto.Longitude) - - // This will have gym defenders in it... - if len(gymData.Url) > 0 { - gym.SetUrl(null.StringFrom(gymData.Url)) - } - gym.SetName(null.StringFrom(gymData.Name)) - - if gymData.Description == "" { - gym.SetDescription(null.NewString("", false)) - } else { - gym.SetDescription(null.StringFrom(gymData.Description)) - } - - type pokemonGymDefender struct { - PokemonId int `json:"pokemon_id,omitempty"` - Form int `json:"form,omitempty"` - Costume int `json:"costume,omitempty"` - Gender int `json:"gender"` - Shiny bool `json:"shiny,omitempty"` - TempEvolution int `json:"temp_evolution,omitempty"` - TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` - Alignment int `json:"alignment,omitempty"` - Badge int `json:"badge,omitempty"` - Background *int64 `json:"background,omitempty"` - DeployedMs int64 `json:"deployed_ms,omitempty"` - DeployedTime int64 `json:"deployed_time,omitempty"` - BattlesWon int32 `json:"battles_won"` - BattlesLost int32 `json:"battles_lost"` - TimesFed int32 `json:"times_fed"` - MotivationNow util.RoundedFloat4 `json:"motivation_now"` - CpNow int32 `json:"cp_now"` - CpWhenDeployed int32 `json:"cp_when_deployed"` - } - - var defenders []pokemonGymDefender - now := time.Now() - for _, protoDefender := range gymData.GymStatusAndDefenders.GymDefender { - motivatedPokemon := protoDefender.MotivatedPokemon - pokemonDisplay := motivatedPokemon.Pokemon.PokemonDisplay - deploymentTotals := protoDefender.DeploymentTotals - defender := pokemonGymDefender{ - DeployedMs: protoDefender.DeploymentTotals.DeploymentDurationMs, - DeployedTime: now. - Add(-1 * time.Millisecond * time.Duration(deploymentTotals.DeploymentDurationMs)). - Unix(), // This will only be approximately correct - BattlesLost: deploymentTotals.BattlesLost, - BattlesWon: deploymentTotals.BattlesWon, - TimesFed: deploymentTotals.TimesFed, - PokemonId: int(protoDefender.MotivatedPokemon.Pokemon.PokemonId), - Form: int(pokemonDisplay.Form), - Costume: int(pokemonDisplay.Costume), - Gender: int(pokemonDisplay.Gender), - TempEvolution: int(pokemonDisplay.CurrentTempEvolution), - TempEvolutionFinishMs: pokemonDisplay.TemporaryEvolutionFinishMs, - Alignment: int(pokemonDisplay.Alignment), - Badge: int(pokemonDisplay.PokemonBadge), - Background: util.ExtractBackgroundFromDisplay(pokemonDisplay), - Shiny: pokemonDisplay.Shiny, - MotivationNow: util.RoundedFloat4(motivatedPokemon.MotivationNow), - CpNow: motivatedPokemon.CpNow, - CpWhenDeployed: motivatedPokemon.CpWhenDeployed, - } - defenders = append(defenders, defender) - } - bDefenders, _ := json.Marshal(defenders) - gym.SetDefenders(null.StringFrom(string(bDefenders))) - // log.Debugf("Gym %s defenders %s ", gym.Id, string(bDefenders)) - - return gym -} - -func (gym *Gym) updateGymFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto, skipName bool) *Gym { - gym.SetId(fortData.Id) - gym.SetLat(fortData.Latitude) - gym.SetLon(fortData.Longitude) - - if len(fortData.Image) > 0 { - gym.SetUrl(null.StringFrom(fortData.Image[0].Url)) - } - if !skipName { - gym.SetName(null.StringFrom(fortData.Name)) - } - - if gym.Deleted { - log.Debugf("Cleared Gym with id '%s' is found again in GMF, therefore kept deleted", gym.Id) - } - - return gym -} - -func (gym *Gym) updateGymFromRsvpProto(fortData *pogo.GetEventRsvpsOutProto) *Gym { - type rsvpTimeslot struct { - Timeslot int64 `json:"timeslot"` - GoingCount int32 `json:"going_count"` - MaybeCount int32 `json:"maybe_count"` - } - - timeslots := make([]rsvpTimeslot, 0) - - for _, timeslot := range fortData.RsvpTimeslots { - if timeslot.GoingCount > 0 || timeslot.MaybeCount > 0 { - timeslots = append(timeslots, rsvpTimeslot{ - Timeslot: timeslot.TimeSlot, - GoingCount: timeslot.GoingCount, - MaybeCount: timeslot.MaybeCount, - }) - } - } - - if len(timeslots) == 0 { - gym.SetRsvps(null.NewString("", false)) - } else { - slices.SortFunc(timeslots, func(a, b rsvpTimeslot) int { - return cmp.Compare(a.Timeslot, b.Timeslot) - }) - - bRsvps, _ := json.Marshal(timeslots) - gym.SetRsvps(null.StringFrom(string(bRsvps))) - } - - return gym -} - -// hasChangesGym compares two Gym structs -// Float tolerance: Lat, Lon -func hasChangesGym(old *Gym, new *Gym) bool { - return old.Id != new.Id || - old.Name != new.Name || - old.Url != new.Url || - old.LastModifiedTimestamp != new.LastModifiedTimestamp || - old.RaidEndTimestamp != new.RaidEndTimestamp || - old.RaidSpawnTimestamp != new.RaidSpawnTimestamp || - old.RaidBattleTimestamp != new.RaidBattleTimestamp || - old.Updated != new.Updated || - old.RaidPokemonId != new.RaidPokemonId || - old.GuardingPokemonId != new.GuardingPokemonId || - old.AvailableSlots != new.AvailableSlots || - old.TeamId != new.TeamId || - old.RaidLevel != new.RaidLevel || - old.Enabled != new.Enabled || - old.ExRaidEligible != new.ExRaidEligible || - // old.InBattle != new.InBattle || - old.RaidPokemonMove1 != new.RaidPokemonMove1 || - old.RaidPokemonMove2 != new.RaidPokemonMove2 || - old.RaidPokemonForm != new.RaidPokemonForm || - old.RaidPokemonAlignment != new.RaidPokemonAlignment || - old.RaidPokemonCp != new.RaidPokemonCp || - old.RaidIsExclusive != new.RaidIsExclusive || - old.CellId != new.CellId || - old.Deleted != new.Deleted || - old.TotalCp != new.TotalCp || - old.FirstSeenTimestamp != new.FirstSeenTimestamp || - old.RaidPokemonGender != new.RaidPokemonGender || - old.SponsorId != new.SponsorId || - old.PartnerId != new.PartnerId || - old.RaidPokemonCostume != new.RaidPokemonCostume || - old.RaidPokemonEvolution != new.RaidPokemonEvolution || - old.ArScanEligible != new.ArScanEligible || - old.PowerUpLevel != new.PowerUpLevel || - old.PowerUpPoints != new.PowerUpPoints || - old.PowerUpEndTimestamp != new.PowerUpEndTimestamp || - old.Description != new.Description || - old.Rsvps != new.Rsvps || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) - -} - -// hasChangesInternalGym compares two Gym structs for changes that will be stored in memory -// Float tolerance: Lat, Lon -func hasInternalChangesGym(old *Gym, new *Gym) bool { - return old.InBattle != new.InBattle || - old.Defenders != new.Defenders -} - -type GymDetailsWebhook struct { - Id string `json:"id"` - Name string `json:"name"` - Url string `json:"url"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Team int64 `json:"team"` - GuardPokemonId int64 `json:"guard_pokemon_id"` - SlotsAvailable int64 `json:"slots_available"` - ExRaidEligible int64 `json:"ex_raid_eligible"` - InBattle bool `json:"in_battle"` - SponsorId int64 `json:"sponsor_id"` - PartnerId int64 `json:"partner_id"` - PowerUpPoints int64 `json:"power_up_points"` - PowerUpLevel int64 `json:"power_up_level"` - PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` - ArScanEligible int64 `json:"ar_scan_eligible"` - Defenders any `json:"defenders"` -} - -type RaidWebhook struct { - GymId string `json:"gym_id"` - GymName string `json:"gym_name"` - GymUrl string `json:"gym_url"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - TeamId int64 `json:"team_id"` - Spawn int64 `json:"spawn"` - Start int64 `json:"start"` - End int64 `json:"end"` - Level int64 `json:"level"` - PokemonId int64 `json:"pokemon_id"` - Cp int64 `json:"cp"` - Gender int64 `json:"gender"` - Form int64 `json:"form"` - Alignment int64 `json:"alignment"` - Costume int64 `json:"costume"` - Evolution int64 `json:"evolution"` - Move1 int64 `json:"move_1"` - Move2 int64 `json:"move_2"` - ExRaidEligible int64 `json:"ex_raid_eligible"` - IsExclusive int64 `json:"is_exclusive"` - SponsorId int64 `json:"sponsor_id"` - PartnerId string `json:"partner_id"` - PowerUpPoints int64 `json:"power_up_points"` - PowerUpLevel int64 `json:"power_up_level"` - PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` - ArScanEligible int64 `json:"ar_scan_eligible"` - Rsvps json.RawMessage `json:"rsvps"` -} - -func createGymFortWebhooks(gym *Gym) { - fort := InitWebHookFortFromGym(gym) - if gym.newRecord { - CreateFortWebHooks(nil, fort, NEW) - } else { - // Build old fort from saved old values - oldFort := &FortWebhook{ - Type: GYM.String(), - Id: gym.Id, - Name: gym.oldValues.Name.Ptr(), - ImageUrl: gym.oldValues.Url.Ptr(), - Description: gym.oldValues.Description.Ptr(), - Location: Location{Latitude: gym.oldValues.Lat, Longitude: gym.oldValues.Lon}, - } - CreateFortWebHooks(oldFort, fort, EDIT) - } -} - -func createGymWebhooks(gym *Gym, areas []geo.AreaName) { - if gym.newRecord || - (gym.oldValues.AvailableSlots != gym.AvailableSlots || gym.oldValues.TeamId != gym.TeamId || gym.oldValues.InBattle != gym.InBattle) { - gymDetails := GymDetailsWebhook{ - Id: gym.Id, - Name: gym.Name.ValueOrZero(), - Url: gym.Url.ValueOrZero(), - Latitude: gym.Lat, - Longitude: gym.Lon, - Team: gym.TeamId.ValueOrZero(), - GuardPokemonId: gym.GuardingPokemonId.ValueOrZero(), - SlotsAvailable: func() int64 { - if gym.AvailableSlots.Valid { - return gym.AvailableSlots.Int64 - } else { - return 6 - } - }(), - ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), - InBattle: func() bool { return gym.InBattle.ValueOrZero() != 0 }(), - Defenders: func() any { - if gym.Defenders.Valid { - return json.RawMessage(gym.Defenders.ValueOrZero()) - } else { - return nil - } - }(), - } - - webhooksSender.AddMessage(webhooks.GymDetails, gymDetails, areas) - } - - if gym.RaidSpawnTimestamp.ValueOrZero() > 0 && - (gym.newRecord || gym.oldValues.RaidLevel != gym.RaidLevel || - gym.oldValues.RaidPokemonId != gym.RaidPokemonId || - gym.oldValues.RaidSpawnTimestamp != gym.RaidSpawnTimestamp || gym.oldValues.Rsvps != gym.Rsvps) { - raidBattleTime := gym.RaidBattleTimestamp.ValueOrZero() - raidEndTime := gym.RaidEndTimestamp.ValueOrZero() - now := time.Now().Unix() - - if (raidBattleTime > now && gym.RaidLevel.ValueOrZero() > 0) || - (raidEndTime > now && gym.RaidPokemonId.ValueOrZero() > 0) { - gymName := "Unknown" - if gym.Name.Valid { - gymName = gym.Name.String - } - - var rsvps json.RawMessage - if gym.Rsvps.Valid { - rsvps = json.RawMessage(gym.Rsvps.ValueOrZero()) - } - - raidHook := RaidWebhook{ - GymId: gym.Id, - GymName: gymName, - GymUrl: gym.Url.ValueOrZero(), - Latitude: gym.Lat, - Longitude: gym.Lon, - TeamId: gym.TeamId.ValueOrZero(), - Spawn: gym.RaidSpawnTimestamp.ValueOrZero(), - Start: gym.RaidBattleTimestamp.ValueOrZero(), - End: gym.RaidEndTimestamp.ValueOrZero(), - Level: gym.RaidLevel.ValueOrZero(), - PokemonId: gym.RaidPokemonId.ValueOrZero(), - Cp: gym.RaidPokemonCp.ValueOrZero(), - Gender: gym.RaidPokemonGender.ValueOrZero(), - Form: gym.RaidPokemonForm.ValueOrZero(), - Alignment: gym.RaidPokemonAlignment.ValueOrZero(), - Costume: gym.RaidPokemonCostume.ValueOrZero(), - Evolution: gym.RaidPokemonEvolution.ValueOrZero(), - Move1: gym.RaidPokemonMove1.ValueOrZero(), - Move2: gym.RaidPokemonMove2.ValueOrZero(), - ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), - IsExclusive: gym.RaidIsExclusive.ValueOrZero(), - SponsorId: gym.SponsorId.ValueOrZero(), - PartnerId: gym.PartnerId.ValueOrZero(), - PowerUpPoints: gym.PowerUpPoints.ValueOrZero(), - PowerUpLevel: gym.PowerUpLevel.ValueOrZero(), - PowerUpEndTimestamp: gym.PowerUpEndTimestamp.ValueOrZero(), - ArScanEligible: gym.ArScanEligible.ValueOrZero(), - Rsvps: rsvps, - } - - webhooksSender.AddMessage(webhooks.Raid, raidHook, areas) - statsCollector.UpdateRaidCount(areas, gym.RaidLevel.ValueOrZero()) - } - } -} - -func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { - now := time.Now().Unix() - if !gym.IsNewRecord() && !gym.IsDirty() && !gym.IsInternalDirty() { - // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. - if gym.Updated > now-GetUpdateThreshold(900) { - // if a gym is unchanged and was seen recently, skip saving - return - } - } - gym.Updated = now - - if gym.IsDirty() { - if gym.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ - "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) - - statsCollector.IncDbQuery("insert gym", err) - if err != nil { - log.Errorf("insert gym: %s", err) - return - } - - _, _ = res, err - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ - "lat = :lat, "+ - "lon = :lon, "+ - "name = :name, "+ - "url = :url, "+ - "last_modified_timestamp = :last_modified_timestamp, "+ - "raid_end_timestamp = :raid_end_timestamp, "+ - "raid_spawn_timestamp = :raid_spawn_timestamp, "+ - "raid_battle_timestamp = :raid_battle_timestamp, "+ - "updated = :updated, "+ - "raid_pokemon_id = :raid_pokemon_id, "+ - "guarding_pokemon_id = :guarding_pokemon_id, "+ - "guarding_pokemon_display = :guarding_pokemon_display, "+ - "available_slots = :available_slots, "+ - "team_id = :team_id, "+ - "raid_level = :raid_level, "+ - "enabled = :enabled, "+ - "ex_raid_eligible = :ex_raid_eligible, "+ - "in_battle = :in_battle, "+ - "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ - "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ - "raid_pokemon_form = :raid_pokemon_form, "+ - "raid_pokemon_alignment = :raid_pokemon_alignment, "+ - "raid_pokemon_cp = :raid_pokemon_cp, "+ - "raid_is_exclusive = :raid_is_exclusive, "+ - "cell_id = :cell_id, "+ - "deleted = :deleted, "+ - "total_cp = :total_cp, "+ - "raid_pokemon_gender = :raid_pokemon_gender, "+ - "sponsor_id = :sponsor_id, "+ - "partner_id = :partner_id, "+ - "raid_pokemon_costume = :raid_pokemon_costume, "+ - "raid_pokemon_evolution = :raid_pokemon_evolution, "+ - "ar_scan_eligible = :ar_scan_eligible, "+ - "power_up_level = :power_up_level, "+ - "power_up_points = :power_up_points, "+ - "power_up_end_timestamp = :power_up_end_timestamp,"+ - "description = :description,"+ - "defenders = :defenders,"+ - "rsvps = :rsvps "+ - "WHERE id = :id", gym, - ) - statsCollector.IncDbQuery("update gym", err) - if err != nil { - log.Errorf("Update gym %s", err) - } - _, _ = res, err - } - } - - //gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) - areas := MatchStatsGeofence(gym.Lat, gym.Lon) - createGymWebhooks(gym, areas) - createGymFortWebhooks(gym) - updateRaidStats(gym, areas) - if dbDebugEnabled { - gym.changedFields = gym.changedFields[:0] - } - if gym.IsNewRecord() { - gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) - gym.newRecord = false - } - gym.ClearDirty() -} - -func updateGymGetMapFortCache(gym *Gym, skipName bool) { - storedGetMapFort := getMapFortsCache.Get(gym.Id) - if storedGetMapFort != nil { - getMapFort := storedGetMapFort.Value() - getMapFortsCache.Delete(gym.Id) - gym.updateGymFromGetMapFortsOutProto(getMapFort, skipName) - log.Debugf("Updated Gym using stored getMapFort: %s", gym.Id) - } -} - -func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { - gym, unlock, err := getOrCreateGymRecord(ctx, db, fort.Id) - if err != nil { - return err.Error() - } - defer unlock() - - gym.updateGymFromFortProto(fort) - - updateGymGetMapFortCache(gym, true) - saveGymRecord(ctx, db, gym) - - return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} - -func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymInfo *pogo.GymGetInfoOutProto) string { - gym, unlock, err := getOrCreateGymRecord(ctx, db, gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) - if err != nil { - return err.Error() - } - defer unlock() - - gym.updateGymFromGymInfoOutProto(gymInfo) - - updateGymGetMapFortCache(gym, true) - saveGymRecord(ctx, db, gym) - return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} - -func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { - gym, unlock, err := getGymRecordForUpdate(ctx, db, mapFort.Id) - if err != nil { - return false, err.Error() - } - - // we missed it in Pokestop & Gym. Lets save it to cache - if gym == nil { - return false, "" - } - defer unlock() - - gym.updateGymFromGetMapFortsOutProto(mapFort, false) - saveGymRecord(ctx, db, gym) - return true, fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} - -func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pogo.RaidDetails, resp *pogo.GetEventRsvpsOutProto) string { - gym, unlock, err := getGymRecordForUpdate(ctx, db, req.FortId) - if err != nil { - return err.Error() - } - - if gym == nil { - // Do not add RSVP details to unknown gyms - return fmt.Sprintf("%s Gym not present", req.FortId) - } - defer unlock() - - gym.updateGymFromRsvpProto(resp) - - saveGymRecord(ctx, db, gym) - - return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} - -func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { - gym, unlock, err := getGymRecordForUpdate(ctx, db, fortId) - if err != nil { - return err.Error() - } - - if gym == nil { - // Do not add RSVP details to unknown gyms - return fmt.Sprintf("%s Gym not present", fortId) - } - defer unlock() - - if gym.Rsvps.Valid { - gym.SetRsvps(null.NewString("", false)) - - saveGymRecord(ctx, db, gym) - } - - return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) -} diff --git a/decoder/gym_decode.go b/decoder/gym_decode.go new file mode 100644 index 00000000..3bec72aa --- /dev/null +++ b/decoder/gym_decode.go @@ -0,0 +1,311 @@ +package decoder + +import ( + "cmp" + "encoding/json" + "slices" + "strings" + "time" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" + "golbat/util" +) + +func escapeLike(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `%`, `\%`) + s = strings.ReplaceAll(s, `_`, `\_`) + return s +} + +func calculatePowerUpPoints(fortData *pogo.PokemonFortProto) (null.Int, null.Int) { + now := time.Now().Unix() + powerUpLevelExpirationMs := int64(fortData.PowerUpLevelExpirationMs) / 1000 + powerUpPoints := int64(fortData.PowerUpProgressPoints) + powerUpLevel := null.IntFrom(0) + powerUpEndTimestamp := null.NewInt(0, false) + if powerUpPoints < 50 { + powerUpLevel = null.IntFrom(0) + } else if powerUpPoints < 100 && powerUpLevelExpirationMs > now { + powerUpLevel = null.IntFrom(1) + powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) + } else if powerUpPoints < 150 && powerUpLevelExpirationMs > now { + powerUpLevel = null.IntFrom(2) + powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) + } else if powerUpLevelExpirationMs > now { + powerUpLevel = null.IntFrom(3) + powerUpEndTimestamp = null.IntFrom(powerUpLevelExpirationMs) + } else { + powerUpLevel = null.IntFrom(0) + } + + return powerUpLevel, powerUpEndTimestamp +} + +func (gym *Gym) updateGymFromFort(fortData *pogo.PokemonFortProto, cellId uint64) *Gym { + type pokemonDisplay struct { + Form int `json:"form,omitempty"` + Costume int `json:"costume,omitempty"` + Gender int `json:"gender"` + Shiny bool `json:"shiny,omitempty"` + TempEvolution int `json:"temp_evolution,omitempty"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` + Alignment int `json:"alignment,omitempty"` + Badge int `json:"badge,omitempty"` + Background *int64 `json:"background,omitempty"` + } + gym.SetId(fortData.FortId) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) + gym.SetEnabled(null.IntFrom(util.BoolToInt[int64](fortData.Enabled))) + gym.SetGuardingPokemonId(null.IntFrom(int64(fortData.GuardPokemonId))) + if fortData.GuardPokemonDisplay == nil { + gym.SetGuardingPokemonDisplay(null.NewString("", false)) + } else { + display, _ := json.Marshal(pokemonDisplay{ + Form: int(fortData.GuardPokemonDisplay.Form), + Costume: int(fortData.GuardPokemonDisplay.Costume), + Gender: int(fortData.GuardPokemonDisplay.Gender), + Shiny: fortData.GuardPokemonDisplay.Shiny, + TempEvolution: int(fortData.GuardPokemonDisplay.CurrentTempEvolution), + TempEvolutionFinishMs: fortData.GuardPokemonDisplay.TemporaryEvolutionFinishMs, + Alignment: int(fortData.GuardPokemonDisplay.Alignment), + Badge: int(fortData.GuardPokemonDisplay.PokemonBadge), + Background: util.ExtractBackgroundFromDisplay(fortData.GuardPokemonDisplay), + }) + gym.SetGuardingPokemonDisplay(null.StringFrom(string(display))) + } + gym.SetTeamId(null.IntFrom(int64(fortData.Team))) + if fortData.GymDisplay != nil { + gym.SetAvailableSlots(null.IntFrom(int64(fortData.GymDisplay.SlotsAvailable))) + } else { + gym.SetAvailableSlots(null.IntFrom(6)) // this may be an incorrect assumption + } + gym.SetLastModifiedTimestamp(null.IntFrom(fortData.LastModifiedMs / 1000)) + gym.SetExRaidEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsExRaidEligible))) + + if fortData.ImageUrl != "" { + gym.SetUrl(null.StringFrom(fortData.ImageUrl)) + } + gym.SetInBattle(null.IntFrom(util.BoolToInt[int64](fortData.IsInBattle))) + gym.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) + gym.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) + + powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) + gym.SetPowerUpLevel(powerUpLevel) + gym.SetPowerUpEndTimestamp(powerUpEndTimestamp) + + if fortData.PartnerId == "" { + gym.SetPartnerId(null.NewString("", false)) + } else { + gym.SetPartnerId(null.StringFrom(fortData.PartnerId)) + } + + if fortData.ImageUrl != "" { + gym.SetUrl(null.StringFrom(fortData.ImageUrl)) + } + if fortData.Team == 0 { // check!! + gym.SetTotalCp(null.IntFrom(0)) + } else { + if fortData.GymDisplay != nil { + totalCp := int64(fortData.GymDisplay.TotalGymCp) + if gym.TotalCp.Int64-totalCp > 100 || totalCp-gym.TotalCp.Int64 > 100 { + gym.SetTotalCp(null.IntFrom(totalCp)) + } + } else { + gym.SetTotalCp(null.IntFrom(0)) + } + } + + if fortData.RaidInfo != nil { + gym.SetRaidEndTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidEndMs) / 1000)) + gym.SetRaidSpawnTimestamp(null.IntFrom(int64(fortData.RaidInfo.RaidSpawnMs) / 1000)) + raidBattleTimestamp := int64(fortData.RaidInfo.RaidBattleMs) / 1000 + + if gym.RaidBattleTimestamp.ValueOrZero() != raidBattleTimestamp { + // We are reporting a new raid, clear rsvp data + gym.SetRsvps(null.NewString("", false)) + } + gym.SetRaidBattleTimestamp(null.IntFrom(raidBattleTimestamp)) + + gym.SetRaidLevel(null.IntFrom(int64(fortData.RaidInfo.RaidLevel))) + if fortData.RaidInfo.RaidPokemon != nil { + gym.SetRaidPokemonId(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonId))) + gym.SetRaidPokemonMove1(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move1))) + gym.SetRaidPokemonMove2(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Move2))) + gym.SetRaidPokemonForm(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Form))) + gym.SetRaidPokemonAlignment(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Alignment))) + gym.SetRaidPokemonCp(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.Cp))) + gym.SetRaidPokemonGender(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Gender))) + gym.SetRaidPokemonCostume(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.Costume))) + gym.SetRaidPokemonEvolution(null.IntFrom(int64(fortData.RaidInfo.RaidPokemon.PokemonDisplay.CurrentTempEvolution))) + } else { + gym.SetRaidPokemonId(null.IntFrom(0)) + gym.SetRaidPokemonMove1(null.IntFrom(0)) + gym.SetRaidPokemonMove2(null.IntFrom(0)) + gym.SetRaidPokemonForm(null.IntFrom(0)) + gym.SetRaidPokemonAlignment(null.IntFrom(0)) + gym.SetRaidPokemonCp(null.IntFrom(0)) + gym.SetRaidPokemonGender(null.IntFrom(0)) + gym.SetRaidPokemonCostume(null.IntFrom(0)) + gym.SetRaidPokemonEvolution(null.IntFrom(0)) + } + + gym.SetRaidIsExclusive(null.IntFrom(0)) //null.IntFrom(util.BoolToInt[int64](fortData.RaidInfo.IsExclusive)) + } + + gym.SetCellId(null.IntFrom(int64(cellId))) + + if gym.Deleted { + gym.SetDeleted(false) + log.Warnf("Cleared Gym with id '%s' is found again in GMO, therefore un-deleted", gym.Id) + // Restore in fort tracker if enabled + if fortTracker != nil { + fortTracker.RestoreFort(gym.Id, cellId, true, time.Now().Unix()) + } + } + + return gym +} + +func (gym *Gym) updateGymFromFortProto(fortData *pogo.FortDetailsOutProto) *Gym { + gym.SetId(fortData.Id) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) + if len(fortData.ImageUrl) > 0 { + gym.SetUrl(null.StringFrom(fortData.ImageUrl[0])) + } + gym.SetName(null.StringFrom(fortData.Name)) + + return gym +} + +func (gym *Gym) updateGymFromGymInfoOutProto(gymData *pogo.GymGetInfoOutProto) *Gym { + gym.SetId(gymData.GymStatusAndDefenders.PokemonFortProto.FortId) + gym.SetLat(gymData.GymStatusAndDefenders.PokemonFortProto.Latitude) + gym.SetLon(gymData.GymStatusAndDefenders.PokemonFortProto.Longitude) + + // This will have gym defenders in it... + if len(gymData.Url) > 0 { + gym.SetUrl(null.StringFrom(gymData.Url)) + } + gym.SetName(null.StringFrom(gymData.Name)) + + if gymData.Description == "" { + gym.SetDescription(null.NewString("", false)) + } else { + gym.SetDescription(null.StringFrom(gymData.Description)) + } + + type pokemonGymDefender struct { + PokemonId int `json:"pokemon_id,omitempty"` + Form int `json:"form,omitempty"` + Costume int `json:"costume,omitempty"` + Gender int `json:"gender"` + Shiny bool `json:"shiny,omitempty"` + TempEvolution int `json:"temp_evolution,omitempty"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` + Alignment int `json:"alignment,omitempty"` + Badge int `json:"badge,omitempty"` + Background *int64 `json:"background,omitempty"` + DeployedMs int64 `json:"deployed_ms,omitempty"` + DeployedTime int64 `json:"deployed_time,omitempty"` + BattlesWon int32 `json:"battles_won"` + BattlesLost int32 `json:"battles_lost"` + TimesFed int32 `json:"times_fed"` + MotivationNow util.RoundedFloat4 `json:"motivation_now"` + CpNow int32 `json:"cp_now"` + CpWhenDeployed int32 `json:"cp_when_deployed"` + } + + var defenders []pokemonGymDefender + now := time.Now() + for _, protoDefender := range gymData.GymStatusAndDefenders.GymDefender { + motivatedPokemon := protoDefender.MotivatedPokemon + pokemonDisplay := motivatedPokemon.Pokemon.PokemonDisplay + deploymentTotals := protoDefender.DeploymentTotals + defender := pokemonGymDefender{ + DeployedMs: protoDefender.DeploymentTotals.DeploymentDurationMs, + DeployedTime: now. + Add(-1 * time.Millisecond * time.Duration(deploymentTotals.DeploymentDurationMs)). + Unix(), // This will only be approximately correct + BattlesLost: deploymentTotals.BattlesLost, + BattlesWon: deploymentTotals.BattlesWon, + TimesFed: deploymentTotals.TimesFed, + PokemonId: int(protoDefender.MotivatedPokemon.Pokemon.PokemonId), + Form: int(pokemonDisplay.Form), + Costume: int(pokemonDisplay.Costume), + Gender: int(pokemonDisplay.Gender), + TempEvolution: int(pokemonDisplay.CurrentTempEvolution), + TempEvolutionFinishMs: pokemonDisplay.TemporaryEvolutionFinishMs, + Alignment: int(pokemonDisplay.Alignment), + Badge: int(pokemonDisplay.PokemonBadge), + Background: util.ExtractBackgroundFromDisplay(pokemonDisplay), + Shiny: pokemonDisplay.Shiny, + MotivationNow: util.RoundedFloat4(motivatedPokemon.MotivationNow), + CpNow: motivatedPokemon.CpNow, + CpWhenDeployed: motivatedPokemon.CpWhenDeployed, + } + defenders = append(defenders, defender) + } + bDefenders, _ := json.Marshal(defenders) + gym.SetDefenders(null.StringFrom(string(bDefenders))) + // log.Debugf("Gym %s defenders %s ", gym.Id, string(bDefenders)) + + return gym +} + +func (gym *Gym) updateGymFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto, skipName bool) *Gym { + gym.SetId(fortData.Id) + gym.SetLat(fortData.Latitude) + gym.SetLon(fortData.Longitude) + + if len(fortData.Image) > 0 { + gym.SetUrl(null.StringFrom(fortData.Image[0].Url)) + } + if !skipName { + gym.SetName(null.StringFrom(fortData.Name)) + } + + if gym.Deleted { + log.Debugf("Cleared Gym with id '%s' is found again in GMF, therefore kept deleted", gym.Id) + } + + return gym +} + +func (gym *Gym) updateGymFromRsvpProto(fortData *pogo.GetEventRsvpsOutProto) *Gym { + type rsvpTimeslot struct { + Timeslot int64 `json:"timeslot"` + GoingCount int32 `json:"going_count"` + MaybeCount int32 `json:"maybe_count"` + } + + timeslots := make([]rsvpTimeslot, 0) + + for _, timeslot := range fortData.RsvpTimeslots { + if timeslot.GoingCount > 0 || timeslot.MaybeCount > 0 { + timeslots = append(timeslots, rsvpTimeslot{ + Timeslot: timeslot.TimeSlot, + GoingCount: timeslot.GoingCount, + MaybeCount: timeslot.MaybeCount, + }) + } + } + + if len(timeslots) == 0 { + gym.SetRsvps(null.NewString("", false)) + } else { + slices.SortFunc(timeslots, func(a, b rsvpTimeslot) int { + return cmp.Compare(a.Timeslot, b.Timeslot) + }) + + bRsvps, _ := json.Marshal(timeslots) + gym.SetRsvps(null.StringFrom(string(bRsvps))) + } + + return gym +} diff --git a/decoder/gym_process.go b/decoder/gym_process.go new file mode 100644 index 00000000..1c34104c --- /dev/null +++ b/decoder/gym_process.go @@ -0,0 +1,97 @@ +package decoder + +import ( + "context" + "fmt" + + "github.com/guregu/null/v6" + + "golbat/db" + "golbat/pogo" +) + +func UpdateGymRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { + gym, unlock, err := getOrCreateGymRecord(ctx, db, fort.Id) + if err != nil { + return err.Error() + } + defer unlock() + + gym.updateGymFromFortProto(fort) + + updateGymGetMapFortCache(gym, true) + saveGymRecord(ctx, db, gym) + + return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} + +func UpdateGymRecordWithGymInfoProto(ctx context.Context, db db.DbDetails, gymInfo *pogo.GymGetInfoOutProto) string { + gym, unlock, err := getOrCreateGymRecord(ctx, db, gymInfo.GymStatusAndDefenders.PokemonFortProto.FortId) + if err != nil { + return err.Error() + } + defer unlock() + + gym.updateGymFromGymInfoOutProto(gymInfo) + + updateGymGetMapFortCache(gym, true) + saveGymRecord(ctx, db, gym) + return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} + +func UpdateGymRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { + gym, unlock, err := getGymRecordForUpdate(ctx, db, mapFort.Id) + if err != nil { + return false, err.Error() + } + + // we missed it in Pokestop & Gym. Lets save it to cache + if gym == nil { + return false, "" + } + defer unlock() + + gym.updateGymFromGetMapFortsOutProto(mapFort, false) + saveGymRecord(ctx, db, gym) + return true, fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} + +func UpdateGymRecordWithRsvpProto(ctx context.Context, db db.DbDetails, req *pogo.RaidDetails, resp *pogo.GetEventRsvpsOutProto) string { + gym, unlock, err := getGymRecordForUpdate(ctx, db, req.FortId) + if err != nil { + return err.Error() + } + + if gym == nil { + // Do not add RSVP details to unknown gyms + return fmt.Sprintf("%s Gym not present", req.FortId) + } + defer unlock() + + gym.updateGymFromRsvpProto(resp) + + saveGymRecord(ctx, db, gym) + + return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} + +func ClearGymRsvp(ctx context.Context, db db.DbDetails, fortId string) string { + gym, unlock, err := getGymRecordForUpdate(ctx, db, fortId) + if err != nil { + return err.Error() + } + + if gym == nil { + // Do not add RSVP details to unknown gyms + return fmt.Sprintf("%s Gym not present", fortId) + } + defer unlock() + + if gym.Rsvps.Valid { + gym.SetRsvps(null.NewString("", false)) + + saveGymRecord(ctx, db, gym) + } + + return fmt.Sprintf("%s %s", gym.Id, gym.Name.ValueOrZero()) +} diff --git a/decoder/gym_state.go b/decoder/gym_state.go new file mode 100644 index 00000000..ce235085 --- /dev/null +++ b/decoder/gym_state.go @@ -0,0 +1,430 @@ +package decoder + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "time" + + "golbat/config" + "golbat/db" + "golbat/geo" + "golbat/webhooks" + + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" +) + +func loadGymFromDatabase(ctx context.Context, db db.DbDetails, fortId string, gym *Gym) error { + err := db.GeneralDb.GetContext(ctx, gym, "SELECT id, lat, lon, name, url, last_modified_timestamp, raid_end_timestamp, raid_spawn_timestamp, raid_battle_timestamp, updated, raid_pokemon_id, guarding_pokemon_id, guarding_pokemon_display, available_slots, team_id, raid_level, enabled, ex_raid_eligible, in_battle, raid_pokemon_move_1, raid_pokemon_move_2, raid_pokemon_form, raid_pokemon_alignment, raid_pokemon_cp, raid_is_exclusive, cell_id, deleted, total_cp, first_seen_timestamp, raid_pokemon_gender, sponsor_id, partner_id, raid_pokemon_costume, raid_pokemon_evolution, ar_scan_eligible, power_up_level, power_up_points, power_up_end_timestamp, description, defenders, rsvps FROM gym WHERE id = ?", fortId) + statsCollector.IncDbQuery("select gym", err) + return err +} + +// PeekGymRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekGymRecord(fortId string) (*Gym, func(), error) { + if item := gymCache.Get(fortId); item != nil { + gym := item.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil + } + return nil, nil, nil +} + +// GetGymRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func GetGymRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + // Check cache first + if item := gymCache.Get(fortId); item != nil { + gym := item.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil + } + + dbGym := Gym{} + err := loadGymFromDatabase(ctx, db, fortId, &dbGym) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbGym.ClearDirty() + + // Atomically cache the loaded Gym - if another goroutine raced us, + // we'll get their Gym and use that instead (ensuring same mutex) + existingGym, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { + // Only called if key doesn't exist - our Pokestop wins + if config.Config.TestFortInMemory { + fortRtreeUpdateGymOnGet(&dbGym) + } + return &dbGym + }) + + gym := existingGym.Value() + gym.Lock() + return gym, func() { gym.Unlock() }, nil +} + +// getGymRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Gym. +// Caller MUST call returned unlock function if non-nil. +func getGymRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + gym, unlock, err := GetGymRecordReadOnly(ctx, db, fortId) + if err != nil || gym == nil { + return nil, nil, err + } + gym.snapshotOldValues() + return gym, unlock, nil +} + +// getOrCreateGymRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateGymRecord(ctx context.Context, db db.DbDetails, fortId string) (*Gym, func(), error) { + // Create new Gym atomically - function only called if key doesn't exist + gymItem, _ := gymCache.GetOrSetFunc(fortId, func() *Gym { + return &Gym{Id: fortId, newRecord: true} + }) + + gym := gymItem.Value() + gym.Lock() + + if gym.newRecord { + // We should attempt to load from database + err := loadGymFromDatabase(ctx, db, fortId, gym) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + gym.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + gym.newRecord = false + gym.ClearDirty() + if config.Config.TestFortInMemory { + fortRtreeUpdateGymOnGet(gym) + } + } + } + + gym.snapshotOldValues() + return gym, func() { gym.Unlock() }, nil +} + +// hasChangesGym compares two Gym structs +// Float tolerance: Lat, Lon +func hasChangesGym(old *Gym, new *Gym) bool { + return old.Id != new.Id || + old.Name != new.Name || + old.Url != new.Url || + old.LastModifiedTimestamp != new.LastModifiedTimestamp || + old.RaidEndTimestamp != new.RaidEndTimestamp || + old.RaidSpawnTimestamp != new.RaidSpawnTimestamp || + old.RaidBattleTimestamp != new.RaidBattleTimestamp || + old.Updated != new.Updated || + old.RaidPokemonId != new.RaidPokemonId || + old.GuardingPokemonId != new.GuardingPokemonId || + old.AvailableSlots != new.AvailableSlots || + old.TeamId != new.TeamId || + old.RaidLevel != new.RaidLevel || + old.Enabled != new.Enabled || + old.ExRaidEligible != new.ExRaidEligible || + // old.InBattle != new.InBattle || + old.RaidPokemonMove1 != new.RaidPokemonMove1 || + old.RaidPokemonMove2 != new.RaidPokemonMove2 || + old.RaidPokemonForm != new.RaidPokemonForm || + old.RaidPokemonAlignment != new.RaidPokemonAlignment || + old.RaidPokemonCp != new.RaidPokemonCp || + old.RaidIsExclusive != new.RaidIsExclusive || + old.CellId != new.CellId || + old.Deleted != new.Deleted || + old.TotalCp != new.TotalCp || + old.FirstSeenTimestamp != new.FirstSeenTimestamp || + old.RaidPokemonGender != new.RaidPokemonGender || + old.SponsorId != new.SponsorId || + old.PartnerId != new.PartnerId || + old.RaidPokemonCostume != new.RaidPokemonCostume || + old.RaidPokemonEvolution != new.RaidPokemonEvolution || + old.ArScanEligible != new.ArScanEligible || + old.PowerUpLevel != new.PowerUpLevel || + old.PowerUpPoints != new.PowerUpPoints || + old.PowerUpEndTimestamp != new.PowerUpEndTimestamp || + old.Description != new.Description || + old.Rsvps != new.Rsvps || + !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || + !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) + +} + +// hasChangesInternalGym compares two Gym structs for changes that will be stored in memory +// Float tolerance: Lat, Lon +func hasInternalChangesGym(old *Gym, new *Gym) bool { + return old.InBattle != new.InBattle || + old.Defenders != new.Defenders +} + +type GymDetailsWebhook struct { + Id string `json:"id"` + Name string `json:"name"` + Url string `json:"url"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Team int64 `json:"team"` + GuardPokemonId int64 `json:"guard_pokemon_id"` + SlotsAvailable int64 `json:"slots_available"` + ExRaidEligible int64 `json:"ex_raid_eligible"` + InBattle bool `json:"in_battle"` + SponsorId int64 `json:"sponsor_id"` + PartnerId int64 `json:"partner_id"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + ArScanEligible int64 `json:"ar_scan_eligible"` + Defenders any `json:"defenders"` +} + +type RaidWebhook struct { + GymId string `json:"gym_id"` + GymName string `json:"gym_name"` + GymUrl string `json:"gym_url"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + TeamId int64 `json:"team_id"` + Spawn int64 `json:"spawn"` + Start int64 `json:"start"` + End int64 `json:"end"` + Level int64 `json:"level"` + PokemonId int64 `json:"pokemon_id"` + Cp int64 `json:"cp"` + Gender int64 `json:"gender"` + Form int64 `json:"form"` + Alignment int64 `json:"alignment"` + Costume int64 `json:"costume"` + Evolution int64 `json:"evolution"` + Move1 int64 `json:"move_1"` + Move2 int64 `json:"move_2"` + ExRaidEligible int64 `json:"ex_raid_eligible"` + IsExclusive int64 `json:"is_exclusive"` + SponsorId int64 `json:"sponsor_id"` + PartnerId string `json:"partner_id"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + ArScanEligible int64 `json:"ar_scan_eligible"` + Rsvps json.RawMessage `json:"rsvps"` +} + +func createGymFortWebhooks(gym *Gym) { + fort := InitWebHookFortFromGym(gym) + if gym.newRecord { + CreateFortWebHooks(nil, fort, NEW) + } else { + // Build old fort from saved old values + oldFort := &FortWebhook{ + Type: GYM.String(), + Id: gym.Id, + Name: gym.oldValues.Name.Ptr(), + ImageUrl: gym.oldValues.Url.Ptr(), + Description: gym.oldValues.Description.Ptr(), + Location: Location{Latitude: gym.oldValues.Lat, Longitude: gym.oldValues.Lon}, + } + CreateFortWebHooks(oldFort, fort, EDIT) + } +} + +func createGymWebhooks(gym *Gym, areas []geo.AreaName) { + if gym.newRecord || + (gym.oldValues.AvailableSlots != gym.AvailableSlots || gym.oldValues.TeamId != gym.TeamId || gym.oldValues.InBattle != gym.InBattle) { + gymDetails := GymDetailsWebhook{ + Id: gym.Id, + Name: gym.Name.ValueOrZero(), + Url: gym.Url.ValueOrZero(), + Latitude: gym.Lat, + Longitude: gym.Lon, + Team: gym.TeamId.ValueOrZero(), + GuardPokemonId: gym.GuardingPokemonId.ValueOrZero(), + SlotsAvailable: func() int64 { + if gym.AvailableSlots.Valid { + return gym.AvailableSlots.Int64 + } else { + return 6 + } + }(), + ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), + InBattle: func() bool { return gym.InBattle.ValueOrZero() != 0 }(), + Defenders: func() any { + if gym.Defenders.Valid { + return json.RawMessage(gym.Defenders.ValueOrZero()) + } else { + return nil + } + }(), + } + + webhooksSender.AddMessage(webhooks.GymDetails, gymDetails, areas) + } + + if gym.RaidSpawnTimestamp.ValueOrZero() > 0 && + (gym.newRecord || gym.oldValues.RaidLevel != gym.RaidLevel || + gym.oldValues.RaidPokemonId != gym.RaidPokemonId || + gym.oldValues.RaidSpawnTimestamp != gym.RaidSpawnTimestamp || gym.oldValues.Rsvps != gym.Rsvps) { + raidBattleTime := gym.RaidBattleTimestamp.ValueOrZero() + raidEndTime := gym.RaidEndTimestamp.ValueOrZero() + now := time.Now().Unix() + + if (raidBattleTime > now && gym.RaidLevel.ValueOrZero() > 0) || + (raidEndTime > now && gym.RaidPokemonId.ValueOrZero() > 0) { + gymName := "Unknown" + if gym.Name.Valid { + gymName = gym.Name.String + } + + var rsvps json.RawMessage + if gym.Rsvps.Valid { + rsvps = json.RawMessage(gym.Rsvps.ValueOrZero()) + } + + raidHook := RaidWebhook{ + GymId: gym.Id, + GymName: gymName, + GymUrl: gym.Url.ValueOrZero(), + Latitude: gym.Lat, + Longitude: gym.Lon, + TeamId: gym.TeamId.ValueOrZero(), + Spawn: gym.RaidSpawnTimestamp.ValueOrZero(), + Start: gym.RaidBattleTimestamp.ValueOrZero(), + End: gym.RaidEndTimestamp.ValueOrZero(), + Level: gym.RaidLevel.ValueOrZero(), + PokemonId: gym.RaidPokemonId.ValueOrZero(), + Cp: gym.RaidPokemonCp.ValueOrZero(), + Gender: gym.RaidPokemonGender.ValueOrZero(), + Form: gym.RaidPokemonForm.ValueOrZero(), + Alignment: gym.RaidPokemonAlignment.ValueOrZero(), + Costume: gym.RaidPokemonCostume.ValueOrZero(), + Evolution: gym.RaidPokemonEvolution.ValueOrZero(), + Move1: gym.RaidPokemonMove1.ValueOrZero(), + Move2: gym.RaidPokemonMove2.ValueOrZero(), + ExRaidEligible: gym.ExRaidEligible.ValueOrZero(), + IsExclusive: gym.RaidIsExclusive.ValueOrZero(), + SponsorId: gym.SponsorId.ValueOrZero(), + PartnerId: gym.PartnerId.ValueOrZero(), + PowerUpPoints: gym.PowerUpPoints.ValueOrZero(), + PowerUpLevel: gym.PowerUpLevel.ValueOrZero(), + PowerUpEndTimestamp: gym.PowerUpEndTimestamp.ValueOrZero(), + ArScanEligible: gym.ArScanEligible.ValueOrZero(), + Rsvps: rsvps, + } + + webhooksSender.AddMessage(webhooks.Raid, raidHook, areas) + statsCollector.UpdateRaidCount(areas, gym.RaidLevel.ValueOrZero()) + } + } +} + +func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { + now := time.Now().Unix() + if !gym.IsNewRecord() && !gym.IsDirty() && !gym.IsInternalDirty() { + // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. + if gym.Updated > now-GetUpdateThreshold(900) { + // if a gym is unchanged and was seen recently, skip saving + return + } + } + gym.Updated = now + + if gym.IsDirty() { + if gym.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Gym", gym.Id, gym.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO gym (id,lat,lon,name,url,last_modified_timestamp,raid_end_timestamp,raid_spawn_timestamp,raid_battle_timestamp,updated,raid_pokemon_id,guarding_pokemon_id,guarding_pokemon_display,available_slots,team_id,raid_level,enabled,ex_raid_eligible,in_battle,raid_pokemon_move_1,raid_pokemon_move_2,raid_pokemon_form,raid_pokemon_alignment,raid_pokemon_cp,raid_is_exclusive,cell_id,deleted,total_cp,first_seen_timestamp,raid_pokemon_gender,sponsor_id,partner_id,raid_pokemon_costume,raid_pokemon_evolution,ar_scan_eligible,power_up_level,power_up_points,power_up_end_timestamp,description, defenders, rsvps) "+ + "VALUES (:id,:lat,:lon,:name,:url,UNIX_TIMESTAMP(),:raid_end_timestamp,:raid_spawn_timestamp,:raid_battle_timestamp,:updated,:raid_pokemon_id,:guarding_pokemon_id,:guarding_pokemon_display,:available_slots,:team_id,:raid_level,:enabled,:ex_raid_eligible,:in_battle,:raid_pokemon_move_1,:raid_pokemon_move_2,:raid_pokemon_form,:raid_pokemon_alignment,:raid_pokemon_cp,:raid_is_exclusive,:cell_id,0,:total_cp,UNIX_TIMESTAMP(),:raid_pokemon_gender,:sponsor_id,:partner_id,:raid_pokemon_costume,:raid_pokemon_evolution,:ar_scan_eligible,:power_up_level,:power_up_points,:power_up_end_timestamp,:description, :defenders, :rsvps)", gym) + + statsCollector.IncDbQuery("insert gym", err) + if err != nil { + log.Errorf("insert gym: %s", err) + return + } + + _, _ = res, err + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Gym", gym.Id, gym.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, "UPDATE gym SET "+ + "lat = :lat, "+ + "lon = :lon, "+ + "name = :name, "+ + "url = :url, "+ + "last_modified_timestamp = :last_modified_timestamp, "+ + "raid_end_timestamp = :raid_end_timestamp, "+ + "raid_spawn_timestamp = :raid_spawn_timestamp, "+ + "raid_battle_timestamp = :raid_battle_timestamp, "+ + "updated = :updated, "+ + "raid_pokemon_id = :raid_pokemon_id, "+ + "guarding_pokemon_id = :guarding_pokemon_id, "+ + "guarding_pokemon_display = :guarding_pokemon_display, "+ + "available_slots = :available_slots, "+ + "team_id = :team_id, "+ + "raid_level = :raid_level, "+ + "enabled = :enabled, "+ + "ex_raid_eligible = :ex_raid_eligible, "+ + "in_battle = :in_battle, "+ + "raid_pokemon_move_1 = :raid_pokemon_move_1, "+ + "raid_pokemon_move_2 = :raid_pokemon_move_2, "+ + "raid_pokemon_form = :raid_pokemon_form, "+ + "raid_pokemon_alignment = :raid_pokemon_alignment, "+ + "raid_pokemon_cp = :raid_pokemon_cp, "+ + "raid_is_exclusive = :raid_is_exclusive, "+ + "cell_id = :cell_id, "+ + "deleted = :deleted, "+ + "total_cp = :total_cp, "+ + "raid_pokemon_gender = :raid_pokemon_gender, "+ + "sponsor_id = :sponsor_id, "+ + "partner_id = :partner_id, "+ + "raid_pokemon_costume = :raid_pokemon_costume, "+ + "raid_pokemon_evolution = :raid_pokemon_evolution, "+ + "ar_scan_eligible = :ar_scan_eligible, "+ + "power_up_level = :power_up_level, "+ + "power_up_points = :power_up_points, "+ + "power_up_end_timestamp = :power_up_end_timestamp,"+ + "description = :description,"+ + "defenders = :defenders,"+ + "rsvps = :rsvps "+ + "WHERE id = :id", gym, + ) + statsCollector.IncDbQuery("update gym", err) + if err != nil { + log.Errorf("Update gym %s", err) + } + _, _ = res, err + } + } + + //gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) + areas := MatchStatsGeofence(gym.Lat, gym.Lon) + createGymWebhooks(gym, areas) + createGymFortWebhooks(gym) + updateRaidStats(gym, areas) + if dbDebugEnabled { + gym.changedFields = gym.changedFields[:0] + } + if gym.IsNewRecord() { + gymCache.Set(gym.Id, gym, ttlcache.DefaultTTL) + gym.newRecord = false + } + gym.ClearDirty() +} + +func updateGymGetMapFortCache(gym *Gym, skipName bool) { + storedGetMapFort := getMapFortsCache.Get(gym.Id) + if storedGetMapFort != nil { + getMapFort := storedGetMapFort.Value() + getMapFortsCache.Delete(gym.Id) + gym.updateGymFromGetMapFortsOutProto(getMapFort, skipName) + log.Debugf("Updated Gym using stored getMapFort: %s", gym.Id) + } +} diff --git a/decoder/incident.go b/decoder/incident.go index 38b5a992..7149a978 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -1,19 +1,9 @@ package decoder import ( - "context" - "database/sql" - "errors" "sync" - "time" - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - null "gopkg.in/guregu/null.v4" - - "golbat/db" - "golbat/pogo" - "golbat/webhooks" + null "github.com/guregu/null/v6" ) // Incident struct. @@ -223,269 +213,3 @@ func (incident *Incident) SetSlot3Form(v null.Int) { incident.dirty = true } } - -func loadIncidentFromDatabase(ctx context.Context, db db.DbDetails, incidentId string, incident *Incident) error { - err := db.GeneralDb.GetContext(ctx, incident, - "SELECT id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form "+ - "FROM incident WHERE incident.id = ?", incidentId) - statsCollector.IncDbQuery("select incident", err) - return err -} - -// peekIncidentRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func peekIncidentRecord(incidentId string) (*Incident, func(), error) { - if item := incidentCache.Get(incidentId); item != nil { - incident := item.Value() - incident.Lock() - return incident, func() { incident.Unlock() }, nil - } - return nil, nil, nil -} - -// getIncidentRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getIncidentRecordReadOnly(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { - // Check cache first - if item := incidentCache.Get(incidentId); item != nil { - incident := item.Value() - incident.Lock() - return incident, func() { incident.Unlock() }, nil - } - - dbIncident := Incident{} - err := loadIncidentFromDatabase(ctx, db, incidentId, &dbIncident) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbIncident.ClearDirty() - - // Atomically cache the loaded Incident - if another goroutine raced us, - // we'll get their Incident and use that instead (ensuring same mutex) - existingIncident, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { - return &dbIncident - }) - - incident := existingIncident.Value() - incident.Lock() - return incident, func() { incident.Unlock() }, nil -} - -// getIncidentRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Caller MUST call returned unlock function if non-nil. -func getIncidentRecordForUpdate(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { - incident, unlock, err := getIncidentRecordReadOnly(ctx, db, incidentId) - if err != nil || incident == nil { - return nil, nil, err - } - incident.snapshotOldValues() - return incident, unlock, nil -} - -// getOrCreateIncidentRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreateIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string, pokestopId string) (*Incident, func(), error) { - // Create new Incident atomically - function only called if key doesn't exist - incidentItem, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { - return &Incident{Id: incidentId, PokestopId: pokestopId, newRecord: true} - }) - - incident := incidentItem.Value() - incident.Lock() - - if incident.newRecord { - // We should attempt to load from database - err := loadIncidentFromDatabase(ctx, db, incidentId, incident) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - incident.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - incident.newRecord = false - incident.ClearDirty() - } - } - - incident.snapshotOldValues() - return incident, func() { incident.Unlock() }, nil -} - -func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident) { - // Skip save if not dirty and not new - if !incident.IsDirty() && !incident.IsNewRecord() { - return - } - - incident.Updated = time.Now().Unix() - - if incident.IsNewRecord() { - res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ - "VALUES (:id, :pokestop_id, :start, :expiration, :display_type, :style, :character, :updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, :slot_2_form, :slot_3_pokemon_id, :slot_3_form)", incident) - - if err != nil { - log.Errorf("insert incident: %s", err) - return - } - statsCollector.IncDbQuery("insert incident", err) - _, _ = res, err - } else { - res, err := db.GeneralDb.NamedExec("UPDATE incident SET "+ - "start = :start, "+ - "expiration = :expiration, "+ - "display_type = :display_type, "+ - "style = :style, "+ - "`character` = :character, "+ - "updated = :updated, "+ - "confirmed = :confirmed, "+ - "slot_1_pokemon_id = :slot_1_pokemon_id, "+ - "slot_1_form = :slot_1_form, "+ - "slot_2_pokemon_id = :slot_2_pokemon_id, "+ - "slot_2_form = :slot_2_form, "+ - "slot_3_pokemon_id = :slot_3_pokemon_id, "+ - "slot_3_form = :slot_3_form "+ - "WHERE id = :id", incident, - ) - statsCollector.IncDbQuery("update incident", err) - if err != nil { - log.Errorf("Update incident %s", err) - } - _, _ = res, err - } - - createIncidentWebhooks(ctx, db, incident) - - var stopLat, stopLon float64 - stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) - if stop != nil { - stopLat, stopLon = stop.Lat, stop.Lon - unlock() - } - - areas := MatchStatsGeofence(stopLat, stopLon) - updateIncidentStats(incident, areas) - - incident.ClearDirty() - if incident.IsNewRecord() { - incident.newRecord = false - incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) - } -} - -func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Incident) { - old := &incident.oldValues - isNew := incident.IsNewRecord() - - if isNew || (old.ExpirationTime != incident.ExpirationTime || old.Character != incident.Character || old.Confirmed != incident.Confirmed || old.Slot1PokemonId != incident.Slot1PokemonId) { - var pokestopName, stopUrl string - var stopLat, stopLon float64 - var stopEnabled bool - stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) - if stop != nil { - pokestopName = stop.Name.ValueOrZero() - stopLat, stopLon = stop.Lat, stop.Lon - stopUrl = stop.Url.ValueOrZero() - stopEnabled = stop.Enabled.ValueOrZero() - unlock() - } - if pokestopName == "" { - pokestopName = "Unknown" - } - - var lineup []webhookLineup - if incident.Slot1PokemonId.Valid { - lineup = []webhookLineup{ - { - Slot: 1, - PokemonId: incident.Slot1PokemonId, - Form: incident.Slot1Form, - }, - { - Slot: 2, - PokemonId: incident.Slot2PokemonId, - Form: incident.Slot2Form, - }, - { - Slot: 3, - PokemonId: incident.Slot3PokemonId, - Form: incident.Slot3Form, - }, - } - } - - incidentHook := IncidentWebhook{ - Id: incident.Id, - PokestopId: incident.PokestopId, - Latitude: stopLat, - Longitude: stopLon, - PokestopName: pokestopName, - Url: stopUrl, - Enabled: stopEnabled, - Start: incident.StartTime, - IncidentExpireTimestamp: incident.ExpirationTime, - Expiration: incident.ExpirationTime, - DisplayType: incident.DisplayType, - Style: incident.Style, - GruntType: incident.Character, - Character: incident.Character, - Updated: incident.Updated, - Confirmed: incident.Confirmed, - Lineup: lineup, - } - - areas := MatchStatsGeofence(stop.Lat, stop.Lon) - webhooksSender.AddMessage(webhooks.Invasion, incidentHook, areas) - statsCollector.UpdateIncidentCount(areas) - } -} - -func (incident *Incident) updateFromPokestopIncidentDisplay(pokestopDisplay *pogo.PokestopIncidentDisplayProto) { - incident.SetId(pokestopDisplay.IncidentId) - incident.SetStartTime(int64(pokestopDisplay.IncidentStartMs / 1000)) - incident.SetExpirationTime(int64(pokestopDisplay.IncidentExpirationMs / 1000)) - incident.SetDisplayType(int16(pokestopDisplay.IncidentDisplayType)) - if (incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE)) && incident.Confirmed { - log.Debugf("Incident has already been confirmed as a decoy: %s", incident.Id) - return - } - characterDisplay := pokestopDisplay.GetCharacterDisplay() - if characterDisplay != nil { - // team := pokestopDisplay.Open - incident.SetStyle(int16(characterDisplay.Style)) - incident.SetCharacter(int16(characterDisplay.Character)) - } else { - incident.SetStyle(0) - incident.SetCharacter(0) - } -} - -func (incident *Incident) updateFromOpenInvasionCombatSessionOut(protoRes *pogo.OpenInvasionCombatSessionOutProto) { - incident.SetSlot1PokemonId(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokedexId.Number()), true)) - incident.SetSlot1Form(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokemonDisplay.Form.Number()), true)) - for i, pokemon := range protoRes.Combat.Opponent.ReservePokemon { - if i == 0 { - incident.SetSlot2PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) - incident.SetSlot2Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) - } else if i == 1 { - incident.SetSlot3PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) - incident.SetSlot3Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) - } - } - incident.SetConfirmed(true) -} - -func (incident *Incident) updateFromStartIncidentOut(proto *pogo.StartIncidentOutProto) { - incident.SetCharacter(int16(proto.GetIncident().GetStep()[0].GetPokestopDialogue().GetDialogueLine()[0].GetCharacter())) - if incident.Character == int16(pogo.EnumWrapper_CHARACTER_GIOVANNI) || - incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || - incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE) { - incident.SetConfirmed(true) - } - incident.SetStartTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentStartMs() / 1000)) - incident.SetExpirationTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentExpirationMs() / 1000)) -} diff --git a/decoder/incident_decode.go b/decoder/incident_decode.go new file mode 100644 index 00000000..046105fc --- /dev/null +++ b/decoder/incident_decode.go @@ -0,0 +1,54 @@ +package decoder + +import ( + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" +) + +func (incident *Incident) updateFromPokestopIncidentDisplay(pokestopDisplay *pogo.PokestopIncidentDisplayProto) { + incident.SetId(pokestopDisplay.IncidentId) + incident.SetStartTime(int64(pokestopDisplay.IncidentStartMs / 1000)) + incident.SetExpirationTime(int64(pokestopDisplay.IncidentExpirationMs / 1000)) + incident.SetDisplayType(int16(pokestopDisplay.IncidentDisplayType)) + if (incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE)) && incident.Confirmed { + log.Debugf("Incident has already been confirmed as a decoy: %s", incident.Id) + return + } + characterDisplay := pokestopDisplay.GetCharacterDisplay() + if characterDisplay != nil { + // team := pokestopDisplay.Open + incident.SetStyle(int16(characterDisplay.Style)) + incident.SetCharacter(int16(characterDisplay.Character)) + } else { + incident.SetStyle(0) + incident.SetCharacter(0) + } +} + +func (incident *Incident) updateFromOpenInvasionCombatSessionOut(protoRes *pogo.OpenInvasionCombatSessionOutProto) { + incident.SetSlot1PokemonId(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokedexId.Number()), true)) + incident.SetSlot1Form(null.NewInt(int64(protoRes.Combat.Opponent.ActivePokemon.PokemonDisplay.Form.Number()), true)) + for i, pokemon := range protoRes.Combat.Opponent.ReservePokemon { + if i == 0 { + incident.SetSlot2PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) + incident.SetSlot2Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) + } else if i == 1 { + incident.SetSlot3PokemonId(null.NewInt(int64(pokemon.PokedexId.Number()), true)) + incident.SetSlot3Form(null.NewInt(int64(pokemon.PokemonDisplay.Form.Number()), true)) + } + } + incident.SetConfirmed(true) +} + +func (incident *Incident) updateFromStartIncidentOut(proto *pogo.StartIncidentOutProto) { + incident.SetCharacter(int16(proto.GetIncident().GetStep()[0].GetPokestopDialogue().GetDialogueLine()[0].GetCharacter())) + if incident.Character == int16(pogo.EnumWrapper_CHARACTER_GIOVANNI) || + incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_MALE) || + incident.Character == int16(pogo.EnumWrapper_CHARACTER_DECOY_GRUNT_FEMALE) { + incident.SetConfirmed(true) + } + incident.SetStartTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentStartMs() / 1000)) + incident.SetExpirationTime(int64(proto.Incident.GetCompletionDisplay().GetIncidentExpirationMs() / 1000)) +} diff --git a/decoder/incident_process.go b/decoder/incident_process.go new file mode 100644 index 00000000..4550c7c9 --- /dev/null +++ b/decoder/incident_process.go @@ -0,0 +1,43 @@ +package decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func UpdateIncidentLineup(ctx context.Context, db db.DbDetails, protoReq *pogo.OpenInvasionCombatSessionProto, protoRes *pogo.OpenInvasionCombatSessionOutProto) string { + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, protoReq.IncidentLookup.IncidentId, protoReq.IncidentLookup.FortId) + if err != nil { + return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) + } + defer unlock() + + if incident.newRecord { + log.Debugf("Updating lineup before it was saved: %s", protoReq.IncidentLookup.IncidentId) + } + incident.updateFromOpenInvasionCombatSessionOut(protoRes) + + saveIncidentRecord(ctx, db, incident) + return "" +} + +func ConfirmIncident(ctx context.Context, db db.DbDetails, proto *pogo.StartIncidentOutProto) string { + incident, unlock, err := getOrCreateIncidentRecord(ctx, db, proto.Incident.IncidentId, proto.Incident.FortId) + if err != nil { + return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) + } + defer unlock() + + if incident.newRecord { + log.Debugf("Confirming incident before it was saved: %s", proto.Incident.IncidentId) + } + incident.updateFromStartIncidentOut(proto) + + saveIncidentRecord(ctx, db, incident) + return "" +} diff --git a/decoder/incident_state.go b/decoder/incident_state.go new file mode 100644 index 00000000..d3b8d80f --- /dev/null +++ b/decoder/incident_state.go @@ -0,0 +1,234 @@ +package decoder + +import ( + "context" + "database/sql" + "errors" + "time" + + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/webhooks" +) + +func loadIncidentFromDatabase(ctx context.Context, db db.DbDetails, incidentId string, incident *Incident) error { + err := db.GeneralDb.GetContext(ctx, incident, + "SELECT id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form "+ + "FROM incident WHERE incident.id = ?", incidentId) + statsCollector.IncDbQuery("select incident", err) + return err +} + +// peekIncidentRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekIncidentRecord(incidentId string) (*Incident, func(), error) { + if item := incidentCache.Get(incidentId); item != nil { + incident := item.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil + } + return nil, nil, nil +} + +// getIncidentRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getIncidentRecordReadOnly(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { + // Check cache first + if item := incidentCache.Get(incidentId); item != nil { + incident := item.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil + } + + dbIncident := Incident{} + err := loadIncidentFromDatabase(ctx, db, incidentId, &dbIncident) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbIncident.ClearDirty() + + // Atomically cache the loaded Incident - if another goroutine raced us, + // we'll get their Incident and use that instead (ensuring same mutex) + existingIncident, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { + return &dbIncident + }) + + incident := existingIncident.Value() + incident.Lock() + return incident, func() { incident.Unlock() }, nil +} + +// getIncidentRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getIncidentRecordForUpdate(ctx context.Context, db db.DbDetails, incidentId string) (*Incident, func(), error) { + incident, unlock, err := getIncidentRecordReadOnly(ctx, db, incidentId) + if err != nil || incident == nil { + return nil, nil, err + } + incident.snapshotOldValues() + return incident, unlock, nil +} + +// getOrCreateIncidentRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateIncidentRecord(ctx context.Context, db db.DbDetails, incidentId string, pokestopId string) (*Incident, func(), error) { + // Create new Incident atomically - function only called if key doesn't exist + incidentItem, _ := incidentCache.GetOrSetFunc(incidentId, func() *Incident { + return &Incident{Id: incidentId, PokestopId: pokestopId, newRecord: true} + }) + + incident := incidentItem.Value() + incident.Lock() + + if incident.newRecord { + // We should attempt to load from database + err := loadIncidentFromDatabase(ctx, db, incidentId, incident) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + incident.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + incident.newRecord = false + incident.ClearDirty() + } + } + + incident.snapshotOldValues() + return incident, func() { incident.Unlock() }, nil +} + +func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident) { + // Skip save if not dirty and not new + if !incident.IsDirty() && !incident.IsNewRecord() { + return + } + + incident.Updated = time.Now().Unix() + + if incident.IsNewRecord() { + res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ + "VALUES (:id, :pokestop_id, :start, :expiration, :display_type, :style, :character, :updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, :slot_2_form, :slot_3_pokemon_id, :slot_3_form)", incident) + + if err != nil { + log.Errorf("insert incident: %s", err) + return + } + statsCollector.IncDbQuery("insert incident", err) + _, _ = res, err + } else { + res, err := db.GeneralDb.NamedExec("UPDATE incident SET "+ + "start = :start, "+ + "expiration = :expiration, "+ + "display_type = :display_type, "+ + "style = :style, "+ + "`character` = :character, "+ + "updated = :updated, "+ + "confirmed = :confirmed, "+ + "slot_1_pokemon_id = :slot_1_pokemon_id, "+ + "slot_1_form = :slot_1_form, "+ + "slot_2_pokemon_id = :slot_2_pokemon_id, "+ + "slot_2_form = :slot_2_form, "+ + "slot_3_pokemon_id = :slot_3_pokemon_id, "+ + "slot_3_form = :slot_3_form "+ + "WHERE id = :id", incident, + ) + statsCollector.IncDbQuery("update incident", err) + if err != nil { + log.Errorf("Update incident %s", err) + } + _, _ = res, err + } + + createIncidentWebhooks(ctx, db, incident) + + var stopLat, stopLon float64 + stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) + if stop != nil { + stopLat, stopLon = stop.Lat, stop.Lon + unlock() + } + + areas := MatchStatsGeofence(stopLat, stopLon) + updateIncidentStats(incident, areas) + + incident.ClearDirty() + if incident.IsNewRecord() { + incident.newRecord = false + incidentCache.Set(incident.Id, incident, ttlcache.DefaultTTL) + } +} + +func createIncidentWebhooks(ctx context.Context, db db.DbDetails, incident *Incident) { + old := &incident.oldValues + isNew := incident.IsNewRecord() + + if isNew || (old.ExpirationTime != incident.ExpirationTime || old.Character != incident.Character || old.Confirmed != incident.Confirmed || old.Slot1PokemonId != incident.Slot1PokemonId) { + var pokestopName, stopUrl string + var stopLat, stopLon float64 + var stopEnabled bool + stop, unlock, _ := getPokestopRecordReadOnly(ctx, db, incident.PokestopId) + if stop != nil { + pokestopName = stop.Name.ValueOrZero() + stopLat, stopLon = stop.Lat, stop.Lon + stopUrl = stop.Url.ValueOrZero() + stopEnabled = stop.Enabled.ValueOrZero() + unlock() + } + if pokestopName == "" { + pokestopName = "Unknown" + } + + var lineup []webhookLineup + if incident.Slot1PokemonId.Valid { + lineup = []webhookLineup{ + { + Slot: 1, + PokemonId: incident.Slot1PokemonId, + Form: incident.Slot1Form, + }, + { + Slot: 2, + PokemonId: incident.Slot2PokemonId, + Form: incident.Slot2Form, + }, + { + Slot: 3, + PokemonId: incident.Slot3PokemonId, + Form: incident.Slot3Form, + }, + } + } + + incidentHook := IncidentWebhook{ + Id: incident.Id, + PokestopId: incident.PokestopId, + Latitude: stopLat, + Longitude: stopLon, + PokestopName: pokestopName, + Url: stopUrl, + Enabled: stopEnabled, + Start: incident.StartTime, + IncidentExpireTimestamp: incident.ExpirationTime, + Expiration: incident.ExpirationTime, + DisplayType: incident.DisplayType, + Style: incident.Style, + GruntType: incident.Character, + Character: incident.Character, + Updated: incident.Updated, + Confirmed: incident.Confirmed, + Lineup: lineup, + } + + areas := MatchStatsGeofence(stopLat, stopLon) + webhooksSender.AddMessage(webhooks.Invasion, incidentHook, areas) + statsCollector.UpdateIncidentCount(areas) + } +} diff --git a/decoder/main.go b/decoder/main.go index 21467b7c..51dc4e56 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -1,19 +1,16 @@ package decoder import ( - "context" - "fmt" "math" "runtime" "time" "github.com/UnownHash/gohbem" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" "golbat/config" - "golbat/db" "golbat/geo" "golbat/pogo" "golbat/stats_collector" @@ -249,251 +246,6 @@ func nullFloatAlmostEqual(a, b null.Float, tolerance float64) bool { } } -func UpdateFortBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawFortData) { - // Logic is: - // 1. Filter out pokestops that are unchanged (last modified time) - // 2. Fetch current stops from database - // 3. Generate batch of inserts as needed (with on duplicate saveGymRecord) - - //var stopsToModify []string - - for _, fort := range p { - fortId := fort.Data.FortId - if fort.Data.FortType == pogo.FortType_CHECKPOINT && scanParameters.ProcessPokestops { - pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fortId) - if err != nil { - log.Errorf("getOrCreatePokestopRecord: %s", err) - continue - } - - pokestop.updatePokestopFromFort(fort.Data, fort.Cell, fort.Timestamp/1000) - - // If this is a new pokestop, check if it was converted from a gym and copy shared fields - if pokestop.IsNewRecord() { - gym, gymUnlock, _ := getGymRecordReadOnly(ctx, db, fortId) - if gym != nil { - pokestop.copySharedFieldsFrom(gym) - gymUnlock() - } - } - - savePokestopRecord(ctx, db, pokestop) - unlock() - - incidents := fort.Data.PokestopDisplays - if incidents == nil && fort.Data.PokestopDisplay != nil { - incidents = []*pogo.PokestopIncidentDisplayProto{fort.Data.PokestopDisplay} - } - - if incidents != nil { - for _, incidentProto := range incidents { - incident, unlock, err := getOrCreateIncidentRecord(ctx, db, incidentProto.IncidentId, fortId) - if err != nil { - log.Errorf("getOrCreateIncidentRecord: %s", err) - continue - } - incident.updateFromPokestopIncidentDisplay(incidentProto) - saveIncidentRecord(ctx, db, incident) - unlock() - } - } - } - - if fort.Data.FortType == pogo.FortType_GYM && scanParameters.ProcessGyms { - gym, gymUnlock, err := getOrCreateGymRecord(ctx, db, fortId) - if err != nil { - log.Errorf("getOrCreateGymRecord: %s", err) - continue - } - - gym.updateGymFromFort(fort.Data, fort.Cell) - - // If this is a new gym, check if it was converted from a pokestop and copy shared fields - if gym.IsNewRecord() { - pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, fortId) - if pokestop != nil { - gym.copySharedFieldsFrom(pokestop) - unlock() - } - } - - saveGymRecord(ctx, db, gym) - gymUnlock() - } - } -} - -func UpdateStationBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, p []RawStationData) { - for _, stationProto := range p { - stationId := stationProto.Data.Id - station, unlock, err := getOrCreateStationRecord(ctx, db, stationId) - if err != nil { - log.Errorf("getOrCreateStationRecord: %s", err) - continue - } - station.updateFromStationProto(stationProto.Data, stationProto.Cell) - saveStationRecord(ctx, db, station) - unlock() - } -} - -func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters ScanParameters, wildPokemonList []RawWildPokemonData, nearbyPokemonList []RawNearbyPokemonData, mapPokemonList []RawMapPokemonData, weather []*pogo.ClientWeatherProto, username string) { - weatherLookup := make(map[int64]pogo.GameplayWeatherProto_WeatherCondition) - for _, weatherProto := range weather { - weatherLookup[weatherProto.S2CellId] = weatherProto.GameplayWeather.GameplayCondition - } - - for _, wild := range wildPokemonList { - encounterId := wild.Data.EncounterId - - // spawnpointUpdateFromWild doesn't need Pokemon lock - spawnpointUpdateFromWild(ctx, db, wild.Data, wild.Timestamp) - - if scanParameters.ProcessWild { - // Use read-only getter - we're only checking if update is needed, then queuing - pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) - if err != nil { - log.Errorf("getPokemonRecordReadOnly: %s", err) - continue - } - - updateTime := wild.Timestamp / 1000 - shouldQueue := pokemon == nil || pokemon.wildSignificantUpdate(wild.Data, updateTime) - - if unlock != nil { - unlock() - } - - if shouldQueue { - // The sweeper will process it after timeout if no encounter arrives - pending := &PendingPokemon{ - EncounterId: encounterId, - WildPokemon: wild.Data, - CellId: int64(wild.Cell), - TimestampMs: wild.Timestamp, - UpdateTime: updateTime, - WeatherLookup: weatherLookup, - Username: username, - } - pokemonPendingQueue.AddPending(pending) - } - } - } - - if scanParameters.ProcessNearby { - for _, nearby := range nearbyPokemonList { - encounterId := nearby.Data.EncounterId - - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Printf("getOrCreatePokemonRecord: %s", err) - continue - } - - updateTime := nearby.Timestamp / 1000 - if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { - pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) - } - - unlock() - } - } - - for _, mapPokemon := range mapPokemonList { - encounterId := mapPokemon.Data.EncounterId - - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Printf("getOrCreatePokemonRecord: %s", err) - continue - } - - pokemon.updateFromMap(ctx, db, mapPokemon.Data, int64(mapPokemon.Cell), weatherLookup, mapPokemon.Timestamp, username) - storedDiskEncounter := diskEncounterCache.Get(encounterId) - if storedDiskEncounter != nil { - diskEncounter := storedDiskEncounter.Value() - diskEncounterCache.Delete(encounterId) - pokemon.updatePokemonFromDiskEncounterProto(ctx, db, diskEncounter, username) - //log.Infof("Processed stored disk encounter") - } - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, mapPokemon.Timestamp/1000) - - unlock() - } -} - -func UpdateClientWeatherBatch(ctx context.Context, db db.DbDetails, p []*pogo.ClientWeatherProto, timestampMs int64, account string) (updates []WeatherUpdate) { - hourKey := timestampMs / time.Hour.Milliseconds() - for _, weatherProto := range p { - weather, unlock, err := getOrCreateWeatherRecord(ctx, db, weatherProto.S2CellId) - if err != nil { - log.Printf("getOrCreateWeatherRecord: %s", err) - continue - } - - if weather.newRecord || timestampMs >= weather.UpdatedMs { - state := getWeatherConsensusState(weatherProto.S2CellId, hourKey) - if state != nil { - publish, publishProto := state.applyObservation(hourKey, account, weatherProto) - if publish { - if publishProto == nil { - publishProto = weatherProto - } - weather.UpdatedMs = timestampMs - weather.updateWeatherFromClientWeatherProto(publishProto) - saveWeatherRecord(ctx, db, weather) - if weather.oldValues.GameplayCondition != weather.GameplayCondition { - updates = append(updates, WeatherUpdate{ - S2CellId: publishProto.S2CellId, - NewWeather: int32(publishProto.GetGameplayWeather().GetGameplayCondition()), - }) - } - } - } - } - - unlock() - } - return updates -} - -func UpdateClientMapS2CellBatch(ctx context.Context, db db.DbDetails, cellIds []uint64) { - saveS2CellRecords(ctx, db, cellIds) -} - -func UpdateIncidentLineup(ctx context.Context, db db.DbDetails, protoReq *pogo.OpenInvasionCombatSessionProto, protoRes *pogo.OpenInvasionCombatSessionOutProto) string { - incident, unlock, err := getOrCreateIncidentRecord(ctx, db, protoReq.IncidentLookup.IncidentId, protoReq.IncidentLookup.FortId) - if err != nil { - return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) - } - defer unlock() - - if incident.newRecord { - log.Debugf("Updating lineup before it was saved: %s", protoReq.IncidentLookup.IncidentId) - } - incident.updateFromOpenInvasionCombatSessionOut(protoRes) - - saveIncidentRecord(ctx, db, incident) - return "" -} - -func ConfirmIncident(ctx context.Context, db db.DbDetails, proto *pogo.StartIncidentOutProto) string { - incident, unlock, err := getOrCreateIncidentRecord(ctx, db, proto.Incident.IncidentId, proto.Incident.FortId) - if err != nil { - return fmt.Sprintf("getOrCreateIncidentRecord: %s", err) - } - defer unlock() - - if incident.newRecord { - log.Debugf("Confirming incident before it was saved: %s", proto.Incident.IncidentId) - } - incident.updateFromStartIncidentOut(proto) - - saveIncidentRecord(ctx, db, incident) - return "" -} - func SetWebhooksSender(whSender webhooksSenderInterface) { webhooksSender = whSender } diff --git a/decoder/player.go b/decoder/player.go index 0478ae75..01dea73f 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -9,9 +9,9 @@ import ( "golbat/db" "golbat/pogo" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) // Player struct. Name is the primary key. diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 47096a4d..cc69bdb6 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -1,29 +1,11 @@ package decoder import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" "sync" - "time" - "golbat/config" - "golbat/db" - "golbat/geo" "golbat/grpc" - "golbat/pogo" - "golbat/webhooks" - "github.com/UnownHash/gohbem" - "github.com/golang/geo/s2" - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "google.golang.org/protobuf/proto" - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" ) // Pokemon struct. @@ -480,1495 +462,3 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { } } } - -// peekPokemonRecordReadOnly acquires lock, does NOT take snapshot. -// Use for read-only checks which will not cause a backing database lookup -// Caller must use returned unlock function -func peekPokemonRecordReadOnly(encounterId uint64) (*Pokemon, func(), error) { - if item := pokemonCache.Get(encounterId); item != nil { - pokemon := item.Value() - pokemon.Lock() - return pokemon, func() { pokemon.Unlock() }, nil - } - - return nil, nil, nil -} - -func loadPokemonFromDatabase(ctx context.Context, db db.DbDetails, encounterId uint64, pokemon *Pokemon) error { - err := db.PokemonDb.GetContext(ctx, pokemon, - "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ - "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ - "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ - "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ - "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) - statsCollector.IncDbQuery("select pokemon", err) - - return err -} - -// getPokemonRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks, but will cause a backing database lookup -// Caller MUST call returned unlock function. -func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { - // If we are in-memory only, this is identical to peek - if config.Config.PokemonMemoryOnly { - return peekPokemonRecordReadOnly(encounterId) - } - - // Check cache first - if item := pokemonCache.Get(encounterId); item != nil { - pokemon := item.Value() - pokemon.Lock() - return pokemon, func() { pokemon.Unlock() }, nil - } - - dbPokemon := Pokemon{} - err := loadPokemonFromDatabase(ctx, db, encounterId, &dbPokemon) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbPokemon.ClearDirty() - - // Atomically cache the loaded Pokemon - if another goroutine raced us, - // we'll get their Pokemon and use that instead (ensuring same mutex) - existingPokemon, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - // Only called if key doesn't exist - our Pokemon wins - pokemonRtreeUpdatePokemonOnGet(&dbPokemon) - return &dbPokemon - }) - - pokemon := existingPokemon.Value() - pokemon.Lock() - return pokemon, func() { pokemon.Unlock() }, nil -} - -// getPokemonRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Use when modifying the Pokemon. -// Caller MUST call returned unlock function. -func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { - pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) - if err != nil || pokemon == nil { - return nil, nil, err - } - pokemon.snapshotOldValues() - return pokemon, unlock, nil -} - -// getOrCreatePokemonRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { - // Create new Pokemon atomically - function only called if key doesn't exist - pokemonItem, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { - return &Pokemon{Id: encounterId, newRecord: true} - }) - - pokemon := pokemonItem.Value() - pokemon.Lock() - - if config.Config.PokemonMemoryOnly { - pokemon.snapshotOldValues() - return pokemon, func() { pokemon.Unlock() }, nil - } - - if pokemon.newRecord { - // We should attempt to load from database - err := loadPokemonFromDatabase(ctx, db, encounterId, pokemon) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - pokemon.Unlock() - return nil, nil, err - } - } else { - // We loaded - pokemon.newRecord = false - pokemon.ClearDirty() - pokemonRtreeUpdatePokemonOnGet(pokemon) - } - } - - pokemon.snapshotOldValues() - return pokemon, func() { pokemon.Unlock() }, nil -} - -// hasChangesPokemon compares two Pokemon structs -// Ignored: Username, Iv, Pvp -// Float tolerance: Lat, Lon -// Null Float tolerance: Weight, Height, Capture1, Capture2, Capture3 -func hasChangesPokemon(old *Pokemon, new *Pokemon) bool { - return old.Id != new.Id || - old.PokestopId != new.PokestopId || - old.SpawnId != new.SpawnId || - old.Size != new.Size || - old.ExpireTimestamp != new.ExpireTimestamp || - old.Updated != new.Updated || - old.PokemonId != new.PokemonId || - old.Move1 != new.Move1 || - old.Move2 != new.Move2 || - old.Gender != new.Gender || - old.Cp != new.Cp || - old.AtkIv != new.AtkIv || - old.DefIv != new.DefIv || - old.StaIv != new.StaIv || - old.Form != new.Form || - old.Level != new.Level || - old.IsStrong != new.IsStrong || - old.Weather != new.Weather || - old.Costume != new.Costume || - old.FirstSeenTimestamp != new.FirstSeenTimestamp || - old.Changed != new.Changed || - old.CellId != new.CellId || - old.ExpireTimestampVerified != new.ExpireTimestampVerified || - old.DisplayPokemonId != new.DisplayPokemonId || - old.IsDitto != new.IsDitto || - old.SeenType != new.SeenType || - old.Shiny != new.Shiny || - old.IsEvent != new.IsEvent || - !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || - !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) || - !nullFloatAlmostEqual(old.Weight, new.Weight, floatTolerance) || - !nullFloatAlmostEqual(old.Height, new.Height, floatTolerance) || - !nullFloatAlmostEqual(old.Capture1, new.Capture1, floatTolerance) || - !nullFloatAlmostEqual(old.Capture2, new.Capture2, floatTolerance) || - !nullFloatAlmostEqual(old.Capture3, new.Capture3, floatTolerance) -} - -func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Pokemon, isEncounter, writeDB, webhook bool, now int64) { - if !pokemon.newRecord && !pokemon.IsDirty() { - return - } - - // uncomment to debug excessive writes - //if !pokemon.isNewRecord() && oldPokemon.AtkIv == pokemon.AtkIv && oldPokemon.DefIv == pokemon.DefIv && oldPokemon.StaIv == pokemon.StaIv && oldPokemon.Level == pokemon.Level && oldPokemon.ExpireTimestampVerified == pokemon.ExpireTimestampVerified && oldPokemon.PokemonId == pokemon.PokemonId && oldPokemon.ExpireTimestamp == pokemon.ExpireTimestamp && oldPokemon.PokestopId == pokemon.PokestopId && math.Abs(pokemon.Lat-oldPokemon.Lat) < .000001 && math.Abs(pokemon.Lon-oldPokemon.Lon) < .000001 { - // log.Errorf("Why are we updating this? %s", cmp.Diff(oldPokemon, pokemon, cmp.Options{ - // ignoreNearFloats, ignoreNearNullFloats, - // cmpopts.IgnoreFields(Pokemon{}, "Username", "Iv", "Pvp"), - // })) - //} - - if pokemon.FirstSeenTimestamp == 0 { - pokemon.FirstSeenTimestamp = now - } - - pokemon.Updated = null.IntFrom(now) - if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId || pokemon.oldValues.Cp != pokemon.Cp { - pokemon.Changed = now - } - - changePvpField := false - var pvpResults map[string][]gohbem.PokemonEntry - if ohbem != nil { - // Calculating PVP data - check for changes in pokemon properties that affect PVP rankings - // For new records, always calculate; for existing, check if relevant fields changed - shouldCalculatePvp := pokemon.AtkIv.Valid && (pokemon.isNewRecord() || pokemon.IsDirty()) - if shouldCalculatePvp { - pvp, err := ohbem.QueryPvPRank(int(pokemon.PokemonId), - int(pokemon.Form.ValueOrZero()), - int(pokemon.Costume.ValueOrZero()), - int(pokemon.Gender.ValueOrZero()), - int(pokemon.AtkIv.ValueOrZero()), - int(pokemon.DefIv.ValueOrZero()), - int(pokemon.StaIv.ValueOrZero()), - float64(pokemon.Level.ValueOrZero())) - - if err == nil { - pvpBytes, _ := json.Marshal(pvp) - pokemon.Pvp = null.StringFrom(string(pvpBytes)) - changePvpField = true - pvpResults = pvp - } - } - if !pokemon.AtkIv.Valid && pokemon.isNewRecord() { - pokemon.Pvp = null.NewString("", false) - changePvpField = true - } - } - - var oldSeenType string - if !pokemon.oldValues.SeenType.Valid { - oldSeenType = "n/a" - } else { - oldSeenType = pokemon.oldValues.SeenType.ValueOrZero() - } - - log.Debugf("Updating pokemon [%d] from %s->%s - newRecord: %t", pokemon.Id, oldSeenType, pokemon.SeenType.ValueOrZero(), pokemon.isNewRecord()) - //log.Println(cmp.Diff(oldPokemon, pokemon)) - - if writeDB && !config.Config.PokemonMemoryOnly { - if isEncounter && config.Config.PokemonInternalToDb { - unboosted, boosted, strong := pokemon.locateAllScans() - if unboosted != nil && boosted != nil { - unboosted.RemoveDittoAuxInfo() - boosted.RemoveDittoAuxInfo() - } - if strong != nil { - strong.RemoveDittoAuxInfo() - } - marshaled, err := proto.Marshal(&pokemon.internal) - if err == nil { - pokemon.GolbatInternal = marshaled - } else { - log.Errorf("[POKEMON] Failed to marshal internal data for %d, data may be lost: %s", pokemon.Id, err) - } - } - if pokemon.isNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) - } - pvpField, pvpValue := "", "" - if changePvpField { - pvpField, pvpValue = "pvp, ", ":pvp, " - } - res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("INSERT INTO pokemon (id, pokemon_id, lat, lon,"+ - "spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, move_1, move_2,"+ - "gender, form, cp, level, strong, weather, costume, weight, height, size,"+ - "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id,"+ - "expire_timestamp_verified, shiny, username, %s is_event, seen_type) "+ - "VALUES (\"%d\", :pokemon_id, :lat, :lon, :spawn_id, :expire_timestamp, :atk_iv, :def_iv, :sta_iv,"+ - ":golbat_internal, :iv, :move_1, :move_2, :gender, :form, :cp, :level, :strong, :weather, :costume,"+ - ":weight, :height, :size, :display_pokemon_id, :is_ditto, :pokestop_id, :updated,"+ - ":first_seen_timestamp, :changed, :cell_id, :expire_timestamp_verified, :shiny, :username, %s :is_event,"+ - ":seen_type)", pvpField, pokemon.Id, pvpValue), pokemon) - - statsCollector.IncDbQuery("insert pokemon", err) - if err != nil { - log.Errorf("insert pokemon: [%d] %s", pokemon.Id, err) - log.Errorf("Full structure: %+v", pokemon) - pokemonCache.Delete(pokemon.Id) - // Force reload of pokemon from database - return - } - - rows, rowsErr := res.RowsAffected() - log.Debugf("Inserting pokemon [%d] after insert res = %d %v", pokemon.Id, rows, rowsErr) - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) - } - pvpUpdate := "" - if changePvpField { - pvpUpdate = "pvp = :pvp, " - } - res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("UPDATE pokemon SET "+ - "pokestop_id = :pokestop_id, "+ - "spawn_id = :spawn_id, "+ - "lat = :lat, "+ - "lon = :lon, "+ - "weight = :weight, "+ - "height = :height, "+ - "size = :size, "+ - "expire_timestamp = :expire_timestamp, "+ - "updated = :updated, "+ - "pokemon_id = :pokemon_id, "+ - "move_1 = :move_1, "+ - "move_2 = :move_2, "+ - "gender = :gender, "+ - "cp = :cp, "+ - "atk_iv = :atk_iv, "+ - "def_iv = :def_iv, "+ - "sta_iv = :sta_iv, "+ - "golbat_internal = :golbat_internal,"+ - "iv = :iv,"+ - "form = :form, "+ - "level = :level, "+ - "strong = :strong, "+ - "weather = :weather, "+ - "costume = :costume, "+ - "first_seen_timestamp = :first_seen_timestamp, "+ - "changed = :changed, "+ - "cell_id = :cell_id, "+ - "expire_timestamp_verified = :expire_timestamp_verified, "+ - "display_pokemon_id = :display_pokemon_id, "+ - "is_ditto = :is_ditto, "+ - "seen_type = :seen_type, "+ - "shiny = :shiny, "+ - "username = :username, "+ - "%s"+ - "is_event = :is_event "+ - "WHERE id = \"%d\"", pvpUpdate, pokemon.Id), pokemon, - ) - statsCollector.IncDbQuery("update pokemon", err) - if err != nil { - log.Errorf("Update pokemon [%d] %s", pokemon.Id, err) - log.Errorf("Full structure: %+v", pokemon) - pokemonCache.Delete(pokemon.Id) - // Force reload of pokemon from database - - return - } - rows, rowsErr := res.RowsAffected() - log.Debugf("Updating pokemon [%d] after update res = %d %v", pokemon.Id, rows, rowsErr) - } - } - - // Update pokemon rtree - if pokemon.isNewRecord() { - addPokemonToTree(pokemon) - } else if pokemon.Lat != pokemon.oldValues.Lat || pokemon.Lon != pokemon.oldValues.Lon { - // Position changed - update R-tree by removing from old position and adding to new - removePokemonFromTree(pokemon.Id, pokemon.oldValues.Lat, pokemon.oldValues.Lon) - addPokemonToTree(pokemon) - } - - updatePokemonLookup(pokemon, changePvpField, pvpResults) - - areas := MatchStatsGeofence(pokemon.Lat, pokemon.Lon) - if webhook { - createPokemonWebhooks(ctx, db, pokemon, areas) - } - updatePokemonStats(pokemon, areas, now) - - if dbDebugEnabled { - pokemon.changedFields = pokemon.changedFields[:0] - } - pokemon.newRecord = false // After saving, it's no longer a new record - pokemon.ClearDirty() - - pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache - - if db.UsePokemonCache { - pokemonCache.Set(pokemon.Id, pokemon, pokemon.remainingDuration(now)) - } -} - -type PokemonWebhook struct { - SpawnpointId string `json:"spawnpoint_id"` - PokestopId string `json:"pokestop_id"` - PokestopName *string `json:"pokestop_name"` - EncounterId string `json:"encounter_id"` - PokemonId int16 `json:"pokemon_id"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - DisappearTime int64 `json:"disappear_time"` - DisappearTimeVerified bool `json:"disappear_time_verified"` - FirstSeen int64 `json:"first_seen"` - LastModifiedTime null.Int `json:"last_modified_time"` - Gender null.Int `json:"gender"` - Cp null.Int `json:"cp"` - Form null.Int `json:"form"` - Costume null.Int `json:"costume"` - IndividualAttack null.Int `json:"individual_attack"` - IndividualDefense null.Int `json:"individual_defense"` - IndividualStamina null.Int `json:"individual_stamina"` - PokemonLevel null.Int `json:"pokemon_level"` - Move1 null.Int `json:"move_1"` - Move2 null.Int `json:"move_2"` - Weight null.Float `json:"weight"` - Size null.Int `json:"size"` - Height null.Float `json:"height"` - Weather null.Int `json:"weather"` - Capture1 float64 `json:"capture_1"` - Capture2 float64 `json:"capture_2"` - Capture3 float64 `json:"capture_3"` - Shiny null.Bool `json:"shiny"` - Username null.String `json:"username"` - DisplayPokemonId null.Int `json:"display_pokemon_id"` - IsEvent int8 `json:"is_event"` - SeenType null.String `json:"seen_type"` - Pvp json.RawMessage `json:"pvp"` -} - -func createPokemonWebhooks(ctx context.Context, db db.DbDetails, pokemon *Pokemon, areas []geo.AreaName) { - if pokemon.isNewRecord() || - pokemon.oldValues.PokemonId != pokemon.PokemonId || - pokemon.oldValues.Weather != pokemon.Weather || - pokemon.oldValues.Cp != pokemon.Cp { - - spawnpointId := "None" - if pokemon.SpawnId.Valid { - spawnpointId = strconv.FormatInt(pokemon.SpawnId.ValueOrZero(), 16) - } - - pokestopId := "None" - if pokemon.PokestopId.Valid { - pokestopId = pokemon.PokestopId.ValueOrZero() - } - - var pokestopName *string - if pokemon.PokestopId.Valid { - pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokemon.PokestopId.String) - name := "Unknown" - if pokestop != nil { - name = pokestop.Name.ValueOrZero() - unlock() - } - pokestopName = &name - } - - var pvp json.RawMessage - if pokemon.Pvp.Valid { - pvp = json.RawMessage(pokemon.Pvp.ValueOrZero()) - } - - pokemonHook := PokemonWebhook{ - SpawnpointId: spawnpointId, - PokestopId: pokestopId, - PokestopName: pokestopName, - EncounterId: strconv.FormatUint(pokemon.Id, 10), - PokemonId: pokemon.PokemonId, - Latitude: pokemon.Lat, - Longitude: pokemon.Lon, - DisappearTime: pokemon.ExpireTimestamp.ValueOrZero(), - DisappearTimeVerified: pokemon.ExpireTimestampVerified, - FirstSeen: pokemon.FirstSeenTimestamp, - LastModifiedTime: pokemon.Updated, - Gender: pokemon.Gender, - Cp: pokemon.Cp, - Form: pokemon.Form, - Costume: pokemon.Costume, - IndividualAttack: pokemon.AtkIv, - IndividualDefense: pokemon.DefIv, - IndividualStamina: pokemon.StaIv, - PokemonLevel: pokemon.Level, - Move1: pokemon.Move1, - Move2: pokemon.Move2, - Weight: pokemon.Weight, - Size: pokemon.Size, - Height: pokemon.Height, - Weather: pokemon.Weather, - Capture1: pokemon.Capture1.ValueOrZero(), - Capture2: pokemon.Capture2.ValueOrZero(), - Capture3: pokemon.Capture3.ValueOrZero(), - Shiny: pokemon.Shiny, - Username: pokemon.Username, - DisplayPokemonId: pokemon.DisplayPokemonId, - IsEvent: pokemon.IsEvent, - SeenType: pokemon.SeenType, - Pvp: pvp, - } - - if pokemon.AtkIv.Valid && pokemon.DefIv.Valid && pokemon.StaIv.Valid { - webhooksSender.AddMessage(webhooks.PokemonIV, pokemonHook, areas) - } else { - webhooksSender.AddMessage(webhooks.PokemonNoIV, pokemonHook, areas) - } - } -} - -func (pokemon *Pokemon) populateInternal() { - if len(pokemon.GolbatInternal) == 0 || len(pokemon.internal.ScanHistory) != 0 { - return - } - err := proto.Unmarshal(pokemon.GolbatInternal, &pokemon.internal) - if err != nil { - log.Warnf("Failed to parse internal data for %d: %s", pokemon.Id, err) - pokemon.internal.Reset() - } -} - -func (pokemon *Pokemon) locateScan(isStrong bool, isBoosted bool) (*grpc.PokemonScan, bool) { - pokemon.populateInternal() - var bestMatching *grpc.PokemonScan - for _, entry := range pokemon.internal.ScanHistory { - if entry.Strong != isStrong { - continue - } - if entry.Weather != int32(pogo.GameplayWeatherProto_NONE) == isBoosted { - return entry, true - } else { - bestMatching = entry - } - } - return bestMatching, false -} - -func (pokemon *Pokemon) locateAllScans() (unboosted, boosted, strong *grpc.PokemonScan) { - pokemon.populateInternal() - for _, entry := range pokemon.internal.ScanHistory { - if entry.Strong { - strong = entry - } else if entry.Weather != int32(pogo.GameplayWeatherProto_NONE) { - boosted = entry - } else { - unboosted = entry - } - } - return -} - -func (pokemon *Pokemon) isNewRecord() bool { - return pokemon.newRecord -} - -func (pokemon *Pokemon) remainingDuration(now int64) time.Duration { - remaining := ttlcache.DefaultTTL - if pokemon.ExpireTimestampVerified { - timeLeft := 60 + pokemon.ExpireTimestamp.ValueOrZero() - now - if timeLeft > 1 { - remaining = time.Duration(timeLeft) * time.Second - } - } - return remaining -} - -func (pokemon *Pokemon) addWildPokemon(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, timestampMs int64, trustworthyTimestamp bool) { - if wildPokemon.EncounterId != pokemon.Id { - panic("Unmatched EncounterId") - } - pokemon.SetLat(wildPokemon.Latitude) - pokemon.SetLon(wildPokemon.Longitude) - - spawnId, err := strconv.ParseInt(wildPokemon.SpawnPointId, 16, 64) - if err != nil { - panic(err) - } - pokemon.SetSpawnId(null.IntFrom(spawnId)) - - pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, trustworthyTimestamp) - pokemon.setPokemonDisplay(int16(wildPokemon.Pokemon.PokemonId), wildPokemon.Pokemon.PokemonDisplay) -} - -// wildSignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and -// should be written. -func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto, time int64) bool { - pokemonDisplay := wildPokemon.Pokemon.PokemonDisplay - // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired - - return pokemon.SeenType.ValueOrZero() == SeenType_Cell || - pokemon.SeenType.ValueOrZero() == SeenType_NearbyStop || - pokemon.PokemonId != int16(wildPokemon.Pokemon.PokemonId) || - pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || - pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || - pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || - pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) || - (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) -} - -// wildSignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and -// should be written. -func (pokemon *Pokemon) nearbySignificantUpdate(wildPokemon *pogo.NearbyPokemonProto, time int64) bool { - pokemonDisplay := wildPokemon.PokemonDisplay - // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired - - pokemonChanged := pokemon.PokemonId != int16(pokemonDisplay.DisplayId) || - pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || - pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || - pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || - pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) - - if pokemonChanged { - return true - } - - hasExpired := (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) - - if hasExpired { - return true - } - - if pokemon.SeenType.ValueOrZero() == SeenType_Cell { - return true - } - - // if it's at a nearby stop, or encounter and no other details have changed update is not worthwhile - return false -} - -func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - pokemon.SetIsEvent(0) - switch pokemon.SeenType.ValueOrZero() { - case "", SeenType_Cell, SeenType_NearbyStop: - pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) - } - pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs, true) - pokemon.recomputeCpIfNeeded(ctx, db, weather) - pokemon.SetUsername(null.StringFrom(username)) - pokemon.SetCellId(null.IntFrom(cellId)) -} - -func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapPokemon *pogo.MapPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - - if !pokemon.isNewRecord() { - // Do not ever overwrite lure details based on seeing it again in the GMO - return - } - - pokemon.SetIsEvent(0) - - pokemon.Id = mapPokemon.EncounterId - - spawnpointId := mapPokemon.SpawnpointId - - pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, spawnpointId) - if pokestop == nil { - // Unrecognised pokestop - return - } - pokemon.SetPokestopId(null.StringFrom(pokestop.Id)) - pokemon.SetLat(pokestop.Lat) - pokemon.SetLon(pokestop.Lon) - pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) - unlock() - - if mapPokemon.PokemonDisplay != nil { - pokemon.setPokemonDisplay(int16(mapPokemon.PokedexTypeId), mapPokemon.PokemonDisplay) - pokemon.recomputeCpIfNeeded(ctx, db, weather) - // The mapPokemon and nearbyPokemon GMOs don't contain actual shininess. - // shiny = mapPokemon.pokemonDisplay.shiny - } else { - log.Warnf("[POKEMON] MapPokemonProto missing PokemonDisplay for %d", pokemon.Id) - } - if !pokemon.Username.Valid { - pokemon.SetUsername(null.StringFrom(username)) - } - - if mapPokemon.ExpirationTimeMs > 0 && !pokemon.ExpireTimestampVerified { - pokemon.SetExpireTimestamp(null.IntFrom(mapPokemon.ExpirationTimeMs / 1000)) - pokemon.SetExpireTimestampVerified(true) - // if we have cached an encounter for this pokemon, update the TTL. - encounterCache.UpdateTTL(pokemon.Id, pokemon.remainingDuration(timestampMs/1000)) - } else { - pokemon.SetExpireTimestampVerified(false) - } - - pokemon.SetCellId(null.IntFrom(cellId)) -} - -func (pokemon *Pokemon) calculateIv(a int64, d int64, s int64) { - if pokemon.AtkIv.ValueOrZero() != a || pokemon.DefIv.ValueOrZero() != d || pokemon.StaIv.ValueOrZero() != s || - !pokemon.AtkIv.Valid || !pokemon.DefIv.Valid || !pokemon.StaIv.Valid { - pokemon.AtkIv = null.IntFrom(a) - pokemon.DefIv = null.IntFrom(d) - pokemon.StaIv = null.IntFrom(s) - pokemon.Iv = null.FloatFrom(float64(a+d+s) / .45) - pokemon.dirty = true - } -} - -func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, nearbyPokemon *pogo.NearbyPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { - pokemon.SetIsEvent(0) - pokestopId := nearbyPokemon.FortId - pokemon.setPokemonDisplay(int16(nearbyPokemon.PokedexNumber), nearbyPokemon.PokemonDisplay) - pokemon.recomputeCpIfNeeded(ctx, db, weather) - pokemon.SetUsername(null.StringFrom(username)) - - var lat, lon float64 - overrideLatLon := pokemon.isNewRecord() - useCellLatLon := true - if pokestopId != "" { - switch pokemon.SeenType.ValueOrZero() { - case "", SeenType_Cell: - overrideLatLon = true // a better estimate is available - case SeenType_NearbyStop: - default: - return - } - pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokestopId) - if pokestop == nil { - // Unrecognised pokestop, rollback changes - overrideLatLon = pokemon.isNewRecord() - } else { - pokemon.SetSeenType(null.StringFrom(SeenType_NearbyStop)) - pokemon.SetPokestopId(null.StringFrom(pokestopId)) - lat, lon = pokestop.Lat, pokestop.Lon - useCellLatLon = false - unlock() - } - } - if useCellLatLon { - // Cell Pokemon - if !overrideLatLon && pokemon.SeenType.ValueOrZero() != SeenType_Cell { - // do not downgrade to nearby cell - return - } - - s2cell := s2.CellFromCellID(s2.CellID(cellId)) - lat = s2cell.CapBound().RectBound().Center().Lat.Degrees() - lon = s2cell.CapBound().RectBound().Center().Lng.Degrees() - - pokemon.SetSeenType(null.StringFrom(SeenType_Cell)) - } - if overrideLatLon { - pokemon.SetLat(lat) - pokemon.SetLon(lon) - } else { - midpoint := s2.LatLngFromPoint(s2.Point{s2.PointFromLatLng(s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon)). - Add(s2.PointFromLatLng(s2.LatLngFromDegrees(lat, lon)).Vector)}) - pokemon.SetLat(midpoint.Lat.Degrees()) - pokemon.SetLon(midpoint.Lng.Degrees()) - } - pokemon.SetCellId(null.IntFrom(cellId)) - pokemon.setUnknownTimestamp(timestampMs / 1000) -} - -const SeenType_Cell string = "nearby_cell" // Pokemon was seen in a cell (without accurate location) -const SeenType_NearbyStop string = "nearby_stop" // Pokemon was seen at a nearby Pokestop, location set to lon, lat of pokestop -const SeenType_Wild string = "wild" // Pokemon was seen in the wild, accurate location but with no IV details -const SeenType_Encounter string = "encounter" // Pokemon has been encountered giving exact details of current IV -const SeenType_LureWild string = "lure_wild" // Pokemon was seen at a lure -const SeenType_LureEncounter string = "lure_encounter" // Pokemon has been encountered at a lure -const SeenType_TappableEncounter string = "tappable_encounter" // Pokemon has been encountered from tappable -const SeenType_TappableLureEncounter string = "tappable_lure_encounter" // Pokemon has been encountered from a lured tappable - -// setExpireTimestampFromSpawnpoint sets the current Pokemon object ExpireTimeStamp, and ExpireTimeStampVerified from the Spawnpoint -// information held. -// db - the database connection to be used -// timestampMs - the timestamp to be used for calculations -// trustworthyTimestamp - whether this timestamp is fully trustworthy (ie comes from GMO server time) -func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db db.DbDetails, timestampMs int64, trustworthyTimestamp bool) { - if !trustworthyTimestamp && pokemon.ExpireTimestampVerified { - // If our time is not trustworthy, and we have already set a time from some other source (eg a GMO) - // don't modify it - - return - } - - spawnId := pokemon.SpawnId.ValueOrZero() - if spawnId == 0 { - return - } - - pokemon.ExpireTimestampVerified = false - spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) - if spawnPoint != nil && spawnPoint.DespawnSec.Valid { - despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) - unlock() - - date := time.Unix(timestampMs/1000, 0) - secondOfHour := date.Second() + date.Minute()*60 - - despawnOffset := despawnSecond - secondOfHour - if despawnOffset < 0 { - despawnOffset += 3600 - } - pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) - pokemon.SetExpireTimestampVerified(true) - } else { - if unlock != nil { - unlock() - } - pokemon.setUnknownTimestamp(timestampMs / 1000) - } -} - -func (pokemon *Pokemon) setUnknownTimestamp(now int64) { - if !pokemon.ExpireTimestamp.Valid { - pokemon.SetExpireTimestamp(null.IntFrom(now + 20*60)) // should be configurable, add on 20min - } else { - if pokemon.ExpireTimestamp.Int64 < now { - pokemon.SetExpireTimestamp(null.IntFrom(now + 10*60)) // should be configurable, add on 10min - } - } -} - -func checkScans(old *grpc.PokemonScan, new *grpc.PokemonScan) error { - if old == nil || old.CompressedIv() == new.CompressedIv() { - return nil - } - return errors.New(fmt.Sprintf("Unexpected IV mismatch %s != %s", old, new)) -} - -func (pokemon *Pokemon) setDittoAttributes(mode string, isDitto bool, old, new *grpc.PokemonScan) { - if isDitto { - log.Debugf("[POKEMON] %d: %s Ditto found %s -> %s", pokemon.Id, mode, old, new) - pokemon.SetIsDitto(true) - pokemon.SetDisplayPokemonId(null.IntFrom(int64(pokemon.PokemonId))) - pokemon.SetPokemonId(int16(pogo.HoloPokemonId_DITTO)) - } else { - log.Debugf("[POKEMON] %d: %s not Ditto found %s -> %s", pokemon.Id, mode, old, new) - } -} -func (pokemon *Pokemon) resetDittoAttributes(mode string, old, aux, new *grpc.PokemonScan) (*grpc.PokemonScan, error) { - log.Debugf("[POKEMON] %d: %s Ditto was reset %s (%s) -> %s", pokemon.Id, mode, old, aux, new) - pokemon.SetIsDitto(false) - pokemon.SetDisplayPokemonId(null.NewInt(0, false)) - pokemon.SetPokemonId(int16(pokemon.DisplayPokemonId.Int64)) - return new, checkScans(old, new) -} - -// As far as I'm concerned, wild Ditto only depends on species but not costume/gender/form -var dittoDisguises sync.Map - -func confirmDitto(scan *grpc.PokemonScan) { - now := time.Now() - lastSeen, exists := dittoDisguises.Swap(scan.Pokemon, now) - if exists { - log.Debugf("[DITTO] Disguise %s reseen after %s", scan, now.Sub(lastSeen.(time.Time))) - } else { - var sb strings.Builder - sb.WriteString("[DITTO] New disguise ") - sb.WriteString(scan.String()) - sb.WriteString(" found. Current disguises ") - dittoDisguises.Range(func(disguise, lastSeen interface{}) bool { - sb.WriteString(strconv.FormatInt(int64(disguise.(int32)), 10)) - sb.WriteString(" (") - sb.WriteString(now.Sub(lastSeen.(time.Time)).String()) - sb.WriteString(") ") - return true - }) - log.Info(sb.String()) - } -} - -// detectDitto returns the IV/level set that should be used for persisting to db/seen if caught. -// error is set if something unexpected happened and the scan history should be cleared. -func (pokemon *Pokemon) detectDitto(scan *grpc.PokemonScan) (*grpc.PokemonScan, error) { - unboostedScan, boostedScan, strongScan := pokemon.locateAllScans() - if scan.Strong { - if strongScan != nil { - expectedLevel := strongScan.Level - isBoosted := scan.Weather != int32(pogo.GameplayWeatherProto_NONE) - if strongScan.Weather != int32(pogo.GameplayWeatherProto_NONE) != isBoosted { - if isBoosted { - expectedLevel += 5 - } else { - expectedLevel -= 5 - } - } - if scan.Level != expectedLevel || scan.CompressedIv() != strongScan.CompressedIv() { - return scan, errors.New(fmt.Sprintf("Unexpected strong Pokemon (Ditto?), %s -> %s", - strongScan, scan)) - } - } - return scan, nil - } - - // Here comes the Ditto logic. Embrace yourself :) - // Ditto weather can be split into 4 categories: - // - 00: No weather boost - // - 0P: No weather boost but Ditto is actually boosted by partly cloudy causing seen IV to be boosted [atypical] - // - B0: Weather boosts disguise but not Ditto causing seen IV to be unboosted [atypical] - // - PP: Weather being partly cloudy boosts both disguise and Ditto - // - // We will also use 0N/BN/PN to denote a normal non-Ditto spawn with corresponding weather boosts. - // Disguise IV depends on Ditto weather boost instead, and caught Ditto is boosted only in PP state. - if pokemon.IsDitto { - var unboostedLevel int32 - if boostedScan != nil { - unboostedLevel = boostedScan.Level - 5 - } else if unboostedScan != nil { - unboostedLevel = unboostedScan.Level - } else { - pokemon.resetDittoAttributes("?", nil, nil, scan) - return scan, errors.New("Missing past scans. Ditto will be reset") - } - // If IsDitto = true, then the IV sets in history are ALWAYS confirmed - scan.Confirmed = true - switch scan.Weather { - case int32(pogo.GameplayWeatherProto_NONE): - if scan.CellWeather == int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { - switch scan.Level { - case unboostedLevel: - return pokemon.resetDittoAttributes("0N", unboostedScan, boostedScan, scan) - case unboostedLevel + 5: - // For a confirmed Ditto, we persist IV in inactive only in 0P state - // when disguise is boosted, it has same IV as Ditto - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return unboostedScan, checkScans(boostedScan, scan) - } - return scan, errors.New(fmt.Sprintf("Unexpected 0P Ditto level change, %s/%s -> %s", - unboostedScan, boostedScan, scan)) - } - return scan, checkScans(unboostedScan, scan) - case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): - return scan, checkScans(boostedScan, scan) - } - switch scan.Level { - case unboostedLevel: - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, checkScans(unboostedScan, scan) - case unboostedLevel + 5: - return pokemon.resetDittoAttributes("BN", boostedScan, unboostedScan, scan) - } - return scan, errors.New(fmt.Sprintf("Unexpected B0 Ditto level change, %s/%s -> %s", - unboostedScan, boostedScan, scan)) - } - - isBoosted := scan.Weather != int32(pogo.GameplayWeatherProto_NONE) - var matchingScan *grpc.PokemonScan - if unboostedScan != nil || boostedScan != nil { - if unboostedScan != nil && boostedScan != nil { // if we have both IVs then they must be correct - if unboostedScan.Level == scan.Level { - if isBoosted { - pokemon.setDittoAttributes(">B0", true, unboostedScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, nil - } - return scan, checkScans(unboostedScan, scan) - } else if boostedScan.Level == scan.Level { - if isBoosted { - return scan, checkScans(boostedScan, scan) - } - pokemon.setDittoAttributes(">0P", true, boostedScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return unboostedScan, nil - } - return scan, errors.New(fmt.Sprintf("Unexpected third level found %s, %s vs %s", - unboostedScan, boostedScan, scan)) - } - - levelAdjustment := int32(0) - if isBoosted { - if boostedScan != nil { - matchingScan = boostedScan - } else { - matchingScan = unboostedScan - levelAdjustment = 5 - } - } else { - if unboostedScan != nil { - matchingScan = unboostedScan - } else { - matchingScan = boostedScan - levelAdjustment = -5 - } - } - // There are 10 total possible transitions among these states, i.e. all 12 of them except for 0P <-> PP. - // A Ditto in 00/PP state is undetectable. We try to detect them in the remaining possibilities. - // Now we try to detect all 10 possible conditions where we could identify Ditto with certainty - switch scan.Level - (matchingScan.Level + levelAdjustment) { - case 0: - // the Pokémon has been encountered before, but we find an unexpected level when reencountering it => Ditto - // note that at this point the level should have been already readjusted according to the new weather boost - case 5: - switch scan.Weather { - case int32(pogo.GameplayWeatherProto_NONE): - switch matchingScan.Weather { - case int32(pogo.GameplayWeatherProto_NONE): - pokemon.setDittoAttributes("00/0N>0P", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return unboostedScan, nil - case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): - if err := checkScans(matchingScan, scan); err != nil { - return scan, err - } - pokemon.setDittoAttributes("PN>0P", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - return unboostedScan, nil - } - if err := checkScans(matchingScan, scan); err != nil { - return scan, err - } - if scan.CellWeather != int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { - if scan.MustHaveRerolled(matchingScan) { - pokemon.setDittoAttributes("B0>00/[0N]", false, matchingScan, scan) - } else { - // set Ditto as it is most likely B0>00 if species did not reroll - pokemon.setDittoAttributes("B0>[00]/0N", true, matchingScan, scan) - } - scan.Confirmed = true - } else if matchingScan.Confirmed || scan.MustBeBoosted() { - pokemon.setDittoAttributes("BN>0P", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - return unboostedScan, nil - // scan.MustBeUnboosted() need not be checked since matchingScan would not have been in B0 - } else { - // in case of BN>0P, we set Ditto to be a hidden 0P state, hoping we rediscover later - // setting 0P Ditto would also mean that we have a Ditto with unconfirmed IV which is a bad idea - if _, possible := dittoDisguises.Load(scan.Pokemon); possible { - if _, possible := dittoDisguises.Load(matchingScan.Pokemon); !possible { - // this guess is most likely to be correct except when Ditto pool just rerolled - pokemon.setDittoAttributes("BN>[0P] or B0>0N", true, matchingScan, scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return unboostedScan, nil - } - } - pokemon.setDittoAttributes("BN>0P or B0>[0N]", false, matchingScan, scan) - } - matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) - case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): - // we can never be sure if this is a Ditto or rerolling into non-Ditto - if scan.MustHaveRerolled(matchingScan) { - pokemon.setDittoAttributes("B0>PP/[PN]", false, matchingScan, scan) - } else { - pokemon.setDittoAttributes("B0>[PP]/PN", true, matchingScan, scan) - } - matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) - default: - pokemon.setDittoAttributes("B0>BN", false, matchingScan, scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) - } - return scan, nil - case -5: - switch scan.Weather { - case int32(pogo.GameplayWeatherProto_NONE): - // we can never be sure if this is a Ditto or rerolling into non-Ditto - if scan.MustHaveRerolled(matchingScan) { - pokemon.setDittoAttributes("0P>00/[0N]", false, matchingScan, scan) - } else { - pokemon.setDittoAttributes("0P>[00]/0N", true, matchingScan, scan) - } - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return scan, nil - case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): - pokemon.setDittoAttributes("0P>PN", false, matchingScan, scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - return scan, checkScans(matchingScan, scan) - } - if matchingScan.Weather != int32(pogo.GameplayWeatherProto_NONE) { - pokemon.setDittoAttributes("BN/PP/PN>B0", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, nil - } - if err := checkScans(matchingScan, scan); err != nil { - return scan, err - } - if scan.MustBeBoosted() { - pokemon.setDittoAttributes("0P>BN", false, matchingScan, scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - } else if matchingScan.Confirmed || // this covers scan.MustBeUnboosted() - matchingScan.CellWeather != int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { - pokemon.setDittoAttributes("00/0N>B0", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - scan.Confirmed = true - } else { - // same rationale as BN>0P or B0>[0N] - if _, possible := dittoDisguises.Load(scan.Pokemon); possible { - if _, possible := dittoDisguises.Load(matchingScan.Pokemon); !possible { - // this guess is most likely to be correct except when Ditto pool just rerolled - pokemon.setDittoAttributes("0N>[B0] or 0P>BN", true, matchingScan, scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, nil - } - } - pokemon.setDittoAttributes("0N>B0 or 0P>[BN]", false, matchingScan, scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - } - return scan, nil - case 10: - pokemon.setDittoAttributes("B0>0P", true, matchingScan, scan) - confirmDitto(scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - return matchingScan, nil // unboostedScan is a wrong guess in this case - case -10: - pokemon.setDittoAttributes("0P>B0", true, matchingScan, scan) - confirmDitto(scan) - matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - return scan, nil - default: - return scan, errors.New(fmt.Sprintf("Unexpected level %s -> %s", matchingScan, scan)) - } - } - if isBoosted { - if scan.MustBeUnboosted() { - pokemon.setDittoAttributes("B0", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_NONE) - scan.Confirmed = true - return scan, checkScans(unboostedScan, scan) - } - scan.Confirmed = scan.MustBeBoosted() - return scan, checkScans(boostedScan, scan) - } else if scan.MustBeBoosted() { - pokemon.setDittoAttributes("0P", true, matchingScan, scan) - confirmDitto(scan) - scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - scan.Confirmed = true - return unboostedScan, checkScans(boostedScan, scan) - } - scan.Confirmed = scan.MustBeUnboosted() - return scan, checkScans(unboostedScan, scan) -} - -func (pokemon *Pokemon) clearIv(cp bool) { - if pokemon.AtkIv.Valid || pokemon.DefIv.Valid || pokemon.StaIv.Valid || pokemon.Iv.Valid { - pokemon.dirty = true - } - pokemon.AtkIv = null.NewInt(0, false) - pokemon.DefIv = null.NewInt(0, false) - pokemon.StaIv = null.NewInt(0, false) - pokemon.Iv = null.NewFloat(0, false) - if cp { - switch pokemon.SeenType.ValueOrZero() { - case SeenType_LureEncounter: - pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) - case SeenType_Encounter: - pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) - } - pokemon.SetCp(null.NewInt(0, false)) - pokemon.SetPvp(null.NewString("", false)) - } -} - -// caller should setPokemonDisplay prior to calling this -func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails, proto *pogo.PokemonProto, username string) { - pokemon.SetUsername(null.StringFrom(username)) - pokemon.SetShiny(null.BoolFrom(proto.PokemonDisplay.Shiny)) - pokemon.SetCp(null.IntFrom(int64(proto.Cp))) - pokemon.SetMove1(null.IntFrom(int64(proto.Move1))) - pokemon.SetMove2(null.IntFrom(int64(proto.Move2))) - pokemon.SetHeight(null.FloatFrom(float64(proto.HeightM))) - pokemon.SetSize(null.IntFrom(int64(proto.Size))) - pokemon.SetWeight(null.FloatFrom(float64(proto.WeightKg))) - - scan := grpc.PokemonScan{ - Weather: int32(pokemon.Weather.Int64), - Strong: pokemon.IsStrong.Bool, - Attack: proto.IndividualAttack, - Defense: proto.IndividualDefense, - Stamina: proto.IndividualStamina, - CellWeather: int32(pokemon.Weather.Int64), - Pokemon: int32(proto.PokemonId), - Costume: int32(proto.PokemonDisplay.Costume), - Gender: int32(proto.PokemonDisplay.Gender), - Form: int32(proto.PokemonDisplay.Form), - } - if scan.CellWeather == int32(pogo.GameplayWeatherProto_NONE) { - weather, unlock, err := peekWeatherRecord(weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon)) - if weather == nil || !weather.GameplayCondition.Valid { - log.Warnf("Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) - } else { - scan.CellWeather = int32(weather.GameplayCondition.Int64) - } - if unlock != nil { - unlock() - } - } - if proto.CpMultiplier < 0.734 { - scan.Level = int32((58.215688455154954*proto.CpMultiplier-2.7012478057856497)*proto.CpMultiplier + 1.3220677708486794) - } else if proto.CpMultiplier < .795 { - scan.Level = int32(171.34093607855277*proto.CpMultiplier - 94.95626666368578) - } else { - scan.Level = int32(199.99995231630976*proto.CpMultiplier - 117.55996066890287) - } - - caughtIv, err := pokemon.detectDitto(&scan) - if err != nil { - caughtIv = &scan - log.Errorf("[POKEMON] Unexpected %d: %s", pokemon.Id, err) - } - if caughtIv == nil { // this can only happen for a 0P Ditto - pokemon.SetLevel(null.IntFrom(int64(scan.Level - 5))) - pokemon.clearIv(false) - } else { - pokemon.SetLevel(null.IntFrom(int64(caughtIv.Level))) - pokemon.calculateIv(int64(caughtIv.Attack), int64(caughtIv.Defense), int64(caughtIv.Stamina)) - } - if err == nil { - newScans := make([]*grpc.PokemonScan, len(pokemon.internal.ScanHistory)+1) - entriesCount := 0 - for _, oldEntry := range pokemon.internal.ScanHistory { - if oldEntry.Strong != scan.Strong || !oldEntry.Strong && - oldEntry.Weather == int32(pogo.GameplayWeatherProto_NONE) != - (scan.Weather == int32(pogo.GameplayWeatherProto_NONE)) { - newScans[entriesCount] = oldEntry - entriesCount++ - } - } - newScans[entriesCount] = &scan - pokemon.internal.ScanHistory = newScans[:entriesCount+1] - } else { - // undo possible changes - scan.Confirmed = false - scan.Weather = int32(pokemon.Weather.Int64) - pokemon.internal.ScanHistory = make([]*grpc.PokemonScan, 1) - pokemon.internal.ScanHistory[0] = &scan - } -} - -func (pokemon *Pokemon) updatePokemonFromEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.EncounterOutProto, username string, timestampMs int64) { - pokemon.SetIsEvent(0) - pokemon.addWildPokemon(ctx, db, encounterData.Pokemon, timestampMs, false) - // tappable encounter can also be available in seen as normal encounter once tapped - if pokemon.isSeenFromTappable() { - pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) - } - pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon.Pokemon, username) - - if pokemon.CellId.Valid == false { - centerCoord := s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon) - cellID := s2.CellIDFromLatLng(centerCoord).Parent(15) - pokemon.SetCellId(null.IntFrom(int64(cellID))) - } -} - -func (pokemon *Pokemon) isSeenFromTappable() bool { - return pokemon.SeenType.ValueOrZero() != SeenType_TappableEncounter && pokemon.SeenType.ValueOrZero() != SeenType_TappableLureEncounter -} - -func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.DiskEncounterOutProto, username string) { - pokemon.SetIsEvent(0) - pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) - pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) - pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) -} - -func (pokemon *Pokemon) updatePokemonFromTappableEncounterProto(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounterData *pogo.TappableEncounterProto, username string, timestampMs int64) { - pokemon.SetIsEvent(0) - pokemon.SetLat(request.LocationHintLat) - pokemon.SetLon(request.LocationHintLng) - - if spawnpointId := request.GetLocation().GetSpawnpointId(); spawnpointId != "" { - pokemon.SetSeenType(null.StringFrom(SeenType_TappableEncounter)) - - spawnId, err := strconv.ParseInt(spawnpointId, 16, 64) - if err != nil { - panic(err) - } - - pokemon.SetSpawnId(null.IntFrom(spawnId)) - pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, false) - } else if fortId := request.GetLocation().GetFortId(); fortId != "" { - pokemon.SetSeenType(null.StringFrom(SeenType_TappableLureEncounter)) - - pokemon.SetPokestopId(null.StringFrom(fortId)) - // we don't know any despawn times from lured/fort tappables - pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) - pokemon.SetExpireTimestampVerified(false) - } - if !pokemon.Username.Valid { - pokemon.SetUsername(null.StringFrom(username)) - } - pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) - pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) -} - -func (pokemon *Pokemon) setPokemonDisplay(pokemonId int16, display *pogo.PokemonDisplayProto) { - if !pokemon.isNewRecord() { - // If we would like to support detect A/B spawn in the future, fill in more code here from Chuck - var oldId int16 - if pokemon.IsDitto { - oldId = int16(pokemon.DisplayPokemonId.ValueOrZero()) - } else { - oldId = pokemon.PokemonId - } - if oldId != pokemonId || pokemon.Form != null.IntFrom(int64(display.Form)) || - pokemon.Costume != null.IntFrom(int64(display.Costume)) || - pokemon.Gender != null.IntFrom(int64(display.Gender)) || - pokemon.IsStrong.ValueOrZero() != display.IsStrongPokemon { - log.Debugf("Pokemon %d changed from (%d,%d,%d,%d,%t) to (%d,%d,%d,%d,%t)", pokemon.Id, oldId, - pokemon.Form.ValueOrZero(), pokemon.Costume.ValueOrZero(), pokemon.Gender.ValueOrZero(), - pokemon.IsStrong.ValueOrZero(), - pokemonId, display.Form, display.Costume, display.Gender, display.IsStrongPokemon) - pokemon.SetWeight(null.NewFloat(0, false)) - pokemon.SetHeight(null.NewFloat(0, false)) - pokemon.SetSize(null.NewInt(0, false)) - pokemon.SetMove1(null.NewInt(0, false)) - pokemon.SetMove2(null.NewInt(0, false)) - pokemon.SetCp(null.NewInt(0, false)) - pokemon.SetShiny(null.NewBool(false, false)) - pokemon.SetIsDitto(false) - pokemon.SetDisplayPokemonId(null.NewInt(0, false)) - pokemon.SetPvp(null.NewString("", false)) - } - } - if pokemon.isNewRecord() || !pokemon.IsDitto { - pokemon.SetPokemonId(pokemonId) - } - pokemon.SetGender(null.IntFrom(int64(display.Gender))) - pokemon.SetForm(null.IntFrom(int64(display.Form))) - pokemon.SetCostume(null.IntFrom(int64(display.Costume))) - if !pokemon.isNewRecord() { - pokemon.repopulateIv(int64(display.WeatherBoostedCondition), display.IsStrongPokemon) - } - pokemon.SetWeather(null.IntFrom(int64(display.WeatherBoostedCondition))) - pokemon.SetIsStrong(null.BoolFrom(display.IsStrongPokemon)) -} - -func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { - var isBoosted bool - if !pokemon.IsDitto { - isBoosted = weather != int64(pogo.GameplayWeatherProto_NONE) - if isStrong == pokemon.IsStrong.ValueOrZero() && - pokemon.Weather.ValueOrZero() != int64(pogo.GameplayWeatherProto_NONE) == isBoosted { - return - } - } else if isStrong { - log.Errorf("Strong Ditto??? I can't handle this fml %d", pokemon.Id) - pokemon.clearIv(true) - return - } else { - isBoosted = weather == int64(pogo.GameplayWeatherProto_PARTLY_CLOUDY) - // both Ditto and disguise are boosted and Ditto was not boosted: none -> boosted - // or both Ditto and disguise were boosted and Ditto is not boosted: boosted -> none - if pokemon.Weather.ValueOrZero() == int64(pogo.GameplayWeatherProto_PARTLY_CLOUDY) == isBoosted { - return - } - } - matchingScan, isBoostedMatches := pokemon.locateScan(isStrong, isBoosted) - var oldAtk, oldDef, oldSta int64 - if matchingScan == nil { - pokemon.SetLevel(null.NewInt(0, false)) - pokemon.clearIv(true) - } else { - oldLevel := pokemon.Level.ValueOrZero() - if pokemon.AtkIv.Valid { - oldAtk = pokemon.AtkIv.Int64 - oldDef = pokemon.DefIv.Int64 - oldSta = pokemon.StaIv.Int64 - } else { - oldAtk = -1 - oldDef = -1 - oldSta = -1 - } - newLevel := int64(matchingScan.Level) - if isBoostedMatches || isStrong { // strong Pokemon IV is unaffected by weather - pokemon.calculateIv(int64(matchingScan.Attack), int64(matchingScan.Defense), int64(matchingScan.Stamina)) - switch pokemon.SeenType.ValueOrZero() { - case SeenType_LureWild: - pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) - case SeenType_Wild: - pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) - } - } else { - pokemon.clearIv(true) - } - if !isBoostedMatches { - if isBoosted { - newLevel += 5 - } else { - newLevel -= 5 - } - } - pokemon.SetLevel(null.IntFrom(newLevel)) - if newLevel != oldLevel || pokemon.AtkIv.Valid && - (pokemon.AtkIv.Int64 != oldAtk || pokemon.DefIv.Int64 != oldDef || pokemon.StaIv.Int64 != oldSta) { - pokemon.SetCp(null.NewInt(0, false)) - pokemon.SetPvp(null.NewString("", false)) - } - } -} - -func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition) { - if pokemon.Cp.Valid || ohbem == nil { - return - } - var displayPokemon int - shouldOverrideIv := false - var overrideIv *grpc.PokemonScan - if pokemon.IsDitto { - displayPokemon = int(pokemon.DisplayPokemonId.Int64) - if pokemon.Weather.Int64 == int64(pogo.GameplayWeatherProto_NONE) { - cellId := weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon) - cellWeather, found := weather[cellId] - if !found { - record, unlock, err := getWeatherRecordReadOnly(ctx, db, cellId) - if err != nil || record == nil || !record.GameplayCondition.Valid { - log.Warnf("[POKEMON] Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) - } else { - log.Warnf("[POKEMON] Weather not found locally for %d at %d", pokemon.Id, cellId) - cellWeather = pogo.GameplayWeatherProto_WeatherCondition(record.GameplayCondition.Int64) - found = true - } - if unlock != nil { - unlock() - } - } - if found && cellWeather == pogo.GameplayWeatherProto_PARTLY_CLOUDY { - shouldOverrideIv = true - scan, isBoostedMatches := pokemon.locateScan(false, false) - if scan != nil && isBoostedMatches { - overrideIv = scan - } - } - } - } else { - displayPokemon = int(pokemon.PokemonId) - } - var cp int - var err error - if shouldOverrideIv { - if overrideIv == nil { - return - } - // You should see boosted IV for 0P Ditto - cp, err = ohbem.CalculateCp(displayPokemon, int(pokemon.Form.ValueOrZero()), 0, - int(overrideIv.Attack), int(overrideIv.Defense), int(overrideIv.Stamina), float64(overrideIv.Level)) - } else { - if !pokemon.AtkIv.Valid || !pokemon.Level.Valid { - return - } - cp, err = ohbem.CalculateCp(displayPokemon, int(pokemon.Form.ValueOrZero()), 0, - int(pokemon.AtkIv.Int64), int(pokemon.DefIv.Int64), int(pokemon.StaIv.Int64), - float64(pokemon.Level.Int64)) - } - if err == nil { - pokemon.SetCp(null.IntFrom(int64(cp))) - } else { - log.Warnf("Pokemon %d %d CP unset due to error %s", pokemon.Id, displayPokemon, err) - } -} - -func UpdatePokemonRecordWithEncounterProto(ctx context.Context, db db.DbDetails, encounter *pogo.EncounterOutProto, username string, timestamp int64) string { - if encounter.Pokemon == nil { - return "No encounter" - } - - encounterId := encounter.Pokemon.EncounterId - - // Remove from pending queue - encounter arrived so no need for delayed wild update - if pokemonPendingQueue != nil { - pokemonPendingQueue.Remove(encounterId) - } - - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Errorf("Error pokemon [%d]: %s", encounterId, err) - return fmt.Sprintf("Error finding pokemon %s", err) - } - defer unlock() - - pokemon.updatePokemonFromEncounterProto(ctx, db, encounter, username, timestamp) - savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, timestamp/1000) - // updateEncounterStats() should only be called for encounters, and called - // even if we have the pokemon record already. - updateEncounterStats(pokemon) - - return fmt.Sprintf("%d %d Pokemon %d CP%d", encounter.Pokemon.EncounterId, encounterId, pokemon.PokemonId, encounter.Pokemon.Pokemon.Cp) -} - -func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDetails, encounter *pogo.DiskEncounterOutProto, username string) string { - if encounter.Pokemon == nil { - return "No encounter" - } - - encounterId := uint64(encounter.Pokemon.PokemonDisplay.DisplayId) - - pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) - if err != nil { - log.Errorf("Error pokemon [%d]: %s", encounterId, err) - return fmt.Sprintf("Error finding pokemon %s", err) - } - - if pokemon == nil || pokemon.isNewRecord() { - // No pokemon found - unlock not set when pokemon is nil - if unlock != nil { - unlock() - } - diskEncounterCache.Set(encounterId, encounter, ttlcache.DefaultTTL) - return fmt.Sprintf("%d Disk encounter without previous GMO - Pokemon stored for later", encounterId) - } - defer unlock() - - pokemon.updatePokemonFromDiskEncounterProto(ctx, db, encounter, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) - // updateEncounterStats() should only be called for encounters, and called - // even if we have the pokemon record already. - updateEncounterStats(pokemon) - - return fmt.Sprintf("%d Disk Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp) -} - -func UpdatePokemonRecordWithTappableEncounter(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounter *pogo.TappableEncounterProto, username string, timestampMs int64) string { - encounterId := request.GetEncounterId() - - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Errorf("Error pokemon [%d]: %s", encounterId, err) - return fmt.Sprintf("Error finding pokemon %s", err) - } - defer unlock() - - pokemon.updatePokemonFromTappableEncounterProto(ctx, db, request, encounter, username, timestampMs) - savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) - // updateEncounterStats() should only be called for encounters, and called - // even if we have the pokemon record already. - updateEncounterStats(pokemon) - - return fmt.Sprintf("%d Tappable Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp) -} diff --git a/decoder/pokemonRtree.go b/decoder/pokemonRtree.go index 48aadc01..16dff824 100644 --- a/decoder/pokemonRtree.go +++ b/decoder/pokemonRtree.go @@ -8,11 +8,11 @@ import ( "golbat/config" "github.com/UnownHash/gohbem" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" "github.com/puzpuzpuz/xsync/v3" log "github.com/sirupsen/logrus" "github.com/tidwall/rtree" - "gopkg.in/guregu/null.v4" ) type PokemonLookupCacheItem struct { diff --git a/decoder/pokemon_decode.go b/decoder/pokemon_decode.go new file mode 100644 index 00000000..e29de264 --- /dev/null +++ b/decoder/pokemon_decode.go @@ -0,0 +1,968 @@ +package decoder + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "golbat/db" + "golbat/grpc" + "golbat/pogo" + + "github.com/golang/geo/s2" + "github.com/guregu/null/v6" + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" +) + +func (pokemon *Pokemon) populateInternal() { + if len(pokemon.GolbatInternal) == 0 || len(pokemon.internal.ScanHistory) != 0 { + return + } + err := proto.Unmarshal(pokemon.GolbatInternal, &pokemon.internal) + if err != nil { + log.Warnf("Failed to parse internal data for %d: %s", pokemon.Id, err) + pokemon.internal.Reset() + } +} + +func (pokemon *Pokemon) locateScan(isStrong bool, isBoosted bool) (*grpc.PokemonScan, bool) { + pokemon.populateInternal() + var bestMatching *grpc.PokemonScan + for _, entry := range pokemon.internal.ScanHistory { + if entry.Strong != isStrong { + continue + } + if entry.Weather != int32(pogo.GameplayWeatherProto_NONE) == isBoosted { + return entry, true + } else { + bestMatching = entry + } + } + return bestMatching, false +} + +func (pokemon *Pokemon) locateAllScans() (unboosted, boosted, strong *grpc.PokemonScan) { + pokemon.populateInternal() + for _, entry := range pokemon.internal.ScanHistory { + if entry.Strong { + strong = entry + } else if entry.Weather != int32(pogo.GameplayWeatherProto_NONE) { + boosted = entry + } else { + unboosted = entry + } + } + return +} + +func (pokemon *Pokemon) isNewRecord() bool { + return pokemon.newRecord +} + +func (pokemon *Pokemon) remainingDuration(now int64) time.Duration { + remaining := ttlcache.DefaultTTL + if pokemon.ExpireTimestampVerified { + timeLeft := 60 + pokemon.ExpireTimestamp.ValueOrZero() - now + if timeLeft > 1 { + remaining = time.Duration(timeLeft) * time.Second + } + } + return remaining +} + +func (pokemon *Pokemon) addWildPokemon(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, timestampMs int64, trustworthyTimestamp bool) { + if wildPokemon.EncounterId != pokemon.Id { + panic("Unmatched EncounterId") + } + pokemon.SetLat(wildPokemon.Latitude) + pokemon.SetLon(wildPokemon.Longitude) + + spawnId, err := strconv.ParseInt(wildPokemon.SpawnPointId, 16, 64) + if err != nil { + panic(err) + } + pokemon.SetSpawnId(null.IntFrom(spawnId)) + + pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, trustworthyTimestamp) + pokemon.setPokemonDisplay(int16(wildPokemon.Pokemon.PokemonId), wildPokemon.Pokemon.PokemonDisplay) +} + +// wildSignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and +// should be written. +func (pokemon *Pokemon) wildSignificantUpdate(wildPokemon *pogo.WildPokemonProto, time int64) bool { + pokemonDisplay := wildPokemon.Pokemon.PokemonDisplay + // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired + + return pokemon.SeenType.ValueOrZero() == SeenType_Cell || + pokemon.SeenType.ValueOrZero() == SeenType_NearbyStop || + pokemon.PokemonId != int16(wildPokemon.Pokemon.PokemonId) || + pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || + pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || + pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || + pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) || + (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) +} + +// nearbySignificantUpdate returns true if the wild pokemon is significantly different from the current pokemon and +// should be written. +func (pokemon *Pokemon) nearbySignificantUpdate(wildPokemon *pogo.NearbyPokemonProto, time int64) bool { + pokemonDisplay := wildPokemon.PokemonDisplay + // We would accept a wild update if the pokemon has changed; or to extend an unknown spawn time that is expired + + pokemonChanged := pokemon.PokemonId != int16(pokemonDisplay.DisplayId) || + pokemon.Form.ValueOrZero() != int64(pokemonDisplay.Form) || + pokemon.Weather.ValueOrZero() != int64(pokemonDisplay.WeatherBoostedCondition) || + pokemon.Costume.ValueOrZero() != int64(pokemonDisplay.Costume) || + pokemon.Gender.ValueOrZero() != int64(pokemonDisplay.Gender) + + if pokemonChanged { + return true + } + + hasExpired := (!pokemon.ExpireTimestampVerified && pokemon.ExpireTimestamp.ValueOrZero() < time) + + if hasExpired { + return true + } + + if pokemon.SeenType.ValueOrZero() == SeenType_Cell { + return true + } + + // if it's at a nearby stop, or encounter and no other details have changed update is not worthwhile + return false +} + +func (pokemon *Pokemon) updateFromWild(ctx context.Context, db db.DbDetails, wildPokemon *pogo.WildPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { + pokemon.SetIsEvent(0) + switch pokemon.SeenType.ValueOrZero() { + case "", SeenType_Cell, SeenType_NearbyStop: + pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) + } + pokemon.addWildPokemon(ctx, db, wildPokemon, timestampMs, true) + pokemon.recomputeCpIfNeeded(ctx, db, weather) + pokemon.SetUsername(null.StringFrom(username)) + pokemon.SetCellId(null.IntFrom(cellId)) +} + +func (pokemon *Pokemon) updateFromMap(ctx context.Context, db db.DbDetails, mapPokemon *pogo.MapPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { + + if !pokemon.isNewRecord() { + // Do not ever overwrite lure details based on seeing it again in the GMO + return + } + + pokemon.SetIsEvent(0) + + pokemon.Id = mapPokemon.EncounterId + + spawnpointId := mapPokemon.SpawnpointId + + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, spawnpointId) + if pokestop == nil { + // Unrecognised pokestop + return + } + pokemon.SetPokestopId(null.StringFrom(pokestop.Id)) + pokemon.SetLat(pokestop.Lat) + pokemon.SetLon(pokestop.Lon) + pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) + unlock() + + if mapPokemon.PokemonDisplay != nil { + pokemon.setPokemonDisplay(int16(mapPokemon.PokedexTypeId), mapPokemon.PokemonDisplay) + pokemon.recomputeCpIfNeeded(ctx, db, weather) + // The mapPokemon and nearbyPokemon GMOs don't contain actual shininess. + // shiny = mapPokemon.pokemonDisplay.shiny + } else { + log.Warnf("[POKEMON] MapPokemonProto missing PokemonDisplay for %d", pokemon.Id) + } + if !pokemon.Username.Valid { + pokemon.SetUsername(null.StringFrom(username)) + } + + if mapPokemon.ExpirationTimeMs > 0 && !pokemon.ExpireTimestampVerified { + pokemon.SetExpireTimestamp(null.IntFrom(mapPokemon.ExpirationTimeMs / 1000)) + pokemon.SetExpireTimestampVerified(true) + // if we have cached an encounter for this pokemon, update the TTL. + encounterCache.UpdateTTL(pokemon.Id, pokemon.remainingDuration(timestampMs/1000)) + } else { + pokemon.SetExpireTimestampVerified(false) + } + + pokemon.SetCellId(null.IntFrom(cellId)) +} + +func (pokemon *Pokemon) calculateIv(a int64, d int64, s int64) { + if pokemon.AtkIv.ValueOrZero() != a || pokemon.DefIv.ValueOrZero() != d || pokemon.StaIv.ValueOrZero() != s || + !pokemon.AtkIv.Valid || !pokemon.DefIv.Valid || !pokemon.StaIv.Valid { + pokemon.AtkIv = null.IntFrom(a) + pokemon.DefIv = null.IntFrom(d) + pokemon.StaIv = null.IntFrom(s) + pokemon.Iv = null.FloatFrom(float64(a+d+s) / .45) + pokemon.dirty = true + } +} + +func (pokemon *Pokemon) updateFromNearby(ctx context.Context, db db.DbDetails, nearbyPokemon *pogo.NearbyPokemonProto, cellId int64, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition, timestampMs int64, username string) { + pokemon.SetIsEvent(0) + pokestopId := nearbyPokemon.FortId + pokemon.setPokemonDisplay(int16(nearbyPokemon.PokedexNumber), nearbyPokemon.PokemonDisplay) + pokemon.recomputeCpIfNeeded(ctx, db, weather) + pokemon.SetUsername(null.StringFrom(username)) + + var lat, lon float64 + overrideLatLon := pokemon.isNewRecord() + useCellLatLon := true + if pokestopId != "" { + switch pokemon.SeenType.ValueOrZero() { + case "", SeenType_Cell: + overrideLatLon = true // a better estimate is available + case SeenType_NearbyStop: + default: + return + } + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokestopId) + if pokestop == nil { + // Unrecognised pokestop, rollback changes + overrideLatLon = pokemon.isNewRecord() + } else { + pokemon.SetSeenType(null.StringFrom(SeenType_NearbyStop)) + pokemon.SetPokestopId(null.StringFrom(pokestopId)) + lat, lon = pokestop.Lat, pokestop.Lon + useCellLatLon = false + unlock() + } + } + if useCellLatLon { + // Cell Pokemon + if !overrideLatLon && pokemon.SeenType.ValueOrZero() != SeenType_Cell { + // do not downgrade to nearby cell + return + } + + s2cell := s2.CellFromCellID(s2.CellID(cellId)) + lat = s2cell.CapBound().RectBound().Center().Lat.Degrees() + lon = s2cell.CapBound().RectBound().Center().Lng.Degrees() + + pokemon.SetSeenType(null.StringFrom(SeenType_Cell)) + } + if overrideLatLon { + pokemon.SetLat(lat) + pokemon.SetLon(lon) + } else { + midpoint := s2.LatLngFromPoint(s2.Point{s2.PointFromLatLng(s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon)). + Add(s2.PointFromLatLng(s2.LatLngFromDegrees(lat, lon)).Vector)}) + pokemon.SetLat(midpoint.Lat.Degrees()) + pokemon.SetLon(midpoint.Lng.Degrees()) + } + pokemon.SetCellId(null.IntFrom(cellId)) + pokemon.setUnknownTimestamp(timestampMs / 1000) +} + +const SeenType_Cell string = "nearby_cell" // Pokemon was seen in a cell (without accurate location) +const SeenType_NearbyStop string = "nearby_stop" // Pokemon was seen at a nearby Pokestop, location set to lon, lat of pokestop +const SeenType_Wild string = "wild" // Pokemon was seen in the wild, accurate location but with no IV details +const SeenType_Encounter string = "encounter" // Pokemon has been encountered giving exact details of current IV +const SeenType_LureWild string = "lure_wild" // Pokemon was seen at a lure +const SeenType_LureEncounter string = "lure_encounter" // Pokemon has been encountered at a lure +const SeenType_TappableEncounter string = "tappable_encounter" // Pokemon has been encountered from tappable +const SeenType_TappableLureEncounter string = "tappable_lure_encounter" // Pokemon has been encountered from a lured tappable + +// setExpireTimestampFromSpawnpoint sets the current Pokemon object ExpireTimeStamp, and ExpireTimeStampVerified from the Spawnpoint +// information held. +// db - the database connection to be used +// timestampMs - the timestamp to be used for calculations +// trustworthyTimestamp - whether this timestamp is fully trustworthy (ie comes from GMO server time) +func (pokemon *Pokemon) setExpireTimestampFromSpawnpoint(ctx context.Context, db db.DbDetails, timestampMs int64, trustworthyTimestamp bool) { + if !trustworthyTimestamp && pokemon.ExpireTimestampVerified { + // If our time is not trustworthy, and we have already set a time from some other source (eg a GMO) + // don't modify it + + return + } + + spawnId := pokemon.SpawnId.ValueOrZero() + if spawnId == 0 { + return + } + + pokemon.ExpireTimestampVerified = false + spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) + if spawnPoint != nil && spawnPoint.DespawnSec.Valid { + despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) + unlock() + + date := time.Unix(timestampMs/1000, 0) + secondOfHour := date.Second() + date.Minute()*60 + + despawnOffset := despawnSecond - secondOfHour + if despawnOffset < 0 { + despawnOffset += 3600 + } + pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) + pokemon.SetExpireTimestampVerified(true) + } else { + if unlock != nil { + unlock() + } + pokemon.setUnknownTimestamp(timestampMs / 1000) + } +} + +func (pokemon *Pokemon) setUnknownTimestamp(now int64) { + if !pokemon.ExpireTimestamp.Valid { + pokemon.SetExpireTimestamp(null.IntFrom(now + 20*60)) // should be configurable, add on 20min + } else { + if pokemon.ExpireTimestamp.Int64 < now { + pokemon.SetExpireTimestamp(null.IntFrom(now + 10*60)) // should be configurable, add on 10min + } + } +} + +func checkScans(old *grpc.PokemonScan, new *grpc.PokemonScan) error { + if old == nil || old.CompressedIv() == new.CompressedIv() { + return nil + } + return errors.New(fmt.Sprintf("Unexpected IV mismatch %s != %s", old, new)) +} + +func (pokemon *Pokemon) setDittoAttributes(mode string, isDitto bool, old, new *grpc.PokemonScan) { + if isDitto { + log.Debugf("[POKEMON] %d: %s Ditto found %s -> %s", pokemon.Id, mode, old, new) + pokemon.SetIsDitto(true) + pokemon.SetDisplayPokemonId(null.IntFrom(int64(pokemon.PokemonId))) + pokemon.SetPokemonId(int16(pogo.HoloPokemonId_DITTO)) + } else { + log.Debugf("[POKEMON] %d: %s not Ditto found %s -> %s", pokemon.Id, mode, old, new) + } +} +func (pokemon *Pokemon) resetDittoAttributes(mode string, old, aux, new *grpc.PokemonScan) (*grpc.PokemonScan, error) { + log.Debugf("[POKEMON] %d: %s Ditto was reset %s (%s) -> %s", pokemon.Id, mode, old, aux, new) + pokemon.SetIsDitto(false) + pokemon.SetDisplayPokemonId(null.NewInt(0, false)) + pokemon.SetPokemonId(int16(pokemon.DisplayPokemonId.Int64)) + return new, checkScans(old, new) +} + +// As far as I'm concerned, wild Ditto only depends on species but not costume/gender/form +var dittoDisguises sync.Map + +func confirmDitto(scan *grpc.PokemonScan) { + now := time.Now() + lastSeen, exists := dittoDisguises.Swap(scan.Pokemon, now) + if exists { + log.Debugf("[DITTO] Disguise %s reseen after %s", scan, now.Sub(lastSeen.(time.Time))) + } else { + var sb strings.Builder + sb.WriteString("[DITTO] New disguise ") + sb.WriteString(scan.String()) + sb.WriteString(" found. Current disguises ") + dittoDisguises.Range(func(disguise, lastSeen interface{}) bool { + sb.WriteString(strconv.FormatInt(int64(disguise.(int32)), 10)) + sb.WriteString(" (") + sb.WriteString(now.Sub(lastSeen.(time.Time)).String()) + sb.WriteString(") ") + return true + }) + log.Info(sb.String()) + } +} + +// detectDitto returns the IV/level set that should be used for persisting to db/seen if caught. +// error is set if something unexpected happened and the scan history should be cleared. +func (pokemon *Pokemon) detectDitto(scan *grpc.PokemonScan) (*grpc.PokemonScan, error) { + unboostedScan, boostedScan, strongScan := pokemon.locateAllScans() + if scan.Strong { + if strongScan != nil { + expectedLevel := strongScan.Level + isBoosted := scan.Weather != int32(pogo.GameplayWeatherProto_NONE) + if strongScan.Weather != int32(pogo.GameplayWeatherProto_NONE) != isBoosted { + if isBoosted { + expectedLevel += 5 + } else { + expectedLevel -= 5 + } + } + if scan.Level != expectedLevel || scan.CompressedIv() != strongScan.CompressedIv() { + return scan, errors.New(fmt.Sprintf("Unexpected strong Pokemon (Ditto?), %s -> %s", + strongScan, scan)) + } + } + return scan, nil + } + + // Here comes the Ditto logic. Embrace yourself :) + // Ditto weather can be split into 4 categories: + // - 00: No weather boost + // - 0P: No weather boost but Ditto is actually boosted by partly cloudy causing seen IV to be boosted [atypical] + // - B0: Weather boosts disguise but not Ditto causing seen IV to be unboosted [atypical] + // - PP: Weather being partly cloudy boosts both disguise and Ditto + // + // We will also use 0N/BN/PN to denote a normal non-Ditto spawn with corresponding weather boosts. + // Disguise IV depends on Ditto weather boost instead, and caught Ditto is boosted only in PP state. + if pokemon.IsDitto { + var unboostedLevel int32 + if boostedScan != nil { + unboostedLevel = boostedScan.Level - 5 + } else if unboostedScan != nil { + unboostedLevel = unboostedScan.Level + } else { + pokemon.resetDittoAttributes("?", nil, nil, scan) + return scan, errors.New("Missing past scans. Ditto will be reset") + } + // If IsDitto = true, then the IV sets in history are ALWAYS confirmed + scan.Confirmed = true + switch scan.Weather { + case int32(pogo.GameplayWeatherProto_NONE): + if scan.CellWeather == int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { + switch scan.Level { + case unboostedLevel: + return pokemon.resetDittoAttributes("0N", unboostedScan, boostedScan, scan) + case unboostedLevel + 5: + // For a confirmed Ditto, we persist IV in inactive only in 0P state + // when disguise is boosted, it has same IV as Ditto + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return unboostedScan, checkScans(boostedScan, scan) + } + return scan, errors.New(fmt.Sprintf("Unexpected 0P Ditto level change, %s/%s -> %s", + unboostedScan, boostedScan, scan)) + } + return scan, checkScans(unboostedScan, scan) + case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): + return scan, checkScans(boostedScan, scan) + } + switch scan.Level { + case unboostedLevel: + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, checkScans(unboostedScan, scan) + case unboostedLevel + 5: + return pokemon.resetDittoAttributes("BN", boostedScan, unboostedScan, scan) + } + return scan, errors.New(fmt.Sprintf("Unexpected B0 Ditto level change, %s/%s -> %s", + unboostedScan, boostedScan, scan)) + } + + isBoosted := scan.Weather != int32(pogo.GameplayWeatherProto_NONE) + var matchingScan *grpc.PokemonScan + if unboostedScan != nil || boostedScan != nil { + if unboostedScan != nil && boostedScan != nil { // if we have both IVs then they must be correct + if unboostedScan.Level == scan.Level { + if isBoosted { + pokemon.setDittoAttributes(">B0", true, unboostedScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, nil + } + return scan, checkScans(unboostedScan, scan) + } else if boostedScan.Level == scan.Level { + if isBoosted { + return scan, checkScans(boostedScan, scan) + } + pokemon.setDittoAttributes(">0P", true, boostedScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return unboostedScan, nil + } + return scan, errors.New(fmt.Sprintf("Unexpected third level found %s, %s vs %s", + unboostedScan, boostedScan, scan)) + } + + levelAdjustment := int32(0) + if isBoosted { + if boostedScan != nil { + matchingScan = boostedScan + } else { + matchingScan = unboostedScan + levelAdjustment = 5 + } + } else { + if unboostedScan != nil { + matchingScan = unboostedScan + } else { + matchingScan = boostedScan + levelAdjustment = -5 + } + } + // There are 10 total possible transitions among these states, i.e. all 12 of them except for 0P <-> PP. + // A Ditto in 00/PP state is undetectable. We try to detect them in the remaining possibilities. + // Now we try to detect all 10 possible conditions where we could identify Ditto with certainty + switch scan.Level - (matchingScan.Level + levelAdjustment) { + case 0: + // the Pokémon has been encountered before, but we find an unexpected level when reencountering it => Ditto + // note that at this point the level should have been already readjusted according to the new weather boost + case 5: + switch scan.Weather { + case int32(pogo.GameplayWeatherProto_NONE): + switch matchingScan.Weather { + case int32(pogo.GameplayWeatherProto_NONE): + pokemon.setDittoAttributes("00/0N>0P", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return unboostedScan, nil + case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): + if err := checkScans(matchingScan, scan); err != nil { + return scan, err + } + pokemon.setDittoAttributes("PN>0P", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + return unboostedScan, nil + } + if err := checkScans(matchingScan, scan); err != nil { + return scan, err + } + if scan.CellWeather != int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { + if scan.MustHaveRerolled(matchingScan) { + pokemon.setDittoAttributes("B0>00/[0N]", false, matchingScan, scan) + } else { + // set Ditto as it is most likely B0>00 if species did not reroll + pokemon.setDittoAttributes("B0>[00]/0N", true, matchingScan, scan) + } + scan.Confirmed = true + } else if matchingScan.Confirmed || scan.MustBeBoosted() { + pokemon.setDittoAttributes("BN>0P", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + return unboostedScan, nil + // scan.MustBeUnboosted() need not be checked since matchingScan would not have been in B0 + } else { + // in case of BN>0P, we set Ditto to be a hidden 0P state, hoping we rediscover later + // setting 0P Ditto would also mean that we have a Ditto with unconfirmed IV which is a bad idea + if _, possible := dittoDisguises.Load(scan.Pokemon); possible { + if _, possible := dittoDisguises.Load(matchingScan.Pokemon); !possible { + // this guess is most likely to be correct except when Ditto pool just rerolled + pokemon.setDittoAttributes("BN>[0P] or B0>0N", true, matchingScan, scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return unboostedScan, nil + } + } + pokemon.setDittoAttributes("BN>0P or B0>[0N]", false, matchingScan, scan) + } + matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) + case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): + // we can never be sure if this is a Ditto or rerolling into non-Ditto + if scan.MustHaveRerolled(matchingScan) { + pokemon.setDittoAttributes("B0>PP/[PN]", false, matchingScan, scan) + } else { + pokemon.setDittoAttributes("B0>[PP]/PN", true, matchingScan, scan) + } + matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) + default: + pokemon.setDittoAttributes("B0>BN", false, matchingScan, scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) + } + return scan, nil + case -5: + switch scan.Weather { + case int32(pogo.GameplayWeatherProto_NONE): + // we can never be sure if this is a Ditto or rerolling into non-Ditto + if scan.MustHaveRerolled(matchingScan) { + pokemon.setDittoAttributes("0P>00/[0N]", false, matchingScan, scan) + } else { + pokemon.setDittoAttributes("0P>[00]/0N", true, matchingScan, scan) + } + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return scan, nil + case int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY): + pokemon.setDittoAttributes("0P>PN", false, matchingScan, scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + return scan, checkScans(matchingScan, scan) + } + if matchingScan.Weather != int32(pogo.GameplayWeatherProto_NONE) { + pokemon.setDittoAttributes("BN/PP/PN>B0", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, nil + } + if err := checkScans(matchingScan, scan); err != nil { + return scan, err + } + if scan.MustBeBoosted() { + pokemon.setDittoAttributes("0P>BN", false, matchingScan, scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + } else if matchingScan.Confirmed || // this covers scan.MustBeUnboosted() + matchingScan.CellWeather != int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) { + pokemon.setDittoAttributes("00/0N>B0", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + scan.Confirmed = true + } else { + // same rationale as BN>0P or B0>[0N] + if _, possible := dittoDisguises.Load(scan.Pokemon); possible { + if _, possible := dittoDisguises.Load(matchingScan.Pokemon); !possible { + // this guess is most likely to be correct except when Ditto pool just rerolled + pokemon.setDittoAttributes("0N>[B0] or 0P>BN", true, matchingScan, scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, nil + } + } + pokemon.setDittoAttributes("0N>B0 or 0P>[BN]", false, matchingScan, scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + } + return scan, nil + case 10: + pokemon.setDittoAttributes("B0>0P", true, matchingScan, scan) + confirmDitto(scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_NONE) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + return matchingScan, nil // unboostedScan is a wrong guess in this case + case -10: + pokemon.setDittoAttributes("0P>B0", true, matchingScan, scan) + confirmDitto(scan) + matchingScan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + return scan, nil + default: + return scan, errors.New(fmt.Sprintf("Unexpected level %s -> %s", matchingScan, scan)) + } + } + if isBoosted { + if scan.MustBeUnboosted() { + pokemon.setDittoAttributes("B0", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_NONE) + scan.Confirmed = true + return scan, checkScans(unboostedScan, scan) + } + scan.Confirmed = scan.MustBeBoosted() + return scan, checkScans(boostedScan, scan) + } else if scan.MustBeBoosted() { + pokemon.setDittoAttributes("0P", true, matchingScan, scan) + confirmDitto(scan) + scan.Weather = int32(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + scan.Confirmed = true + return unboostedScan, checkScans(boostedScan, scan) + } + scan.Confirmed = scan.MustBeUnboosted() + return scan, checkScans(unboostedScan, scan) +} + +func (pokemon *Pokemon) clearIv(cp bool) { + if pokemon.AtkIv.Valid || pokemon.DefIv.Valid || pokemon.StaIv.Valid || pokemon.Iv.Valid { + pokemon.dirty = true + } + pokemon.AtkIv = null.NewInt(0, false) + pokemon.DefIv = null.NewInt(0, false) + pokemon.StaIv = null.NewInt(0, false) + pokemon.Iv = null.NewFloat(0, false) + if cp { + switch pokemon.SeenType.ValueOrZero() { + case SeenType_LureEncounter: + pokemon.SetSeenType(null.StringFrom(SeenType_LureWild)) + case SeenType_Encounter: + pokemon.SetSeenType(null.StringFrom(SeenType_Wild)) + } + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) + } +} + +// caller should setPokemonDisplay prior to calling this +func (pokemon *Pokemon) addEncounterPokemon(ctx context.Context, db db.DbDetails, proto *pogo.PokemonProto, username string) { + pokemon.SetUsername(null.StringFrom(username)) + pokemon.SetShiny(null.BoolFrom(proto.PokemonDisplay.Shiny)) + pokemon.SetCp(null.IntFrom(int64(proto.Cp))) + pokemon.SetMove1(null.IntFrom(int64(proto.Move1))) + pokemon.SetMove2(null.IntFrom(int64(proto.Move2))) + pokemon.SetHeight(null.FloatFrom(float64(proto.HeightM))) + pokemon.SetSize(null.IntFrom(int64(proto.Size))) + pokemon.SetWeight(null.FloatFrom(float64(proto.WeightKg))) + + scan := grpc.PokemonScan{ + Weather: int32(pokemon.Weather.Int64), + Strong: pokemon.IsStrong.Bool, + Attack: proto.IndividualAttack, + Defense: proto.IndividualDefense, + Stamina: proto.IndividualStamina, + CellWeather: int32(pokemon.Weather.Int64), + Pokemon: int32(proto.PokemonId), + Costume: int32(proto.PokemonDisplay.Costume), + Gender: int32(proto.PokemonDisplay.Gender), + Form: int32(proto.PokemonDisplay.Form), + } + if scan.CellWeather == int32(pogo.GameplayWeatherProto_NONE) { + weather, unlock, err := peekWeatherRecord(weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon)) + if weather == nil || !weather.GameplayCondition.Valid { + log.Warnf("Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) + } else { + scan.CellWeather = int32(weather.GameplayCondition.Int64) + } + if unlock != nil { + unlock() + } + } + if proto.CpMultiplier < 0.734 { + scan.Level = int32((58.215688455154954*proto.CpMultiplier-2.7012478057856497)*proto.CpMultiplier + 1.3220677708486794) + } else if proto.CpMultiplier < .795 { + scan.Level = int32(171.34093607855277*proto.CpMultiplier - 94.95626666368578) + } else { + scan.Level = int32(199.99995231630976*proto.CpMultiplier - 117.55996066890287) + } + + caughtIv, err := pokemon.detectDitto(&scan) + if err != nil { + caughtIv = &scan + log.Errorf("[POKEMON] Unexpected %d: %s", pokemon.Id, err) + } + if caughtIv == nil { // this can only happen for a 0P Ditto + pokemon.SetLevel(null.IntFrom(int64(scan.Level - 5))) + pokemon.clearIv(false) + } else { + pokemon.SetLevel(null.IntFrom(int64(caughtIv.Level))) + pokemon.calculateIv(int64(caughtIv.Attack), int64(caughtIv.Defense), int64(caughtIv.Stamina)) + } + if err == nil { + newScans := make([]*grpc.PokemonScan, len(pokemon.internal.ScanHistory)+1) + entriesCount := 0 + for _, oldEntry := range pokemon.internal.ScanHistory { + if oldEntry.Strong != scan.Strong || !oldEntry.Strong && + oldEntry.Weather == int32(pogo.GameplayWeatherProto_NONE) != + (scan.Weather == int32(pogo.GameplayWeatherProto_NONE)) { + newScans[entriesCount] = oldEntry + entriesCount++ + } + } + newScans[entriesCount] = &scan + pokemon.internal.ScanHistory = newScans[:entriesCount+1] + } else { + // undo possible changes + scan.Confirmed = false + scan.Weather = int32(pokemon.Weather.Int64) + pokemon.internal.ScanHistory = make([]*grpc.PokemonScan, 1) + pokemon.internal.ScanHistory[0] = &scan + } +} + +func (pokemon *Pokemon) updatePokemonFromEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.EncounterOutProto, username string, timestampMs int64) { + pokemon.SetIsEvent(0) + pokemon.addWildPokemon(ctx, db, encounterData.Pokemon, timestampMs, false) + // tappable encounter can also be available in seen as normal encounter once tapped + if pokemon.isSeenFromTappable() { + pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) + } + pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon.Pokemon, username) + + if pokemon.CellId.Valid == false { + centerCoord := s2.LatLngFromDegrees(pokemon.Lat, pokemon.Lon) + cellID := s2.CellIDFromLatLng(centerCoord).Parent(15) + pokemon.SetCellId(null.IntFrom(int64(cellID))) + } +} + +func (pokemon *Pokemon) isSeenFromTappable() bool { + return pokemon.SeenType.ValueOrZero() != SeenType_TappableEncounter && pokemon.SeenType.ValueOrZero() != SeenType_TappableLureEncounter +} + +func (pokemon *Pokemon) updatePokemonFromDiskEncounterProto(ctx context.Context, db db.DbDetails, encounterData *pogo.DiskEncounterOutProto, username string) { + pokemon.SetIsEvent(0) + pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) + pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) + pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) +} + +func (pokemon *Pokemon) updatePokemonFromTappableEncounterProto(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounterData *pogo.TappableEncounterProto, username string, timestampMs int64) { + pokemon.SetIsEvent(0) + pokemon.SetLat(request.LocationHintLat) + pokemon.SetLon(request.LocationHintLng) + + if spawnpointId := request.GetLocation().GetSpawnpointId(); spawnpointId != "" { + pokemon.SetSeenType(null.StringFrom(SeenType_TappableEncounter)) + + spawnId, err := strconv.ParseInt(spawnpointId, 16, 64) + if err != nil { + panic(err) + } + + pokemon.SetSpawnId(null.IntFrom(spawnId)) + pokemon.setExpireTimestampFromSpawnpoint(ctx, db, timestampMs, false) + } else if fortId := request.GetLocation().GetFortId(); fortId != "" { + pokemon.SetSeenType(null.StringFrom(SeenType_TappableLureEncounter)) + + pokemon.SetPokestopId(null.StringFrom(fortId)) + // we don't know any despawn times from lured/fort tappables + pokemon.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) + pokemon.SetExpireTimestampVerified(false) + } + if !pokemon.Username.Valid { + pokemon.SetUsername(null.StringFrom(username)) + } + pokemon.setPokemonDisplay(int16(encounterData.Pokemon.PokemonId), encounterData.Pokemon.PokemonDisplay) + pokemon.addEncounterPokemon(ctx, db, encounterData.Pokemon, username) +} + +func (pokemon *Pokemon) setPokemonDisplay(pokemonId int16, display *pogo.PokemonDisplayProto) { + if !pokemon.isNewRecord() { + // If we would like to support detect A/B spawn in the future, fill in more code here from Chuck + var oldId int16 + if pokemon.IsDitto { + oldId = int16(pokemon.DisplayPokemonId.ValueOrZero()) + } else { + oldId = pokemon.PokemonId + } + if oldId != pokemonId || pokemon.Form != null.IntFrom(int64(display.Form)) || + pokemon.Costume != null.IntFrom(int64(display.Costume)) || + pokemon.Gender != null.IntFrom(int64(display.Gender)) || + pokemon.IsStrong.ValueOrZero() != display.IsStrongPokemon { + log.Debugf("Pokemon %d changed from (%d,%d,%d,%d,%t) to (%d,%d,%d,%d,%t)", pokemon.Id, oldId, + pokemon.Form.ValueOrZero(), pokemon.Costume.ValueOrZero(), pokemon.Gender.ValueOrZero(), + pokemon.IsStrong.ValueOrZero(), + pokemonId, display.Form, display.Costume, display.Gender, display.IsStrongPokemon) + pokemon.SetWeight(null.NewFloat(0, false)) + pokemon.SetHeight(null.NewFloat(0, false)) + pokemon.SetSize(null.NewInt(0, false)) + pokemon.SetMove1(null.NewInt(0, false)) + pokemon.SetMove2(null.NewInt(0, false)) + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetShiny(null.NewBool(false, false)) + pokemon.SetIsDitto(false) + pokemon.SetDisplayPokemonId(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) + } + } + if pokemon.isNewRecord() || !pokemon.IsDitto { + pokemon.SetPokemonId(pokemonId) + } + pokemon.SetGender(null.IntFrom(int64(display.Gender))) + pokemon.SetForm(null.IntFrom(int64(display.Form))) + pokemon.SetCostume(null.IntFrom(int64(display.Costume))) + if !pokemon.isNewRecord() { + pokemon.repopulateIv(int64(display.WeatherBoostedCondition), display.IsStrongPokemon) + } + pokemon.SetWeather(null.IntFrom(int64(display.WeatherBoostedCondition))) + pokemon.SetIsStrong(null.BoolFrom(display.IsStrongPokemon)) +} + +func (pokemon *Pokemon) repopulateIv(weather int64, isStrong bool) { + var isBoosted bool + if !pokemon.IsDitto { + isBoosted = weather != int64(pogo.GameplayWeatherProto_NONE) + if isStrong == pokemon.IsStrong.ValueOrZero() && + pokemon.Weather.ValueOrZero() != int64(pogo.GameplayWeatherProto_NONE) == isBoosted { + return + } + } else if isStrong { + log.Errorf("Strong Ditto??? I can't handle this fml %d", pokemon.Id) + pokemon.clearIv(true) + return + } else { + isBoosted = weather == int64(pogo.GameplayWeatherProto_PARTLY_CLOUDY) + // both Ditto and disguise are boosted and Ditto was not boosted: none -> boosted + // or both Ditto and disguise were boosted and Ditto is not boosted: boosted -> none + if pokemon.Weather.ValueOrZero() == int64(pogo.GameplayWeatherProto_PARTLY_CLOUDY) == isBoosted { + return + } + } + matchingScan, isBoostedMatches := pokemon.locateScan(isStrong, isBoosted) + var oldAtk, oldDef, oldSta int64 + if matchingScan == nil { + pokemon.SetLevel(null.NewInt(0, false)) + pokemon.clearIv(true) + } else { + oldLevel := pokemon.Level.ValueOrZero() + if pokemon.AtkIv.Valid { + oldAtk = pokemon.AtkIv.Int64 + oldDef = pokemon.DefIv.Int64 + oldSta = pokemon.StaIv.Int64 + } else { + oldAtk = -1 + oldDef = -1 + oldSta = -1 + } + newLevel := int64(matchingScan.Level) + if isBoostedMatches || isStrong { // strong Pokemon IV is unaffected by weather + pokemon.calculateIv(int64(matchingScan.Attack), int64(matchingScan.Defense), int64(matchingScan.Stamina)) + switch pokemon.SeenType.ValueOrZero() { + case SeenType_LureWild: + pokemon.SetSeenType(null.StringFrom(SeenType_LureEncounter)) + case SeenType_Wild: + pokemon.SetSeenType(null.StringFrom(SeenType_Encounter)) + } + } else { + pokemon.clearIv(true) + } + if !isBoostedMatches { + if isBoosted { + newLevel += 5 + } else { + newLevel -= 5 + } + } + pokemon.SetLevel(null.IntFrom(newLevel)) + if newLevel != oldLevel || pokemon.AtkIv.Valid && + (pokemon.AtkIv.Int64 != oldAtk || pokemon.DefIv.Int64 != oldDef || pokemon.StaIv.Int64 != oldSta) { + pokemon.SetCp(null.NewInt(0, false)) + pokemon.SetPvp(null.NewString("", false)) + } + } +} + +func (pokemon *Pokemon) recomputeCpIfNeeded(ctx context.Context, db db.DbDetails, weather map[int64]pogo.GameplayWeatherProto_WeatherCondition) { + if pokemon.Cp.Valid || ohbem == nil { + return + } + var displayPokemon int + shouldOverrideIv := false + var overrideIv *grpc.PokemonScan + if pokemon.IsDitto { + displayPokemon = int(pokemon.DisplayPokemonId.Int64) + if pokemon.Weather.Int64 == int64(pogo.GameplayWeatherProto_NONE) { + cellId := weatherCellIdFromLatLon(pokemon.Lat, pokemon.Lon) + cellWeather, found := weather[cellId] + if !found { + record, unlock, err := getWeatherRecordReadOnly(ctx, db, cellId) + if err != nil || record == nil || !record.GameplayCondition.Valid { + log.Warnf("[POKEMON] Failed to obtain weather for Pokemon %d: %s", pokemon.Id, err) + } else { + log.Warnf("[POKEMON] Weather not found locally for %d at %d", pokemon.Id, cellId) + cellWeather = pogo.GameplayWeatherProto_WeatherCondition(record.GameplayCondition.Int64) + found = true + } + if unlock != nil { + unlock() + } + } + if found && cellWeather == pogo.GameplayWeatherProto_PARTLY_CLOUDY { + shouldOverrideIv = true + scan, isBoostedMatches := pokemon.locateScan(false, false) + if scan != nil && isBoostedMatches { + overrideIv = scan + } + } + } + } else { + displayPokemon = int(pokemon.PokemonId) + } + var cp int + var err error + if shouldOverrideIv { + if overrideIv == nil { + return + } + // You should see boosted IV for 0P Ditto + cp, err = ohbem.CalculateCp(displayPokemon, int(pokemon.Form.ValueOrZero()), 0, + int(overrideIv.Attack), int(overrideIv.Defense), int(overrideIv.Stamina), float64(overrideIv.Level)) + } else { + if !pokemon.AtkIv.Valid || !pokemon.Level.Valid { + return + } + cp, err = ohbem.CalculateCp(displayPokemon, int(pokemon.Form.ValueOrZero()), 0, + int(pokemon.AtkIv.Int64), int(pokemon.DefIv.Int64), int(pokemon.StaIv.Int64), + float64(pokemon.Level.Int64)) + } + if err == nil { + pokemon.SetCp(null.IntFrom(int64(cp))) + } else { + log.Warnf("Pokemon %d %d CP unset due to error %s", pokemon.Id, displayPokemon, err) + } +} diff --git a/decoder/pokemon_process.go b/decoder/pokemon_process.go new file mode 100644 index 00000000..0d9585ad --- /dev/null +++ b/decoder/pokemon_process.go @@ -0,0 +1,92 @@ +package decoder + +import ( + "context" + "fmt" + "time" + + "golbat/db" + "golbat/pogo" + + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" +) + +func UpdatePokemonRecordWithEncounterProto(ctx context.Context, db db.DbDetails, encounter *pogo.EncounterOutProto, username string, timestamp int64) string { + if encounter.Pokemon == nil { + return "No encounter" + } + + encounterId := encounter.Pokemon.EncounterId + + // Remove from pending queue - encounter arrived so no need for delayed wild update + if pokemonPendingQueue != nil { + pokemonPendingQueue.Remove(encounterId) + } + + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Errorf("Error pokemon [%d]: %s", encounterId, err) + return fmt.Sprintf("Error finding pokemon %s", err) + } + defer unlock() + + pokemon.updatePokemonFromEncounterProto(ctx, db, encounter, username, timestamp) + savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, timestamp/1000) + // updateEncounterStats() should only be called for encounters, and called + // even if we have the pokemon record already. + updateEncounterStats(pokemon) + + return fmt.Sprintf("%d %d Pokemon %d CP%d", encounter.Pokemon.EncounterId, encounterId, pokemon.PokemonId, encounter.Pokemon.Pokemon.Cp) +} + +func UpdatePokemonRecordWithDiskEncounterProto(ctx context.Context, db db.DbDetails, encounter *pogo.DiskEncounterOutProto, username string) string { + if encounter.Pokemon == nil { + return "No encounter" + } + + encounterId := uint64(encounter.Pokemon.PokemonDisplay.DisplayId) + + pokemon, unlock, err := getPokemonRecordForUpdate(ctx, db, encounterId) + if err != nil { + log.Errorf("Error pokemon [%d]: %s", encounterId, err) + return fmt.Sprintf("Error finding pokemon %s", err) + } + + if pokemon == nil || pokemon.isNewRecord() { + // No pokemon found - unlock not set when pokemon is nil + if unlock != nil { + unlock() + } + diskEncounterCache.Set(encounterId, encounter, ttlcache.DefaultTTL) + return fmt.Sprintf("%d Disk encounter without previous GMO - Pokemon stored for later", encounterId) + } + defer unlock() + + pokemon.updatePokemonFromDiskEncounterProto(ctx, db, encounter, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) + // updateEncounterStats() should only be called for encounters, and called + // even if we have the pokemon record already. + updateEncounterStats(pokemon) + + return fmt.Sprintf("%d Disk Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp) +} + +func UpdatePokemonRecordWithTappableEncounter(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, encounter *pogo.TappableEncounterProto, username string, timestampMs int64) string { + encounterId := request.GetEncounterId() + + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Errorf("Error pokemon [%d]: %s", encounterId, err) + return fmt.Sprintf("Error finding pokemon %s", err) + } + defer unlock() + + pokemon.updatePokemonFromTappableEncounterProto(ctx, db, request, encounter, username, timestampMs) + savePokemonRecordAsAtTime(ctx, db, pokemon, true, true, true, time.Now().Unix()) + // updateEncounterStats() should only be called for encounters, and called + // even if we have the pokemon record already. + updateEncounterStats(pokemon) + + return fmt.Sprintf("%d Tappable Pokemon %d CP%d", encounterId, pokemon.PokemonId, encounter.Pokemon.Cp) +} diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go new file mode 100644 index 00000000..e135dbbe --- /dev/null +++ b/decoder/pokemon_state.go @@ -0,0 +1,486 @@ +package decoder + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strconv" + + "golbat/config" + "golbat/db" + "golbat/geo" + "golbat/webhooks" + + "github.com/UnownHash/gohbem" + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + "google.golang.org/protobuf/proto" +) + +// peekPokemonRecordReadOnly acquires lock, does NOT take snapshot. +// Use for read-only checks which will not cause a backing database lookup +// Caller must use returned unlock function +func peekPokemonRecordReadOnly(encounterId uint64) (*Pokemon, func(), error) { + if item := pokemonCache.Get(encounterId); item != nil { + pokemon := item.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil + } + + return nil, nil, nil +} + +func loadPokemonFromDatabase(ctx context.Context, db db.DbDetails, encounterId uint64, pokemon *Pokemon) error { + err := db.PokemonDb.GetContext(ctx, pokemon, + "SELECT id, pokemon_id, lat, lon, spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, "+ + "move_1, move_2, gender, form, cp, level, strong, weather, costume, weight, height, size, "+ + "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id, "+ + "expire_timestamp_verified, shiny, username, pvp, is_event, seen_type "+ + "FROM pokemon WHERE id = ?", strconv.FormatUint(encounterId, 10)) + statsCollector.IncDbQuery("select pokemon", err) + + return err +} + +// getPokemonRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks, but will cause a backing database lookup +// Caller MUST call returned unlock function. +func getPokemonRecordReadOnly(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + // If we are in-memory only, this is identical to peek + if config.Config.PokemonMemoryOnly { + return peekPokemonRecordReadOnly(encounterId) + } + + // Check cache first + if item := pokemonCache.Get(encounterId); item != nil { + pokemon := item.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil + } + + dbPokemon := Pokemon{} + err := loadPokemonFromDatabase(ctx, db, encounterId, &dbPokemon) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbPokemon.ClearDirty() + + // Atomically cache the loaded Pokemon - if another goroutine raced us, + // we'll get their Pokemon and use that instead (ensuring same mutex) + existingPokemon, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + // Only called if key doesn't exist - our Pokemon wins + pokemonRtreeUpdatePokemonOnGet(&dbPokemon) + return &dbPokemon + }) + + pokemon := existingPokemon.Value() + pokemon.Lock() + return pokemon, func() { pokemon.Unlock() }, nil +} + +// getPokemonRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Pokemon. +// Caller MUST call returned unlock function. +func getPokemonRecordForUpdate(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + pokemon, unlock, err := getPokemonRecordReadOnly(ctx, db, encounterId) + if err != nil || pokemon == nil { + return nil, nil, err + } + pokemon.snapshotOldValues() + return pokemon, unlock, nil +} + +// getOrCreatePokemonRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreatePokemonRecord(ctx context.Context, db db.DbDetails, encounterId uint64) (*Pokemon, func(), error) { + // Create new Pokemon atomically - function only called if key doesn't exist + pokemonItem, _ := pokemonCache.GetOrSetFunc(encounterId, func() *Pokemon { + return &Pokemon{Id: encounterId, newRecord: true} + }) + + pokemon := pokemonItem.Value() + pokemon.Lock() + + if config.Config.PokemonMemoryOnly { + pokemon.snapshotOldValues() + return pokemon, func() { pokemon.Unlock() }, nil + } + + if pokemon.newRecord { + // We should attempt to load from database + err := loadPokemonFromDatabase(ctx, db, encounterId, pokemon) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + pokemon.Unlock() + return nil, nil, err + } + } else { + // We loaded + pokemon.newRecord = false + pokemon.ClearDirty() + pokemonRtreeUpdatePokemonOnGet(pokemon) + } + } + + pokemon.snapshotOldValues() + return pokemon, func() { pokemon.Unlock() }, nil +} + +// hasChangesPokemon compares two Pokemon structs +// Ignored: Username, Iv, Pvp +// Float tolerance: Lat, Lon +// Null Float tolerance: Weight, Height, Capture1, Capture2, Capture3 +func hasChangesPokemon(old *Pokemon, new *Pokemon) bool { + return old.Id != new.Id || + old.PokestopId != new.PokestopId || + old.SpawnId != new.SpawnId || + old.Size != new.Size || + old.ExpireTimestamp != new.ExpireTimestamp || + old.Updated != new.Updated || + old.PokemonId != new.PokemonId || + old.Move1 != new.Move1 || + old.Move2 != new.Move2 || + old.Gender != new.Gender || + old.Cp != new.Cp || + old.AtkIv != new.AtkIv || + old.DefIv != new.DefIv || + old.StaIv != new.StaIv || + old.Form != new.Form || + old.Level != new.Level || + old.IsStrong != new.IsStrong || + old.Weather != new.Weather || + old.Costume != new.Costume || + old.FirstSeenTimestamp != new.FirstSeenTimestamp || + old.Changed != new.Changed || + old.CellId != new.CellId || + old.ExpireTimestampVerified != new.ExpireTimestampVerified || + old.DisplayPokemonId != new.DisplayPokemonId || + old.IsDitto != new.IsDitto || + old.SeenType != new.SeenType || + old.Shiny != new.Shiny || + old.IsEvent != new.IsEvent || + !floatAlmostEqual(old.Lat, new.Lat, floatTolerance) || + !floatAlmostEqual(old.Lon, new.Lon, floatTolerance) || + !nullFloatAlmostEqual(old.Weight, new.Weight, floatTolerance) || + !nullFloatAlmostEqual(old.Height, new.Height, floatTolerance) || + !nullFloatAlmostEqual(old.Capture1, new.Capture1, floatTolerance) || + !nullFloatAlmostEqual(old.Capture2, new.Capture2, floatTolerance) || + !nullFloatAlmostEqual(old.Capture3, new.Capture3, floatTolerance) +} + +func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Pokemon, isEncounter, writeDB, webhook bool, now int64) { + if !pokemon.newRecord && !pokemon.IsDirty() { + return + } + + // uncomment to debug excessive writes + //if !pokemon.isNewRecord() && oldPokemon.AtkIv == pokemon.AtkIv && oldPokemon.DefIv == pokemon.DefIv && oldPokemon.StaIv == pokemon.StaIv && oldPokemon.Level == pokemon.Level && oldPokemon.ExpireTimestampVerified == pokemon.ExpireTimestampVerified && oldPokemon.PokemonId == pokemon.PokemonId && oldPokemon.ExpireTimestamp == pokemon.ExpireTimestamp && oldPokemon.PokestopId == pokemon.PokestopId && math.Abs(pokemon.Lat-oldPokemon.Lat) < .000001 && math.Abs(pokemon.Lon-oldPokemon.Lon) < .000001 { + // log.Errorf("Why are we updating this? %s", cmp.Diff(oldPokemon, pokemon, cmp.Options{ + // ignoreNearFloats, ignoreNearNullFloats, + // cmpopts.IgnoreFields(Pokemon{}, "Username", "Iv", "Pvp"), + // })) + //} + + if pokemon.FirstSeenTimestamp == 0 { + pokemon.FirstSeenTimestamp = now + } + + pokemon.Updated = null.IntFrom(now) + if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId || pokemon.oldValues.Cp != pokemon.Cp { + pokemon.Changed = now + } + + changePvpField := false + var pvpResults map[string][]gohbem.PokemonEntry + if ohbem != nil { + // Calculating PVP data - check for changes in pokemon properties that affect PVP rankings + // For new records, always calculate; for existing, check if relevant fields changed + shouldCalculatePvp := pokemon.AtkIv.Valid && (pokemon.isNewRecord() || pokemon.IsDirty()) + if shouldCalculatePvp { + pvp, err := ohbem.QueryPvPRank(int(pokemon.PokemonId), + int(pokemon.Form.ValueOrZero()), + int(pokemon.Costume.ValueOrZero()), + int(pokemon.Gender.ValueOrZero()), + int(pokemon.AtkIv.ValueOrZero()), + int(pokemon.DefIv.ValueOrZero()), + int(pokemon.StaIv.ValueOrZero()), + float64(pokemon.Level.ValueOrZero())) + + if err == nil { + pvpBytes, _ := json.Marshal(pvp) + pokemon.Pvp = null.StringFrom(string(pvpBytes)) + changePvpField = true + pvpResults = pvp + } + } + if !pokemon.AtkIv.Valid && pokemon.isNewRecord() { + pokemon.Pvp = null.NewString("", false) + changePvpField = true + } + } + + var oldSeenType string + if !pokemon.oldValues.SeenType.Valid { + oldSeenType = "n/a" + } else { + oldSeenType = pokemon.oldValues.SeenType.ValueOrZero() + } + + log.Debugf("Updating pokemon [%d] from %s->%s - newRecord: %t", pokemon.Id, oldSeenType, pokemon.SeenType.ValueOrZero(), pokemon.isNewRecord()) + //log.Println(cmp.Diff(oldPokemon, pokemon)) + + if writeDB && !config.Config.PokemonMemoryOnly { + if isEncounter && config.Config.PokemonInternalToDb { + unboosted, boosted, strong := pokemon.locateAllScans() + if unboosted != nil && boosted != nil { + unboosted.RemoveDittoAuxInfo() + boosted.RemoveDittoAuxInfo() + } + if strong != nil { + strong.RemoveDittoAuxInfo() + } + marshaled, err := proto.Marshal(&pokemon.internal) + if err == nil { + pokemon.GolbatInternal = marshaled + } else { + log.Errorf("[POKEMON] Failed to marshal internal data for %d, data may be lost: %s", pokemon.Id, err) + } + } + if pokemon.isNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } + pvpField, pvpValue := "", "" + if changePvpField { + pvpField, pvpValue = "pvp, ", ":pvp, " + } + res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("INSERT INTO pokemon (id, pokemon_id, lat, lon,"+ + "spawn_id, expire_timestamp, atk_iv, def_iv, sta_iv, golbat_internal, iv, move_1, move_2,"+ + "gender, form, cp, level, strong, weather, costume, weight, height, size,"+ + "display_pokemon_id, is_ditto, pokestop_id, updated, first_seen_timestamp, changed, cell_id,"+ + "expire_timestamp_verified, shiny, username, %s is_event, seen_type) "+ + "VALUES (\"%d\", :pokemon_id, :lat, :lon, :spawn_id, :expire_timestamp, :atk_iv, :def_iv, :sta_iv,"+ + ":golbat_internal, :iv, :move_1, :move_2, :gender, :form, :cp, :level, :strong, :weather, :costume,"+ + ":weight, :height, :size, :display_pokemon_id, :is_ditto, :pokestop_id, :updated,"+ + ":first_seen_timestamp, :changed, :cell_id, :expire_timestamp_verified, :shiny, :username, %s :is_event,"+ + ":seen_type)", pvpField, pokemon.Id, pvpValue), pokemon) + + statsCollector.IncDbQuery("insert pokemon", err) + if err != nil { + log.Errorf("insert pokemon: [%d] %s", pokemon.Id, err) + log.Errorf("Full structure: %+v", pokemon) + pokemonCache.Delete(pokemon.Id) + // Force reload of pokemon from database + return + } + + rows, rowsErr := res.RowsAffected() + log.Debugf("Inserting pokemon [%d] after insert res = %d %v", pokemon.Id, rows, rowsErr) + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } + pvpUpdate := "" + if changePvpField { + pvpUpdate = "pvp = :pvp, " + } + res, err := db.PokemonDb.NamedExecContext(ctx, fmt.Sprintf("UPDATE pokemon SET "+ + "pokestop_id = :pokestop_id, "+ + "spawn_id = :spawn_id, "+ + "lat = :lat, "+ + "lon = :lon, "+ + "weight = :weight, "+ + "height = :height, "+ + "size = :size, "+ + "expire_timestamp = :expire_timestamp, "+ + "updated = :updated, "+ + "pokemon_id = :pokemon_id, "+ + "move_1 = :move_1, "+ + "move_2 = :move_2, "+ + "gender = :gender, "+ + "cp = :cp, "+ + "atk_iv = :atk_iv, "+ + "def_iv = :def_iv, "+ + "sta_iv = :sta_iv, "+ + "golbat_internal = :golbat_internal,"+ + "iv = :iv,"+ + "form = :form, "+ + "level = :level, "+ + "strong = :strong, "+ + "weather = :weather, "+ + "costume = :costume, "+ + "first_seen_timestamp = :first_seen_timestamp, "+ + "changed = :changed, "+ + "cell_id = :cell_id, "+ + "expire_timestamp_verified = :expire_timestamp_verified, "+ + "display_pokemon_id = :display_pokemon_id, "+ + "is_ditto = :is_ditto, "+ + "seen_type = :seen_type, "+ + "shiny = :shiny, "+ + "username = :username, "+ + "%s"+ + "is_event = :is_event "+ + "WHERE id = \"%d\"", pvpUpdate, pokemon.Id), pokemon, + ) + statsCollector.IncDbQuery("update pokemon", err) + if err != nil { + log.Errorf("Update pokemon [%d] %s", pokemon.Id, err) + log.Errorf("Full structure: %+v", pokemon) + pokemonCache.Delete(pokemon.Id) + // Force reload of pokemon from database + + return + } + rows, rowsErr := res.RowsAffected() + log.Debugf("Updating pokemon [%d] after update res = %d %v", pokemon.Id, rows, rowsErr) + } + } + + // Update pokemon rtree + if pokemon.isNewRecord() { + addPokemonToTree(pokemon) + } else if pokemon.Lat != pokemon.oldValues.Lat || pokemon.Lon != pokemon.oldValues.Lon { + // Position changed - update R-tree by removing from old position and adding to new + removePokemonFromTree(pokemon.Id, pokemon.oldValues.Lat, pokemon.oldValues.Lon) + addPokemonToTree(pokemon) + } + + updatePokemonLookup(pokemon, changePvpField, pvpResults) + + areas := MatchStatsGeofence(pokemon.Lat, pokemon.Lon) + if webhook { + createPokemonWebhooks(ctx, db, pokemon, areas) + } + updatePokemonStats(pokemon, areas, now) + + if dbDebugEnabled { + pokemon.changedFields = pokemon.changedFields[:0] + } + pokemon.newRecord = false // After saving, it's no longer a new record + pokemon.ClearDirty() + + pokemon.Pvp = null.NewString("", false) // Reset PVP field to avoid keeping it in memory cache + + if db.UsePokemonCache { + pokemonCache.Set(pokemon.Id, pokemon, pokemon.remainingDuration(now)) + } +} + +type PokemonWebhook struct { + SpawnpointId string `json:"spawnpoint_id"` + PokestopId string `json:"pokestop_id"` + PokestopName *string `json:"pokestop_name"` + EncounterId string `json:"encounter_id"` + PokemonId int16 `json:"pokemon_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + DisappearTime int64 `json:"disappear_time"` + DisappearTimeVerified bool `json:"disappear_time_verified"` + FirstSeen int64 `json:"first_seen"` + LastModifiedTime null.Int `json:"last_modified_time"` + Gender null.Int `json:"gender"` + Cp null.Int `json:"cp"` + Form null.Int `json:"form"` + Costume null.Int `json:"costume"` + IndividualAttack null.Int `json:"individual_attack"` + IndividualDefense null.Int `json:"individual_defense"` + IndividualStamina null.Int `json:"individual_stamina"` + PokemonLevel null.Int `json:"pokemon_level"` + Move1 null.Int `json:"move_1"` + Move2 null.Int `json:"move_2"` + Weight null.Float `json:"weight"` + Size null.Int `json:"size"` + Height null.Float `json:"height"` + Weather null.Int `json:"weather"` + Capture1 float64 `json:"capture_1"` + Capture2 float64 `json:"capture_2"` + Capture3 float64 `json:"capture_3"` + Shiny null.Bool `json:"shiny"` + Username null.String `json:"username"` + DisplayPokemonId null.Int `json:"display_pokemon_id"` + IsEvent int8 `json:"is_event"` + SeenType null.String `json:"seen_type"` + Pvp json.RawMessage `json:"pvp"` +} + +func createPokemonWebhooks(ctx context.Context, db db.DbDetails, pokemon *Pokemon, areas []geo.AreaName) { + if pokemon.isNewRecord() || + pokemon.oldValues.PokemonId != pokemon.PokemonId || + pokemon.oldValues.Weather != pokemon.Weather || + pokemon.oldValues.Cp != pokemon.Cp { + + spawnpointId := "None" + if pokemon.SpawnId.Valid { + spawnpointId = strconv.FormatInt(pokemon.SpawnId.ValueOrZero(), 16) + } + + pokestopId := "None" + if pokemon.PokestopId.Valid { + pokestopId = pokemon.PokestopId.ValueOrZero() + } + + var pokestopName *string + if pokemon.PokestopId.Valid { + pokestop, unlock, _ := getPokestopRecordReadOnly(ctx, db, pokemon.PokestopId.String) + name := "Unknown" + if pokestop != nil { + name = pokestop.Name.ValueOrZero() + unlock() + } + pokestopName = &name + } + + var pvp json.RawMessage + if pokemon.Pvp.Valid { + pvp = json.RawMessage(pokemon.Pvp.ValueOrZero()) + } + + pokemonHook := PokemonWebhook{ + SpawnpointId: spawnpointId, + PokestopId: pokestopId, + PokestopName: pokestopName, + EncounterId: strconv.FormatUint(pokemon.Id, 10), + PokemonId: pokemon.PokemonId, + Latitude: pokemon.Lat, + Longitude: pokemon.Lon, + DisappearTime: pokemon.ExpireTimestamp.ValueOrZero(), + DisappearTimeVerified: pokemon.ExpireTimestampVerified, + FirstSeen: pokemon.FirstSeenTimestamp, + LastModifiedTime: pokemon.Updated, + Gender: pokemon.Gender, + Cp: pokemon.Cp, + Form: pokemon.Form, + Costume: pokemon.Costume, + IndividualAttack: pokemon.AtkIv, + IndividualDefense: pokemon.DefIv, + IndividualStamina: pokemon.StaIv, + PokemonLevel: pokemon.Level, + Move1: pokemon.Move1, + Move2: pokemon.Move2, + Weight: pokemon.Weight, + Size: pokemon.Size, + Height: pokemon.Height, + Weather: pokemon.Weather, + Capture1: pokemon.Capture1.ValueOrZero(), + Capture2: pokemon.Capture2.ValueOrZero(), + Capture3: pokemon.Capture3.ValueOrZero(), + Shiny: pokemon.Shiny, + Username: pokemon.Username, + DisplayPokemonId: pokemon.DisplayPokemonId, + IsEvent: pokemon.IsEvent, + SeenType: pokemon.SeenType, + Pvp: pvp, + } + + if pokemon.AtkIv.Valid && pokemon.DefIv.Valid && pokemon.StaIv.Valid { + webhooksSender.AddMessage(webhooks.PokemonIV, pokemonHook, areas) + } else { + webhooksSender.AddMessage(webhooks.PokemonNoIV, pokemonHook, areas) + } + } +} diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 8e25200f..cad7e262 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1,26 +1,9 @@ package decoder import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" - "strings" "sync" - "time" - "github.com/jellydator/ttlcache/v3" - "github.com/paulmach/orb/geojson" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" - - "golbat/config" - "golbat/db" - "golbat/pogo" - "golbat/tz" - "golbat/util" - "golbat/webhooks" + "github.com/guregu/null/v6" ) // Pokestop struct. @@ -593,997 +576,3 @@ func (p *Pokestop) SetShowcaseRankings(v null.String) { } } } - -type QuestWebhook struct { - PokestopId string `json:"pokestop_id"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - PokestopName string `json:"pokestop_name"` - Type null.Int `json:"type"` - Target null.Int `json:"target"` - Template null.String `json:"template"` - Title null.String `json:"title"` - Conditions json.RawMessage `json:"conditions"` - Rewards json.RawMessage `json:"rewards"` - Updated int64 `json:"updated"` - ArScanEligible int64 `json:"ar_scan_eligible"` - PokestopUrl string `json:"pokestop_url"` - WithAr bool `json:"with_ar"` -} - -type PokestopWebhook struct { - PokestopId string `json:"pokestop_id"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Name string `json:"name"` - Url string `json:"url"` - LureExpiration int64 `json:"lure_expiration"` - LastModified int64 `json:"last_modified"` - Enabled bool `json:"enabled"` - LureId int16 `json:"lure_id"` - ArScanEligible int64 `json:"ar_scan_eligible"` - PowerUpLevel int64 `json:"power_up_level"` - PowerUpPoints int64 `json:"power_up_points"` - PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` - Updated int64 `json:"updated"` - ShowcaseFocus null.String `json:"showcase_focus"` - ShowcasePokemonId null.Int `json:"showcase_pokemon_id"` - ShowcasePokemonFormId null.Int `json:"showcase_pokemon_form_id"` - ShowcasePokemonTypeId null.Int `json:"showcase_pokemon_type_id"` - ShowcaseRankingStandard null.Int `json:"showcase_ranking_standard"` - ShowcaseExpiry null.Int `json:"showcase_expiry"` - ShowcaseRankings json.RawMessage `json:"showcase_rankings"` -} - -func loadPokestopFromDatabase(ctx context.Context, db db.DbDetails, fortId string, pokestop *Pokestop) error { - err := db.GeneralDb.GetContext(ctx, pokestop, - `SELECT pokestop.id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, - pokestop.updated, quest_type, quest_timestamp, quest_target, quest_conditions, - quest_rewards, quest_template, quest_title, - alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, - alternative_quest_conditions, alternative_quest_rewards, - alternative_quest_template, alternative_quest_title, cell_id, deleted, lure_id, sponsor_id, partner_id, - ar_scan_eligible, power_up_points, power_up_level, power_up_end_timestamp, - quest_expiry, alternative_quest_expiry, description, showcase_pokemon_id, showcase_pokemon_form_id, - showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings - FROM pokestop - WHERE pokestop.id = ? `, fortId) - statsCollector.IncDbQuery("select pokestop", err) - return err -} - -// PeekPokestopRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func PeekPokestopRecord(fortId string) (*Pokestop, func(), error) { - if item := pokestopCache.Get(fortId); item != nil { - pokestop := item.Value() - pokestop.Lock() - return pokestop, func() { pokestop.Unlock() }, nil - } - return nil, nil, nil -} - -// getPokestopRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { - // Check cache first - if item := pokestopCache.Get(fortId); item != nil { - pokestop := item.Value() - pokestop.Lock() - return pokestop, func() { pokestop.Unlock() }, nil - } - - dbPokestop := Pokestop{} - err := loadPokestopFromDatabase(ctx, db, fortId, &dbPokestop) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - - dbPokestop.ClearDirty() - - // Atomically cache the loaded Pokestop - if another goroutine raced us, - // we'll get their Pokestop and use that instead (ensuring same mutex) - existingPokestop, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { - // Only called if key doesn't exist - our Pokestop wins - if config.Config.TestFortInMemory { - fortRtreeUpdatePokestopOnGet(&dbPokestop) - } - return &dbPokestop - }) - - pokestop := existingPokestop.Value() - pokestop.Lock() - return pokestop, func() { pokestop.Unlock() }, nil -} - -// getPokestopRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Use when modifying the Pokestop. -// Caller MUST call returned unlock function if non-nil. -func getPokestopRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { - pokestop, unlock, err := getPokestopRecordReadOnly(ctx, db, fortId) - if err != nil || pokestop == nil { - return nil, nil, err - } - pokestop.snapshotOldValues() - return pokestop, unlock, nil -} - -// getOrCreatePokestopRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { - // Create new Pokestop atomically - function only called if key doesn't exist - pokestopItem, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { - return &Pokestop{Id: fortId, newRecord: true} - }) - - pokestop := pokestopItem.Value() - pokestop.Lock() - - if pokestop.newRecord { - // We should attempt to load from database - err := loadPokestopFromDatabase(ctx, db, fortId, pokestop) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - pokestop.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - pokestop.newRecord = false - pokestop.ClearDirty() - if config.Config.TestFortInMemory { - fortRtreeUpdatePokestopOnGet(pokestop) - } - } - } - - pokestop.snapshotOldValues() - return pokestop, func() { pokestop.Unlock() }, nil -} - -var LureTime int64 = 1800 - -func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, cellId uint64, now int64) *Pokestop { - stop.SetId(fortData.FortId) - stop.SetLat(fortData.Latitude) - stop.SetLon(fortData.Longitude) - - stop.SetPartnerId(null.NewString(fortData.PartnerId, fortData.PartnerId != "")) - stop.SetSponsorId(null.IntFrom(int64(fortData.Sponsor))) - stop.SetEnabled(null.BoolFrom(fortData.Enabled)) - stop.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) - stop.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) - powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) - stop.SetPowerUpLevel(powerUpLevel) - stop.SetPowerUpEndTimestamp(powerUpEndTimestamp) - - // lasModifiedMs is also modified when incident happens - lastModifiedTimestamp := fortData.LastModifiedMs / 1000 - stop.SetLastModifiedTimestamp(null.IntFrom(lastModifiedTimestamp)) - - if len(fortData.ActiveFortModifier) > 0 { - lureId := int16(fortData.ActiveFortModifier[0]) - if lureId >= 501 && lureId <= 510 { - lureEnd := lastModifiedTimestamp + LureTime - oldLureEnd := stop.LureExpireTimestamp.ValueOrZero() - if stop.LureId != lureId { - stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) - stop.SetLureId(lureId) - } else { - // wait some time after lure end before a restart in case of timing issue - if now > oldLureEnd+30 { - for now > lureEnd { - lureEnd += LureTime - } - // lure needs to be restarted - stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) - } - } - } - } - - if fortData.ImageUrl != "" { - stop.SetUrl(null.StringFrom(fortData.ImageUrl)) - } - stop.SetCellId(null.IntFrom(int64(cellId))) - - if stop.Deleted { - stop.SetDeleted(false) - log.Warnf("Cleared Stop with id '%s' is found again in GMO, therefore un-deleted", stop.Id) - // Restore in fort tracker if enabled - if fortTracker != nil { - fortTracker.RestoreFort(stop.Id, cellId, false, time.Now().Unix()) - } - } - return stop -} - -func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOutProto, haveAr bool) string { - - if questProto.ChallengeQuest == nil { - log.Debugf("Received blank quest") - return "Blank quest" - } - questData := questProto.ChallengeQuest.Quest - questTitle := questProto.ChallengeQuest.QuestDisplay.Description - questType := int64(questData.QuestType) - questTarget := int64(questData.Goal.Target) - questTemplate := strings.ToLower(questData.TemplateId) - - conditions := []map[string]any{} - rewards := []map[string]any{} - - for _, conditionData := range questData.Goal.Condition { - condition := make(map[string]any) - infoData := make(map[string]any) - condition["type"] = int(conditionData.Type) - switch conditionData.Type { - case pogo.QuestConditionProto_WITH_BADGE_TYPE: - info := conditionData.GetWithBadgeType() - infoData["amount"] = info.Amount - infoData["badge_rank"] = info.BadgeRank - badgeTypeById := []int{} - for _, badge := range info.BadgeType { - badgeTypeById = append(badgeTypeById, int(badge)) - } - infoData["badge_types"] = badgeTypeById - - case pogo.QuestConditionProto_WITH_ITEM: - info := conditionData.GetWithItem() - if int(info.Item) != 0 { - infoData["item_id"] = int(info.Item) - } - case pogo.QuestConditionProto_WITH_RAID_LEVEL: - info := conditionData.GetWithRaidLevel() - raidLevelById := []int{} - for _, raidLevel := range info.RaidLevel { - raidLevelById = append(raidLevelById, int(raidLevel)) - } - infoData["raid_levels"] = raidLevelById - case pogo.QuestConditionProto_WITH_POKEMON_TYPE: - info := conditionData.GetWithPokemonType() - pokemonTypesById := []int{} - for _, t := range info.PokemonType { - pokemonTypesById = append(pokemonTypesById, int(t)) - } - infoData["pokemon_type_ids"] = pokemonTypesById - case pogo.QuestConditionProto_WITH_POKEMON_CATEGORY: - info := conditionData.GetWithPokemonCategory() - if info.CategoryName != "" { - infoData["category_name"] = info.CategoryName - } - pokemonById := []int{} - for _, pokemon := range info.PokemonIds { - pokemonById = append(pokemonById, int(pokemon)) - } - infoData["pokemon_ids"] = pokemonById - case pogo.QuestConditionProto_WITH_WIN_RAID_STATUS: - case pogo.QuestConditionProto_WITH_THROW_TYPE: - info := conditionData.GetWithThrowType() - if int(info.GetThrowType()) != 0 { // TODO: RDM has ThrowType here, ensure it is the same thing - infoData["throw_type_id"] = int(info.GetThrowType()) - } - infoData["hit"] = info.GetHit() - case pogo.QuestConditionProto_WITH_THROW_TYPE_IN_A_ROW: - info := conditionData.GetWithThrowType() - if int(info.GetThrowType()) != 0 { - infoData["throw_type_id"] = int(info.GetThrowType()) - } - infoData["hit"] = info.GetHit() - case pogo.QuestConditionProto_WITH_LOCATION: - info := conditionData.GetWithLocation() - infoData["cell_ids"] = info.S2CellId - case pogo.QuestConditionProto_WITH_DISTANCE: - info := conditionData.GetWithDistance() - infoData["distance"] = info.DistanceKm - case pogo.QuestConditionProto_WITH_POKEMON_ALIGNMENT: - info := conditionData.GetWithPokemonAlignment() - alignmentIds := []int{} - for _, alignment := range info.Alignment { - alignmentIds = append(alignmentIds, int(alignment)) - } - infoData["alignment_ids"] = alignmentIds - case pogo.QuestConditionProto_WITH_INVASION_CHARACTER: - info := conditionData.GetWithInvasionCharacter() - characterCategoryIds := []int{} - for _, characterCategory := range info.Category { - characterCategoryIds = append(characterCategoryIds, int(characterCategory)) - } - infoData["character_category_ids"] = characterCategoryIds - case pogo.QuestConditionProto_WITH_NPC_COMBAT: - info := conditionData.GetWithNpcCombat() - infoData["win"] = info.RequiresWin - infoData["template_ids"] = info.CombatNpcTrainerId - case pogo.QuestConditionProto_WITH_PLAYER_LEVEL: - info := conditionData.GetWithPlayerLevel() - infoData["level"] = info.Level - case pogo.QuestConditionProto_WITH_BUDDY: - info := conditionData.GetWithBuddy() - if info != nil { - infoData["min_buddy_level"] = int(info.MinBuddyLevel) - infoData["must_be_on_map"] = info.MustBeOnMap - } else { - infoData["min_buddy_level"] = 0 - infoData["must_be_on_map"] = false - } - case pogo.QuestConditionProto_WITH_DAILY_BUDDY_AFFECTION: - info := conditionData.GetWithDailyBuddyAffection() - infoData["min_buddy_affection_earned_today"] = info.MinBuddyAffectionEarnedToday - case pogo.QuestConditionProto_WITH_TEMP_EVO_POKEMON: - info := conditionData.GetWithTempEvoId() - tempEvoIds := []int{} - for _, evolution := range info.MegaForm { - tempEvoIds = append(tempEvoIds, int(evolution)) - } - infoData["raid_pokemon_evolutions"] = tempEvoIds - case pogo.QuestConditionProto_WITH_ITEM_TYPE: - info := conditionData.GetWithItemType() - itemTypes := []int{} - for _, itemType := range info.ItemType { - itemTypes = append(itemTypes, int(itemType)) - } - infoData["item_type_ids"] = itemTypes - case pogo.QuestConditionProto_WITH_RAID_ELAPSED_TIME: - info := conditionData.GetWithElapsedTime() - infoData["time"] = int64(info.ElapsedTimeMs) / 1000 - case pogo.QuestConditionProto_WITH_WIN_GYM_BATTLE_STATUS: - case pogo.QuestConditionProto_WITH_SUPER_EFFECTIVE_CHARGE: - case pogo.QuestConditionProto_WITH_UNIQUE_POKESTOP: - case pogo.QuestConditionProto_WITH_QUEST_CONTEXT: - case pogo.QuestConditionProto_WITH_WIN_BATTLE_STATUS: - case pogo.QuestConditionProto_WITH_CURVE_BALL: - case pogo.QuestConditionProto_WITH_NEW_FRIEND: - case pogo.QuestConditionProto_WITH_DAYS_IN_A_ROW: - case pogo.QuestConditionProto_WITH_WEATHER_BOOST: - case pogo.QuestConditionProto_WITH_DAILY_CAPTURE_BONUS: - case pogo.QuestConditionProto_WITH_DAILY_SPIN_BONUS: - case pogo.QuestConditionProto_WITH_UNIQUE_POKEMON: - case pogo.QuestConditionProto_WITH_BUDDY_INTERESTING_POI: - case pogo.QuestConditionProto_WITH_POKEMON_LEVEL: - case pogo.QuestConditionProto_WITH_SINGLE_DAY: - case pogo.QuestConditionProto_WITH_UNIQUE_POKEMON_TEAM: - case pogo.QuestConditionProto_WITH_MAX_CP: - case pogo.QuestConditionProto_WITH_LUCKY_POKEMON: - case pogo.QuestConditionProto_WITH_LEGENDARY_POKEMON: - case pogo.QuestConditionProto_WITH_GBL_RANK: - case pogo.QuestConditionProto_WITH_CATCHES_IN_A_ROW: - case pogo.QuestConditionProto_WITH_ENCOUNTER_TYPE: - case pogo.QuestConditionProto_WITH_COMBAT_TYPE: - case pogo.QuestConditionProto_WITH_GEOTARGETED_POI: - case pogo.QuestConditionProto_WITH_FRIEND_LEVEL: - case pogo.QuestConditionProto_WITH_STICKER: - case pogo.QuestConditionProto_WITH_POKEMON_CP: - case pogo.QuestConditionProto_WITH_RAID_LOCATION: - case pogo.QuestConditionProto_WITH_FRIENDS_RAID: - case pogo.QuestConditionProto_WITH_POKEMON_COSTUME: - default: - break - } - - if infoData != nil { - condition["info"] = infoData - } - conditions = append(conditions, condition) - } - - for _, rewardData := range questData.QuestRewards { - reward := make(map[string]any) - infoData := make(map[string]any) - reward["type"] = int(rewardData.Type) - switch rewardData.Type { - case pogo.QuestRewardProto_EXPERIENCE: - infoData["amount"] = rewardData.GetExp() - case pogo.QuestRewardProto_ITEM: - info := rewardData.GetItem() - infoData["amount"] = info.Amount - infoData["item_id"] = int(info.Item) - case pogo.QuestRewardProto_STARDUST: - infoData["amount"] = rewardData.GetStardust() - case pogo.QuestRewardProto_CANDY: - info := rewardData.GetCandy() - infoData["amount"] = info.Amount - infoData["pokemon_id"] = int(info.PokemonId) - case pogo.QuestRewardProto_XL_CANDY: - info := rewardData.GetXlCandy() - infoData["amount"] = info.Amount - infoData["pokemon_id"] = int(info.PokemonId) - case pogo.QuestRewardProto_POKEMON_ENCOUNTER: - info := rewardData.GetPokemonEncounter() - if info.IsHiddenDitto { - infoData["pokemon_id"] = 132 - infoData["pokemon_id_display"] = int(info.GetPokemonId()) - } else { - infoData["pokemon_id"] = int(info.GetPokemonId()) - } - if info.ShinyProbability > 0.0 { - infoData["shiny_probability"] = info.ShinyProbability - } - if display := info.PokemonDisplay; display != nil { - if costumeId := int(display.Costume); costumeId != 0 { - infoData["costume_id"] = costumeId - } - if formId := int(display.Form); formId != 0 { - infoData["form_id"] = formId - } - if genderId := int(display.Gender); genderId != 0 { - infoData["gender_id"] = genderId - } - if display.Shiny { - infoData["shiny"] = display.Shiny - } - if background := util.ExtractBackgroundFromDisplay(display); background != nil { - infoData["background"] = background - } - if breadMode := int(display.BreadModeEnum); breadMode != 0 { - infoData["bread_mode"] = breadMode - } - } else { - - } - case pogo.QuestRewardProto_POKECOIN: - infoData["amount"] = rewardData.GetPokecoin() - case pogo.QuestRewardProto_STICKER: - info := rewardData.GetSticker() - infoData["amount"] = info.Amount - infoData["sticker_id"] = info.StickerId - case pogo.QuestRewardProto_MEGA_RESOURCE: - info := rewardData.GetMegaResource() - infoData["amount"] = info.Amount - infoData["pokemon_id"] = int(info.PokemonId) - case pogo.QuestRewardProto_AVATAR_CLOTHING: - case pogo.QuestRewardProto_QUEST: - case pogo.QuestRewardProto_LEVEL_CAP: - case pogo.QuestRewardProto_INCIDENT: - case pogo.QuestRewardProto_PLAYER_ATTRIBUTE: - default: - break - - } - reward["info"] = infoData - rewards = append(rewards, reward) - } - - questConditions, _ := json.Marshal(conditions) - questRewards, _ := json.Marshal(rewards) - questTimestamp := time.Now().Unix() - - questExpiry := null.NewInt(0, false) - - stopTimezone := tz.SearchTimezone(stop.Lat, stop.Lon) - if stopTimezone != "" { - loc, err := time.LoadLocation(stopTimezone) - if err != nil { - log.Warnf("Unrecognised time zone %s at %f,%f", stopTimezone, stop.Lat, stop.Lon) - } else { - year, month, day := time.Now().In(loc).Date() - t := time.Date(year, month, day, 0, 0, 0, 0, loc).AddDate(0, 0, 1) - unixTime := t.Unix() - questExpiry = null.IntFrom(unixTime) - } - } - - if questExpiry.Valid == false { - questExpiry = null.IntFrom(time.Now().Unix() + 24*60*60) // Set expiry to 24 hours from now - } - - if !haveAr { - stop.SetAlternativeQuestType(null.IntFrom(questType)) - stop.SetAlternativeQuestTarget(null.IntFrom(questTarget)) - stop.SetAlternativeQuestTemplate(null.StringFrom(questTemplate)) - stop.SetAlternativeQuestTitle(null.StringFrom(questTitle)) - stop.SetAlternativeQuestConditions(null.StringFrom(string(questConditions))) - stop.SetAlternativeQuestRewards(null.StringFrom(string(questRewards))) - stop.SetAlternativeQuestTimestamp(null.IntFrom(questTimestamp)) - stop.SetAlternativeQuestExpiry(questExpiry) - } else { - stop.SetQuestType(null.IntFrom(questType)) - stop.SetQuestTarget(null.IntFrom(questTarget)) - stop.SetQuestTemplate(null.StringFrom(questTemplate)) - stop.SetQuestTitle(null.StringFrom(questTitle)) - stop.SetQuestConditions(null.StringFrom(string(questConditions))) - stop.SetQuestRewards(null.StringFrom(string(questRewards))) - stop.SetQuestTimestamp(null.IntFrom(questTimestamp)) - stop.SetQuestExpiry(questExpiry) - } - - return questTitle -} - -func (stop *Pokestop) updatePokestopFromFortDetailsProto(fortData *pogo.FortDetailsOutProto) *Pokestop { - stop.SetId(fortData.Id) - stop.SetLat(fortData.Latitude) - stop.SetLon(fortData.Longitude) - if len(fortData.ImageUrl) > 0 { - stop.SetUrl(null.StringFrom(fortData.ImageUrl[0])) - } - stop.SetName(null.StringFrom(fortData.Name)) - - if fortData.Description == "" { - stop.SetDescription(null.NewString("", false)) - } else { - stop.SetDescription(null.StringFrom(fortData.Description)) - } - - if fortData.Modifier != nil && len(fortData.Modifier) > 0 { - // DeployingPlayerCodename contains the name of the player if we want that - lureId := int16(fortData.Modifier[0].ModifierType) - lureExpiry := fortData.Modifier[0].ExpirationTimeMs / 1000 - - stop.SetLureId(lureId) - stop.SetLureExpireTimestamp(null.IntFrom(lureExpiry)) - } - - return stop -} - -func (stop *Pokestop) updatePokestopFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto) *Pokestop { - stop.SetId(fortData.Id) - stop.SetLat(fortData.Latitude) - stop.SetLon(fortData.Longitude) - - if len(fortData.Image) > 0 { - stop.SetUrl(null.StringFrom(fortData.Image[0].Url)) - } - stop.SetName(null.StringFrom(fortData.Name)) - if stop.Deleted { - log.Debugf("Cleared Stop with id '%s' is found again in GMF, therefore kept deleted", stop.Id) - } - return stop -} - -func (stop *Pokestop) updatePokestopFromGetContestDataOutProto(contest *pogo.ContestProto) { - stop.SetShowcaseRankingStandard(null.IntFrom(int64(contest.GetMetric().GetRankingStandard()))) - stop.SetShowcaseExpiry(null.IntFrom(contest.GetSchedule().GetContestCycle().GetEndTimeMs() / 1000)) - - focusStore := createFocusStoreFromContestProto(contest) - - if len(focusStore) > 1 { - log.Warnf("SHOWCASE: we got more than one showcase focus: %v", focusStore) - } - - for key, focus := range focusStore { - focus["type"] = key - jsonBytes, err := json.Marshal(focus) - if err != nil { - log.Errorf("SHOWCASE: Stop '%s' - Focus '%v' marshalling failed: %s", stop.Id, focus, err) - } - stop.SetShowcaseFocus(null.StringFrom(string(jsonBytes))) - // still support old format - probably still required to filter in external tools - stop.extractShowcasePokemonInfoDeprecated(key, focus) - } -} - -func (stop *Pokestop) updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) { - type contestEntry struct { - Rank int `json:"rank"` - Score float64 `json:"score"` - PokemonId int `json:"pokemon_id"` - Form int `json:"form"` - Costume int `json:"costume"` - Gender int `json:"gender"` - Shiny bool `json:"shiny"` - TempEvolution int `json:"temp_evolution"` - TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms"` - Alignment int `json:"alignment"` - Badge int `json:"badge"` - Background *int64 `json:"background,omitempty"` - } - type contestJson struct { - TotalEntries int `json:"total_entries"` - LastUpdate int64 `json:"last_update"` - ContestEntries []contestEntry `json:"contest_entries"` - } - - j := contestJson{LastUpdate: time.Now().Unix()} - j.TotalEntries = int(contestData.TotalEntries) - - for _, entry := range contestData.GetContestEntries() { - rank := entry.GetRank() - if rank > 3 { - break - } - j.ContestEntries = append(j.ContestEntries, contestEntry{ - Rank: int(rank), - Score: entry.GetScore(), - PokemonId: int(entry.GetPokedexId()), - Form: int(entry.GetPokemonDisplay().Form), - Costume: int(entry.GetPokemonDisplay().Costume), - Gender: int(entry.GetPokemonDisplay().Gender), - Shiny: entry.GetPokemonDisplay().Shiny, - TempEvolution: int(entry.GetPokemonDisplay().CurrentTempEvolution), - TempEvolutionFinishMs: entry.GetPokemonDisplay().TemporaryEvolutionFinishMs, - Alignment: int(entry.GetPokemonDisplay().Alignment), - Badge: int(entry.GetPokemonDisplay().PokemonBadge), - Background: util.ExtractBackgroundFromDisplay(entry.PokemonDisplay), - }) - - } - jsonString, _ := json.Marshal(j) - stop.SetShowcaseRankings(null.StringFrom(string(jsonString))) -} - -func createPokestopFortWebhooks(stop *Pokestop) { - fort := InitWebHookFortFromPokestop(stop) - if stop.newRecord { - CreateFortWebHooks(nil, fort, NEW) - } else { - // Build old fort from saved old values - oldFort := &FortWebhook{ - Type: POKESTOP.String(), - Id: stop.Id, - Name: stop.oldValues.Name.Ptr(), - ImageUrl: stop.oldValues.Url.Ptr(), - Description: stop.oldValues.Description.Ptr(), - Location: Location{Latitude: stop.oldValues.Lat, Longitude: stop.oldValues.Lon}, - } - CreateFortWebHooks(oldFort, fort, EDIT) - } -} - -func createPokestopWebhooks(stop *Pokestop) { - - areas := MatchStatsGeofence(stop.Lat, stop.Lon) - - pokestopName := "Unknown" - if stop.Name.Valid { - pokestopName = stop.Name.String - } - - if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldValues.AlternativeQuestType) { - questHook := QuestWebhook{ - PokestopId: stop.Id, - Latitude: stop.Lat, - Longitude: stop.Lon, - PokestopName: pokestopName, - Type: stop.AlternativeQuestType, - Target: stop.AlternativeQuestTarget, - Template: stop.AlternativeQuestTemplate, - Title: stop.AlternativeQuestTitle, - Conditions: json.RawMessage(stop.AlternativeQuestConditions.ValueOrZero()), - Rewards: json.RawMessage(stop.AlternativeQuestRewards.ValueOrZero()), - Updated: stop.Updated, - ArScanEligible: stop.ArScanEligible.ValueOrZero(), - PokestopUrl: stop.Url.ValueOrZero(), - WithAr: false, - } - webhooksSender.AddMessage(webhooks.Quest, questHook, areas) - } - - if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldValues.QuestType) { - questHook := QuestWebhook{ - PokestopId: stop.Id, - Latitude: stop.Lat, - Longitude: stop.Lon, - PokestopName: pokestopName, - Type: stop.QuestType, - Target: stop.QuestTarget, - Template: stop.QuestTemplate, - Title: stop.QuestTitle, - Conditions: json.RawMessage(stop.QuestConditions.ValueOrZero()), - Rewards: json.RawMessage(stop.QuestRewards.ValueOrZero()), - Updated: stop.Updated, - ArScanEligible: stop.ArScanEligible.ValueOrZero(), - PokestopUrl: stop.Url.ValueOrZero(), - WithAr: true, - } - webhooksSender.AddMessage(webhooks.Quest, questHook, areas) - } - if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldValues.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldValues.PowerUpEndTimestamp)) { - var showcaseRankings json.RawMessage - if stop.ShowcaseRankings.Valid { - showcaseRankings = json.RawMessage(stop.ShowcaseRankings.ValueOrZero()) - } - - pokestopHook := PokestopWebhook{ - PokestopId: stop.Id, - Latitude: stop.Lat, - Longitude: stop.Lon, - Name: pokestopName, - Url: stop.Url.ValueOrZero(), - LureExpiration: stop.LureExpireTimestamp.ValueOrZero(), - LastModified: stop.LastModifiedTimestamp.ValueOrZero(), - Enabled: stop.Enabled.ValueOrZero(), - LureId: stop.LureId, - ArScanEligible: stop.ArScanEligible.ValueOrZero(), - PowerUpLevel: stop.PowerUpLevel.ValueOrZero(), - PowerUpPoints: stop.PowerUpPoints.ValueOrZero(), - PowerUpEndTimestamp: stop.PowerUpEndTimestamp.ValueOrZero(), - Updated: stop.Updated, - ShowcaseFocus: stop.ShowcaseFocus, - ShowcasePokemonId: stop.ShowcasePokemon, - ShowcasePokemonFormId: stop.ShowcasePokemonForm, - ShowcasePokemonTypeId: stop.ShowcasePokemonType, - ShowcaseRankingStandard: stop.ShowcaseRankingStandard, - ShowcaseExpiry: stop.ShowcaseExpiry, - ShowcaseRankings: showcaseRankings, - } - - webhooksSender.AddMessage(webhooks.Pokestop, pokestopHook, areas) - } -} - -func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop) { - now := time.Now().Unix() - if !pokestop.IsNewRecord() && !pokestop.IsDirty() { - // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. - if pokestop.Updated > now-GetUpdateThreshold(900) { - // if a pokestop is unchanged, but we did see it again after 15 minutes, then save again - return - } - } - pokestop.Updated = now - - if pokestop.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Pokestop", pokestop.Id, pokestop.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, ` - INSERT INTO pokestop ( - id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, - quest_timestamp, quest_target, quest_conditions, quest_rewards, quest_template, quest_title, - alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, - alternative_quest_conditions, alternative_quest_rewards, alternative_quest_template, - alternative_quest_title, cell_id, lure_id, sponsor_id, partner_id, ar_scan_eligible, - power_up_points, power_up_level, power_up_end_timestamp, updated, first_seen_timestamp, - quest_expiry, alternative_quest_expiry, description, showcase_focus, showcase_pokemon_id, - showcase_pokemon_form_id, showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings - ) - VALUES ( - :id, :lat, :lon, :name, :url, :enabled, :lure_expire_timestamp, :last_modified_timestamp, :quest_type, - :quest_timestamp, :quest_target, :quest_conditions, :quest_rewards, :quest_template, :quest_title, - :alternative_quest_type, :alternative_quest_timestamp, :alternative_quest_target, - :alternative_quest_conditions, :alternative_quest_rewards, :alternative_quest_template, - :alternative_quest_title, :cell_id, :lure_id, :sponsor_id, :partner_id, :ar_scan_eligible, - :power_up_points, :power_up_level, :power_up_end_timestamp, - UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), - :quest_expiry, :alternative_quest_expiry, :description, :showcase_focus, :showcase_pokemon_id, - :showcase_pokemon_form_id, :showcase_pokemon_type_id, :showcase_ranking_standard, :showcase_expiry, :showcase_rankings)`, - pokestop) - - statsCollector.IncDbQuery("insert pokestop", err) - //log.Debugf("Insert pokestop %s %+v", pokestop.Id, pokestop) - if err != nil { - log.Errorf("insert pokestop: %s", err) - return - } - - _, _ = res, err - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, ` - UPDATE pokestop SET - lat = :lat, - lon = :lon, - name = :name, - url = :url, - enabled = :enabled, - lure_expire_timestamp = :lure_expire_timestamp, - last_modified_timestamp = :last_modified_timestamp, - updated = :updated, - quest_type = :quest_type, - quest_timestamp = :quest_timestamp, - quest_target = :quest_target, - quest_conditions = :quest_conditions, - quest_rewards = :quest_rewards, - quest_template = :quest_template, - quest_title = :quest_title, - alternative_quest_type = :alternative_quest_type, - alternative_quest_timestamp = :alternative_quest_timestamp, - alternative_quest_target = :alternative_quest_target, - alternative_quest_conditions = :alternative_quest_conditions, - alternative_quest_rewards = :alternative_quest_rewards, - alternative_quest_template = :alternative_quest_template, - alternative_quest_title = :alternative_quest_title, - cell_id = :cell_id, - lure_id = :lure_id, - deleted = :deleted, - sponsor_id = :sponsor_id, - partner_id = :partner_id, - ar_scan_eligible = :ar_scan_eligible, - power_up_points = :power_up_points, - power_up_level = :power_up_level, - power_up_end_timestamp = :power_up_end_timestamp, - quest_expiry = :quest_expiry, - alternative_quest_expiry = :alternative_quest_expiry, - description = :description, - showcase_focus = :showcase_focus, - showcase_pokemon_id = :showcase_pokemon_id, - showcase_pokemon_form_id = :showcase_pokemon_form_id, - showcase_pokemon_type_id = :showcase_pokemon_type_id, - showcase_ranking_standard = :showcase_ranking_standard, - showcase_expiry = :showcase_expiry, - showcase_rankings = :showcase_rankings - WHERE id = :id`, - pokestop, - ) - statsCollector.IncDbQuery("update pokestop", err) - //log.Debugf("Update pokestop %s %+v", pokestop.Id, pokestop) - if err != nil { - log.Errorf("update pokestop %s: %s", pokestop.Id, err) - return - } - _ = res - } - //pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) - if dbDebugEnabled { - pokestop.changedFields = pokestop.changedFields[:0] - } - - createPokestopWebhooks(pokestop) - createPokestopFortWebhooks(pokestop) - if pokestop.IsNewRecord() { - pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) - pokestop.newRecord = false - } - pokestop.ClearDirty() - -} - -func updatePokestopGetMapFortCache(pokestop *Pokestop) { - storedGetMapFort := getMapFortsCache.Get(pokestop.Id) - if storedGetMapFort != nil { - getMapFort := storedGetMapFort.Value() - getMapFortsCache.Delete(pokestop.Id) - pokestop.updatePokestopFromGetMapFortsOutProto(getMapFort) - log.Debugf("Updated Gym using stored getMapFort: %s", pokestop.Id) - } -} - -func UpdatePokestopRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { - pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fort.Id) - if err != nil { - log.Printf("Update pokestop %s", err) - return fmt.Sprintf("Error %s", err) - } - defer unlock() - - pokestop.updatePokestopFromFortDetailsProto(fort) - - updatePokestopGetMapFortCache(pokestop) - savePokestopRecord(ctx, db, pokestop) - return fmt.Sprintf("%s %s", fort.Id, fort.Name) -} - -func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.FortSearchOutProto, haveAr bool) string { - haveArStr := "NoAR" - if haveAr { - haveArStr = "AR" - } - - if quest.ChallengeQuest == nil { - statsCollector.IncDecodeQuest("error", "no_quest") - return fmt.Sprintf("%s %s Blank quest", quest.FortId, haveArStr) - } - - statsCollector.IncDecodeQuest("ok", haveArStr) - - pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, quest.FortId) - if err != nil { - log.Printf("Update quest %s", err) - return fmt.Sprintf("error %s", err) - } - defer unlock() - - questTitle := pokestop.updatePokestopFromQuestProto(quest, haveAr) - - updatePokestopGetMapFortCache(pokestop) - savePokestopRecord(ctx, db, pokestop) - - areas := MatchStatsGeofence(pokestop.Lat, pokestop.Lon) - updateQuestStats(pokestop, haveAr, areas) - - return fmt.Sprintf("%s %s %s", quest.FortId, haveArStr, questTitle) -} - -func ClearQuestsWithinGeofence(ctx context.Context, dbDetails db.DbDetails, geofence *geojson.Feature) { - started := time.Now() - rows, err := db.RemoveQuests(ctx, dbDetails, geofence) - if err != nil { - log.Errorf("ClearQuest: Error removing quests: %s", err) - return - } - ClearPokestopCache() - log.Infof("ClearQuest: Removed quests from %d pokestops in %s", rows, time.Since(started)) -} - -func GetQuestStatusWithGeofence(dbDetails db.DbDetails, geofence *geojson.Feature) db.QuestStatus { - res, err := db.GetQuestStatus(dbDetails, geofence) - if err != nil { - log.Errorf("QuestStatus: Error retrieving quests: %s", err) - return db.QuestStatus{} - } - return res -} - -func UpdatePokestopRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { - pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, mapFort.Id) - if err != nil { - log.Printf("Update pokestop %s", err) - return false, fmt.Sprintf("Error %s", err) - } - - if pokestop == nil { - return false, "" - } - defer unlock() - - pokestop.updatePokestopFromGetMapFortsOutProto(mapFort) - savePokestopRecord(ctx, db, pokestop) - return true, fmt.Sprintf("%s %s", mapFort.Id, mapFort.Name) -} - -func GetPokestopPositions(details db.DbDetails, geofence *geojson.Feature) ([]db.QuestLocation, error) { - return db.GetPokestopPositions(details, geofence) -} - -func UpdatePokestopWithContestData(ctx context.Context, db db.DbDetails, request *pogo.GetContestDataProto, contestData *pogo.GetContestDataOutProto) string { - if contestData.ContestIncident == nil || len(contestData.ContestIncident.Contests) == 0 { - return "No contests found" - } - - var fortId string - if request != nil { - fortId = request.FortId - } else { - fortId = getFortIdFromContest(contestData.ContestIncident.Contests[0].ContestId) - } - - if fortId == "" { - return "No fortId found" - } - - if len(contestData.ContestIncident.Contests) > 1 { - log.Errorf("More than one contest found") - return fmt.Sprintf("More than one contest found in %s", fortId) - } - - contest := contestData.ContestIncident.Contests[0] - - pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) - if err != nil { - log.Printf("Get pokestop %s", err) - return "Error getting pokestop" - } - - if pokestop == nil { - log.Infof("Contest data for pokestop %s not found", fortId) - return fmt.Sprintf("Contest data for pokestop %s not found", fortId) - } - defer unlock() - - pokestop.updatePokestopFromGetContestDataOutProto(contest) - savePokestopRecord(ctx, db, pokestop) - - return fmt.Sprintf("Contest %s", fortId) -} - -func getFortIdFromContest(id string) string { - return strings.Split(id, "-")[0] -} - -func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDetails, request *pogo.GetPokemonSizeLeaderboardEntryProto, contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) string { - fortId := getFortIdFromContest(request.GetContestId()) - - pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) - if err != nil { - log.Printf("Get pokestop %s", err) - return "Error getting pokestop" - } - - if pokestop == nil { - log.Infof("Contest data for pokestop %s not found", fortId) - return fmt.Sprintf("Contest data for pokestop %s not found", fortId) - } - defer unlock() - - pokestop.updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData) - savePokestopRecord(ctx, db, pokestop) - - return fmt.Sprintf("Contest Detail %s", fortId) -} diff --git a/decoder/pokestop_decode.go b/decoder/pokestop_decode.go new file mode 100644 index 00000000..87941c73 --- /dev/null +++ b/decoder/pokestop_decode.go @@ -0,0 +1,475 @@ +package decoder + +import ( + "encoding/json" + "strings" + "time" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" + "golbat/tz" + "golbat/util" +) + +var LureTime int64 = 1800 + +func (stop *Pokestop) updatePokestopFromFort(fortData *pogo.PokemonFortProto, cellId uint64, now int64) *Pokestop { + stop.SetId(fortData.FortId) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) + + stop.SetPartnerId(null.NewString(fortData.PartnerId, fortData.PartnerId != "")) + stop.SetSponsorId(null.IntFrom(int64(fortData.Sponsor))) + stop.SetEnabled(null.BoolFrom(fortData.Enabled)) + stop.SetArScanEligible(null.IntFrom(util.BoolToInt[int64](fortData.IsArScanEligible))) + stop.SetPowerUpPoints(null.IntFrom(int64(fortData.PowerUpProgressPoints))) + powerUpLevel, powerUpEndTimestamp := calculatePowerUpPoints(fortData) + stop.SetPowerUpLevel(powerUpLevel) + stop.SetPowerUpEndTimestamp(powerUpEndTimestamp) + + // lasModifiedMs is also modified when incident happens + lastModifiedTimestamp := fortData.LastModifiedMs / 1000 + stop.SetLastModifiedTimestamp(null.IntFrom(lastModifiedTimestamp)) + + if len(fortData.ActiveFortModifier) > 0 { + lureId := int16(fortData.ActiveFortModifier[0]) + if lureId >= 501 && lureId <= 510 { + lureEnd := lastModifiedTimestamp + LureTime + oldLureEnd := stop.LureExpireTimestamp.ValueOrZero() + if stop.LureId != lureId { + stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) + stop.SetLureId(lureId) + } else { + // wait some time after lure end before a restart in case of timing issue + if now > oldLureEnd+30 { + for now > lureEnd { + lureEnd += LureTime + } + // lure needs to be restarted + stop.SetLureExpireTimestamp(null.IntFrom(lureEnd)) + } + } + } + } + + if fortData.ImageUrl != "" { + stop.SetUrl(null.StringFrom(fortData.ImageUrl)) + } + stop.SetCellId(null.IntFrom(int64(cellId))) + + if stop.Deleted { + stop.SetDeleted(false) + log.Warnf("Cleared Stop with id '%s' is found again in GMO, therefore un-deleted", stop.Id) + // Restore in fort tracker if enabled + if fortTracker != nil { + fortTracker.RestoreFort(stop.Id, cellId, false, time.Now().Unix()) + } + } + return stop +} + +func (stop *Pokestop) updatePokestopFromQuestProto(questProto *pogo.FortSearchOutProto, haveAr bool) string { + + if questProto.ChallengeQuest == nil { + log.Debugf("Received blank quest") + return "Blank quest" + } + questData := questProto.ChallengeQuest.Quest + questTitle := questProto.ChallengeQuest.QuestDisplay.Description + questType := int64(questData.QuestType) + questTarget := int64(questData.Goal.Target) + questTemplate := strings.ToLower(questData.TemplateId) + + conditions := []map[string]any{} + rewards := []map[string]any{} + + for _, conditionData := range questData.Goal.Condition { + condition := make(map[string]any) + infoData := make(map[string]any) + condition["type"] = int(conditionData.Type) + switch conditionData.Type { + case pogo.QuestConditionProto_WITH_BADGE_TYPE: + info := conditionData.GetWithBadgeType() + infoData["amount"] = info.Amount + infoData["badge_rank"] = info.BadgeRank + badgeTypeById := []int{} + for _, badge := range info.BadgeType { + badgeTypeById = append(badgeTypeById, int(badge)) + } + infoData["badge_types"] = badgeTypeById + + case pogo.QuestConditionProto_WITH_ITEM: + info := conditionData.GetWithItem() + if int(info.Item) != 0 { + infoData["item_id"] = int(info.Item) + } + case pogo.QuestConditionProto_WITH_RAID_LEVEL: + info := conditionData.GetWithRaidLevel() + raidLevelById := []int{} + for _, raidLevel := range info.RaidLevel { + raidLevelById = append(raidLevelById, int(raidLevel)) + } + infoData["raid_levels"] = raidLevelById + case pogo.QuestConditionProto_WITH_POKEMON_TYPE: + info := conditionData.GetWithPokemonType() + pokemonTypesById := []int{} + for _, t := range info.PokemonType { + pokemonTypesById = append(pokemonTypesById, int(t)) + } + infoData["pokemon_type_ids"] = pokemonTypesById + case pogo.QuestConditionProto_WITH_POKEMON_CATEGORY: + info := conditionData.GetWithPokemonCategory() + if info.CategoryName != "" { + infoData["category_name"] = info.CategoryName + } + pokemonById := []int{} + for _, pokemon := range info.PokemonIds { + pokemonById = append(pokemonById, int(pokemon)) + } + infoData["pokemon_ids"] = pokemonById + case pogo.QuestConditionProto_WITH_WIN_RAID_STATUS: + case pogo.QuestConditionProto_WITH_THROW_TYPE: + info := conditionData.GetWithThrowType() + if int(info.GetThrowType()) != 0 { // TODO: RDM has ThrowType here, ensure it is the same thing + infoData["throw_type_id"] = int(info.GetThrowType()) + } + infoData["hit"] = info.GetHit() + case pogo.QuestConditionProto_WITH_THROW_TYPE_IN_A_ROW: + info := conditionData.GetWithThrowType() + if int(info.GetThrowType()) != 0 { + infoData["throw_type_id"] = int(info.GetThrowType()) + } + infoData["hit"] = info.GetHit() + case pogo.QuestConditionProto_WITH_LOCATION: + info := conditionData.GetWithLocation() + infoData["cell_ids"] = info.S2CellId + case pogo.QuestConditionProto_WITH_DISTANCE: + info := conditionData.GetWithDistance() + infoData["distance"] = info.DistanceKm + case pogo.QuestConditionProto_WITH_POKEMON_ALIGNMENT: + info := conditionData.GetWithPokemonAlignment() + alignmentIds := []int{} + for _, alignment := range info.Alignment { + alignmentIds = append(alignmentIds, int(alignment)) + } + infoData["alignment_ids"] = alignmentIds + case pogo.QuestConditionProto_WITH_INVASION_CHARACTER: + info := conditionData.GetWithInvasionCharacter() + characterCategoryIds := []int{} + for _, characterCategory := range info.Category { + characterCategoryIds = append(characterCategoryIds, int(characterCategory)) + } + infoData["character_category_ids"] = characterCategoryIds + case pogo.QuestConditionProto_WITH_NPC_COMBAT: + info := conditionData.GetWithNpcCombat() + infoData["win"] = info.RequiresWin + infoData["template_ids"] = info.CombatNpcTrainerId + case pogo.QuestConditionProto_WITH_PLAYER_LEVEL: + info := conditionData.GetWithPlayerLevel() + infoData["level"] = info.Level + case pogo.QuestConditionProto_WITH_BUDDY: + info := conditionData.GetWithBuddy() + if info != nil { + infoData["min_buddy_level"] = int(info.MinBuddyLevel) + infoData["must_be_on_map"] = info.MustBeOnMap + } else { + infoData["min_buddy_level"] = 0 + infoData["must_be_on_map"] = false + } + case pogo.QuestConditionProto_WITH_DAILY_BUDDY_AFFECTION: + info := conditionData.GetWithDailyBuddyAffection() + infoData["min_buddy_affection_earned_today"] = info.MinBuddyAffectionEarnedToday + case pogo.QuestConditionProto_WITH_TEMP_EVO_POKEMON: + info := conditionData.GetWithTempEvoId() + tempEvoIds := []int{} + for _, evolution := range info.MegaForm { + tempEvoIds = append(tempEvoIds, int(evolution)) + } + infoData["raid_pokemon_evolutions"] = tempEvoIds + case pogo.QuestConditionProto_WITH_ITEM_TYPE: + info := conditionData.GetWithItemType() + itemTypes := []int{} + for _, itemType := range info.ItemType { + itemTypes = append(itemTypes, int(itemType)) + } + infoData["item_type_ids"] = itemTypes + case pogo.QuestConditionProto_WITH_RAID_ELAPSED_TIME: + info := conditionData.GetWithElapsedTime() + infoData["time"] = int64(info.ElapsedTimeMs) / 1000 + case pogo.QuestConditionProto_WITH_WIN_GYM_BATTLE_STATUS: + case pogo.QuestConditionProto_WITH_SUPER_EFFECTIVE_CHARGE: + case pogo.QuestConditionProto_WITH_UNIQUE_POKESTOP: + case pogo.QuestConditionProto_WITH_QUEST_CONTEXT: + case pogo.QuestConditionProto_WITH_WIN_BATTLE_STATUS: + case pogo.QuestConditionProto_WITH_CURVE_BALL: + case pogo.QuestConditionProto_WITH_NEW_FRIEND: + case pogo.QuestConditionProto_WITH_DAYS_IN_A_ROW: + case pogo.QuestConditionProto_WITH_WEATHER_BOOST: + case pogo.QuestConditionProto_WITH_DAILY_CAPTURE_BONUS: + case pogo.QuestConditionProto_WITH_DAILY_SPIN_BONUS: + case pogo.QuestConditionProto_WITH_UNIQUE_POKEMON: + case pogo.QuestConditionProto_WITH_BUDDY_INTERESTING_POI: + case pogo.QuestConditionProto_WITH_POKEMON_LEVEL: + case pogo.QuestConditionProto_WITH_SINGLE_DAY: + case pogo.QuestConditionProto_WITH_UNIQUE_POKEMON_TEAM: + case pogo.QuestConditionProto_WITH_MAX_CP: + case pogo.QuestConditionProto_WITH_LUCKY_POKEMON: + case pogo.QuestConditionProto_WITH_LEGENDARY_POKEMON: + case pogo.QuestConditionProto_WITH_GBL_RANK: + case pogo.QuestConditionProto_WITH_CATCHES_IN_A_ROW: + case pogo.QuestConditionProto_WITH_ENCOUNTER_TYPE: + case pogo.QuestConditionProto_WITH_COMBAT_TYPE: + case pogo.QuestConditionProto_WITH_GEOTARGETED_POI: + case pogo.QuestConditionProto_WITH_FRIEND_LEVEL: + case pogo.QuestConditionProto_WITH_STICKER: + case pogo.QuestConditionProto_WITH_POKEMON_CP: + case pogo.QuestConditionProto_WITH_RAID_LOCATION: + case pogo.QuestConditionProto_WITH_FRIENDS_RAID: + case pogo.QuestConditionProto_WITH_POKEMON_COSTUME: + default: + break + } + + if infoData != nil { + condition["info"] = infoData + } + conditions = append(conditions, condition) + } + + for _, rewardData := range questData.QuestRewards { + reward := make(map[string]any) + infoData := make(map[string]any) + reward["type"] = int(rewardData.Type) + switch rewardData.Type { + case pogo.QuestRewardProto_EXPERIENCE: + infoData["amount"] = rewardData.GetExp() + case pogo.QuestRewardProto_ITEM: + info := rewardData.GetItem() + infoData["amount"] = info.Amount + infoData["item_id"] = int(info.Item) + case pogo.QuestRewardProto_STARDUST: + infoData["amount"] = rewardData.GetStardust() + case pogo.QuestRewardProto_CANDY: + info := rewardData.GetCandy() + infoData["amount"] = info.Amount + infoData["pokemon_id"] = int(info.PokemonId) + case pogo.QuestRewardProto_XL_CANDY: + info := rewardData.GetXlCandy() + infoData["amount"] = info.Amount + infoData["pokemon_id"] = int(info.PokemonId) + case pogo.QuestRewardProto_POKEMON_ENCOUNTER: + info := rewardData.GetPokemonEncounter() + if info.IsHiddenDitto { + infoData["pokemon_id"] = 132 + infoData["pokemon_id_display"] = int(info.GetPokemonId()) + } else { + infoData["pokemon_id"] = int(info.GetPokemonId()) + } + if info.ShinyProbability > 0.0 { + infoData["shiny_probability"] = info.ShinyProbability + } + if display := info.PokemonDisplay; display != nil { + if costumeId := int(display.Costume); costumeId != 0 { + infoData["costume_id"] = costumeId + } + if formId := int(display.Form); formId != 0 { + infoData["form_id"] = formId + } + if genderId := int(display.Gender); genderId != 0 { + infoData["gender_id"] = genderId + } + if display.Shiny { + infoData["shiny"] = display.Shiny + } + if background := util.ExtractBackgroundFromDisplay(display); background != nil { + infoData["background"] = background + } + if breadMode := int(display.BreadModeEnum); breadMode != 0 { + infoData["bread_mode"] = breadMode + } + } else { + + } + case pogo.QuestRewardProto_POKECOIN: + infoData["amount"] = rewardData.GetPokecoin() + case pogo.QuestRewardProto_STICKER: + info := rewardData.GetSticker() + infoData["amount"] = info.Amount + infoData["sticker_id"] = info.StickerId + case pogo.QuestRewardProto_MEGA_RESOURCE: + info := rewardData.GetMegaResource() + infoData["amount"] = info.Amount + infoData["pokemon_id"] = int(info.PokemonId) + case pogo.QuestRewardProto_AVATAR_CLOTHING: + case pogo.QuestRewardProto_QUEST: + case pogo.QuestRewardProto_LEVEL_CAP: + case pogo.QuestRewardProto_INCIDENT: + case pogo.QuestRewardProto_PLAYER_ATTRIBUTE: + default: + break + + } + reward["info"] = infoData + rewards = append(rewards, reward) + } + + questConditions, _ := json.Marshal(conditions) + questRewards, _ := json.Marshal(rewards) + questTimestamp := time.Now().Unix() + + questExpiry := null.NewInt(0, false) + + stopTimezone := tz.SearchTimezone(stop.Lat, stop.Lon) + if stopTimezone != "" { + loc, err := time.LoadLocation(stopTimezone) + if err != nil { + log.Warnf("Unrecognised time zone %s at %f,%f", stopTimezone, stop.Lat, stop.Lon) + } else { + year, month, day := time.Now().In(loc).Date() + t := time.Date(year, month, day, 0, 0, 0, 0, loc).AddDate(0, 0, 1) + unixTime := t.Unix() + questExpiry = null.IntFrom(unixTime) + } + } + + if questExpiry.Valid == false { + questExpiry = null.IntFrom(time.Now().Unix() + 24*60*60) // Set expiry to 24 hours from now + } + + if !haveAr { + stop.SetAlternativeQuestType(null.IntFrom(questType)) + stop.SetAlternativeQuestTarget(null.IntFrom(questTarget)) + stop.SetAlternativeQuestTemplate(null.StringFrom(questTemplate)) + stop.SetAlternativeQuestTitle(null.StringFrom(questTitle)) + stop.SetAlternativeQuestConditions(null.StringFrom(string(questConditions))) + stop.SetAlternativeQuestRewards(null.StringFrom(string(questRewards))) + stop.SetAlternativeQuestTimestamp(null.IntFrom(questTimestamp)) + stop.SetAlternativeQuestExpiry(questExpiry) + } else { + stop.SetQuestType(null.IntFrom(questType)) + stop.SetQuestTarget(null.IntFrom(questTarget)) + stop.SetQuestTemplate(null.StringFrom(questTemplate)) + stop.SetQuestTitle(null.StringFrom(questTitle)) + stop.SetQuestConditions(null.StringFrom(string(questConditions))) + stop.SetQuestRewards(null.StringFrom(string(questRewards))) + stop.SetQuestTimestamp(null.IntFrom(questTimestamp)) + stop.SetQuestExpiry(questExpiry) + } + + return questTitle +} + +func (stop *Pokestop) updatePokestopFromFortDetailsProto(fortData *pogo.FortDetailsOutProto) *Pokestop { + stop.SetId(fortData.Id) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) + if len(fortData.ImageUrl) > 0 { + stop.SetUrl(null.StringFrom(fortData.ImageUrl[0])) + } + stop.SetName(null.StringFrom(fortData.Name)) + + if fortData.Description == "" { + stop.SetDescription(null.NewString("", false)) + } else { + stop.SetDescription(null.StringFrom(fortData.Description)) + } + + if fortData.Modifier != nil && len(fortData.Modifier) > 0 { + // DeployingPlayerCodename contains the name of the player if we want that + lureId := int16(fortData.Modifier[0].ModifierType) + lureExpiry := fortData.Modifier[0].ExpirationTimeMs / 1000 + + stop.SetLureId(lureId) + stop.SetLureExpireTimestamp(null.IntFrom(lureExpiry)) + } + + return stop +} + +func (stop *Pokestop) updatePokestopFromGetMapFortsOutProto(fortData *pogo.GetMapFortsOutProto_FortProto) *Pokestop { + stop.SetId(fortData.Id) + stop.SetLat(fortData.Latitude) + stop.SetLon(fortData.Longitude) + + if len(fortData.Image) > 0 { + stop.SetUrl(null.StringFrom(fortData.Image[0].Url)) + } + stop.SetName(null.StringFrom(fortData.Name)) + if stop.Deleted { + log.Debugf("Cleared Stop with id '%s' is found again in GMF, therefore kept deleted", stop.Id) + } + return stop +} + +func (stop *Pokestop) updatePokestopFromGetContestDataOutProto(contest *pogo.ContestProto) { + stop.SetShowcaseRankingStandard(null.IntFrom(int64(contest.GetMetric().GetRankingStandard()))) + stop.SetShowcaseExpiry(null.IntFrom(contest.GetSchedule().GetContestCycle().GetEndTimeMs() / 1000)) + + focusStore := createFocusStoreFromContestProto(contest) + + if len(focusStore) > 1 { + log.Warnf("SHOWCASE: we got more than one showcase focus: %v", focusStore) + } + + for key, focus := range focusStore { + focus["type"] = key + jsonBytes, err := json.Marshal(focus) + if err != nil { + log.Errorf("SHOWCASE: Stop '%s' - Focus '%v' marshalling failed: %s", stop.Id, focus, err) + } + stop.SetShowcaseFocus(null.StringFrom(string(jsonBytes))) + // still support old format - probably still required to filter in external tools + stop.extractShowcasePokemonInfoDeprecated(key, focus) + } +} + +func (stop *Pokestop) updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) { + type contestEntry struct { + Rank int `json:"rank"` + Score float64 `json:"score"` + PokemonId int `json:"pokemon_id"` + Form int `json:"form"` + Costume int `json:"costume"` + Gender int `json:"gender"` + Shiny bool `json:"shiny"` + TempEvolution int `json:"temp_evolution"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms"` + Alignment int `json:"alignment"` + Badge int `json:"badge"` + Background *int64 `json:"background,omitempty"` + } + type contestJson struct { + TotalEntries int `json:"total_entries"` + LastUpdate int64 `json:"last_update"` + ContestEntries []contestEntry `json:"contest_entries"` + } + + j := contestJson{LastUpdate: time.Now().Unix()} + j.TotalEntries = int(contestData.TotalEntries) + + for _, entry := range contestData.GetContestEntries() { + rank := entry.GetRank() + if rank > 3 { + break + } + j.ContestEntries = append(j.ContestEntries, contestEntry{ + Rank: int(rank), + Score: entry.GetScore(), + PokemonId: int(entry.GetPokedexId()), + Form: int(entry.GetPokemonDisplay().Form), + Costume: int(entry.GetPokemonDisplay().Costume), + Gender: int(entry.GetPokemonDisplay().Gender), + Shiny: entry.GetPokemonDisplay().Shiny, + TempEvolution: int(entry.GetPokemonDisplay().CurrentTempEvolution), + TempEvolutionFinishMs: entry.GetPokemonDisplay().TemporaryEvolutionFinishMs, + Alignment: int(entry.GetPokemonDisplay().Alignment), + Badge: int(entry.GetPokemonDisplay().PokemonBadge), + Background: util.ExtractBackgroundFromDisplay(entry.PokemonDisplay), + }) + + } + jsonString, _ := json.Marshal(j) + stop.SetShowcaseRankings(null.StringFrom(string(jsonString))) +} diff --git a/decoder/pokestop_process.go b/decoder/pokestop_process.go new file mode 100644 index 00000000..69dd78da --- /dev/null +++ b/decoder/pokestop_process.go @@ -0,0 +1,167 @@ +package decoder + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/paulmach/orb/geojson" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func UpdatePokestopRecordWithFortDetailsOutProto(ctx context.Context, db db.DbDetails, fort *pogo.FortDetailsOutProto) string { + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, fort.Id) + if err != nil { + log.Printf("Update pokestop %s", err) + return fmt.Sprintf("Error %s", err) + } + defer unlock() + + pokestop.updatePokestopFromFortDetailsProto(fort) + + updatePokestopGetMapFortCache(pokestop) + savePokestopRecord(ctx, db, pokestop) + return fmt.Sprintf("%s %s", fort.Id, fort.Name) +} + +func UpdatePokestopWithQuest(ctx context.Context, db db.DbDetails, quest *pogo.FortSearchOutProto, haveAr bool) string { + haveArStr := "NoAR" + if haveAr { + haveArStr = "AR" + } + + if quest.ChallengeQuest == nil { + statsCollector.IncDecodeQuest("error", "no_quest") + return fmt.Sprintf("%s %s Blank quest", quest.FortId, haveArStr) + } + + statsCollector.IncDecodeQuest("ok", haveArStr) + + pokestop, unlock, err := getOrCreatePokestopRecord(ctx, db, quest.FortId) + if err != nil { + log.Printf("Update quest %s", err) + return fmt.Sprintf("error %s", err) + } + defer unlock() + + questTitle := pokestop.updatePokestopFromQuestProto(quest, haveAr) + + updatePokestopGetMapFortCache(pokestop) + savePokestopRecord(ctx, db, pokestop) + + areas := MatchStatsGeofence(pokestop.Lat, pokestop.Lon) + updateQuestStats(pokestop, haveAr, areas) + + return fmt.Sprintf("%s %s %s", quest.FortId, haveArStr, questTitle) +} + +func ClearQuestsWithinGeofence(ctx context.Context, dbDetails db.DbDetails, geofence *geojson.Feature) { + started := time.Now() + rows, err := db.RemoveQuests(ctx, dbDetails, geofence) + if err != nil { + log.Errorf("ClearQuest: Error removing quests: %s", err) + return + } + ClearPokestopCache() + log.Infof("ClearQuest: Removed quests from %d pokestops in %s", rows, time.Since(started)) +} + +func GetQuestStatusWithGeofence(dbDetails db.DbDetails, geofence *geojson.Feature) db.QuestStatus { + res, err := db.GetQuestStatus(dbDetails, geofence) + if err != nil { + log.Errorf("QuestStatus: Error retrieving quests: %s", err) + return db.QuestStatus{} + } + return res +} + +func UpdatePokestopRecordWithGetMapFortsOutProto(ctx context.Context, db db.DbDetails, mapFort *pogo.GetMapFortsOutProto_FortProto) (bool, string) { + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, mapFort.Id) + if err != nil { + log.Printf("Update pokestop %s", err) + return false, fmt.Sprintf("Error %s", err) + } + + if pokestop == nil { + return false, "" + } + defer unlock() + + pokestop.updatePokestopFromGetMapFortsOutProto(mapFort) + savePokestopRecord(ctx, db, pokestop) + return true, fmt.Sprintf("%s %s", mapFort.Id, mapFort.Name) +} + +func GetPokestopPositions(details db.DbDetails, geofence *geojson.Feature) ([]db.QuestLocation, error) { + return db.GetPokestopPositions(details, geofence) +} + +func UpdatePokestopWithContestData(ctx context.Context, db db.DbDetails, request *pogo.GetContestDataProto, contestData *pogo.GetContestDataOutProto) string { + if contestData.ContestIncident == nil || len(contestData.ContestIncident.Contests) == 0 { + return "No contests found" + } + + var fortId string + if request != nil { + fortId = request.FortId + } else { + fortId = getFortIdFromContest(contestData.ContestIncident.Contests[0].ContestId) + } + + if fortId == "" { + return "No fortId found" + } + + if len(contestData.ContestIncident.Contests) > 1 { + log.Errorf("More than one contest found") + return fmt.Sprintf("More than one contest found in %s", fortId) + } + + contest := contestData.ContestIncident.Contests[0] + + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) + if err != nil { + log.Printf("Get pokestop %s", err) + return "Error getting pokestop" + } + + if pokestop == nil { + log.Infof("Contest data for pokestop %s not found", fortId) + return fmt.Sprintf("Contest data for pokestop %s not found", fortId) + } + defer unlock() + + pokestop.updatePokestopFromGetContestDataOutProto(contest) + savePokestopRecord(ctx, db, pokestop) + + return fmt.Sprintf("Contest %s", fortId) +} + +func getFortIdFromContest(id string) string { + return strings.Split(id, "-")[0] +} + +func UpdatePokestopWithPokemonSizeContestEntry(ctx context.Context, db db.DbDetails, request *pogo.GetPokemonSizeLeaderboardEntryProto, contestData *pogo.GetPokemonSizeLeaderboardEntryOutProto) string { + fortId := getFortIdFromContest(request.GetContestId()) + + pokestop, unlock, err := getPokestopRecordForUpdate(ctx, db, fortId) + if err != nil { + log.Printf("Get pokestop %s", err) + return "Error getting pokestop" + } + + if pokestop == nil { + log.Infof("Contest data for pokestop %s not found", fortId) + return fmt.Sprintf("Contest data for pokestop %s not found", fortId) + } + defer unlock() + + pokestop.updatePokestopFromGetPokemonSizeContestEntryOutProto(contestData) + savePokestopRecord(ctx, db, pokestop) + + return fmt.Sprintf("Contest Detail %s", fortId) +} diff --git a/decoder/pokestop_showcase.go b/decoder/pokestop_showcase.go index 42876295..44fba2ba 100644 --- a/decoder/pokestop_showcase.go +++ b/decoder/pokestop_showcase.go @@ -3,8 +3,8 @@ package decoder import ( "golbat/pogo" + "github.com/guregu/null/v6" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) type contestFocusType string diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go new file mode 100644 index 00000000..f2116179 --- /dev/null +++ b/decoder/pokestop_state.go @@ -0,0 +1,397 @@ +package decoder + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "time" + + "github.com/guregu/null/v6" + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + + "golbat/config" + "golbat/db" + "golbat/webhooks" +) + +func loadPokestopFromDatabase(ctx context.Context, db db.DbDetails, fortId string, pokestop *Pokestop) error { + err := db.GeneralDb.GetContext(ctx, pokestop, + `SELECT pokestop.id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, + pokestop.updated, quest_type, quest_timestamp, quest_target, quest_conditions, + quest_rewards, quest_template, quest_title, + alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, + alternative_quest_conditions, alternative_quest_rewards, + alternative_quest_template, alternative_quest_title, cell_id, deleted, lure_id, sponsor_id, partner_id, + ar_scan_eligible, power_up_points, power_up_level, power_up_end_timestamp, + quest_expiry, alternative_quest_expiry, description, showcase_pokemon_id, showcase_pokemon_form_id, + showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings + FROM pokestop + WHERE pokestop.id = ? `, fortId) + statsCollector.IncDbQuery("select pokestop", err) + return err +} + +// PeekPokestopRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekPokestopRecord(fortId string) (*Pokestop, func(), error) { + if item := pokestopCache.Get(fortId); item != nil { + pokestop := item.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil + } + return nil, nil, nil +} + +// getPokestopRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getPokestopRecordReadOnly(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + // Check cache first + if item := pokestopCache.Get(fortId); item != nil { + pokestop := item.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil + } + + dbPokestop := Pokestop{} + err := loadPokestopFromDatabase(ctx, db, fortId, &dbPokestop) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + + dbPokestop.ClearDirty() + + // Atomically cache the loaded Pokestop - if another goroutine raced us, + // we'll get their Pokestop and use that instead (ensuring same mutex) + existingPokestop, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + // Only called if key doesn't exist - our Pokestop wins + if config.Config.TestFortInMemory { + fortRtreeUpdatePokestopOnGet(&dbPokestop) + } + return &dbPokestop + }) + + pokestop := existingPokestop.Value() + pokestop.Lock() + return pokestop, func() { pokestop.Unlock() }, nil +} + +// getPokestopRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Use when modifying the Pokestop. +// Caller MUST call returned unlock function if non-nil. +func getPokestopRecordForUpdate(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + pokestop, unlock, err := getPokestopRecordReadOnly(ctx, db, fortId) + if err != nil || pokestop == nil { + return nil, nil, err + } + pokestop.snapshotOldValues() + return pokestop, unlock, nil +} + +// getOrCreatePokestopRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreatePokestopRecord(ctx context.Context, db db.DbDetails, fortId string) (*Pokestop, func(), error) { + // Create new Pokestop atomically - function only called if key doesn't exist + pokestopItem, _ := pokestopCache.GetOrSetFunc(fortId, func() *Pokestop { + return &Pokestop{Id: fortId, newRecord: true} + }) + + pokestop := pokestopItem.Value() + pokestop.Lock() + + if pokestop.newRecord { + // We should attempt to load from database + err := loadPokestopFromDatabase(ctx, db, fortId, pokestop) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + pokestop.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + pokestop.newRecord = false + pokestop.ClearDirty() + if config.Config.TestFortInMemory { + fortRtreeUpdatePokestopOnGet(pokestop) + } + } + } + + pokestop.snapshotOldValues() + return pokestop, func() { pokestop.Unlock() }, nil +} + +type QuestWebhook struct { + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + PokestopName string `json:"pokestop_name"` + Type null.Int `json:"type"` + Target null.Int `json:"target"` + Template null.String `json:"template"` + Title null.String `json:"title"` + Conditions json.RawMessage `json:"conditions"` + Rewards json.RawMessage `json:"rewards"` + Updated int64 `json:"updated"` + ArScanEligible int64 `json:"ar_scan_eligible"` + PokestopUrl string `json:"pokestop_url"` + WithAr bool `json:"with_ar"` +} + +type PokestopWebhook struct { + PokestopId string `json:"pokestop_id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + Url string `json:"url"` + LureExpiration int64 `json:"lure_expiration"` + LastModified int64 `json:"last_modified"` + Enabled bool `json:"enabled"` + LureId int16 `json:"lure_id"` + ArScanEligible int64 `json:"ar_scan_eligible"` + PowerUpLevel int64 `json:"power_up_level"` + PowerUpPoints int64 `json:"power_up_points"` + PowerUpEndTimestamp int64 `json:"power_up_end_timestamp"` + Updated int64 `json:"updated"` + ShowcaseFocus null.String `json:"showcase_focus"` + ShowcasePokemonId null.Int `json:"showcase_pokemon_id"` + ShowcasePokemonFormId null.Int `json:"showcase_pokemon_form_id"` + ShowcasePokemonTypeId null.Int `json:"showcase_pokemon_type_id"` + ShowcaseRankingStandard null.Int `json:"showcase_ranking_standard"` + ShowcaseExpiry null.Int `json:"showcase_expiry"` + ShowcaseRankings json.RawMessage `json:"showcase_rankings"` +} + +func createPokestopFortWebhooks(stop *Pokestop) { + fort := InitWebHookFortFromPokestop(stop) + if stop.newRecord { + CreateFortWebHooks(nil, fort, NEW) + } else { + // Build old fort from saved old values + oldFort := &FortWebhook{ + Type: POKESTOP.String(), + Id: stop.Id, + Name: stop.oldValues.Name.Ptr(), + ImageUrl: stop.oldValues.Url.Ptr(), + Description: stop.oldValues.Description.Ptr(), + Location: Location{Latitude: stop.oldValues.Lat, Longitude: stop.oldValues.Lon}, + } + CreateFortWebHooks(oldFort, fort, EDIT) + } +} + +func createPokestopWebhooks(stop *Pokestop) { + + areas := MatchStatsGeofence(stop.Lat, stop.Lon) + + pokestopName := "Unknown" + if stop.Name.Valid { + pokestopName = stop.Name.String + } + + if stop.AlternativeQuestType.Valid && (stop.newRecord || stop.AlternativeQuestType != stop.oldValues.AlternativeQuestType) { + questHook := QuestWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Type: stop.AlternativeQuestType, + Target: stop.AlternativeQuestTarget, + Template: stop.AlternativeQuestTemplate, + Title: stop.AlternativeQuestTitle, + Conditions: json.RawMessage(stop.AlternativeQuestConditions.ValueOrZero()), + Rewards: json.RawMessage(stop.AlternativeQuestRewards.ValueOrZero()), + Updated: stop.Updated, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PokestopUrl: stop.Url.ValueOrZero(), + WithAr: false, + } + webhooksSender.AddMessage(webhooks.Quest, questHook, areas) + } + + if stop.QuestType.Valid && (stop.newRecord || stop.QuestType != stop.oldValues.QuestType) { + questHook := QuestWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + PokestopName: pokestopName, + Type: stop.QuestType, + Target: stop.QuestTarget, + Template: stop.QuestTemplate, + Title: stop.QuestTitle, + Conditions: json.RawMessage(stop.QuestConditions.ValueOrZero()), + Rewards: json.RawMessage(stop.QuestRewards.ValueOrZero()), + Updated: stop.Updated, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PokestopUrl: stop.Url.ValueOrZero(), + WithAr: true, + } + webhooksSender.AddMessage(webhooks.Quest, questHook, areas) + } + if (stop.newRecord && (stop.LureId != 0 || stop.PowerUpEndTimestamp.ValueOrZero() != 0)) || (!stop.newRecord && ((stop.LureExpireTimestamp != stop.oldValues.LureExpireTimestamp && stop.LureId != 0) || stop.PowerUpEndTimestamp != stop.oldValues.PowerUpEndTimestamp)) { + var showcaseRankings json.RawMessage + if stop.ShowcaseRankings.Valid { + showcaseRankings = json.RawMessage(stop.ShowcaseRankings.ValueOrZero()) + } + + pokestopHook := PokestopWebhook{ + PokestopId: stop.Id, + Latitude: stop.Lat, + Longitude: stop.Lon, + Name: pokestopName, + Url: stop.Url.ValueOrZero(), + LureExpiration: stop.LureExpireTimestamp.ValueOrZero(), + LastModified: stop.LastModifiedTimestamp.ValueOrZero(), + Enabled: stop.Enabled.ValueOrZero(), + LureId: stop.LureId, + ArScanEligible: stop.ArScanEligible.ValueOrZero(), + PowerUpLevel: stop.PowerUpLevel.ValueOrZero(), + PowerUpPoints: stop.PowerUpPoints.ValueOrZero(), + PowerUpEndTimestamp: stop.PowerUpEndTimestamp.ValueOrZero(), + Updated: stop.Updated, + ShowcaseFocus: stop.ShowcaseFocus, + ShowcasePokemonId: stop.ShowcasePokemon, + ShowcasePokemonFormId: stop.ShowcasePokemonForm, + ShowcasePokemonTypeId: stop.ShowcasePokemonType, + ShowcaseRankingStandard: stop.ShowcaseRankingStandard, + ShowcaseExpiry: stop.ShowcaseExpiry, + ShowcaseRankings: showcaseRankings, + } + + webhooksSender.AddMessage(webhooks.Pokestop, pokestopHook, areas) + } +} + +func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop) { + now := time.Now().Unix() + if !pokestop.IsNewRecord() && !pokestop.IsDirty() { + // default debounce is 15 minutes (900s). If reduce_updates is enabled, use 12 hours. + if pokestop.Updated > now-GetUpdateThreshold(900) { + // if a pokestop is unchanged, but we did see it again after 15 minutes, then save again + return + } + } + pokestop.Updated = now + + if pokestop.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Pokestop", pokestop.Id, pokestop.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, ` + INSERT INTO pokestop ( + id, lat, lon, name, url, enabled, lure_expire_timestamp, last_modified_timestamp, quest_type, + quest_timestamp, quest_target, quest_conditions, quest_rewards, quest_template, quest_title, + alternative_quest_type, alternative_quest_timestamp, alternative_quest_target, + alternative_quest_conditions, alternative_quest_rewards, alternative_quest_template, + alternative_quest_title, cell_id, lure_id, sponsor_id, partner_id, ar_scan_eligible, + power_up_points, power_up_level, power_up_end_timestamp, updated, first_seen_timestamp, + quest_expiry, alternative_quest_expiry, description, showcase_focus, showcase_pokemon_id, + showcase_pokemon_form_id, showcase_pokemon_type_id, showcase_ranking_standard, showcase_expiry, showcase_rankings + ) + VALUES ( + :id, :lat, :lon, :name, :url, :enabled, :lure_expire_timestamp, :last_modified_timestamp, :quest_type, + :quest_timestamp, :quest_target, :quest_conditions, :quest_rewards, :quest_template, :quest_title, + :alternative_quest_type, :alternative_quest_timestamp, :alternative_quest_target, + :alternative_quest_conditions, :alternative_quest_rewards, :alternative_quest_template, + :alternative_quest_title, :cell_id, :lure_id, :sponsor_id, :partner_id, :ar_scan_eligible, + :power_up_points, :power_up_level, :power_up_end_timestamp, + UNIX_TIMESTAMP(), UNIX_TIMESTAMP(), + :quest_expiry, :alternative_quest_expiry, :description, :showcase_focus, :showcase_pokemon_id, + :showcase_pokemon_form_id, :showcase_pokemon_type_id, :showcase_ranking_standard, :showcase_expiry, :showcase_rankings)`, + pokestop) + + statsCollector.IncDbQuery("insert pokestop", err) + //log.Debugf("Insert pokestop %s %+v", pokestop.Id, pokestop) + if err != nil { + log.Errorf("insert pokestop: %s", err) + return + } + + _, _ = res, err + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Pokestop", pokestop.Id, pokestop.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, ` + UPDATE pokestop SET + lat = :lat, + lon = :lon, + name = :name, + url = :url, + enabled = :enabled, + lure_expire_timestamp = :lure_expire_timestamp, + last_modified_timestamp = :last_modified_timestamp, + updated = :updated, + quest_type = :quest_type, + quest_timestamp = :quest_timestamp, + quest_target = :quest_target, + quest_conditions = :quest_conditions, + quest_rewards = :quest_rewards, + quest_template = :quest_template, + quest_title = :quest_title, + alternative_quest_type = :alternative_quest_type, + alternative_quest_timestamp = :alternative_quest_timestamp, + alternative_quest_target = :alternative_quest_target, + alternative_quest_conditions = :alternative_quest_conditions, + alternative_quest_rewards = :alternative_quest_rewards, + alternative_quest_template = :alternative_quest_template, + alternative_quest_title = :alternative_quest_title, + cell_id = :cell_id, + lure_id = :lure_id, + deleted = :deleted, + sponsor_id = :sponsor_id, + partner_id = :partner_id, + ar_scan_eligible = :ar_scan_eligible, + power_up_points = :power_up_points, + power_up_level = :power_up_level, + power_up_end_timestamp = :power_up_end_timestamp, + quest_expiry = :quest_expiry, + alternative_quest_expiry = :alternative_quest_expiry, + description = :description, + showcase_focus = :showcase_focus, + showcase_pokemon_id = :showcase_pokemon_id, + showcase_pokemon_form_id = :showcase_pokemon_form_id, + showcase_pokemon_type_id = :showcase_pokemon_type_id, + showcase_ranking_standard = :showcase_ranking_standard, + showcase_expiry = :showcase_expiry, + showcase_rankings = :showcase_rankings + WHERE id = :id`, + pokestop, + ) + statsCollector.IncDbQuery("update pokestop", err) + //log.Debugf("Update pokestop %s %+v", pokestop.Id, pokestop) + if err != nil { + log.Errorf("update pokestop %s: %s", pokestop.Id, err) + return + } + _ = res + } + //pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + if dbDebugEnabled { + pokestop.changedFields = pokestop.changedFields[:0] + } + + createPokestopWebhooks(pokestop) + createPokestopFortWebhooks(pokestop) + if pokestop.IsNewRecord() { + pokestopCache.Set(pokestop.Id, pokestop, ttlcache.DefaultTTL) + pokestop.newRecord = false + } + pokestop.ClearDirty() + +} + +func updatePokestopGetMapFortCache(pokestop *Pokestop) { + storedGetMapFort := getMapFortsCache.Get(pokestop.Id) + if storedGetMapFort != nil { + getMapFort := storedGetMapFort.Value() + getMapFortsCache.Delete(pokestop.Id) + pokestop.updatePokestopFromGetMapFortsOutProto(getMapFort) + log.Debugf("Updated Gym using stored getMapFort: %s", pokestop.Id) + } +} diff --git a/decoder/routes.go b/decoder/routes.go index e707b806..214d7812 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -1,21 +1,9 @@ package decoder import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" "sync" - "time" - "golbat/db" - "golbat/pogo" - "golbat/util" - - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" ) // Route struct. @@ -292,238 +280,3 @@ func (r *Route) SetWaypoints(v string) { } } } - -func loadRouteFromDatabase(ctx context.Context, db db.DbDetails, routeId string, route *Route) error { - err := db.GeneralDb.GetContext(ctx, route, - `SELECT * FROM route WHERE route.id = ?`, routeId) - statsCollector.IncDbQuery("select route", err) - return err -} - -// peekRouteRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func peekRouteRecord(routeId string) (*Route, func(), error) { - if item := routeCache.Get(routeId); item != nil { - route := item.Value() - route.Lock() - return route, func() { route.Unlock() }, nil - } - return nil, nil, nil -} - -// getRouteRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getRouteRecordReadOnly(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { - // Check cache first - if item := routeCache.Get(routeId); item != nil { - route := item.Value() - route.Lock() - return route, func() { route.Unlock() }, nil - } - - dbRoute := Route{} - err := loadRouteFromDatabase(ctx, db, routeId, &dbRoute) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbRoute.ClearDirty() - - // Atomically cache the loaded Route - if another goroutine raced us, - // we'll get their Route and use that instead (ensuring same mutex) - existingRoute, _ := routeCache.GetOrSetFunc(routeId, func() *Route { - return &dbRoute - }) - - route := existingRoute.Value() - route.Lock() - return route, func() { route.Unlock() }, nil -} - -// getRouteRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Caller MUST call returned unlock function if non-nil. -func getRouteRecordForUpdate(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { - route, unlock, err := getRouteRecordReadOnly(ctx, db, routeId) - if err != nil || route == nil { - return nil, nil, err - } - route.snapshotOldValues() - return route, unlock, nil -} - -// getOrCreateRouteRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreateRouteRecord(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { - // Create new Route atomically - function only called if key doesn't exist - routeItem, _ := routeCache.GetOrSetFunc(routeId, func() *Route { - return &Route{Id: routeId, newRecord: true} - }) - - route := routeItem.Value() - route.Lock() - - if route.newRecord { - // We should attempt to load from database - err := loadRouteFromDatabase(ctx, db, routeId, route) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - route.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - route.newRecord = false - route.ClearDirty() - } - } - - route.snapshotOldValues() - return route, func() { route.Unlock() }, nil -} - -func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { - // Skip save if not dirty and not new, unless 15-minute debounce expired - if !route.IsDirty() && !route.IsNewRecord() { - if route.Updated > time.Now().Unix()-GetUpdateThreshold(900) { - // if a route is unchanged, but we did see it again after 15 minutes, then save again - return nil - } - } - - route.Updated = time.Now().Unix() - - if route.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Route", route.Id, route.changedFields) - } - _, err := db.GeneralDb.NamedExecContext(ctx, - ` - INSERT INTO route ( - id, name, shortcode, description, distance_meters, - duration_seconds, end_fort_id, end_image, - end_lat, end_lon, image, image_border_color, - reversible, start_fort_id, start_image, - start_lat, start_lon, tags, type, - updated, version, waypoints - ) - VALUES - ( - :id, :name, :shortcode, :description, :distance_meters, - :duration_seconds, :end_fort_id, - :end_image, :end_lat, :end_lon, :image, - :image_border_color, :reversible, - :start_fort_id, :start_image, :start_lat, - :start_lon, :tags, :type, :updated, - :version, :waypoints - ) - `, - route, - ) - - statsCollector.IncDbQuery("insert route", err) - if err != nil { - return fmt.Errorf("insert route error: %w", err) - } - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) - } - _, err := db.GeneralDb.NamedExecContext(ctx, - ` - UPDATE route SET - name = :name, - shortcode = :shortcode, - description = :description, - distance_meters = :distance_meters, - duration_seconds = :duration_seconds, - end_fort_id = :end_fort_id, - end_image = :end_image, - end_lat = :end_lat, - end_lon = :end_lon, - image = :image, - image_border_color = :image_border_color, - reversible = :reversible, - start_fort_id = :start_fort_id, - start_image = :start_image, - start_lat = :start_lat, - start_lon = :start_lon, - tags = :tags, - type = :type, - updated = :updated, - version = :version, - waypoints = :waypoints - WHERE id = :id`, - route, - ) - - statsCollector.IncDbQuery("update route", err) - if err != nil { - return fmt.Errorf("update route error %w", err) - } - } - - if dbDebugEnabled { - route.changedFields = route.changedFields[:0] - } - route.ClearDirty() - if route.IsNewRecord() { - routeCache.Set(route.Id, route, ttlcache.DefaultTTL) - route.newRecord = false - } - return nil -} - -func (route *Route) updateFromSharedRouteProto(sharedRouteProto *pogo.SharedRouteProto) { - route.SetName(sharedRouteProto.GetName()) - if sharedRouteProto.GetShortCode() != "" { - route.SetShortcode(sharedRouteProto.GetShortCode()) - } - description := sharedRouteProto.GetDescription() - // NOTE: Some descriptions have more than 255 runes, which won't fit in our - // varchar(255). - if truncateStr, truncated := util.TruncateUTF8(description, 255); truncated { - log.Warnf("truncating description for route id '%s'. Orig description: %s", - route.Id, - description, - ) - description = truncateStr - } - route.SetDescription(description) - route.SetDistanceMeters(sharedRouteProto.GetRouteDistanceMeters()) - route.SetDurationSeconds(sharedRouteProto.GetRouteDurationSeconds()) - route.SetEndFortId(sharedRouteProto.GetEndPoi().GetAnchor().GetFortId()) - route.SetEndImage(sharedRouteProto.GetEndPoi().GetImageUrl()) - route.SetEndLat(sharedRouteProto.GetEndPoi().GetAnchor().GetLatDegrees()) - route.SetEndLon(sharedRouteProto.GetEndPoi().GetAnchor().GetLngDegrees()) - route.SetImage(sharedRouteProto.GetImage().GetImageUrl()) - route.SetImageBorderColor(sharedRouteProto.GetImage().GetBorderColorHex()) - route.SetReversible(sharedRouteProto.GetReversible()) - route.SetStartFortId(sharedRouteProto.GetStartPoi().GetAnchor().GetFortId()) - route.SetStartImage(sharedRouteProto.GetStartPoi().GetImageUrl()) - route.SetStartLat(sharedRouteProto.GetStartPoi().GetAnchor().GetLatDegrees()) - route.SetStartLon(sharedRouteProto.GetStartPoi().GetAnchor().GetLngDegrees()) - route.SetType(int8(sharedRouteProto.GetType())) - route.SetVersion(sharedRouteProto.GetVersion()) - waypoints, _ := json.Marshal(sharedRouteProto.GetWaypoints()) - route.SetWaypoints(string(waypoints)) - - if len(sharedRouteProto.GetTags()) > 0 { - tags, _ := json.Marshal(sharedRouteProto.GetTags()) - route.SetTags(null.StringFrom(string(tags))) - } -} - -func UpdateRouteRecordWithSharedRouteProto(ctx context.Context, db db.DbDetails, sharedRouteProto *pogo.SharedRouteProto) error { - route, unlock, err := getOrCreateRouteRecord(ctx, db, sharedRouteProto.GetId()) - if err != nil { - return err - } - defer unlock() - - route.updateFromSharedRouteProto(sharedRouteProto) - saveError := saveRouteRecord(ctx, db, route) - return saveError -} diff --git a/decoder/routes_decode.go b/decoder/routes_decode.go new file mode 100644 index 00000000..cb72fd5a --- /dev/null +++ b/decoder/routes_decode.go @@ -0,0 +1,51 @@ +package decoder + +import ( + "encoding/json" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" + "golbat/util" +) + +func (route *Route) updateFromSharedRouteProto(sharedRouteProto *pogo.SharedRouteProto) { + route.SetName(sharedRouteProto.GetName()) + if sharedRouteProto.GetShortCode() != "" { + route.SetShortcode(sharedRouteProto.GetShortCode()) + } + description := sharedRouteProto.GetDescription() + // NOTE: Some descriptions have more than 255 runes, which won't fit in our + // varchar(255). + if truncateStr, truncated := util.TruncateUTF8(description, 255); truncated { + log.Warnf("truncating description for route id '%s'. Orig description: %s", + route.Id, + description, + ) + description = truncateStr + } + route.SetDescription(description) + route.SetDistanceMeters(sharedRouteProto.GetRouteDistanceMeters()) + route.SetDurationSeconds(sharedRouteProto.GetRouteDurationSeconds()) + route.SetEndFortId(sharedRouteProto.GetEndPoi().GetAnchor().GetFortId()) + route.SetEndImage(sharedRouteProto.GetEndPoi().GetImageUrl()) + route.SetEndLat(sharedRouteProto.GetEndPoi().GetAnchor().GetLatDegrees()) + route.SetEndLon(sharedRouteProto.GetEndPoi().GetAnchor().GetLngDegrees()) + route.SetImage(sharedRouteProto.GetImage().GetImageUrl()) + route.SetImageBorderColor(sharedRouteProto.GetImage().GetBorderColorHex()) + route.SetReversible(sharedRouteProto.GetReversible()) + route.SetStartFortId(sharedRouteProto.GetStartPoi().GetAnchor().GetFortId()) + route.SetStartImage(sharedRouteProto.GetStartPoi().GetImageUrl()) + route.SetStartLat(sharedRouteProto.GetStartPoi().GetAnchor().GetLatDegrees()) + route.SetStartLon(sharedRouteProto.GetStartPoi().GetAnchor().GetLngDegrees()) + route.SetType(int8(sharedRouteProto.GetType())) + route.SetVersion(sharedRouteProto.GetVersion()) + waypoints, _ := json.Marshal(sharedRouteProto.GetWaypoints()) + route.SetWaypoints(string(waypoints)) + + if len(sharedRouteProto.GetTags()) > 0 { + tags, _ := json.Marshal(sharedRouteProto.GetTags()) + route.SetTags(null.StringFrom(string(tags))) + } +} diff --git a/decoder/routes_process.go b/decoder/routes_process.go new file mode 100644 index 00000000..a0c5c83b --- /dev/null +++ b/decoder/routes_process.go @@ -0,0 +1,20 @@ +package decoder + +import ( + "context" + + "golbat/db" + "golbat/pogo" +) + +func UpdateRouteRecordWithSharedRouteProto(ctx context.Context, db db.DbDetails, sharedRouteProto *pogo.SharedRouteProto) error { + route, unlock, err := getOrCreateRouteRecord(ctx, db, sharedRouteProto.GetId()) + if err != nil { + return err + } + defer unlock() + + route.updateFromSharedRouteProto(sharedRouteProto) + saveError := saveRouteRecord(ctx, db, route) + return saveError +} diff --git a/decoder/routes_state.go b/decoder/routes_state.go new file mode 100644 index 00000000..8538cc86 --- /dev/null +++ b/decoder/routes_state.go @@ -0,0 +1,196 @@ +package decoder + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/jellydator/ttlcache/v3" + + "golbat/db" +) + +func loadRouteFromDatabase(ctx context.Context, db db.DbDetails, routeId string, route *Route) error { + err := db.GeneralDb.GetContext(ctx, route, + `SELECT * FROM route WHERE route.id = ?`, routeId) + statsCollector.IncDbQuery("select route", err) + return err +} + +// peekRouteRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekRouteRecord(routeId string) (*Route, func(), error) { + if item := routeCache.Get(routeId); item != nil { + route := item.Value() + route.Lock() + return route, func() { route.Unlock() }, nil + } + return nil, nil, nil +} + +// getRouteRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getRouteRecordReadOnly(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + // Check cache first + if item := routeCache.Get(routeId); item != nil { + route := item.Value() + route.Lock() + return route, func() { route.Unlock() }, nil + } + + dbRoute := Route{} + err := loadRouteFromDatabase(ctx, db, routeId, &dbRoute) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbRoute.ClearDirty() + + // Atomically cache the loaded Route - if another goroutine raced us, + // we'll get their Route and use that instead (ensuring same mutex) + existingRoute, _ := routeCache.GetOrSetFunc(routeId, func() *Route { + return &dbRoute + }) + + route := existingRoute.Value() + route.Lock() + return route, func() { route.Unlock() }, nil +} + +// getRouteRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getRouteRecordForUpdate(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + route, unlock, err := getRouteRecordReadOnly(ctx, db, routeId) + if err != nil || route == nil { + return nil, nil, err + } + route.snapshotOldValues() + return route, unlock, nil +} + +// getOrCreateRouteRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateRouteRecord(ctx context.Context, db db.DbDetails, routeId string) (*Route, func(), error) { + // Create new Route atomically - function only called if key doesn't exist + routeItem, _ := routeCache.GetOrSetFunc(routeId, func() *Route { + return &Route{Id: routeId, newRecord: true} + }) + + route := routeItem.Value() + route.Lock() + + if route.newRecord { + // We should attempt to load from database + err := loadRouteFromDatabase(ctx, db, routeId, route) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + route.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + route.newRecord = false + route.ClearDirty() + } + } + + route.snapshotOldValues() + return route, func() { route.Unlock() }, nil +} + +func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { + // Skip save if not dirty and not new, unless 15-minute debounce expired + if !route.IsDirty() && !route.IsNewRecord() { + if route.Updated > time.Now().Unix()-GetUpdateThreshold(900) { + // if a route is unchanged, but we did see it again after 15 minutes, then save again + return nil + } + } + + route.Updated = time.Now().Unix() + + if route.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Route", route.Id, route.changedFields) + } + _, err := db.GeneralDb.NamedExecContext(ctx, + ` + INSERT INTO route ( + id, name, shortcode, description, distance_meters, + duration_seconds, end_fort_id, end_image, + end_lat, end_lon, image, image_border_color, + reversible, start_fort_id, start_image, + start_lat, start_lon, tags, type, + updated, version, waypoints + ) + VALUES + ( + :id, :name, :shortcode, :description, :distance_meters, + :duration_seconds, :end_fort_id, + :end_image, :end_lat, :end_lon, :image, + :image_border_color, :reversible, + :start_fort_id, :start_image, :start_lat, + :start_lon, :tags, :type, :updated, + :version, :waypoints + ) + `, + route, + ) + + statsCollector.IncDbQuery("insert route", err) + if err != nil { + return fmt.Errorf("insert route error: %w", err) + } + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Route", route.Id, route.changedFields) + } + _, err := db.GeneralDb.NamedExecContext(ctx, + ` + UPDATE route SET + name = :name, + shortcode = :shortcode, + description = :description, + distance_meters = :distance_meters, + duration_seconds = :duration_seconds, + end_fort_id = :end_fort_id, + end_image = :end_image, + end_lat = :end_lat, + end_lon = :end_lon, + image = :image, + image_border_color = :image_border_color, + reversible = :reversible, + start_fort_id = :start_fort_id, + start_image = :start_image, + start_lat = :start_lat, + start_lon = :start_lon, + tags = :tags, + type = :type, + updated = :updated, + version = :version, + waypoints = :waypoints + WHERE id = :id`, + route, + ) + + statsCollector.IncDbQuery("update route", err) + if err != nil { + return fmt.Errorf("update route error %w", err) + } + } + + if dbDebugEnabled { + route.changedFields = route.changedFields[:0] + } + route.ClearDirty() + if route.IsNewRecord() { + routeCache.Set(route.Id, route, ttlcache.DefaultTTL) + route.newRecord = false + } + return nil +} diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 4f59ebaf..736b302f 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -9,9 +9,9 @@ import ( "golbat/db" "github.com/golang/geo/s2" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) type S2Cell struct { diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 7e1d8811..f858cc85 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -11,9 +11,9 @@ import ( "golbat/db" "golbat/pogo" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) // Spawnpoint struct. diff --git a/decoder/station.go b/decoder/station.go index bbf5c79a..9feea5c1 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -1,22 +1,9 @@ package decoder import ( - "context" - "database/sql" - "encoding/json" - "errors" - "fmt" "sync" - "time" - "golbat/db" - "golbat/pogo" - "golbat/util" - "golbat/webhooks" - - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" ) // Station struct. @@ -372,376 +359,3 @@ func (station *Station) SetStationedPokemon(v null.String) { } } } - -type StationWebhook struct { - Id string `json:"id"` - Latitude float64 `json:"latitude"` - Longitude float64 `json:"longitude"` - Name string `json:"name"` - StartTime int64 `json:"start_time"` - EndTime int64 `json:"end_time"` - IsBattleAvailable bool `json:"is_battle_available"` - BattleLevel null.Int `json:"battle_level"` - BattleStart null.Int `json:"battle_start"` - BattleEnd null.Int `json:"battle_end"` - BattlePokemonId null.Int `json:"battle_pokemon_id"` - BattlePokemonForm null.Int `json:"battle_pokemon_form"` - BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` - BattlePokemonGender null.Int `json:"battle_pokemon_gender"` - BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` - BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` - BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` - BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` - TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` - TotalStationedGmax null.Int `json:"total_stationed_gmax"` - Updated int64 `json:"updated"` -} - -func loadStationFromDatabase(ctx context.Context, db db.DbDetails, stationId string, station *Station) error { - err := db.GeneralDb.GetContext(ctx, station, - `SELECT id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon - FROM station WHERE id = ?`, stationId) - statsCollector.IncDbQuery("select station", err) - return err -} - -// peekStationRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func peekStationRecord(stationId string) (*Station, func(), error) { - if item := stationCache.Get(stationId); item != nil { - station := item.Value() - station.Lock() - return station, func() { station.Unlock() }, nil - } - return nil, nil, nil -} - -// getStationRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { - // Check cache first - if item := stationCache.Get(stationId); item != nil { - station := item.Value() - station.Lock() - return station, func() { station.Unlock() }, nil - } - - dbStation := Station{} - err := loadStationFromDatabase(ctx, db, stationId, &dbStation) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbStation.ClearDirty() - - // Atomically cache the loaded Station - if another goroutine raced us, - // we'll get their Station and use that instead (ensuring same mutex) - existingStation, _ := stationCache.GetOrSetFunc(stationId, func() *Station { - return &dbStation - }) - - station := existingStation.Value() - station.Lock() - return station, func() { station.Unlock() }, nil -} - -// getStationRecordForUpdate acquires lock AND takes snapshot for webhook comparison. -// Caller MUST call returned unlock function if non-nil. -func getStationRecordForUpdate(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { - station, unlock, err := getStationRecordReadOnly(ctx, db, stationId) - if err != nil || station == nil { - return nil, nil, err - } - station.snapshotOldValues() - return station, unlock, nil -} - -// getOrCreateStationRecord gets existing or creates new, locked with snapshot. -// Caller MUST call returned unlock function. -func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { - // Create new Station atomically - function only called if key doesn't exist - stationItem, _ := stationCache.GetOrSetFunc(stationId, func() *Station { - return &Station{Id: stationId, newRecord: true} - }) - - station := stationItem.Value() - station.Lock() - - if station.newRecord { - // We should attempt to load from database - err := loadStationFromDatabase(ctx, db, stationId, station) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - station.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - station.newRecord = false - station.ClearDirty() - } - } - - station.snapshotOldValues() - return station, func() { station.Unlock() }, nil -} - -func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { - now := time.Now().Unix() - - // Skip save if not dirty and was updated recently (15-min debounce) - if !station.IsDirty() && !station.IsNewRecord() { - if station.Updated > now-GetUpdateThreshold(900) { - return - } - } - - station.Updated = now - - if station.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Station", station.Id, station.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, - ` - INSERT INTO station (id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon) - VALUES (:id,:lat,:lon,:name,:cell_id,:start_time,:end_time,:cooldown_complete,:is_battle_available,:is_inactive,:updated,:battle_level,:battle_start,:battle_end,:battle_pokemon_id,:battle_pokemon_form,:battle_pokemon_costume,:battle_pokemon_gender,:battle_pokemon_alignment,:battle_pokemon_bread_mode,:battle_pokemon_move_1,:battle_pokemon_move_2,:battle_pokemon_stamina,:battle_pokemon_cp_multiplier,:total_stationed_pokemon,:total_stationed_gmax,:stationed_pokemon) - `, station) - - statsCollector.IncDbQuery("insert station", err) - if err != nil { - log.Errorf("insert station: %s", err) - return - } - _, _ = res, err - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Station", station.Id, station.changedFields) - } - res, err := db.GeneralDb.NamedExecContext(ctx, ` - UPDATE station - SET - lat = :lat, - lon = :lon, - name = :name, - cell_id = :cell_id, - start_time = :start_time, - end_time = :end_time, - cooldown_complete = :cooldown_complete, - is_battle_available = :is_battle_available, - is_inactive = :is_inactive, - updated = :updated, - battle_level = :battle_level, - battle_start = :battle_start, - battle_end = :battle_end, - battle_pokemon_id = :battle_pokemon_id, - battle_pokemon_form = :battle_pokemon_form, - battle_pokemon_costume = :battle_pokemon_costume, - battle_pokemon_gender = :battle_pokemon_gender, - battle_pokemon_alignment = :battle_pokemon_alignment, - battle_pokemon_bread_mode = :battle_pokemon_bread_mode, - battle_pokemon_move_1 = :battle_pokemon_move_1, - battle_pokemon_move_2 = :battle_pokemon_move_2, - battle_pokemon_stamina = :battle_pokemon_stamina, - battle_pokemon_cp_multiplier = :battle_pokemon_cp_multiplier, - total_stationed_pokemon = :total_stationed_pokemon, - total_stationed_gmax = :total_stationed_gmax, - stationed_pokemon = :stationed_pokemon - WHERE id = :id - `, station, - ) - statsCollector.IncDbQuery("update station", err) - if err != nil { - log.Errorf("Update station %s", err) - } - _, _ = res, err - } - - if dbDebugEnabled { - station.changedFields = station.changedFields[:0] - } - station.ClearDirty() - createStationWebhooks(station) - if station.IsNewRecord() { - stationCache.Set(station.Id, station, ttlcache.DefaultTTL) - station.newRecord = false - } -} - -func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, cellId uint64) *Station { - station.SetId(stationProto.Id) - name := stationProto.Name - // NOTE: Some names have more than 255 runes, which won't fit in our - // varchar(255). - if truncateStr, truncated := util.TruncateUTF8(stationProto.Name, 255); truncated { - log.Warnf("truncating name for station id '%s'. Orig name: %s", - stationProto.Id, - stationProto.Name, - ) - name = truncateStr - } - station.SetName(name) - station.SetLat(stationProto.Lat) - station.SetLon(stationProto.Lng) - station.SetStartTime(stationProto.StartTimeMs / 1000) - station.SetEndTime(stationProto.EndTimeMs / 1000) - station.SetCooldownComplete(stationProto.CooldownCompleteMs) - station.SetIsBattleAvailable(stationProto.IsBreadBattleAvailable) - if battleDetails := stationProto.BattleDetails; battleDetails != nil { - station.SetBattleLevel(null.IntFrom(int64(battleDetails.BattleLevel))) - station.SetBattleStart(null.IntFrom(battleDetails.BattleWindowStartMs / 1000)) - station.SetBattleEnd(null.IntFrom(battleDetails.BattleWindowEndMs / 1000)) - if pokemon := battleDetails.BattlePokemon; pokemon != nil { - station.SetBattlePokemonId(null.IntFrom(int64(pokemon.PokemonId))) - station.SetBattlePokemonMove1(null.IntFrom(int64(pokemon.Move1))) - station.SetBattlePokemonMove2(null.IntFrom(int64(pokemon.Move2))) - station.SetBattlePokemonForm(null.IntFrom(int64(pokemon.PokemonDisplay.Form))) - station.SetBattlePokemonCostume(null.IntFrom(int64(pokemon.PokemonDisplay.Costume))) - station.SetBattlePokemonGender(null.IntFrom(int64(pokemon.PokemonDisplay.Gender))) - station.SetBattlePokemonAlignment(null.IntFrom(int64(pokemon.PokemonDisplay.Alignment))) - station.SetBattlePokemonBreadMode(null.IntFrom(int64(pokemon.PokemonDisplay.BreadModeEnum))) - station.SetBattlePokemonStamina(null.IntFrom(int64(pokemon.Stamina))) - station.SetBattlePokemonCpMultiplier(null.FloatFrom(float64(pokemon.CpMultiplier))) - if rewardPokemon := battleDetails.RewardPokemon; rewardPokemon != nil && pokemon.PokemonId != rewardPokemon.PokemonId { - log.Infof("[DYNAMAX] Pokemon reward differs from battle: Battle %v - Reward %v", pokemon, rewardPokemon) - } - } - } - station.SetCellId(int64(cellId)) - return station -} - -func (station *Station) updateFromGetStationedPokemonDetailsOutProto(stationProto *pogo.GetStationedPokemonDetailsOutProto) *Station { - type stationedPokemonDetail struct { - PokemonId int `json:"pokemon_id"` - Form int `json:"form"` - Costume int `json:"costume"` - Gender int `json:"gender"` - Shiny bool `json:"shiny,omitempty"` - TempEvolution int `json:"temp_evolution,omitempty"` - TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` - Alignment int `json:"alignment,omitempty"` - Badge int `json:"badge,omitempty"` - Background *int64 `json:"background,omitempty"` - BreadMode int `json:"bread_mode"` - } - - var stationedPokemon []stationedPokemonDetail - stationedGmax := int64(0) - for _, stationedPokemonDetails := range stationProto.StationedPokemons { - pokemon := stationedPokemonDetails.Pokemon - display := pokemon.PokemonDisplay - stationedPokemon = append(stationedPokemon, stationedPokemonDetail{ - PokemonId: int(pokemon.PokemonId), - Form: int(display.Form), - Costume: int(display.Costume), - Gender: int(display.Gender), - Shiny: display.Shiny, - TempEvolution: int(display.CurrentTempEvolution), - TempEvolutionFinishMs: display.TemporaryEvolutionFinishMs, - Alignment: int(display.Alignment), - Badge: int(display.PokemonBadge), - Background: util.ExtractBackgroundFromDisplay(display), - BreadMode: int(display.BreadModeEnum), - }) - if display.BreadModeEnum == pogo.BreadModeEnum_BREAD_DOUGH_MODE || display.BreadModeEnum == pogo.BreadModeEnum_BREAD_DOUGH_MODE_2 { - stationedGmax++ - } - } - jsonString, _ := json.Marshal(stationedPokemon) - station.SetStationedPokemon(null.StringFrom(string(jsonString))) - station.SetTotalStationedPokemon(null.IntFrom(int64(stationProto.TotalNumStationedPokemon))) - station.SetTotalStationedGmax(null.IntFrom(stationedGmax)) - return station -} - -func (station *Station) resetStationedPokemonFromStationDetailsNotFound() *Station { - jsonString, _ := json.Marshal([]string{}) - station.SetStationedPokemon(null.StringFrom(string(jsonString))) - station.SetTotalStationedPokemon(null.IntFrom(0)) - station.SetTotalStationedGmax(null.IntFrom(0)) - return station -} - -func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto) string { - stationId := request.StationId - - station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) - if err != nil { - log.Printf("Get station %s", err) - return "Error getting station" - } - - if station == nil { - log.Infof("Stationed pokemon details for station %s not found", stationId) - return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) - } - defer unlock() - - station.resetStationedPokemonFromStationDetailsNotFound() - saveStationRecord(ctx, db, station) - return fmt.Sprintf("StationedPokemonDetails %s", stationId) -} - -func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto, stationDetails *pogo.GetStationedPokemonDetailsOutProto) string { - stationId := request.StationId - - station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) - if err != nil { - log.Printf("Get station %s", err) - return "Error getting station" - } - - if station == nil { - log.Infof("Stationed pokemon details for station %s not found", stationId) - return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) - } - defer unlock() - - station.updateFromGetStationedPokemonDetailsOutProto(stationDetails) - saveStationRecord(ctx, db, station) - return fmt.Sprintf("StationedPokemonDetails %s", stationId) -} - -func createStationWebhooks(station *Station) { - old := &station.oldValues - isNew := station.IsNewRecord() - - if isNew || station.BattlePokemonId.Valid && (old.EndTime != station.EndTime || - old.BattleEnd != station.BattleEnd || - old.BattlePokemonId != station.BattlePokemonId || - old.BattlePokemonForm != station.BattlePokemonForm || - old.BattlePokemonCostume != station.BattlePokemonCostume || - old.BattlePokemonGender != station.BattlePokemonGender || - old.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { - stationHook := StationWebhook{ - Id: station.Id, - Latitude: station.Lat, - Longitude: station.Lon, - Name: station.Name, - StartTime: station.StartTime, - EndTime: station.EndTime, - IsBattleAvailable: station.IsBattleAvailable, - BattleLevel: station.BattleLevel, - BattleStart: station.BattleStart, - BattleEnd: station.BattleEnd, - BattlePokemonId: station.BattlePokemonId, - BattlePokemonForm: station.BattlePokemonForm, - BattlePokemonCostume: station.BattlePokemonCostume, - BattlePokemonGender: station.BattlePokemonGender, - BattlePokemonAlignment: station.BattlePokemonAlignment, - BattlePokemonBreadMode: station.BattlePokemonBreadMode, - BattlePokemonMove1: station.BattlePokemonMove1, - BattlePokemonMove2: station.BattlePokemonMove2, - TotalStationedPokemon: station.TotalStationedPokemon, - TotalStationedGmax: station.TotalStationedGmax, - Updated: station.Updated, - } - areas := MatchStatsGeofence(station.Lat, station.Lon) - webhooksSender.AddMessage(webhooks.MaxBattle, stationHook, areas) - statsCollector.UpdateMaxBattleCount(areas, station.BattleLevel.ValueOrZero()) - } -} diff --git a/decoder/station_decode.go b/decoder/station_decode.go new file mode 100644 index 00000000..da341b9a --- /dev/null +++ b/decoder/station_decode.go @@ -0,0 +1,106 @@ +package decoder + +import ( + "encoding/json" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/pogo" + "golbat/util" +) + +func (station *Station) updateFromStationProto(stationProto *pogo.StationProto, cellId uint64) *Station { + station.SetId(stationProto.Id) + name := stationProto.Name + // NOTE: Some names have more than 255 runes, which won't fit in our + // varchar(255). + if truncateStr, truncated := util.TruncateUTF8(stationProto.Name, 255); truncated { + log.Warnf("truncating name for station id '%s'. Orig name: %s", + stationProto.Id, + stationProto.Name, + ) + name = truncateStr + } + station.SetName(name) + station.SetLat(stationProto.Lat) + station.SetLon(stationProto.Lng) + station.SetStartTime(stationProto.StartTimeMs / 1000) + station.SetEndTime(stationProto.EndTimeMs / 1000) + station.SetCooldownComplete(stationProto.CooldownCompleteMs) + station.SetIsBattleAvailable(stationProto.IsBreadBattleAvailable) + if battleDetails := stationProto.BattleDetails; battleDetails != nil { + station.SetBattleLevel(null.IntFrom(int64(battleDetails.BattleLevel))) + station.SetBattleStart(null.IntFrom(battleDetails.BattleWindowStartMs / 1000)) + station.SetBattleEnd(null.IntFrom(battleDetails.BattleWindowEndMs / 1000)) + if pokemon := battleDetails.BattlePokemon; pokemon != nil { + station.SetBattlePokemonId(null.IntFrom(int64(pokemon.PokemonId))) + station.SetBattlePokemonMove1(null.IntFrom(int64(pokemon.Move1))) + station.SetBattlePokemonMove2(null.IntFrom(int64(pokemon.Move2))) + station.SetBattlePokemonForm(null.IntFrom(int64(pokemon.PokemonDisplay.Form))) + station.SetBattlePokemonCostume(null.IntFrom(int64(pokemon.PokemonDisplay.Costume))) + station.SetBattlePokemonGender(null.IntFrom(int64(pokemon.PokemonDisplay.Gender))) + station.SetBattlePokemonAlignment(null.IntFrom(int64(pokemon.PokemonDisplay.Alignment))) + station.SetBattlePokemonBreadMode(null.IntFrom(int64(pokemon.PokemonDisplay.BreadModeEnum))) + station.SetBattlePokemonStamina(null.IntFrom(int64(pokemon.Stamina))) + station.SetBattlePokemonCpMultiplier(null.FloatFrom(float64(pokemon.CpMultiplier))) + if rewardPokemon := battleDetails.RewardPokemon; rewardPokemon != nil && pokemon.PokemonId != rewardPokemon.PokemonId { + log.Infof("[DYNAMAX] Pokemon reward differs from battle: Battle %v - Reward %v", pokemon, rewardPokemon) + } + } + } + station.SetCellId(int64(cellId)) + return station +} + +func (station *Station) updateFromGetStationedPokemonDetailsOutProto(stationProto *pogo.GetStationedPokemonDetailsOutProto) *Station { + type stationedPokemonDetail struct { + PokemonId int `json:"pokemon_id"` + Form int `json:"form"` + Costume int `json:"costume"` + Gender int `json:"gender"` + Shiny bool `json:"shiny,omitempty"` + TempEvolution int `json:"temp_evolution,omitempty"` + TempEvolutionFinishMs int64 `json:"temp_evolution_finish_ms,omitempty"` + Alignment int `json:"alignment,omitempty"` + Badge int `json:"badge,omitempty"` + Background *int64 `json:"background,omitempty"` + BreadMode int `json:"bread_mode"` + } + + var stationedPokemon []stationedPokemonDetail + stationedGmax := int64(0) + for _, stationedPokemonDetails := range stationProto.StationedPokemons { + pokemon := stationedPokemonDetails.Pokemon + display := pokemon.PokemonDisplay + stationedPokemon = append(stationedPokemon, stationedPokemonDetail{ + PokemonId: int(pokemon.PokemonId), + Form: int(display.Form), + Costume: int(display.Costume), + Gender: int(display.Gender), + Shiny: display.Shiny, + TempEvolution: int(display.CurrentTempEvolution), + TempEvolutionFinishMs: display.TemporaryEvolutionFinishMs, + Alignment: int(display.Alignment), + Badge: int(display.PokemonBadge), + Background: util.ExtractBackgroundFromDisplay(display), + BreadMode: int(display.BreadModeEnum), + }) + if display.BreadModeEnum == pogo.BreadModeEnum_BREAD_DOUGH_MODE || display.BreadModeEnum == pogo.BreadModeEnum_BREAD_DOUGH_MODE_2 { + stationedGmax++ + } + } + jsonString, _ := json.Marshal(stationedPokemon) + station.SetStationedPokemon(null.StringFrom(string(jsonString))) + station.SetTotalStationedPokemon(null.IntFrom(int64(stationProto.TotalNumStationedPokemon))) + station.SetTotalStationedGmax(null.IntFrom(stationedGmax)) + return station +} + +func (station *Station) resetStationedPokemonFromStationDetailsNotFound() *Station { + jsonString, _ := json.Marshal([]string{}) + station.SetStationedPokemon(null.StringFrom(string(jsonString))) + station.SetTotalStationedPokemon(null.IntFrom(0)) + station.SetTotalStationedGmax(null.IntFrom(0)) + return station +} diff --git a/decoder/station_process.go b/decoder/station_process.go new file mode 100644 index 00000000..05cda289 --- /dev/null +++ b/decoder/station_process.go @@ -0,0 +1,51 @@ +package decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func ResetStationedPokemonWithStationDetailsNotFound(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto) string { + stationId := request.StationId + + station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) + if err != nil { + log.Printf("Get station %s", err) + return "Error getting station" + } + + if station == nil { + log.Infof("Stationed pokemon details for station %s not found", stationId) + return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) + } + defer unlock() + + station.resetStationedPokemonFromStationDetailsNotFound() + saveStationRecord(ctx, db, station) + return fmt.Sprintf("StationedPokemonDetails %s", stationId) +} + +func UpdateStationWithStationDetails(ctx context.Context, db db.DbDetails, request *pogo.GetStationedPokemonDetailsProto, stationDetails *pogo.GetStationedPokemonDetailsOutProto) string { + stationId := request.StationId + + station, unlock, err := getStationRecordForUpdate(ctx, db, stationId) + if err != nil { + log.Printf("Get station %s", err) + return "Error getting station" + } + + if station == nil { + log.Infof("Stationed pokemon details for station %s not found", stationId) + return fmt.Sprintf("Stationed pokemon details for station %s not found", stationId) + } + defer unlock() + + station.updateFromGetStationedPokemonDetailsOutProto(stationDetails) + saveStationRecord(ctx, db, station) + return fmt.Sprintf("StationedPokemonDetails %s", stationId) +} diff --git a/decoder/station_state.go b/decoder/station_state.go new file mode 100644 index 00000000..d4e4e06a --- /dev/null +++ b/decoder/station_state.go @@ -0,0 +1,253 @@ +package decoder + +import ( + "context" + "database/sql" + "errors" + "time" + + "github.com/guregu/null/v6" + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/webhooks" +) + +type StationWebhook struct { + Id string `json:"id"` + Latitude float64 `json:"latitude"` + Longitude float64 `json:"longitude"` + Name string `json:"name"` + StartTime int64 `json:"start_time"` + EndTime int64 `json:"end_time"` + IsBattleAvailable bool `json:"is_battle_available"` + BattleLevel null.Int `json:"battle_level"` + BattleStart null.Int `json:"battle_start"` + BattleEnd null.Int `json:"battle_end"` + BattlePokemonId null.Int `json:"battle_pokemon_id"` + BattlePokemonForm null.Int `json:"battle_pokemon_form"` + BattlePokemonCostume null.Int `json:"battle_pokemon_costume"` + BattlePokemonGender null.Int `json:"battle_pokemon_gender"` + BattlePokemonAlignment null.Int `json:"battle_pokemon_alignment"` + BattlePokemonBreadMode null.Int `json:"battle_pokemon_bread_mode"` + BattlePokemonMove1 null.Int `json:"battle_pokemon_move_1"` + BattlePokemonMove2 null.Int `json:"battle_pokemon_move_2"` + TotalStationedPokemon null.Int `json:"total_stationed_pokemon"` + TotalStationedGmax null.Int `json:"total_stationed_gmax"` + Updated int64 `json:"updated"` +} + +func loadStationFromDatabase(ctx context.Context, db db.DbDetails, stationId string, station *Station) error { + err := db.GeneralDb.GetContext(ctx, station, + `SELECT id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon + FROM station WHERE id = ?`, stationId) + statsCollector.IncDbQuery("select station", err) + return err +} + +// peekStationRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func peekStationRecord(stationId string) (*Station, func(), error) { + if item := stationCache.Get(stationId); item != nil { + station := item.Value() + station.Lock() + return station, func() { station.Unlock() }, nil + } + return nil, nil, nil +} + +// getStationRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getStationRecordReadOnly(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + // Check cache first + if item := stationCache.Get(stationId); item != nil { + station := item.Value() + station.Lock() + return station, func() { station.Unlock() }, nil + } + + dbStation := Station{} + err := loadStationFromDatabase(ctx, db, stationId, &dbStation) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbStation.ClearDirty() + + // Atomically cache the loaded Station - if another goroutine raced us, + // we'll get their Station and use that instead (ensuring same mutex) + existingStation, _ := stationCache.GetOrSetFunc(stationId, func() *Station { + return &dbStation + }) + + station := existingStation.Value() + station.Lock() + return station, func() { station.Unlock() }, nil +} + +// getStationRecordForUpdate acquires lock AND takes snapshot for webhook comparison. +// Caller MUST call returned unlock function if non-nil. +func getStationRecordForUpdate(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + station, unlock, err := getStationRecordReadOnly(ctx, db, stationId) + if err != nil || station == nil { + return nil, nil, err + } + station.snapshotOldValues() + return station, unlock, nil +} + +// getOrCreateStationRecord gets existing or creates new, locked with snapshot. +// Caller MUST call returned unlock function. +func getOrCreateStationRecord(ctx context.Context, db db.DbDetails, stationId string) (*Station, func(), error) { + // Create new Station atomically - function only called if key doesn't exist + stationItem, _ := stationCache.GetOrSetFunc(stationId, func() *Station { + return &Station{Id: stationId, newRecord: true} + }) + + station := stationItem.Value() + station.Lock() + + if station.newRecord { + // We should attempt to load from database + err := loadStationFromDatabase(ctx, db, stationId, station) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + station.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + station.newRecord = false + station.ClearDirty() + } + } + + station.snapshotOldValues() + return station, func() { station.Unlock() }, nil +} + +func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { + now := time.Now().Unix() + + // Skip save if not dirty and was updated recently (15-min debounce) + if !station.IsDirty() && !station.IsNewRecord() { + if station.Updated > now-GetUpdateThreshold(900) { + return + } + } + + station.Updated = now + + if station.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Station", station.Id, station.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, + ` + INSERT INTO station (id, lat, lon, name, cell_id, start_time, end_time, cooldown_complete, is_battle_available, is_inactive, updated, battle_level, battle_start, battle_end, battle_pokemon_id, battle_pokemon_form, battle_pokemon_costume, battle_pokemon_gender, battle_pokemon_alignment, battle_pokemon_bread_mode, battle_pokemon_move_1, battle_pokemon_move_2, battle_pokemon_stamina, battle_pokemon_cp_multiplier, total_stationed_pokemon, total_stationed_gmax, stationed_pokemon) + VALUES (:id,:lat,:lon,:name,:cell_id,:start_time,:end_time,:cooldown_complete,:is_battle_available,:is_inactive,:updated,:battle_level,:battle_start,:battle_end,:battle_pokemon_id,:battle_pokemon_form,:battle_pokemon_costume,:battle_pokemon_gender,:battle_pokemon_alignment,:battle_pokemon_bread_mode,:battle_pokemon_move_1,:battle_pokemon_move_2,:battle_pokemon_stamina,:battle_pokemon_cp_multiplier,:total_stationed_pokemon,:total_stationed_gmax,:stationed_pokemon) + `, station) + + statsCollector.IncDbQuery("insert station", err) + if err != nil { + log.Errorf("insert station: %s", err) + return + } + _, _ = res, err + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Station", station.Id, station.changedFields) + } + res, err := db.GeneralDb.NamedExecContext(ctx, ` + UPDATE station + SET + lat = :lat, + lon = :lon, + name = :name, + cell_id = :cell_id, + start_time = :start_time, + end_time = :end_time, + cooldown_complete = :cooldown_complete, + is_battle_available = :is_battle_available, + is_inactive = :is_inactive, + updated = :updated, + battle_level = :battle_level, + battle_start = :battle_start, + battle_end = :battle_end, + battle_pokemon_id = :battle_pokemon_id, + battle_pokemon_form = :battle_pokemon_form, + battle_pokemon_costume = :battle_pokemon_costume, + battle_pokemon_gender = :battle_pokemon_gender, + battle_pokemon_alignment = :battle_pokemon_alignment, + battle_pokemon_bread_mode = :battle_pokemon_bread_mode, + battle_pokemon_move_1 = :battle_pokemon_move_1, + battle_pokemon_move_2 = :battle_pokemon_move_2, + battle_pokemon_stamina = :battle_pokemon_stamina, + battle_pokemon_cp_multiplier = :battle_pokemon_cp_multiplier, + total_stationed_pokemon = :total_stationed_pokemon, + total_stationed_gmax = :total_stationed_gmax, + stationed_pokemon = :stationed_pokemon + WHERE id = :id + `, station, + ) + statsCollector.IncDbQuery("update station", err) + if err != nil { + log.Errorf("Update station %s", err) + } + _, _ = res, err + } + + if dbDebugEnabled { + station.changedFields = station.changedFields[:0] + } + station.ClearDirty() + createStationWebhooks(station) + if station.IsNewRecord() { + stationCache.Set(station.Id, station, ttlcache.DefaultTTL) + station.newRecord = false + } +} + +func createStationWebhooks(station *Station) { + old := &station.oldValues + isNew := station.IsNewRecord() + + if isNew || station.BattlePokemonId.Valid && (old.EndTime != station.EndTime || + old.BattleEnd != station.BattleEnd || + old.BattlePokemonId != station.BattlePokemonId || + old.BattlePokemonForm != station.BattlePokemonForm || + old.BattlePokemonCostume != station.BattlePokemonCostume || + old.BattlePokemonGender != station.BattlePokemonGender || + old.BattlePokemonBreadMode != station.BattlePokemonBreadMode) { + stationHook := StationWebhook{ + Id: station.Id, + Latitude: station.Lat, + Longitude: station.Lon, + Name: station.Name, + StartTime: station.StartTime, + EndTime: station.EndTime, + IsBattleAvailable: station.IsBattleAvailable, + BattleLevel: station.BattleLevel, + BattleStart: station.BattleStart, + BattleEnd: station.BattleEnd, + BattlePokemonId: station.BattlePokemonId, + BattlePokemonForm: station.BattlePokemonForm, + BattlePokemonCostume: station.BattlePokemonCostume, + BattlePokemonGender: station.BattlePokemonGender, + BattlePokemonAlignment: station.BattlePokemonAlignment, + BattlePokemonBreadMode: station.BattlePokemonBreadMode, + BattlePokemonMove1: station.BattlePokemonMove1, + BattlePokemonMove2: station.BattlePokemonMove2, + TotalStationedPokemon: station.TotalStationedPokemon, + TotalStationedGmax: station.TotalStationedGmax, + Updated: station.Updated, + } + areas := MatchStatsGeofence(station.Lat, station.Lon) + webhooksSender.AddMessage(webhooks.MaxBattle, stationHook, areas) + statsCollector.UpdateMaxBattleCount(areas, station.BattleLevel.ValueOrZero()) + } +} diff --git a/decoder/tappable.go b/decoder/tappable.go index 3a498ef2..57884bfd 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -1,20 +1,9 @@ package decoder import ( - "context" - "database/sql" - "errors" - "fmt" - "strconv" "sync" - "time" - "golbat/db" - "golbat/pogo" - - "github.com/jellydator/ttlcache/v3" - log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" ) // Tappable struct. @@ -166,278 +155,3 @@ func (ta *Tappable) SetExpireTimestampVerified(v bool) { } } } - -func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.DbDetails, tappable *pogo.ProcessTappableOutProto, request *pogo.ProcessTappableProto, timestampMs int64) { - // update from request - ta.Id = request.EncounterId // Id is primary key, don't track as dirty - location := request.GetLocation() - if spawnPointId := location.GetSpawnpointId(); spawnPointId != "" { - spawnId, err := strconv.ParseInt(spawnPointId, 16, 64) - if err != nil { - panic(err) - } - ta.SetSpawnId(null.IntFrom(spawnId)) - } - if fortId := location.GetFortId(); fortId != "" { - ta.SetFortId(null.StringFrom(fortId)) - } - ta.SetType(request.TappableTypeId) - ta.SetLat(request.LocationHintLat) - ta.SetLon(request.LocationHintLng) - ta.setExpireTimestamp(ctx, db, timestampMs) - - // update from tappable - if encounter := tappable.GetEncounter(); encounter != nil { - // tappable is a Pokèmon, encounter is sent in a separate proto - // we store this to link tappable with Pokèmon from encounter proto - ta.SetEncounter(null.IntFrom(int64(encounter.Pokemon.PokemonId))) - } else if reward := tappable.GetReward(); reward != nil { - for _, lootProto := range reward { - for _, itemProto := range lootProto.GetLootItem() { - switch t := itemProto.Type.(type) { - case *pogo.LootItemProto_Item: - ta.SetItemId(null.IntFrom(int64(t.Item))) - ta.SetCount(null.IntFrom(int64(itemProto.Count))) - case *pogo.LootItemProto_Stardust: - log.Warnf("[TAPPABLE] Reward is Stardust: %t", t.Stardust) - case *pogo.LootItemProto_Pokecoin: - log.Warnf("[TAPPABLE] Reward is Pokecoin: %t", t.Pokecoin) - case *pogo.LootItemProto_PokemonCandy: - log.Warnf("[TAPPABLE] Reward is Pokemon Candy: %v", t.PokemonCandy) - case *pogo.LootItemProto_Experience: - log.Warnf("[TAPPABLE] Reward is Experience: %t", t.Experience) - case *pogo.LootItemProto_PokemonEgg: - log.Warnf("[TAPPABLE] Reward is a Pokemon Egg: %v", t.PokemonEgg) - case *pogo.LootItemProto_AvatarTemplateId: - log.Warnf("[TAPPABLE] Reward is an Avatar Template ID: %v", t.AvatarTemplateId) - case *pogo.LootItemProto_StickerId: - log.Warnf("[TAPPABLE] Reward is a Sticker ID: %s", t.StickerId) - case *pogo.LootItemProto_MegaEnergyPokemonId: - log.Warnf("[TAPPABLE] Reward is Mega Energy Pokemon ID: %v", t.MegaEnergyPokemonId) - case *pogo.LootItemProto_XlCandy: - log.Warnf("[TAPPABLE] Reward is XL Candy: %v", t.XlCandy) - case *pogo.LootItemProto_FollowerPokemon: - log.Warnf("[TAPPABLE] Reward is a Follower Pokemon: %v", t.FollowerPokemon) - case *pogo.LootItemProto_NeutralAvatarTemplateId: - log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Template ID: %v", t.NeutralAvatarTemplateId) - case *pogo.LootItemProto_NeutralAvatarItemTemplate: - log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Item Template: %v", t.NeutralAvatarItemTemplate) - case *pogo.LootItemProto_NeutralAvatarItemDisplay: - log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Item Display: %v", t.NeutralAvatarItemDisplay) - default: - log.Warnf("Unknown or unset Type") - } - } - } - } -} - -func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, timestampMs int64) { - ta.SetExpireTimestampVerified(false) - if spawnId := ta.SpawnId.ValueOrZero(); spawnId != 0 { - spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) - if spawnPoint != nil && spawnPoint.DespawnSec.Valid { - despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) - unlock() - - date := time.Unix(timestampMs/1000, 0) - secondOfHour := date.Second() + date.Minute()*60 - - despawnOffset := despawnSecond - secondOfHour - if despawnOffset < 0 { - despawnOffset += 3600 - } - ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) - ta.SetExpireTimestampVerified(true) - } else { - if unlock != nil { - unlock() - } - ta.setUnknownTimestamp(timestampMs / 1000) - } - } else if fortId := ta.FortId.ValueOrZero(); fortId != "" { - // we don't know any despawn times from lured/fort tappables - ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) - } -} - -func (ta *Tappable) setUnknownTimestamp(now int64) { - if !ta.ExpireTimestamp.Valid { - ta.SetExpireTimestamp(null.IntFrom(now + 20*60)) - } else { - if ta.ExpireTimestamp.Int64 < now { - ta.SetExpireTimestamp(null.IntFrom(now + 10*60)) - } - } -} - -func loadTappableFromDatabase(ctx context.Context, db db.DbDetails, id uint64, tappable *Tappable) error { - err := db.GeneralDb.GetContext(ctx, tappable, - `SELECT id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated - FROM tappable WHERE id = ?`, strconv.FormatUint(id, 10)) - statsCollector.IncDbQuery("select tappable", err) - return err -} - -// peekTappableRecord - cache-only lookup, no DB fallback, returns locked. -// Caller MUST call returned unlock function if non-nil. -func peekTappableRecord(id uint64) (*Tappable, func(), error) { - if item := tappableCache.Get(id); item != nil { - tappable := item.Value() - tappable.Lock() - return tappable, func() { tappable.Unlock() }, nil - } - return nil, nil, nil -} - -// getTappableRecordReadOnly acquires lock but does NOT take snapshot. -// Use for read-only checks. Will cause a backing database lookup. -// Caller MUST call returned unlock function if non-nil. -func getTappableRecordReadOnly(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { - // Check cache first - if item := tappableCache.Get(id); item != nil { - tappable := item.Value() - tappable.Lock() - return tappable, func() { tappable.Unlock() }, nil - } - - dbTappable := Tappable{} - err := loadTappableFromDatabase(ctx, db, id, &dbTappable) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil, nil - } - if err != nil { - return nil, nil, err - } - dbTappable.ClearDirty() - - // Atomically cache the loaded Tappable - if another goroutine raced us, - // we'll get their Tappable and use that instead (ensuring same mutex) - existingTappable, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { - return &dbTappable - }) - - tappable := existingTappable.Value() - tappable.Lock() - return tappable, func() { tappable.Unlock() }, nil -} - -// getOrCreateTappableRecord gets existing or creates new, locked. -// Caller MUST call returned unlock function. -func getOrCreateTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { - // Create new Tappable atomically - function only called if key doesn't exist - tappableItem, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { - return &Tappable{Id: id, newRecord: true} - }) - - tappable := tappableItem.Value() - tappable.Lock() - - if tappable.newRecord { - // We should attempt to load from database - err := loadTappableFromDatabase(ctx, db, id, tappable) - if err != nil { - if !errors.Is(err, sql.ErrNoRows) { - tappable.Unlock() - return nil, nil, err - } - } else { - // We loaded from DB - tappable.newRecord = false - tappable.ClearDirty() - } - } - - return tappable, func() { tappable.Unlock() }, nil -} - -// GetTappableRecord is an exported function for API use. -// Returns a tappable if found, nil if not found. -func GetTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, error) { - tappable, unlock, err := getTappableRecordReadOnly(ctx, db, id) - if err != nil { - return nil, err - } - if tappable == nil { - return nil, nil - } - defer unlock() - return tappable, nil -} - -func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tappable) { - // Skip save if not dirty and not new - if !tappable.IsDirty() && !tappable.IsNewRecord() { - return - } - - now := time.Now().Unix() - tappable.Updated = now - - if tappable.IsNewRecord() { - if dbDebugEnabled { - dbDebugLog("INSERT", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) - } - res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` - INSERT INTO tappable ( - id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated - ) VALUES ( - "%d", :lat, :lon, :fort_id, :spawn_id, :type, :pokemon_id, :item_id, :count, :expire_timestamp, :expire_timestamp_verified, :updated - ) - `, tappable.Id), tappable) - statsCollector.IncDbQuery("insert tappable", err) - if err != nil { - log.Errorf("insert tappable %d: %s", tappable.Id, err) - return - } - _ = res - } else { - if dbDebugEnabled { - dbDebugLog("UPDATE", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) - } - res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` - UPDATE tappable SET - lat = :lat, - lon = :lon, - fort_id = :fort_id, - spawn_id = :spawn_id, - type = :type, - pokemon_id = :pokemon_id, - item_id = :item_id, - count = :count, - expire_timestamp = :expire_timestamp, - expire_timestamp_verified = :expire_timestamp_verified, - updated = :updated - WHERE id = "%d" - `, tappable.Id), tappable) - statsCollector.IncDbQuery("update tappable", err) - if err != nil { - log.Errorf("update tappable %d: %s", tappable.Id, err) - return - } - _ = res - } - if dbDebugEnabled { - tappable.changedFields = tappable.changedFields[:0] - } - tappable.ClearDirty() - if tappable.IsNewRecord() { - tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) - tappable.newRecord = false - } -} - -func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { - id := request.GetEncounterId() - - tappable, unlock, err := getOrCreateTappableRecord(ctx, db, id) - if err != nil { - log.Printf("getOrCreateTappableRecord: %s", err) - return "Error getting tappable" - } - defer unlock() - - tappable.updateFromProcessTappableProto(ctx, db, tappableDetails, request, timestampMs) - saveTappableRecord(ctx, db, tappable) - return fmt.Sprintf("ProcessTappableOutProto %d", id) -} diff --git a/decoder/tappable_decode.go b/decoder/tappable_decode.go new file mode 100644 index 00000000..2b6f65e9 --- /dev/null +++ b/decoder/tappable_decode.go @@ -0,0 +1,117 @@ +package decoder + +import ( + "context" + "strconv" + "time" + + "github.com/guregu/null/v6" + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func (ta *Tappable) updateFromProcessTappableProto(ctx context.Context, db db.DbDetails, tappable *pogo.ProcessTappableOutProto, request *pogo.ProcessTappableProto, timestampMs int64) { + // update from request + ta.Id = request.EncounterId // Id is primary key, don't track as dirty + location := request.GetLocation() + if spawnPointId := location.GetSpawnpointId(); spawnPointId != "" { + spawnId, err := strconv.ParseInt(spawnPointId, 16, 64) + if err != nil { + panic(err) + } + ta.SetSpawnId(null.IntFrom(spawnId)) + } + if fortId := location.GetFortId(); fortId != "" { + ta.SetFortId(null.StringFrom(fortId)) + } + ta.SetType(request.TappableTypeId) + ta.SetLat(request.LocationHintLat) + ta.SetLon(request.LocationHintLng) + ta.setExpireTimestamp(ctx, db, timestampMs) + + // update from tappable + if encounter := tappable.GetEncounter(); encounter != nil { + // tappable is a Pokèmon, encounter is sent in a separate proto + // we store this to link tappable with Pokèmon from encounter proto + ta.SetEncounter(null.IntFrom(int64(encounter.Pokemon.PokemonId))) + } else if reward := tappable.GetReward(); reward != nil { + for _, lootProto := range reward { + for _, itemProto := range lootProto.GetLootItem() { + switch t := itemProto.Type.(type) { + case *pogo.LootItemProto_Item: + ta.SetItemId(null.IntFrom(int64(t.Item))) + ta.SetCount(null.IntFrom(int64(itemProto.Count))) + case *pogo.LootItemProto_Stardust: + log.Warnf("[TAPPABLE] Reward is Stardust: %t", t.Stardust) + case *pogo.LootItemProto_Pokecoin: + log.Warnf("[TAPPABLE] Reward is Pokecoin: %t", t.Pokecoin) + case *pogo.LootItemProto_PokemonCandy: + log.Warnf("[TAPPABLE] Reward is Pokemon Candy: %v", t.PokemonCandy) + case *pogo.LootItemProto_Experience: + log.Warnf("[TAPPABLE] Reward is Experience: %t", t.Experience) + case *pogo.LootItemProto_PokemonEgg: + log.Warnf("[TAPPABLE] Reward is a Pokemon Egg: %v", t.PokemonEgg) + case *pogo.LootItemProto_AvatarTemplateId: + log.Warnf("[TAPPABLE] Reward is an Avatar Template ID: %v", t.AvatarTemplateId) + case *pogo.LootItemProto_StickerId: + log.Warnf("[TAPPABLE] Reward is a Sticker ID: %s", t.StickerId) + case *pogo.LootItemProto_MegaEnergyPokemonId: + log.Warnf("[TAPPABLE] Reward is Mega Energy Pokemon ID: %v", t.MegaEnergyPokemonId) + case *pogo.LootItemProto_XlCandy: + log.Warnf("[TAPPABLE] Reward is XL Candy: %v", t.XlCandy) + case *pogo.LootItemProto_FollowerPokemon: + log.Warnf("[TAPPABLE] Reward is a Follower Pokemon: %v", t.FollowerPokemon) + case *pogo.LootItemProto_NeutralAvatarTemplateId: + log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Template ID: %v", t.NeutralAvatarTemplateId) + case *pogo.LootItemProto_NeutralAvatarItemTemplate: + log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Item Template: %v", t.NeutralAvatarItemTemplate) + case *pogo.LootItemProto_NeutralAvatarItemDisplay: + log.Warnf("[TAPPABLE] Reward is a Neutral Avatar Item Display: %v", t.NeutralAvatarItemDisplay) + default: + log.Warnf("Unknown or unset Type") + } + } + } + } +} + +func (ta *Tappable) setExpireTimestamp(ctx context.Context, db db.DbDetails, timestampMs int64) { + ta.SetExpireTimestampVerified(false) + if spawnId := ta.SpawnId.ValueOrZero(); spawnId != 0 { + spawnPoint, unlock, _ := getSpawnpointRecord(ctx, db, spawnId) + if spawnPoint != nil && spawnPoint.DespawnSec.Valid { + despawnSecond := int(spawnPoint.DespawnSec.ValueOrZero()) + unlock() + + date := time.Unix(timestampMs/1000, 0) + secondOfHour := date.Second() + date.Minute()*60 + + despawnOffset := despawnSecond - secondOfHour + if despawnOffset < 0 { + despawnOffset += 3600 + } + ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(despawnOffset))) + ta.SetExpireTimestampVerified(true) + } else { + if unlock != nil { + unlock() + } + ta.setUnknownTimestamp(timestampMs / 1000) + } + } else if fortId := ta.FortId.ValueOrZero(); fortId != "" { + // we don't know any despawn times from lured/fort tappables + ta.SetExpireTimestamp(null.IntFrom(int64(timestampMs)/1000 + int64(120))) + } +} + +func (ta *Tappable) setUnknownTimestamp(now int64) { + if !ta.ExpireTimestamp.Valid { + ta.SetExpireTimestamp(null.IntFrom(now + 20*60)) + } else { + if ta.ExpireTimestamp.Int64 < now { + ta.SetExpireTimestamp(null.IntFrom(now + 10*60)) + } + } +} diff --git a/decoder/tappable_process.go b/decoder/tappable_process.go new file mode 100644 index 00000000..1001ac5e --- /dev/null +++ b/decoder/tappable_process.go @@ -0,0 +1,26 @@ +package decoder + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + + "golbat/db" + "golbat/pogo" +) + +func UpdateTappable(ctx context.Context, db db.DbDetails, request *pogo.ProcessTappableProto, tappableDetails *pogo.ProcessTappableOutProto, timestampMs int64) string { + id := request.GetEncounterId() + + tappable, unlock, err := getOrCreateTappableRecord(ctx, db, id) + if err != nil { + log.Printf("getOrCreateTappableRecord: %s", err) + return "Error getting tappable" + } + defer unlock() + + tappable.updateFromProcessTappableProto(ctx, db, tappableDetails, request, timestampMs) + saveTappableRecord(ctx, db, tappable) + return fmt.Sprintf("ProcessTappableOutProto %d", id) +} diff --git a/decoder/tappable_state.go b/decoder/tappable_state.go new file mode 100644 index 00000000..ff30c42f --- /dev/null +++ b/decoder/tappable_state.go @@ -0,0 +1,157 @@ +package decoder + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strconv" + "time" + + "github.com/jellydator/ttlcache/v3" + log "github.com/sirupsen/logrus" + + "golbat/db" +) + +func loadTappableFromDatabase(ctx context.Context, db db.DbDetails, id uint64, tappable *Tappable) error { + err := db.GeneralDb.GetContext(ctx, tappable, + `SELECT id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated + FROM tappable WHERE id = ?`, strconv.FormatUint(id, 10)) + statsCollector.IncDbQuery("select tappable", err) + return err +} + +// PeekTappableRecord - cache-only lookup, no DB fallback, returns locked. +// Caller MUST call returned unlock function if non-nil. +func PeekTappableRecord(id uint64) (*Tappable, func(), error) { + if item := tappableCache.Get(id); item != nil { + tappable := item.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil + } + return nil, nil, nil +} + +// getTappableRecordReadOnly acquires lock but does NOT take snapshot. +// Use for read-only checks. Will cause a backing database lookup. +// Caller MUST call returned unlock function if non-nil. +func getTappableRecordReadOnly(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { + // Check cache first + if item := tappableCache.Get(id); item != nil { + tappable := item.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil + } + + dbTappable := Tappable{} + err := loadTappableFromDatabase(ctx, db, id, &dbTappable) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil, nil + } + if err != nil { + return nil, nil, err + } + dbTappable.ClearDirty() + + // Atomically cache the loaded Tappable - if another goroutine raced us, + // we'll get their Tappable and use that instead (ensuring same mutex) + existingTappable, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { + return &dbTappable + }) + + tappable := existingTappable.Value() + tappable.Lock() + return tappable, func() { tappable.Unlock() }, nil +} + +// getOrCreateTappableRecord gets existing or creates new, locked. +// Caller MUST call returned unlock function. +func getOrCreateTappableRecord(ctx context.Context, db db.DbDetails, id uint64) (*Tappable, func(), error) { + // Create new Tappable atomically - function only called if key doesn't exist + tappableItem, _ := tappableCache.GetOrSetFunc(id, func() *Tappable { + return &Tappable{Id: id, newRecord: true} + }) + + tappable := tappableItem.Value() + tappable.Lock() + + if tappable.newRecord { + // We should attempt to load from database + err := loadTappableFromDatabase(ctx, db, id, tappable) + if err != nil { + if !errors.Is(err, sql.ErrNoRows) { + tappable.Unlock() + return nil, nil, err + } + } else { + // We loaded from DB + tappable.newRecord = false + tappable.ClearDirty() + } + } + + return tappable, func() { tappable.Unlock() }, nil +} + +func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tappable) { + // Skip save if not dirty and not new + if !tappable.IsDirty() && !tappable.IsNewRecord() { + return + } + + now := time.Now().Unix() + tappable.Updated = now + + if tappable.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) + } + res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` + INSERT INTO tappable ( + id, lat, lon, fort_id, spawn_id, type, pokemon_id, item_id, count, expire_timestamp, expire_timestamp_verified, updated + ) VALUES ( + "%d", :lat, :lon, :fort_id, :spawn_id, :type, :pokemon_id, :item_id, :count, :expire_timestamp, :expire_timestamp_verified, :updated + ) + `, tappable.Id), tappable) + statsCollector.IncDbQuery("insert tappable", err) + if err != nil { + log.Errorf("insert tappable %d: %s", tappable.Id, err) + return + } + _ = res + } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Tappable", strconv.FormatUint(tappable.Id, 10), tappable.changedFields) + } + res, err := details.GeneralDb.NamedExecContext(ctx, fmt.Sprintf(` + UPDATE tappable SET + lat = :lat, + lon = :lon, + fort_id = :fort_id, + spawn_id = :spawn_id, + type = :type, + pokemon_id = :pokemon_id, + item_id = :item_id, + count = :count, + expire_timestamp = :expire_timestamp, + expire_timestamp_verified = :expire_timestamp_verified, + updated = :updated + WHERE id = "%d" + `, tappable.Id), tappable) + statsCollector.IncDbQuery("update tappable", err) + if err != nil { + log.Errorf("update tappable %d: %s", tappable.Id, err) + return + } + _ = res + } + if dbDebugEnabled { + tappable.changedFields = tappable.changedFields[:0] + } + tappable.ClearDirty() + if tappable.IsNewRecord() { + tappableCache.Set(tappable.Id, tappable, ttlcache.DefaultTTL) + tappable.newRecord = false + } +} diff --git a/decoder/weather.go b/decoder/weather.go index 7f346761..b75a64a4 100644 --- a/decoder/weather.go +++ b/decoder/weather.go @@ -11,9 +11,9 @@ import ( "golbat/webhooks" "github.com/golang/geo/s2" + "github.com/guregu/null/v6" "github.com/jellydator/ttlcache/v3" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) // Weather struct. diff --git a/decoder/weather_iv.go b/decoder/weather_iv.go index 7a85d256..2f55df6b 100644 --- a/decoder/weather_iv.go +++ b/decoder/weather_iv.go @@ -13,8 +13,8 @@ import ( "golbat/pogo" "github.com/golang/geo/s2" + "github.com/guregu/null/v6" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) const masterFileURL = "https://raw.githubusercontent.com/WatWowMap/Masterfile-Generator/master/master-latest-rdm.json" diff --git a/go.mod b/go.mod index fc946286..c3b5b86d 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/golang-migrate/migrate/v4 v4.19.1 github.com/golang/geo v0.0.0-20260129164528-943061e2742c github.com/grafana/pyroscope-go v1.2.7 + github.com/guregu/null/v6 v6.0.0 github.com/jellydator/ttlcache/v3 v3.4.0 github.com/jmoiron/sqlx v1.4.0 github.com/knadh/koanf/maps v0.1.2 @@ -21,7 +22,6 @@ require ( github.com/knadh/koanf/providers/file v1.2.1 github.com/knadh/koanf/providers/structs v1.0.0 github.com/knadh/koanf/v2 v2.3.2 - github.com/nmvalera/striped-mutex v0.1.0 github.com/paulmach/orb v0.12.0 github.com/prometheus/client_golang v1.23.2 github.com/puzpuzpuz/xsync/v3 v3.5.1 @@ -32,7 +32,6 @@ require ( github.com/toorop/gin-logrus v0.0.0-20210225092905-2c785434f26f google.golang.org/grpc v1.78.0 google.golang.org/protobuf v1.36.11 - gopkg.in/guregu/null.v4 v4.0.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 ) diff --git a/go.sum b/go.sum index a16aadf3..282db98a 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= +github.com/guregu/null/v6 v6.0.0 h1:N14VRS+4di81i1PXRiprbQJ9EM9gqBa0+KVMeS/QSjQ= +github.com/guregu/null/v6 v6.0.0/go.mod h1:hrMIhIfrOZeLPZhROSn149tpw2gHkidAqxoXNyeX3iQ= github.com/jellydator/ttlcache/v3 v3.4.0 h1:YS4P125qQS0tNhtL6aeYkheEaB/m8HCqdMMP4mnWdTY= github.com/jellydator/ttlcache/v3 v3.4.0/go.mod h1:Hw9EgjymziQD3yGsQdf1FqFdpp7YjFMd4Srg5EJlgD4= github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o= @@ -161,8 +163,6 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/nmvalera/striped-mutex v0.1.0 h1:+fwhVWGAsgkiOqmZGZ0HzDD3p/CEt3S85wf0YnMUpL0= -github.com/nmvalera/striped-mutex v0.1.0/go.mod h1:D22iujuGIgMJ/2xSnQDK0I3kh0t3ZOiao5VPYBbq1O0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= @@ -338,8 +338,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg= -gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/main.go b/main.go index bf486171..f838c496 100644 --- a/main.go +++ b/main.go @@ -6,11 +6,18 @@ import ( "net" "net/http" "net/http/pprof" - "strings" "sync" "time" _ "time/tzdata" + "golbat/config" + db2 "golbat/db" + "golbat/decoder" + "golbat/external" + pb "golbat/grpc" + "golbat/stats_collector" + "golbat/webhooks" + "github.com/gin-gonic/gin" "github.com/go-sql-driver/mysql" "github.com/golang-migrate/migrate/v4" @@ -20,16 +27,6 @@ import ( log "github.com/sirupsen/logrus" ginlogrus "github.com/toorop/gin-logrus" "google.golang.org/grpc" - "google.golang.org/protobuf/proto" - - "golbat/config" - db2 "golbat/db" - "golbat/decoder" - "golbat/external" - pb "golbat/grpc" - "golbat/pogo" - "golbat/stats_collector" - "golbat/webhooks" ) var db *sqlx.DB @@ -387,727 +384,3 @@ func main() { log.Info("Golbat exiting!") } - -func decode(ctx context.Context, method int, protoData *ProtoData) { - getMethodName := func(method int, trimString bool) string { - if val, ok := pogo.Method_name[int32(method)]; ok { - if trimString && strings.HasPrefix(val, "METHOD_") { - return strings.TrimPrefix(val, "METHOD_") - } - return val - } - return fmt.Sprintf("#%d", method) - } - - if method != int(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION) && protoData.Level < 30 { - statsCollector.IncDecodeMethods("error", "low_level", getMethodName(method, true)) - log.Debugf("Insufficient Level %d Did not process hook type %s", protoData.Level, pogo.Method(method)) - return - } - - processed := false - ignore := false - start := time.Now() - result := "" - - switch pogo.Method(method) { - case pogo.Method_METHOD_START_INCIDENT: - result = decodeStartIncident(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_INVASION_OPEN_COMBAT_SESSION: - if protoData.Request != nil { - result = decodeOpenInvasion(ctx, protoData.Request, protoData.Data) - processed = true - } - case pogo.Method_METHOD_FORT_DETAILS: - result = decodeFortDetails(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_GET_MAP_OBJECTS: - result = decodeGMO(ctx, protoData, getScanParameters(protoData)) - processed = true - case pogo.Method_METHOD_GYM_GET_INFO: - result = decodeGetGymInfo(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_ENCOUNTER: - if getScanParameters(protoData).ProcessPokemon { - result = decodeEncounter(ctx, protoData.Data, protoData.Account, protoData.TimestampMs) - } - processed = true - case pogo.Method_METHOD_DISK_ENCOUNTER: - result = decodeDiskEncounter(ctx, protoData.Data, protoData.Account) - processed = true - case pogo.Method_METHOD_FORT_SEARCH: - result = decodeQuest(ctx, protoData.Data, protoData.HaveAr) - processed = true - case pogo.Method_METHOD_GET_PLAYER: - ignore = true - case pogo.Method_METHOD_GET_HOLOHOLO_INVENTORY: - ignore = true - case pogo.Method_METHOD_CREATE_COMBAT_CHALLENGE: - ignore = true - case pogo.Method(pogo.InternalPlatformClientAction_INTERNAL_PROXY_SOCIAL_ACTION): - if protoData.Request != nil { - result = decodeSocialActionWithRequest(protoData.Request, protoData.Data) - processed = true - } - case pogo.Method_METHOD_GET_MAP_FORTS: - result = decodeGetMapForts(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_GET_ROUTES: - result = decodeGetRoutes(ctx, protoData.Data) - processed = true - case pogo.Method_METHOD_GET_CONTEST_DATA: - if getScanParameters(protoData).ProcessPokestops { - // Request helps, but can be decoded without it - result = decodeGetContestData(ctx, protoData.Request, protoData.Data) - } - processed = true - case pogo.Method_METHOD_GET_POKEMON_SIZE_CONTEST_ENTRY: - // Request is essential to decode this - if protoData.Request != nil { - if getScanParameters(protoData).ProcessPokestops { - result = decodeGetPokemonSizeContestEntry(ctx, protoData.Request, protoData.Data) - } - processed = true - } - case pogo.Method_METHOD_GET_STATION_DETAILS: - if getScanParameters(protoData).ProcessStations { - // Request is essential to decode this - result = decodeGetStationDetails(ctx, protoData.Request, protoData.Data) - } - processed = true - case pogo.Method_METHOD_PROCESS_TAPPABLE: - if getScanParameters(protoData).ProcessTappables { - // Request is essential to decode this - result = decodeTappable(ctx, protoData.Request, protoData.Data, protoData.Account, protoData.TimestampMs) - } - processed = true - case pogo.Method_METHOD_GET_EVENT_RSVPS: - if getScanParameters(protoData).ProcessGyms { - result = decodeGetEventRsvp(ctx, protoData.Request, protoData.Data) - } - processed = true - case pogo.Method_METHOD_GET_EVENT_RSVP_COUNT: - if getScanParameters(protoData).ProcessGyms { - result = decodeGetEventRsvpCount(ctx, protoData.Data) - } - processed = true - default: - log.Debugf("Did not know hook type %s", pogo.Method(method)) - } - if !ignore { - elapsed := time.Since(start) - if processed == true { - statsCollector.IncDecodeMethods("ok", "", getMethodName(method, true)) - log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, result) - } else { - log.Debugf("%s/%s %s - %s - %s", protoData.Uuid, protoData.Account, pogo.Method(method), elapsed, "**Did not process**") - statsCollector.IncDecodeMethods("unprocessed", "", getMethodName(method, true)) - } - } -} - -func getScanParameters(protoData *ProtoData) decoder.ScanParameters { - return decoder.FindScanConfiguration(protoData.ScanContext, protoData.Lat, protoData.Lon) -} - -func decodeQuest(ctx context.Context, sDec []byte, haveAr *bool) string { - if haveAr == nil { - statsCollector.IncDecodeQuest("error", "missing_ar_info") - log.Infoln("Cannot determine AR quest - ignoring") - // We should either assume AR quest, or trace inventory like RDM probably - return "No AR quest info" - } - decodedQuest := &pogo.FortSearchOutProto{} - if err := proto.Unmarshal(sDec, decodedQuest); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeQuest("error", "parse") - return "Parse failure" - } - - if decodedQuest.Result != pogo.FortSearchOutProto_SUCCESS { - statsCollector.IncDecodeQuest("error", "non_success") - res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedQuest.Result, - pogo.FortSearchOutProto_Result_name[int32(decodedQuest.Result)]) - return res - } - - return decoder.UpdatePokestopWithQuest(ctx, dbDetails, decodedQuest, *haveAr) - -} - -func decodeSocialActionWithRequest(request []byte, payload []byte) string { - var proxyRequestProto pogo.ProxyRequestProto - - if err := proto.Unmarshal(request, &proxyRequestProto); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeSocialActionWithRequest("error", "request_parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - var proxyResponseProto pogo.ProxyResponseProto - - if err := proto.Unmarshal(payload, &proxyResponseProto); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeSocialActionWithRequest("error", "response_parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED && proxyResponseProto.Status != pogo.ProxyResponseProto_COMPLETED_AND_REASSIGNED { - statsCollector.IncDecodeSocialActionWithRequest("error", "non_success") - return fmt.Sprintf("unsuccessful proxyResponseProto response %d %s", int(proxyResponseProto.Status), proxyResponseProto.Status) - } - - switch pogo.InternalSocialAction(proxyRequestProto.GetAction()) { - case pogo.InternalSocialAction_SOCIAL_ACTION_LIST_FRIEND_STATUS: - statsCollector.IncDecodeSocialActionWithRequest("ok", "list_friend_status") - return decodeGetFriendDetails(proxyResponseProto.Payload) - case pogo.InternalSocialAction_SOCIAL_ACTION_SEARCH_PLAYER: - statsCollector.IncDecodeSocialActionWithRequest("ok", "search_player") - return decodeSearchPlayer(&proxyRequestProto, proxyResponseProto.Payload) - - } - - statsCollector.IncDecodeSocialActionWithRequest("ok", "unknown") - return fmt.Sprintf("Did not process %s", pogo.InternalSocialAction(proxyRequestProto.GetAction()).String()) -} - -func decodeGetFriendDetails(payload []byte) string { - var getFriendDetailsOutProto pogo.InternalGetFriendDetailsOutProto - getFriendDetailsError := proto.Unmarshal(payload, &getFriendDetailsOutProto) - - if getFriendDetailsError != nil { - statsCollector.IncDecodeGetFriendDetails("error", "parse") - log.Errorf("Failed to parse %s", getFriendDetailsError) - return fmt.Sprintf("Failed to parse %s", getFriendDetailsError) - } - - if getFriendDetailsOutProto.GetResult() != pogo.InternalGetFriendDetailsOutProto_SUCCESS || getFriendDetailsOutProto.GetFriend() == nil { - statsCollector.IncDecodeGetFriendDetails("error", "non_success") - return fmt.Sprintf("unsuccessful get friends details") - } - - failures := 0 - - for _, friend := range getFriendDetailsOutProto.GetFriend() { - player := friend.GetPlayer() - - updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, "", player.GetPlayerId()) - if updatePlayerError != nil { - failures++ - } - } - - statsCollector.IncDecodeGetFriendDetails("ok", "") - return fmt.Sprintf("%d players decoded on %d", len(getFriendDetailsOutProto.GetFriend())-failures, len(getFriendDetailsOutProto.GetFriend())) -} - -func decodeSearchPlayer(proxyRequestProto *pogo.ProxyRequestProto, payload []byte) string { - var searchPlayerOutProto pogo.InternalSearchPlayerOutProto - searchPlayerOutError := proto.Unmarshal(payload, &searchPlayerOutProto) - - if searchPlayerOutError != nil { - log.Errorf("Failed to parse %s", searchPlayerOutError) - statsCollector.IncDecodeSearchPlayer("error", "parse") - return fmt.Sprintf("Failed to parse %s", searchPlayerOutError) - } - - if searchPlayerOutProto.GetResult() != pogo.InternalSearchPlayerOutProto_SUCCESS || searchPlayerOutProto.GetPlayer() == nil { - statsCollector.IncDecodeSearchPlayer("error", "non_success") - return fmt.Sprintf("unsuccessful search player response") - } - - var searchPlayerProto pogo.InternalSearchPlayerProto - searchPlayerError := proto.Unmarshal(proxyRequestProto.GetPayload(), &searchPlayerProto) - - if searchPlayerError != nil || searchPlayerProto.GetFriendCode() == "" { - statsCollector.IncDecodeSearchPlayer("error", "parse") - return fmt.Sprintf("Failed to parse %s", searchPlayerError) - } - - player := searchPlayerOutProto.GetPlayer() - updatePlayerError := decoder.UpdatePlayerRecordWithPlayerSummary(dbDetails, player, player.PublicData, searchPlayerProto.GetFriendCode(), "") - if updatePlayerError != nil { - statsCollector.IncDecodeSearchPlayer("error", "update") - return fmt.Sprintf("Failed update player %s", updatePlayerError) - } - - statsCollector.IncDecodeSearchPlayer("ok", "") - return fmt.Sprintf("1 player decoded from SearchPlayerProto") -} - -func decodeFortDetails(ctx context.Context, sDec []byte) string { - decodedFort := &pogo.FortDetailsOutProto{} - if err := proto.Unmarshal(sDec, decodedFort); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeFortDetails("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - switch decodedFort.FortType { - case pogo.FortType_CHECKPOINT: - statsCollector.IncDecodeFortDetails("ok", "pokestop") - return decoder.UpdatePokestopRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) - case pogo.FortType_GYM: - statsCollector.IncDecodeFortDetails("ok", "gym") - return decoder.UpdateGymRecordWithFortDetailsOutProto(ctx, dbDetails, decodedFort) - } - - statsCollector.IncDecodeFortDetails("ok", "unknown") - return "Unknown fort type" -} - -func decodeGetMapForts(ctx context.Context, sDec []byte) string { - decodedMapForts := &pogo.GetMapFortsOutProto{} - if err := proto.Unmarshal(sDec, decodedMapForts); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeGetMapForts("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedMapForts.Status != pogo.GetMapFortsOutProto_SUCCESS { - statsCollector.IncDecodeGetMapForts("error", "non_success") - res := fmt.Sprintf(`GetMapFortsOutProto: Ignored non-success value %d:%s`, decodedMapForts.Status, - pogo.GetMapFortsOutProto_Status_name[int32(decodedMapForts.Status)]) - return res - } - - statsCollector.IncDecodeGetMapForts("ok", "") - var outputString string - processedForts := 0 - - for _, fort := range decodedMapForts.Fort { - status, output := decoder.UpdateFortRecordWithGetMapFortsOutProto(ctx, dbDetails, fort) - if status { - processedForts += 1 - outputString += output + ", " - } - } - - if processedForts > 0 { - return fmt.Sprintf("Updated %d forts: %s", processedForts, outputString) - } - return "No forts updated" -} - -func decodeGetRoutes(ctx context.Context, payload []byte) string { - getRoutesOutProto := &pogo.GetRoutesOutProto{} - if err := proto.Unmarshal(payload, getRoutesOutProto); err != nil { - return fmt.Sprintf("failed to decode GetRoutesOutProto %s", err) - } - - if getRoutesOutProto.Status != pogo.GetRoutesOutProto_SUCCESS { - return fmt.Sprintf("GetRoutesOutProto: Ignored non-success value %d:%s", getRoutesOutProto.Status, getRoutesOutProto.Status.String()) - } - - decodeSuccesses := map[string]bool{} - decodeErrors := map[string]bool{} - - for _, routeMapCell := range getRoutesOutProto.GetRouteMapCell() { - for _, route := range routeMapCell.GetRoute() { - //TODO we need to check the repeated field, for now access last element - routeSubmissionStatus := route.RouteSubmissionStatus[len(route.RouteSubmissionStatus)-1] - if routeSubmissionStatus != nil && routeSubmissionStatus.Status != pogo.RouteSubmissionStatus_PUBLISHED { - log.Warnf("Non published Route found in GetRoutesOutProto, status: %s", routeSubmissionStatus.String()) - continue - } - decodeError := decoder.UpdateRouteRecordWithSharedRouteProto(ctx, dbDetails, route) - if decodeError != nil { - if decodeErrors[route.Id] != true { - decodeErrors[route.Id] = true - } - log.Errorf("Failed to decode route %s", decodeError) - } else if decodeSuccesses[route.Id] != true { - decodeSuccesses[route.Id] = true - } - } - } - - return fmt.Sprintf( - "Decoded %d routes, failed to decode %d routes, from %d cells", - len(decodeSuccesses), - len(decodeErrors), - len(getRoutesOutProto.GetRouteMapCell()), - ) -} - -func decodeGetGymInfo(ctx context.Context, sDec []byte) string { - decodedGymInfo := &pogo.GymGetInfoOutProto{} - if err := proto.Unmarshal(sDec, decodedGymInfo); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeGetGymInfo("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedGymInfo.Result != pogo.GymGetInfoOutProto_SUCCESS { - statsCollector.IncDecodeGetGymInfo("error", "non_success") - res := fmt.Sprintf(`GymGetInfoOutProto: Ignored non-success value %d:%s`, decodedGymInfo.Result, - pogo.GymGetInfoOutProto_Result_name[int32(decodedGymInfo.Result)]) - return res - } - - statsCollector.IncDecodeGetGymInfo("ok", "") - return decoder.UpdateGymRecordWithGymInfoProto(ctx, dbDetails, decodedGymInfo) -} - -func decodeEncounter(ctx context.Context, sDec []byte, username string, timestampMs int64) string { - decodedEncounterInfo := &pogo.EncounterOutProto{} - if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeEncounter("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedEncounterInfo.Status != pogo.EncounterOutProto_ENCOUNTER_SUCCESS { - statsCollector.IncDecodeEncounter("error", "non_success") - res := fmt.Sprintf(`EncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Status, - pogo.EncounterOutProto_Status_name[int32(decodedEncounterInfo.Status)]) - return res - } - - statsCollector.IncDecodeEncounter("ok", "") - return decoder.UpdatePokemonRecordWithEncounterProto(ctx, dbDetails, decodedEncounterInfo, username, timestampMs) -} - -func decodeDiskEncounter(ctx context.Context, sDec []byte, username string) string { - decodedEncounterInfo := &pogo.DiskEncounterOutProto{} - if err := proto.Unmarshal(sDec, decodedEncounterInfo); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeDiskEncounter("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedEncounterInfo.Result != pogo.DiskEncounterOutProto_SUCCESS { - statsCollector.IncDecodeDiskEncounter("error", "non_success") - res := fmt.Sprintf(`DiskEncounterOutProto: Ignored non-success value %d:%s`, decodedEncounterInfo.Result, - pogo.DiskEncounterOutProto_Result_name[int32(decodedEncounterInfo.Result)]) - return res - } - - statsCollector.IncDecodeDiskEncounter("ok", "") - return decoder.UpdatePokemonRecordWithDiskEncounterProto(ctx, dbDetails, decodedEncounterInfo, username) -} - -func decodeStartIncident(ctx context.Context, sDec []byte) string { - decodedIncident := &pogo.StartIncidentOutProto{} - if err := proto.Unmarshal(sDec, decodedIncident); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeStartIncident("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedIncident.Status != pogo.StartIncidentOutProto_SUCCESS { - statsCollector.IncDecodeStartIncident("error", "non_success") - res := fmt.Sprintf(`GiovanniOutProto: Ignored non-success value %d:%s`, decodedIncident.Status, - pogo.StartIncidentOutProto_Status_name[int32(decodedIncident.Status)]) - return res - } - - statsCollector.IncDecodeStartIncident("ok", "") - return decoder.ConfirmIncident(ctx, dbDetails, decodedIncident) -} - -func decodeOpenInvasion(ctx context.Context, request []byte, payload []byte) string { - decodeOpenInvasionRequest := &pogo.OpenInvasionCombatSessionProto{} - - if err := proto.Unmarshal(request, decodeOpenInvasionRequest); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeOpenInvasion("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - if decodeOpenInvasionRequest.IncidentLookup == nil { - return "Invalid OpenInvasionCombatSessionProto received" - } - - decodedOpenInvasionResponse := &pogo.OpenInvasionCombatSessionOutProto{} - if err := proto.Unmarshal(payload, decodedOpenInvasionResponse); err != nil { - log.Errorf("Failed to parse %s", err) - statsCollector.IncDecodeOpenInvasion("error", "parse") - return fmt.Sprintf("Failed to parse %s", err) - } - - if decodedOpenInvasionResponse.Status != pogo.InvasionStatus_SUCCESS { - statsCollector.IncDecodeOpenInvasion("error", "non_success") - res := fmt.Sprintf(`InvasionLineupOutProto: Ignored non-success value %d:%s`, decodedOpenInvasionResponse.Status, - pogo.InvasionStatus_Status_name[int32(decodedOpenInvasionResponse.Status)]) - return res - } - - statsCollector.IncDecodeOpenInvasion("ok", "") - return decoder.UpdateIncidentLineup(ctx, dbDetails, decodeOpenInvasionRequest, decodedOpenInvasionResponse) -} - -func decodeGMO(ctx context.Context, protoData *ProtoData, scanParameters decoder.ScanParameters) string { - decodedGmo := &pogo.GetMapObjectsOutProto{} - - if err := proto.Unmarshal(protoData.Data, decodedGmo); err != nil { - statsCollector.IncDecodeGMO("error", "parse") - log.Errorf("Failed to parse %s", err) - } - - if decodedGmo.Status != pogo.GetMapObjectsOutProto_SUCCESS { - statsCollector.IncDecodeGMO("error", "non_success") - res := fmt.Sprintf(`GetMapObjectsOutProto: Ignored non-success value %d:%s`, decodedGmo.Status, - pogo.GetMapObjectsOutProto_Status_name[int32(decodedGmo.Status)]) - return res - } - - var newForts []decoder.RawFortData - var newStations []decoder.RawStationData - var newWildPokemon []decoder.RawWildPokemonData - var newNearbyPokemon []decoder.RawNearbyPokemonData - var newMapPokemon []decoder.RawMapPokemonData - var newMapCells []uint64 - var cellsToBeCleaned []uint64 - - // track forts per cell for memory-based cleanup (only if tracker enabled) - cellForts := make(map[uint64]*decoder.FortTrackerGMOContents) - - if len(decodedGmo.MapCell) == 0 { - return "Skipping GetMapObjectsOutProto: No map cells found" - } - for _, mapCell := range decodedGmo.MapCell { - // initialize cell forts tracking for every map cell (so empty fort lists are seen as "no forts") - cellForts[mapCell.S2CellId] = &decoder.FortTrackerGMOContents{ - Pokestops: make([]string, 0), - Gyms: make([]string, 0), - Timestamp: mapCell.AsOfTimeMs, - } - // always mark this mapCell to be checked for removed forts. Previously only cells with forts were - // added which meant an empty fort list (all forts removed) was never passed to the tracker. - cellsToBeCleaned = append(cellsToBeCleaned, mapCell.S2CellId) - - if isCellNotEmpty(mapCell) { - newMapCells = append(newMapCells, mapCell.S2CellId) - } - - for _, fort := range mapCell.Fort { - newForts = append(newForts, decoder.RawFortData{Cell: mapCell.S2CellId, Data: fort, Timestamp: mapCell.AsOfTimeMs}) - - // track fort by type for memory-based cleanup (only if tracker enabled) - if cf, ok := cellForts[mapCell.S2CellId]; ok { - switch fort.FortType { - case pogo.FortType_GYM: - cf.Gyms = append(cf.Gyms, fort.FortId) - case pogo.FortType_CHECKPOINT: - cf.Pokestops = append(cf.Pokestops, fort.FortId) - } - } - - if fort.ActivePokemon != nil { - newMapPokemon = append(newMapPokemon, decoder.RawMapPokemonData{Cell: mapCell.S2CellId, Data: fort.ActivePokemon, Timestamp: mapCell.AsOfTimeMs}) - } - } - for _, mon := range mapCell.WildPokemon { - newWildPokemon = append(newWildPokemon, decoder.RawWildPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) - } - for _, mon := range mapCell.NearbyPokemon { - newNearbyPokemon = append(newNearbyPokemon, decoder.RawNearbyPokemonData{Cell: mapCell.S2CellId, Data: mon, Timestamp: mapCell.AsOfTimeMs}) - } - for _, station := range mapCell.Stations { - newStations = append(newStations, decoder.RawStationData{Cell: mapCell.S2CellId, Data: station}) - } - } - - if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { - decoder.UpdateFortBatch(ctx, dbDetails, scanParameters, newForts) - } - var weatherUpdates []decoder.WeatherUpdate - if scanParameters.ProcessWeather { - weatherUpdates = decoder.UpdateClientWeatherBatch(ctx, dbDetails, decodedGmo.ClientWeather, decodedGmo.MapCell[0].AsOfTimeMs, protoData.Account) - } - if scanParameters.ProcessPokemon { - decoder.UpdatePokemonBatch(ctx, dbDetails, scanParameters, newWildPokemon, newNearbyPokemon, newMapPokemon, decodedGmo.ClientWeather, protoData.Account) - if scanParameters.ProcessWeather && scanParameters.ProactiveIVSwitching { - for _, weatherUpdate := range weatherUpdates { - go func(weatherUpdate decoder.WeatherUpdate) { - decoder.ProactiveIVSwitchSem <- true - defer func() { <-decoder.ProactiveIVSwitchSem }() - decoder.ProactiveIVSwitch(ctx, dbDetails, weatherUpdate, scanParameters.ProactiveIVSwitchingToDB, decodedGmo.MapCell[0].AsOfTimeMs/1000) - }(weatherUpdate) - } - } - } - if scanParameters.ProcessStations { - decoder.UpdateStationBatch(ctx, dbDetails, scanParameters, newStations) - } - - if scanParameters.ProcessCells { - decoder.UpdateClientMapS2CellBatch(ctx, dbDetails, newMapCells) - } - - if scanParameters.ProcessGyms || scanParameters.ProcessPokestops { - go func() { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - decoder.CheckRemovedForts(ctx, dbDetails, cellsToBeCleaned, cellForts) - }() - } - - newFortsLen := len(newForts) - newStationsLen := len(newStations) - newWildPokemonLen := len(newWildPokemon) - newNearbyPokemonLen := len(newNearbyPokemon) - newMapPokemonLen := len(newMapPokemon) - newClientWeatherLen := len(decodedGmo.ClientWeather) - newMapCellsLen := len(newMapCells) - - statsCollector.IncDecodeGMO("ok", "") - statsCollector.AddDecodeGMOType("fort", float64(newFortsLen)) - statsCollector.AddDecodeGMOType("station", float64(newStationsLen)) - statsCollector.AddDecodeGMOType("wild_pokemon", float64(newWildPokemonLen)) - statsCollector.AddDecodeGMOType("nearby_pokemon", float64(newNearbyPokemonLen)) - statsCollector.AddDecodeGMOType("map_pokemon", float64(newMapPokemonLen)) - statsCollector.AddDecodeGMOType("weather", float64(newClientWeatherLen)) - statsCollector.AddDecodeGMOType("cell", float64(newMapCellsLen)) - - return fmt.Sprintf("%d cells containing %d forts %d stations %d mon %d nearby", newMapCellsLen, newFortsLen, newStationsLen, newWildPokemonLen, newNearbyPokemonLen) -} - -func isCellNotEmpty(mapCell *pogo.ClientMapCellProto) bool { - return len(mapCell.Stations) > 0 || len(mapCell.Fort) > 0 || len(mapCell.WildPokemon) > 0 || len(mapCell.NearbyPokemon) > 0 || len(mapCell.CatchablePokemon) > 0 -} - -func cellContainsForts(mapCell *pogo.ClientMapCellProto) bool { - return len(mapCell.Fort) > 0 -} - -func decodeGetContestData(ctx context.Context, request []byte, data []byte) string { - var decodedContestData pogo.GetContestDataOutProto - if err := proto.Unmarshal(data, &decodedContestData); err != nil { - log.Errorf("Failed to parse GetContestDataOutProto %s", err) - return fmt.Sprintf("Failed to parse GetContestDataOutProto %s", err) - } - - var decodedContestDataRequest pogo.GetContestDataProto - if request != nil { - if err := proto.Unmarshal(request, &decodedContestDataRequest); err != nil { - log.Errorf("Failed to parse GetContestDataProto %s", err) - return fmt.Sprintf("Failed to parse GetContestDataProto %s", err) - } - } - return decoder.UpdatePokestopWithContestData(ctx, dbDetails, &decodedContestDataRequest, &decodedContestData) -} - -func decodeGetPokemonSizeContestEntry(ctx context.Context, request []byte, data []byte) string { - var decodedPokemonSizeContestEntry pogo.GetPokemonSizeLeaderboardEntryOutProto - if err := proto.Unmarshal(data, &decodedPokemonSizeContestEntry); err != nil { - log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - } - - if decodedPokemonSizeContestEntry.Status != pogo.GetPokemonSizeLeaderboardEntryOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetPokemonSizeLeaderboardEntryOutProto non-success status %s", decodedPokemonSizeContestEntry.Status) - } - - var decodedPokemonSizeContestEntryRequest pogo.GetPokemonSizeLeaderboardEntryProto - if request != nil { - if err := proto.Unmarshal(request, &decodedPokemonSizeContestEntryRequest); err != nil { - log.Errorf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - return fmt.Sprintf("Failed to parse GetPokemonSizeLeaderboardEntryOutProto %s", err) - } - } - - return decoder.UpdatePokestopWithPokemonSizeContestEntry(ctx, dbDetails, &decodedPokemonSizeContestEntryRequest, &decodedPokemonSizeContestEntry) -} - -func decodeGetStationDetails(ctx context.Context, request []byte, data []byte) string { - var decodedGetStationDetails pogo.GetStationedPokemonDetailsOutProto - if err := proto.Unmarshal(data, &decodedGetStationDetails); err != nil { - log.Errorf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) - return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsOutProto %s", err) - } - - var decodedGetStationDetailsRequest pogo.GetStationedPokemonDetailsProto - if request != nil { - if err := proto.Unmarshal(request, &decodedGetStationDetailsRequest); err != nil { - log.Errorf("Failed to parse GetStationedPokemonDetailsProto %s", err) - return fmt.Sprintf("Failed to parse GetStationedPokemonDetailsProto %s", err) - } - } - - if decodedGetStationDetails.Result == pogo.GetStationedPokemonDetailsOutProto_STATION_NOT_FOUND { - // station without stationed pokemon found, therefore we need to reset the columns - return decoder.ResetStationedPokemonWithStationDetailsNotFound(ctx, dbDetails, &decodedGetStationDetailsRequest) - } else if decodedGetStationDetails.Result != pogo.GetStationedPokemonDetailsOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetStationedPokemonDetailsOutProto non-success status %s", decodedGetStationDetails.Result) - } - - return decoder.UpdateStationWithStationDetails(ctx, dbDetails, &decodedGetStationDetailsRequest, &decodedGetStationDetails) -} - -func decodeTappable(ctx context.Context, request, data []byte, username string, timestampMs int64) string { - var tappable pogo.ProcessTappableOutProto - if err := proto.Unmarshal(data, &tappable); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse ProcessTappableOutProto %s", err) - } - - var tappableRequest pogo.ProcessTappableProto - if request != nil { - if err := proto.Unmarshal(request, &tappableRequest); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse ProcessTappableProto %s", err) - } - } - - if tappable.Status != pogo.ProcessTappableOutProto_SUCCESS { - return fmt.Sprintf("Ignored ProcessTappableOutProto non-success status %s", tappable.Status) - } - var result string - if encounter := tappable.GetEncounter(); encounter != nil { - result = decoder.UpdatePokemonRecordWithTappableEncounter(ctx, dbDetails, &tappableRequest, encounter, username, timestampMs) - } - return result + " " + decoder.UpdateTappable(ctx, dbDetails, &tappableRequest, &tappable, timestampMs) -} - -func decodeGetEventRsvp(ctx context.Context, request []byte, data []byte) string { - var rsvp pogo.GetEventRsvpsOutProto - if err := proto.Unmarshal(data, &rsvp); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse GetEventRsvpsOutProto %s", err) - } - - var rsvpRequest pogo.GetEventRsvpsProto - if request != nil { - if err := proto.Unmarshal(request, &rsvpRequest); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse GetEventRsvpsProto %s", err) - } - } - - if rsvp.Status != pogo.GetEventRsvpsOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetEventRsvpsOutProto non-success status %s", rsvp.Status) - } - - switch op := rsvpRequest.EventDetails.(type) { - case *pogo.GetEventRsvpsProto_Raid: - return decoder.UpdateGymRecordWithRsvpProto(ctx, dbDetails, op.Raid, &rsvp) - case *pogo.GetEventRsvpsProto_GmaxBattle: - return "Unsupported GmaxBattle Rsvp received" - } - - return "Failed to parse GetEventRsvpsProto - unknown event type" -} - -func decodeGetEventRsvpCount(ctx context.Context, data []byte) string { - var rsvp pogo.GetEventRsvpCountOutProto - if err := proto.Unmarshal(data, &rsvp); err != nil { - log.Errorf("Failed to parse %s", err) - return fmt.Sprintf("Failed to parse GetEventRsvpCountOutProto %s", err) - } - - if rsvp.Status != pogo.GetEventRsvpCountOutProto_SUCCESS { - return fmt.Sprintf("Ignored GetEventRsvpCountOutProto non-success status %s", rsvp.Status) - } - - var clearLocations []string - for _, rsvpDetails := range rsvp.RsvpDetails { - if rsvpDetails.MaybeCount == 0 && rsvpDetails.GoingCount == 0 { - clearLocations = append(clearLocations, rsvpDetails.LocationId) - decoder.ClearGymRsvp(ctx, dbDetails, rsvpDetails.LocationId) - } - } - - return "Cleared RSVP @ " + strings.Join(clearLocations, ", ") -} diff --git a/routes.go b/routes.go index ff1fda73..0cf7efff 100644 --- a/routes.go +++ b/routes.go @@ -501,14 +501,22 @@ func GetPokestop(c *gin.Context) { return } - c.JSON(http.StatusAccepted, pokestop) + if pokestop == nil { + c.Status(http.StatusNotFound) + return + } + result := decoder.BuildPokestopResult(pokestop) + c.JSON(http.StatusAccepted, result) } func GetGym(c *gin.Context) { gymId := c.Param("gym_id") ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - gym, err := decoder.GetGymRecord(ctx, dbDetails, gymId) + gym, unlock, err := decoder.GetGymRecordReadOnly(ctx, dbDetails, gymId) + if unlock != nil { + defer unlock() + } cancel() if err != nil { log.Warnf("GET /api/gym/id/:gym_id/ Error during post retrieve %v", err) @@ -516,7 +524,12 @@ func GetGym(c *gin.Context) { return } - c.JSON(http.StatusAccepted, gym) + if gym == nil { + c.Status(http.StatusNotFound) + return + } + result := decoder.BuildGymResult(gym) + c.JSON(http.StatusAccepted, result) } // POST /api/gym/query @@ -561,23 +574,29 @@ func GetGyms(c *gin.Context) { } if len(ids) == 0 { - c.JSON(http.StatusOK, []decoder.Gym{}) + c.JSON(http.StatusOK, []decoder.ApiGymResult{}) return } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - out := make([]*decoder.Gym, 0, len(ids)) + out := make([]decoder.ApiGymResult, 0, len(ids)) for _, id := range ids { - g, err := decoder.GetGymRecord(ctx, dbDetails, id) + g, unlock, err := decoder.GetGymRecordReadOnly(ctx, dbDetails, id) if err != nil { + if unlock != nil { + unlock() + } log.Warnf("error retrieving gym %s: %v", id, err) c.Status(http.StatusInternalServerError) return } if g != nil { - out = append(out, g) + out = append(out, decoder.BuildGymResult(g)) + } + if unlock != nil { + unlock() } if ctx.Err() != nil { c.Status(http.StatusInternalServerError) @@ -691,13 +710,16 @@ func SearchGyms(c *gin.Context) { return } - out := make([]*decoder.Gym, 0, len(ids)) + out := make([]decoder.ApiGymResult, 0, len(ids)) for _, id := range ids { if id == "" { continue } - g, err := decoder.GetGymRecord(ctx, dbDetails, id) + g, unlock, err := decoder.GetGymRecordReadOnly(ctx, dbDetails, id) if err != nil { + if unlock != nil { + unlock() + } if errors.Is(err, context.DeadlineExceeded) || errors.Is(ctx.Err(), context.DeadlineExceeded) { log.Warnf("timed out while fetching %s: %v", id, err) c.Status(http.StatusGatewayTimeout) @@ -708,7 +730,10 @@ func SearchGyms(c *gin.Context) { return } if g != nil { - out = append(out, g) + out = append(out, decoder.BuildGymResult(g)) + } + if unlock != nil { + unlock() } if ctx.Err() != nil { c.Status(http.StatusInternalServerError) @@ -727,16 +752,21 @@ func GetTappable(c *gin.Context) { c.Status(http.StatusBadRequest) return } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - tappable, err := decoder.GetTappableRecord(ctx, dbDetails, tappableId) - cancel() + tappable, unlock, err := decoder.PeekTappableRecord(tappableId) + if unlock != nil { + defer unlock() + } if err != nil { log.Warnf("GET /api/tappable/id/:tappable_id/ Error during post retrieve %v", err) c.Status(http.StatusInternalServerError) return } - - c.JSON(http.StatusAccepted, tappable) + if tappable == nil { + c.Status(http.StatusNotFound) + return + } + result := decoder.BuildTappableResult(tappable) + c.JSON(http.StatusAccepted, result) } func GetDevices(c *gin.Context) { diff --git a/stats_collector/noop.go b/stats_collector/noop.go index e50163ea..1a1ff0fa 100644 --- a/stats_collector/noop.go +++ b/stats_collector/noop.go @@ -1,7 +1,7 @@ package stats_collector import ( - "gopkg.in/guregu/null.v4" + "github.com/guregu/null/v6" "golbat/geo" ) diff --git a/stats_collector/prometheus.go b/stats_collector/prometheus.go index e98ea0ae..93547984 100644 --- a/stats_collector/prometheus.go +++ b/stats_collector/prometheus.go @@ -2,13 +2,14 @@ package stats_collector import ( "database/sql" - "golbat/util" "strconv" "sync" "time" + "golbat/util" + + "github.com/guregu/null/v6" "github.com/prometheus/client_golang/prometheus" - "gopkg.in/guregu/null.v4" "golbat/geo" ) diff --git a/stats_collector/stats_collector.go b/stats_collector/stats_collector.go index de14c3f1..9481c41a 100644 --- a/stats_collector/stats_collector.go +++ b/stats_collector/stats_collector.go @@ -6,8 +6,8 @@ import ( "github.com/Depado/ginprom" "github.com/gin-gonic/gin" + "github.com/guregu/null/v6" log "github.com/sirupsen/logrus" - "gopkg.in/guregu/null.v4" ) type StatsCollector interface { From 58c22dbd3a5835b0e2410acb498126aa138ad243 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 31 Jan 2026 14:28:14 +0000 Subject: [PATCH 27/35] Remove json fields, these should not be directly serialised --- decoder/gym.go | 100 +++++++++++++++++++++---------------------- decoder/pokemon.go | 90 +++++++++++++++++++------------------- decoder/pokestop.go | 102 ++++++++++++++++++++++---------------------- decoder/tappable.go | 36 ++++++++-------- 4 files changed, 164 insertions(+), 164 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index c0004247..e966b316 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -9,56 +9,56 @@ import ( // Gym struct. // REMINDER! Keep hasChangesGym updated after making changes type Gym struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - - Id string `db:"id" json:"id"` - Lat float64 `db:"lat" json:"lat"` - Lon float64 `db:"lon" json:"lon"` - Name null.String `db:"name" json:"name"` - Url null.String `db:"url" json:"url"` - LastModifiedTimestamp null.Int `db:"last_modified_timestamp" json:"last_modified_timestamp"` - RaidEndTimestamp null.Int `db:"raid_end_timestamp" json:"raid_end_timestamp"` - RaidSpawnTimestamp null.Int `db:"raid_spawn_timestamp" json:"raid_spawn_timestamp"` - RaidBattleTimestamp null.Int `db:"raid_battle_timestamp" json:"raid_battle_timestamp"` - Updated int64 `db:"updated" json:"updated"` - RaidPokemonId null.Int `db:"raid_pokemon_id" json:"raid_pokemon_id"` - GuardingPokemonId null.Int `db:"guarding_pokemon_id" json:"guarding_pokemon_id"` - GuardingPokemonDisplay null.String `db:"guarding_pokemon_display" json:"guarding_pokemon_display"` - AvailableSlots null.Int `db:"available_slots" json:"available_slots"` - TeamId null.Int `db:"team_id" json:"team_id"` - RaidLevel null.Int `db:"raid_level" json:"raid_level"` - Enabled null.Int `db:"enabled" json:"enabled"` - ExRaidEligible null.Int `db:"ex_raid_eligible" json:"ex_raid_eligible"` - InBattle null.Int `db:"in_battle" json:"in_battle"` - RaidPokemonMove1 null.Int `db:"raid_pokemon_move_1" json:"raid_pokemon_move_1"` - RaidPokemonMove2 null.Int `db:"raid_pokemon_move_2" json:"raid_pokemon_move_2"` - RaidPokemonForm null.Int `db:"raid_pokemon_form" json:"raid_pokemon_form"` - RaidPokemonAlignment null.Int `db:"raid_pokemon_alignment" json:"raid_pokemon_alignment"` - RaidPokemonCp null.Int `db:"raid_pokemon_cp" json:"raid_pokemon_cp"` - RaidIsExclusive null.Int `db:"raid_is_exclusive" json:"raid_is_exclusive"` - CellId null.Int `db:"cell_id" json:"cell_id"` - Deleted bool `db:"deleted" json:"deleted"` - TotalCp null.Int `db:"total_cp" json:"total_cp"` - FirstSeenTimestamp int64 `db:"first_seen_timestamp" json:"first_seen_timestamp"` - RaidPokemonGender null.Int `db:"raid_pokemon_gender" json:"raid_pokemon_gender"` - SponsorId null.Int `db:"sponsor_id" json:"sponsor_id"` - PartnerId null.String `db:"partner_id" json:"partner_id"` - RaidPokemonCostume null.Int `db:"raid_pokemon_costume" json:"raid_pokemon_costume"` - RaidPokemonEvolution null.Int `db:"raid_pokemon_evolution" json:"raid_pokemon_evolution"` - ArScanEligible null.Int `db:"ar_scan_eligible" json:"ar_scan_eligible"` - PowerUpLevel null.Int `db:"power_up_level" json:"power_up_level"` - PowerUpPoints null.Int `db:"power_up_points" json:"power_up_points"` - PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp" json:"power_up_end_timestamp"` - Description null.String `db:"description" json:"description"` - Defenders null.String `db:"defenders" json:"defenders"` - Rsvps null.String `db:"rsvps" json:"rsvps"` - - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving (to db) - internalDirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving (in memory only) - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record - changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) - - oldValues GymOldValues `db:"-" json:"-"` // Old values for webhook comparison + mu sync.Mutex `db:"-"` // Object-level mutex + + Id string `db:"id"` + Lat float64 `db:"lat"` + Lon float64 `db:"lon"` + Name null.String `db:"name"` + Url null.String `db:"url"` + LastModifiedTimestamp null.Int `db:"last_modified_timestamp"` + RaidEndTimestamp null.Int `db:"raid_end_timestamp"` + RaidSpawnTimestamp null.Int `db:"raid_spawn_timestamp"` + RaidBattleTimestamp null.Int `db:"raid_battle_timestamp"` + Updated int64 `db:"updated"` + RaidPokemonId null.Int `db:"raid_pokemon_id"` + GuardingPokemonId null.Int `db:"guarding_pokemon_id"` + GuardingPokemonDisplay null.String `db:"guarding_pokemon_display"` + AvailableSlots null.Int `db:"available_slots"` + TeamId null.Int `db:"team_id"` + RaidLevel null.Int `db:"raid_level"` + Enabled null.Int `db:"enabled"` + ExRaidEligible null.Int `db:"ex_raid_eligible"` + InBattle null.Int `db:"in_battle"` + RaidPokemonMove1 null.Int `db:"raid_pokemon_move_1"` + RaidPokemonMove2 null.Int `db:"raid_pokemon_move_2"` + RaidPokemonForm null.Int `db:"raid_pokemon_form"` + RaidPokemonAlignment null.Int `db:"raid_pokemon_alignment"` + RaidPokemonCp null.Int `db:"raid_pokemon_cp"` + RaidIsExclusive null.Int `db:"raid_is_exclusive"` + CellId null.Int `db:"cell_id"` + Deleted bool `db:"deleted"` + TotalCp null.Int `db:"total_cp"` + FirstSeenTimestamp int64 `db:"first_seen_timestamp"` + RaidPokemonGender null.Int `db:"raid_pokemon_gender"` + SponsorId null.Int `db:"sponsor_id"` + PartnerId null.String `db:"partner_id"` + RaidPokemonCostume null.Int `db:"raid_pokemon_costume"` + RaidPokemonEvolution null.Int `db:"raid_pokemon_evolution"` + ArScanEligible null.Int `db:"ar_scan_eligible"` + PowerUpLevel null.Int `db:"power_up_level"` + PowerUpPoints null.Int `db:"power_up_points"` + PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp"` + Description null.String `db:"description"` + Defenders null.String `db:"defenders"` + Rsvps null.String `db:"rsvps"` + + dirty bool `db:"-"` // Not persisted - tracks if object needs saving (to db) + internalDirty bool `db:"-"` // Not persisted - tracks if object needs saving (in memory only) + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) + + oldValues GymOldValues `db:"-"` // Old values for webhook comparison } // GymOldValues holds old field values for webhook comparison (populated when loading from cache/DB) diff --git a/decoder/pokemon.go b/decoder/pokemon.go index cc69bdb6..b4b9973c 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -17,55 +17,55 @@ import ( // // FirstSeenTimestamp: This field is used in IsNewRecord. It should only be set in savePokemonRecord. type Pokemon struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - - Id uint64 `db:"id" json:"id,string"` - PokestopId null.String `db:"pokestop_id" json:"pokestop_id"` - SpawnId null.Int `db:"spawn_id" json:"spawn_id"` - Lat float64 `db:"lat" json:"lat"` - Lon float64 `db:"lon" json:"lon"` - Weight null.Float `db:"weight" json:"weight"` - Size null.Int `db:"size" json:"size"` - Height null.Float `db:"height" json:"height"` - ExpireTimestamp null.Int `db:"expire_timestamp" json:"expire_timestamp"` - Updated null.Int `db:"updated" json:"updated"` - PokemonId int16 `db:"pokemon_id" json:"pokemon_id"` - Move1 null.Int `db:"move_1" json:"move_1"` - Move2 null.Int `db:"move_2" json:"move_2"` - Gender null.Int `db:"gender" json:"gender"` - Cp null.Int `db:"cp" json:"cp"` - AtkIv null.Int `db:"atk_iv" json:"atk_iv"` - DefIv null.Int `db:"def_iv" json:"def_iv"` - StaIv null.Int `db:"sta_iv" json:"sta_iv"` - GolbatInternal []byte `db:"golbat_internal" json:"golbat_internal"` - Iv null.Float `db:"iv" json:"iv"` - Form null.Int `db:"form" json:"form"` - Level null.Int `db:"level" json:"level"` - IsStrong null.Bool `db:"strong" json:"strong"` - Weather null.Int `db:"weather" json:"weather"` - Costume null.Int `db:"costume" json:"costume"` - FirstSeenTimestamp int64 `db:"first_seen_timestamp" json:"first_seen_timestamp"` - Changed int64 `db:"changed" json:"changed"` - CellId null.Int `db:"cell_id" json:"cell_id"` - ExpireTimestampVerified bool `db:"expire_timestamp_verified" json:"expire_timestamp_verified"` - DisplayPokemonId null.Int `db:"display_pokemon_id" json:"display_pokemon_id"` - IsDitto bool `db:"is_ditto" json:"is_ditto"` - SeenType null.String `db:"seen_type" json:"seen_type"` - Shiny null.Bool `db:"shiny" json:"shiny"` - Username null.String `db:"username" json:"username"` - Capture1 null.Float `db:"capture_1" json:"capture_1"` - Capture2 null.Float `db:"capture_2" json:"capture_2"` - Capture3 null.Float `db:"capture_3" json:"capture_3"` - Pvp null.String `db:"pvp" json:"pvp"` - IsEvent int8 `db:"is_event" json:"is_event"` + mu sync.Mutex `db:"-"` // Object-level mutex + + Id uint64 `db:"id"` + PokestopId null.String `db:"pokestop_id"` + SpawnId null.Int `db:"spawn_id"` + Lat float64 `db:"lat"` + Lon float64 `db:"lon"` + Weight null.Float `db:"weight"` + Size null.Int `db:"size"` + Height null.Float `db:"height"` + ExpireTimestamp null.Int `db:"expire_timestamp"` + Updated null.Int `db:"updated"` + PokemonId int16 `db:"pokemon_id"` + Move1 null.Int `db:"move_1"` + Move2 null.Int `db:"move_2"` + Gender null.Int `db:"gender"` + Cp null.Int `db:"cp"` + AtkIv null.Int `db:"atk_iv"` + DefIv null.Int `db:"def_iv"` + StaIv null.Int `db:"sta_iv"` + GolbatInternal []byte `db:"golbat_internal"` + Iv null.Float `db:"iv"` + Form null.Int `db:"form"` + Level null.Int `db:"level"` + IsStrong null.Bool `db:"strong"` + Weather null.Int `db:"weather"` + Costume null.Int `db:"costume"` + FirstSeenTimestamp int64 `db:"first_seen_timestamp"` + Changed int64 `db:"changed"` + CellId null.Int `db:"cell_id"` + ExpireTimestampVerified bool `db:"expire_timestamp_verified"` + DisplayPokemonId null.Int `db:"display_pokemon_id"` + IsDitto bool `db:"is_ditto"` + SeenType null.String `db:"seen_type"` + Shiny null.Bool `db:"shiny"` + Username null.String `db:"username"` + Capture1 null.Float `db:"capture_1"` + Capture2 null.Float `db:"capture_2"` + Capture3 null.Float `db:"capture_3"` + Pvp null.String `db:"pvp"` + IsEvent int8 `db:"is_event"` internal grpc.PokemonInternal - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` - changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) - oldValues PokemonOldValues `db:"-" json:"-"` // Old values for webhook comparison and stats + oldValues PokemonOldValues `db:"-"` // Old values for webhook comparison and stats } // PokemonOldValues holds old field values for webhook comparison, stats, and R-tree updates diff --git a/decoder/pokestop.go b/decoder/pokestop.go index cad7e262..829ab153 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -8,57 +8,57 @@ import ( // Pokestop struct. type Pokestop struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - - Id string `db:"id" json:"id"` - Lat float64 `db:"lat" json:"lat"` - Lon float64 `db:"lon" json:"lon"` - Name null.String `db:"name" json:"name"` - Url null.String `db:"url" json:"url"` - LureExpireTimestamp null.Int `db:"lure_expire_timestamp" json:"lure_expire_timestamp"` - LastModifiedTimestamp null.Int `db:"last_modified_timestamp" json:"last_modified_timestamp"` - Updated int64 `db:"updated" json:"updated"` - Enabled null.Bool `db:"enabled" json:"enabled"` - QuestType null.Int `db:"quest_type" json:"quest_type"` - QuestTimestamp null.Int `db:"quest_timestamp" json:"quest_timestamp"` - QuestTarget null.Int `db:"quest_target" json:"quest_target"` - QuestConditions null.String `db:"quest_conditions" json:"quest_conditions"` - QuestRewards null.String `db:"quest_rewards" json:"quest_rewards"` - QuestTemplate null.String `db:"quest_template" json:"quest_template"` - QuestTitle null.String `db:"quest_title" json:"quest_title"` - QuestExpiry null.Int `db:"quest_expiry" json:"quest_expiry"` - CellId null.Int `db:"cell_id" json:"cell_id"` - Deleted bool `db:"deleted" json:"deleted"` - LureId int16 `db:"lure_id" json:"lure_id"` - FirstSeenTimestamp int16 `db:"first_seen_timestamp" json:"first_seen_timestamp"` - SponsorId null.Int `db:"sponsor_id" json:"sponsor_id"` - PartnerId null.String `db:"partner_id" json:"partner_id"` - ArScanEligible null.Int `db:"ar_scan_eligible" json:"ar_scan_eligible"` // is an 8 - PowerUpLevel null.Int `db:"power_up_level" json:"power_up_level"` - PowerUpPoints null.Int `db:"power_up_points" json:"power_up_points"` - PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp" json:"power_up_end_timestamp"` - AlternativeQuestType null.Int `db:"alternative_quest_type" json:"alternative_quest_type"` - AlternativeQuestTimestamp null.Int `db:"alternative_quest_timestamp" json:"alternative_quest_timestamp"` - AlternativeQuestTarget null.Int `db:"alternative_quest_target" json:"alternative_quest_target"` - AlternativeQuestConditions null.String `db:"alternative_quest_conditions" json:"alternative_quest_conditions"` - AlternativeQuestRewards null.String `db:"alternative_quest_rewards" json:"alternative_quest_rewards"` - AlternativeQuestTemplate null.String `db:"alternative_quest_template" json:"alternative_quest_template"` - AlternativeQuestTitle null.String `db:"alternative_quest_title" json:"alternative_quest_title"` - AlternativeQuestExpiry null.Int `db:"alternative_quest_expiry" json:"alternative_quest_expiry"` - Description null.String `db:"description" json:"description"` - ShowcaseFocus null.String `db:"showcase_focus" json:"showcase_focus"` - ShowcasePokemon null.Int `db:"showcase_pokemon_id" json:"showcase_pokemon_id"` - ShowcasePokemonForm null.Int `db:"showcase_pokemon_form_id" json:"showcase_pokemon_form_id"` - ShowcasePokemonType null.Int `db:"showcase_pokemon_type_id" json:"showcase_pokemon_type_id"` - ShowcaseRankingStandard null.Int `db:"showcase_ranking_standard" json:"showcase_ranking_standard"` - ShowcaseExpiry null.Int `db:"showcase_expiry" json:"showcase_expiry"` - ShowcaseRankings null.String `db:"showcase_rankings" json:"showcase_rankings"` - - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record - changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) - - oldValues PokestopOldValues `db:"-" json:"-"` // Old values for webhook comparison + mu sync.Mutex `db:"-"` // Object-level mutex + + Id string `db:"id"` + Lat float64 `db:"lat"` + Lon float64 `db:"lon"` + Name null.String `db:"name"` + Url null.String `db:"url"` + LureExpireTimestamp null.Int `db:"lure_expire_timestamp"` + LastModifiedTimestamp null.Int `db:"last_modified_timestamp"` + Updated int64 `db:"updated"` + Enabled null.Bool `db:"enabled"` + QuestType null.Int `db:"quest_type"` + QuestTimestamp null.Int `db:"quest_timestamp"` + QuestTarget null.Int `db:"quest_target"` + QuestConditions null.String `db:"quest_conditions"` + QuestRewards null.String `db:"quest_rewards"` + QuestTemplate null.String `db:"quest_template"` + QuestTitle null.String `db:"quest_title"` + QuestExpiry null.Int `db:"quest_expiry"` + CellId null.Int `db:"cell_id"` + Deleted bool `db:"deleted"` + LureId int16 `db:"lure_id"` + FirstSeenTimestamp int16 `db:"first_seen_timestamp"` + SponsorId null.Int `db:"sponsor_id"` + PartnerId null.String `db:"partner_id"` + ArScanEligible null.Int `db:"ar_scan_eligible"` // is an 8 + PowerUpLevel null.Int `db:"power_up_level"` + PowerUpPoints null.Int `db:"power_up_points"` + PowerUpEndTimestamp null.Int `db:"power_up_end_timestamp"` + AlternativeQuestType null.Int `db:"alternative_quest_type"` + AlternativeQuestTimestamp null.Int `db:"alternative_quest_timestamp"` + AlternativeQuestTarget null.Int `db:"alternative_quest_target"` + AlternativeQuestConditions null.String `db:"alternative_quest_conditions"` + AlternativeQuestRewards null.String `db:"alternative_quest_rewards"` + AlternativeQuestTemplate null.String `db:"alternative_quest_template"` + AlternativeQuestTitle null.String `db:"alternative_quest_title"` + AlternativeQuestExpiry null.Int `db:"alternative_quest_expiry"` + Description null.String `db:"description"` + ShowcaseFocus null.String `db:"showcase_focus"` + ShowcasePokemon null.Int `db:"showcase_pokemon_id"` + ShowcasePokemonForm null.Int `db:"showcase_pokemon_form_id"` + ShowcasePokemonType null.Int `db:"showcase_pokemon_type_id"` + ShowcaseRankingStandard null.Int `db:"showcase_ranking_standard"` + ShowcaseExpiry null.Int `db:"showcase_expiry"` + ShowcaseRankings null.String `db:"showcase_rankings"` + + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) + + oldValues PokestopOldValues `db:"-"` // Old values for webhook comparison } // PokestopOldValues holds old field values for webhook comparison (populated when loading from cache/DB) diff --git a/decoder/tappable.go b/decoder/tappable.go index 57884bfd..3ab8d218 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -9,24 +9,24 @@ import ( // Tappable struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Tappable struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex - - Id uint64 `db:"id" json:"id"` - Lat float64 `db:"lat" json:"lat"` - Lon float64 `db:"lon" json:"lon"` - FortId null.String `db:"fort_id" json:"fort_id"` // either fortId or spawnpointId are given - SpawnId null.Int `db:"spawn_id" json:"spawn_id"` - Type string `db:"type" json:"type"` - Encounter null.Int `db:"pokemon_id" json:"pokemon_id"` - ItemId null.Int `db:"item_id" json:"item_id"` - Count null.Int `db:"count" json:"count"` - ExpireTimestamp null.Int `db:"expire_timestamp" json:"expire_timestamp"` - ExpireTimestampVerified bool `db:"expire_timestamp_verified" json:"expire_timestamp_verified"` - Updated int64 `db:"updated" json:"updated"` - - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record - changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) + mu sync.Mutex `db:"-"` // Object-level mutex + + Id uint64 `db:"id"` + Lat float64 `db:"lat"` + Lon float64 `db:"lon"` + FortId null.String `db:"fort_id"` // either fortId or spawnpointId are given + SpawnId null.Int `db:"spawn_id"` + Type string `db:"type"` + Encounter null.Int `db:"pokemon_id"` + ItemId null.Int `db:"item_id"` + Count null.Int `db:"count"` + ExpireTimestamp null.Int `db:"expire_timestamp"` + ExpireTimestampVerified bool `db:"expire_timestamp_verified"` + Updated int64 `db:"updated"` + + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) } // IsDirty returns true if any field has been modified From b69b5acd813ac74b65ec7acbf68ec018f249bb32 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sat, 31 Jan 2026 14:48:20 +0000 Subject: [PATCH 28/35] Claude review items --- decoder/incident.go | 10 +++++----- decoder/pending_pokemon.go | 6 ++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/decoder/incident.go b/decoder/incident.go index 7149a978..049edf7e 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -3,13 +3,13 @@ package decoder import ( "sync" - null "github.com/guregu/null/v6" + "github.com/guregu/null/v6" ) // Incident struct. // REMINDER! Dirty flag pattern - use setter methods to modify fields type Incident struct { - mu sync.Mutex `db:"-" json:"-"` // Object-level mutex + mu sync.Mutex `db:"-"` // Object-level mutex Id string `db:"id"` PokestopId string `db:"pokestop_id"` @@ -27,10 +27,10 @@ type Incident struct { Slot3PokemonId null.Int `db:"slot_3_pokemon_id"` Slot3Form null.Int `db:"slot_3_form"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record - oldValues IncidentOldValues `db:"-" json:"-"` // Old values for webhook comparison + oldValues IncidentOldValues `db:"-"` // Old values for webhook comparison } // IncidentOldValues holds old field values for webhook comparison and stats diff --git a/decoder/pending_pokemon.go b/decoder/pending_pokemon.go index 60c1153f..39042db3 100644 --- a/decoder/pending_pokemon.go +++ b/decoder/pending_pokemon.go @@ -124,6 +124,12 @@ func (q *PokemonPendingQueue) StartSweeper(ctx context.Context, interval time.Du // processExpired handles pokemon that didn't receive an encounter within the timeout func (q *PokemonPendingQueue) processExpired(ctx context.Context, dbDetails db.DbDetails, expired []*PendingPokemon) { for _, p := range expired { + // Check for shutdown signal between iterations + if ctx.Err() != nil { + log.Debug("Context cancelled, stopping expired pokemon processing") + return + } + processCtx, cancel := context.WithTimeout(ctx, 3*time.Second) pokemon, unlock, err := getOrCreatePokemonRecord(processCtx, dbDetails, p.EncounterId) From 1452c1fd6403d9538b4799b27d95f4f8b223861c Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 17:00:30 +0000 Subject: [PATCH 29/35] Move config options into tuning --- config/config.go | 50 ++++++++++++++++++++++++------------------------ config/reader.go | 8 ++++---- decoder/main.go | 4 ++-- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/config/config.go b/config/config.go index fb66aa37..ab2423d4 100644 --- a/config/config.go +++ b/config/config.go @@ -7,27 +7,25 @@ import ( ) type configDefinition struct { - Port int `koanf:"port"` - GrpcPort int `koanf:"grpc_port"` - Webhooks []Webhook `koanf:"webhooks"` - Database database `koanf:"database"` - Logging logging `koanf:"logging"` - Sentry sentry `koanf:"sentry"` - Pyroscope pyroscope `koanf:"pyroscope"` - Prometheus Prometheus `koanf:"prometheus"` - PokemonMemoryOnly bool `koanf:"pokemon_memory_only"` - PokemonInternalToDb bool `koanf:"pokemon_internal_to_db"` - TestFortInMemory bool `koanf:"test_fort_in_memory"` - Cleanup cleanup `koanf:"cleanup"` - RawBearer string `koanf:"raw_bearer"` - ApiSecret string `koanf:"api_secret"` - Pvp pvp `koanf:"pvp"` - Koji koji `koanf:"koji"` - Tuning tuning `koanf:"tuning"` - Weather weather `koanf:"weather"` - ScanRules []scanRule `koanf:"scan_rules"` - MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` - ReduceUpdates bool `koanf:"reduce_updates"` + Port int `koanf:"port"` + GrpcPort int `koanf:"grpc_port"` + Webhooks []Webhook `koanf:"webhooks"` + Database database `koanf:"database"` + Logging logging `koanf:"logging"` + Sentry sentry `koanf:"sentry"` + Pyroscope pyroscope `koanf:"pyroscope"` + Prometheus Prometheus `koanf:"prometheus"` + PokemonMemoryOnly bool `koanf:"pokemon_memory_only"` + PokemonInternalToDb bool `koanf:"pokemon_internal_to_db"` + TestFortInMemory bool `koanf:"test_fort_in_memory"` + Cleanup cleanup `koanf:"cleanup"` + RawBearer string `koanf:"raw_bearer"` + ApiSecret string `koanf:"api_secret"` + Pvp pvp `koanf:"pvp"` + Koji koji `koanf:"koji"` + Tuning tuning `koanf:"tuning"` + Weather weather `koanf:"weather"` + ScanRules []scanRule `koanf:"scan_rules"` } func (configDefinition configDefinition) GetWebhookInterval() time.Duration { @@ -121,10 +119,12 @@ type database struct { } type tuning struct { - ExtendedTimeout bool `koanf:"extended_timeout"` - MaxPokemonResults int `koanf:"max_pokemon_results"` - MaxPokemonDistance float64 `koanf:"max_pokemon_distance"` - ProfileRoutes bool `koanf:"profile_routes"` + ExtendedTimeout bool `koanf:"extended_timeout"` + MaxPokemonResults int `koanf:"max_pokemon_results"` + MaxPokemonDistance float64 `koanf:"max_pokemon_distance"` + ProfileRoutes bool `koanf:"profile_routes"` + MaxConcurrentProactiveIVSwitch int `koanf:"max_concurrent_proactive_iv_switch"` + ReduceUpdates bool `koanf:"reduce_updates"` } type scanRule struct { diff --git a/config/reader.go b/config/reader.go index e0daa36c..57935895 100644 --- a/config/reader.go +++ b/config/reader.go @@ -51,8 +51,10 @@ func ReadConfig() (configDefinition, error) { MaxPool: 100, }, Tuning: tuning{ - MaxPokemonResults: 3000, - MaxPokemonDistance: 100, + MaxPokemonResults: 3000, + MaxPokemonDistance: 100, + MaxConcurrentProactiveIVSwitch: 6, + ReduceUpdates: false, }, Weather: weather{ ProactiveIVSwitching: true, @@ -61,8 +63,6 @@ func ReadConfig() (configDefinition, error) { Pvp: pvp{ LevelCaps: []int{50, 51}, }, - MaxConcurrentProactiveIVSwitch: 6, - ReduceUpdates: false, }, "koanf"), nil) if defaultErr != nil { fmt.Println(fmt.Errorf("failed to load default config: %w", defaultErr)) diff --git a/decoder/main.go b/decoder/main.go index 51dc4e56..b1e2b589 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -77,7 +77,7 @@ func init() { } func InitProactiveIVSwitchSem() { - ProactiveIVSwitchSem = make(chan bool, config.Config.MaxConcurrentProactiveIVSwitch) + ProactiveIVSwitchSem = make(chan bool, config.Config.Tuning.MaxConcurrentProactiveIVSwitch) } type gohbemLogger struct{} @@ -258,7 +258,7 @@ func SetStatsCollector(collector stats_collector.StatsCollector) { // debounce/last-seen threshold. Pass the default seconds for normal operation // If ReduceUpdates is enabled in the loaded config.Config, this returns 43200 (12 hours). func GetUpdateThreshold(defaultSeconds int64) int64 { - if config.Config.ReduceUpdates { + if config.Config.Tuning.ReduceUpdates { return 43200 // 12 hours } return defaultSeconds From edf6d76e6a5834e02b3f9384d05b32419441e07f Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 21:34:45 +0000 Subject: [PATCH 30/35] Some fields weren't being updated through SetXX --- decoder/gym.go | 10 ++++++++++ decoder/gym_state.go | 2 +- decoder/incident.go | 7 +++++++ decoder/incident_state.go | 2 +- decoder/player.go | 9 ++++++++- decoder/pokemon.go | 20 ++++++++++++++++++++ decoder/pokemon_state.go | 4 ++-- decoder/pokestop.go | 10 ++++++++++ decoder/pokestop_state.go | 2 +- decoder/routes.go | 10 ++++++++++ decoder/routes_state.go | 2 +- decoder/spawnpoint.go | 20 +++++++++++++++++--- decoder/station.go | 10 ++++++++++ decoder/station_state.go | 2 +- decoder/tappable.go | 10 ++++++++++ decoder/tappable_state.go | 2 +- 16 files changed, 110 insertions(+), 12 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index e966b316..b841b730 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -561,3 +561,13 @@ func (gym *Gym) SetRsvps(v null.String) { } } } + +func (gym *Gym) SetUpdated(v int64) { + if gym.Updated != v { + gym.Updated = v + gym.dirty = true + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, "Updated") + } + } +} diff --git a/decoder/gym_state.go b/decoder/gym_state.go index ce235085..e646ac1a 100644 --- a/decoder/gym_state.go +++ b/decoder/gym_state.go @@ -333,7 +333,7 @@ func saveGymRecord(ctx context.Context, db db.DbDetails, gym *Gym) { return } } - gym.Updated = now + gym.SetUpdated(now) if gym.IsDirty() { if gym.IsNewRecord() { diff --git a/decoder/incident.go b/decoder/incident.go index 049edf7e..707ed5ee 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -213,3 +213,10 @@ func (incident *Incident) SetSlot3Form(v null.Int) { incident.dirty = true } } + +func (incident *Incident) SetUpdated(v int64) { + if incident.Updated != v { + incident.Updated = v + incident.dirty = true + } +} diff --git a/decoder/incident_state.go b/decoder/incident_state.go index d3b8d80f..c9da5ba7 100644 --- a/decoder/incident_state.go +++ b/decoder/incident_state.go @@ -111,7 +111,7 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident return } - incident.Updated = time.Now().Unix() + incident.SetUpdated(time.Now().Unix()) if incident.IsNewRecord() { res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ diff --git a/decoder/player.go b/decoder/player.go index 01dea73f..c24054e3 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -633,6 +633,13 @@ func (p *Player) SetCaughtFairy(v null.Int) { } } +func (p *Player) SetLastSeen(v int64) { + if p.LastSeen != v { + p.LastSeen = v + p.dirty = true + } +} + var badgeTypeToPlayerKey = map[pogo.HoloBadgeType]string{ //pogo.HoloBadgeType_BADGE_TRAVEL_KM: "KmWalked", pogo.HoloBadgeType_BADGE_POKEDEX_ENTRIES: "DexGen1", @@ -782,7 +789,7 @@ func savePlayerRecord(db db.DbDetails, player *Player) { return } - player.LastSeen = time.Now().Unix() + player.SetLastSeen(time.Now().Unix()) if player.IsNewRecord() { _, err := db.GeneralDb.NamedExec( diff --git a/decoder/pokemon.go b/decoder/pokemon.go index b4b9973c..81e0f0f8 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -462,3 +462,23 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { } } } + +func (pokemon *Pokemon) SetUpdated(v null.Int) { + if pokemon.Updated != v { + pokemon.Updated = v + pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Updated") + } + } +} + +func (pokemon *Pokemon) SetChanged(v int64) { + if pokemon.Changed != v { + pokemon.Changed = v + pokemon.dirty = true + if dbDebugEnabled { + pokemon.changedFields = append(pokemon.changedFields, "Changed") + } + } +} diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go index e135dbbe..286dcf44 100644 --- a/decoder/pokemon_state.go +++ b/decoder/pokemon_state.go @@ -190,9 +190,9 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po pokemon.FirstSeenTimestamp = now } - pokemon.Updated = null.IntFrom(now) + pokemon.SetUpdated(null.IntFrom(now)) if pokemon.isNewRecord() || pokemon.oldValues.PokemonId != pokemon.PokemonId || pokemon.oldValues.Cp != pokemon.Cp { - pokemon.Changed = now + pokemon.SetChanged(now) } changePvpField := false diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 829ab153..3289cfe4 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -576,3 +576,13 @@ func (p *Pokestop) SetShowcaseRankings(v null.String) { } } } + +func (p *Pokestop) SetUpdated(v int64) { + if p.Updated != v { + p.Updated = v + p.dirty = true + if dbDebugEnabled { + p.changedFields = append(p.changedFields, "Updated") + } + } +} diff --git a/decoder/pokestop_state.go b/decoder/pokestop_state.go index f2116179..9fced113 100644 --- a/decoder/pokestop_state.go +++ b/decoder/pokestop_state.go @@ -276,7 +276,7 @@ func savePokestopRecord(ctx context.Context, db db.DbDetails, pokestop *Pokestop return } } - pokestop.Updated = now + pokestop.SetUpdated(now) if pokestop.IsNewRecord() { if dbDebugEnabled { diff --git a/decoder/routes.go b/decoder/routes.go index 214d7812..2a6aaf75 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -280,3 +280,13 @@ func (r *Route) SetWaypoints(v string) { } } } + +func (r *Route) SetUpdated(v int64) { + if r.Updated != v { + r.Updated = v + r.dirty = true + if dbDebugEnabled { + r.changedFields = append(r.changedFields, "Updated") + } + } +} diff --git a/decoder/routes_state.go b/decoder/routes_state.go index 8538cc86..38a0d918 100644 --- a/decoder/routes_state.go +++ b/decoder/routes_state.go @@ -112,7 +112,7 @@ func saveRouteRecord(ctx context.Context, db db.DbDetails, route *Route) error { } } - route.Updated = time.Now().Unix() + route.SetUpdated(time.Now().Unix()) if route.IsNewRecord() { if dbDebugEnabled { diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index f858cc85..9a3a4a8a 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -119,6 +119,20 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { } } +func (s *Spawnpoint) SetUpdated(v int64) { + if s.Updated != v { + s.Updated = v + s.dirty = true + } +} + +func (s *Spawnpoint) SetLastSeen(v int64) { + if s.LastSeen != v { + s.LastSeen = v + s.dirty = true + } +} + func loadSpawnpointFromDatabase(ctx context.Context, db db.DbDetails, spawnpointId int64, spawnpoint *Spawnpoint) error { err := db.GeneralDb.GetContext(ctx, spawnpoint, "SELECT id, lat, lon, updated, last_seen, despawn_sec FROM spawnpoint WHERE id = ?", spawnpointId) @@ -249,8 +263,8 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi return } - spawnpoint.Updated = time.Now().Unix() // ensure future updates are set correctly - spawnpoint.LastSeen = time.Now().Unix() // ensure future updates are set correctly + spawnpoint.SetUpdated(time.Now().Unix()) // ensure future updates are set correctly + spawnpoint.SetLastSeen(time.Now().Unix()) // ensure future updates are set correctly _, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO spawnpoint (id, lat, lon, updated, last_seen, despawn_sec)"+ "VALUES (:id, :lat, :lon, :updated, :last_seen, :despawn_sec)"+ @@ -281,7 +295,7 @@ func spawnpointSeen(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoint // update at least every 6 hours (21600s). If reduce_updates is enabled, use 12 hours. if now-spawnpoint.LastSeen > GetUpdateThreshold(21600) { - spawnpoint.LastSeen = now + spawnpoint.SetLastSeen(now) _, err := db.GeneralDb.ExecContext(ctx, "UPDATE spawnpoint "+ "SET last_seen=? "+ diff --git a/decoder/station.go b/decoder/station.go index 9feea5c1..d46bd2c2 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -359,3 +359,13 @@ func (station *Station) SetStationedPokemon(v null.String) { } } } + +func (station *Station) SetUpdated(v int64) { + if station.Updated != v { + station.Updated = v + station.dirty = true + if dbDebugEnabled { + station.changedFields = append(station.changedFields, "Updated") + } + } +} diff --git a/decoder/station_state.go b/decoder/station_state.go index d4e4e06a..b038936d 100644 --- a/decoder/station_state.go +++ b/decoder/station_state.go @@ -140,7 +140,7 @@ func saveStationRecord(ctx context.Context, db db.DbDetails, station *Station) { } } - station.Updated = now + station.SetUpdated(now) if station.IsNewRecord() { if dbDebugEnabled { diff --git a/decoder/tappable.go b/decoder/tappable.go index 3ab8d218..94ab1332 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -155,3 +155,13 @@ func (ta *Tappable) SetExpireTimestampVerified(v bool) { } } } + +func (ta *Tappable) SetUpdated(v int64) { + if ta.Updated != v { + ta.Updated = v + ta.dirty = true + if dbDebugEnabled { + ta.changedFields = append(ta.changedFields, "Updated") + } + } +} diff --git a/decoder/tappable_state.go b/decoder/tappable_state.go index ff30c42f..517c22ab 100644 --- a/decoder/tappable_state.go +++ b/decoder/tappable_state.go @@ -101,7 +101,7 @@ func saveTappableRecord(ctx context.Context, details db.DbDetails, tappable *Tap } now := time.Now().Unix() - tappable.Updated = now + tappable.SetUpdated(now) if tappable.IsNewRecord() { if dbDebugEnabled { From 140e36af73b93152c5c24a40e2c355018c27ad93 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 22:00:24 +0000 Subject: [PATCH 31/35] Improved dbupdate logging to show before/after --- decoder/gym.go | 235 ++++++++++++++------------- decoder/incident.go | 51 +++++- decoder/incident_state.go | 6 + decoder/player.go | 332 +++++++++++++++++++++++++++++++++++++- decoder/pokemon.go | 187 ++++++++++----------- decoder/pokemon_state.go | 4 + decoder/pokestop.go | 259 ++++++++++++++--------------- decoder/routes.go | 127 +++++++-------- decoder/s2cell.go | 2 +- decoder/spawnpoint.go | 32 +++- decoder/station.go | 163 +++++++++---------- decoder/tappable.go | 67 ++++---- 12 files changed, 945 insertions(+), 520 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index b841b730..d906a0c0 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -178,176 +179,179 @@ func (gym *Gym) Unlock() { func (gym *Gym) SetId(v string) { if gym.Id != v { - gym.Id = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Id") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Id:%s->%s", gym.Id, v)) } + gym.Id = v + gym.dirty = true } } func (gym *Gym) SetLat(v float64) { if !floatAlmostEqual(gym.Lat, v, floatTolerance) { - gym.Lat = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Lat") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Lat:%f->%f", gym.Lat, v)) } + gym.Lat = v + gym.dirty = true } } func (gym *Gym) SetLon(v float64) { if !floatAlmostEqual(gym.Lon, v, floatTolerance) { - gym.Lon = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Lon") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Lon:%f->%f", gym.Lon, v)) } + gym.Lon = v + gym.dirty = true } } func (gym *Gym) SetName(v null.String) { if gym.Name != v { - gym.Name = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Name") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Name:%v->%v", gym.Name, v)) } + gym.Name = v + gym.dirty = true } } func (gym *Gym) SetUrl(v null.String) { if gym.Url != v { - gym.Url = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Url") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Url:%v->%v", gym.Url, v)) } + gym.Url = v + gym.dirty = true } } func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { if gym.LastModifiedTimestamp != v { - gym.LastModifiedTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "LastModifiedTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("LastModifiedTimestamp:%v->%v", gym.LastModifiedTimestamp, v)) } + gym.LastModifiedTimestamp = v + gym.dirty = true } } func (gym *Gym) SetRaidEndTimestamp(v null.Int) { if gym.RaidEndTimestamp != v { - gym.RaidEndTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidEndTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidEndTimestamp:%v->%v", gym.RaidEndTimestamp, v)) } + gym.RaidEndTimestamp = v + gym.dirty = true } } func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { if gym.RaidSpawnTimestamp != v { - gym.RaidSpawnTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidSpawnTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidSpawnTimestamp:%v->%v", gym.RaidSpawnTimestamp, v)) } + gym.RaidSpawnTimestamp = v + gym.dirty = true } } func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { if gym.RaidBattleTimestamp != v { - gym.RaidBattleTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidBattleTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidBattleTimestamp:%v->%v", gym.RaidBattleTimestamp, v)) } + gym.RaidBattleTimestamp = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonId(v null.Int) { if gym.RaidPokemonId != v { - gym.RaidPokemonId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonId:%v->%v", gym.RaidPokemonId, v)) } + gym.RaidPokemonId = v + gym.dirty = true } } func (gym *Gym) SetGuardingPokemonId(v null.Int) { if gym.GuardingPokemonId != v { - gym.GuardingPokemonId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "GuardingPokemonId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonId:%v->%v", gym.GuardingPokemonId, v)) } + gym.GuardingPokemonId = v + gym.dirty = true } } func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { if gym.GuardingPokemonDisplay != v { - gym.GuardingPokemonDisplay = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "GuardingPokemonDisplay") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonDisplay:%v->%v", gym.GuardingPokemonDisplay, v)) } + gym.GuardingPokemonDisplay = v + gym.dirty = true } } func (gym *Gym) SetAvailableSlots(v null.Int) { if gym.AvailableSlots != v { - gym.AvailableSlots = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "AvailableSlots") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("AvailableSlots:%v->%v", gym.AvailableSlots, v)) } + gym.AvailableSlots = v + gym.dirty = true } } func (gym *Gym) SetTeamId(v null.Int) { if gym.TeamId != v { - gym.TeamId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "TeamId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("TeamId:%v->%v", gym.TeamId, v)) } + gym.TeamId = v + gym.dirty = true } } func (gym *Gym) SetRaidLevel(v null.Int) { if gym.RaidLevel != v { - gym.RaidLevel = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidLevel") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidLevel:%v->%v", gym.RaidLevel, v)) } + gym.RaidLevel = v + gym.dirty = true } } func (gym *Gym) SetEnabled(v null.Int) { if gym.Enabled != v { - gym.Enabled = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Enabled") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Enabled:%v->%v", gym.Enabled, v)) } + gym.Enabled = v + gym.dirty = true } } func (gym *Gym) SetExRaidEligible(v null.Int) { if gym.ExRaidEligible != v { - gym.ExRaidEligible = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "ExRaidEligible") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("ExRaidEligible:%v->%v", gym.ExRaidEligible, v)) } + gym.ExRaidEligible = v + gym.dirty = true } } func (gym *Gym) SetInBattle(v null.Int) { if gym.InBattle != v { + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, fmt.Sprintf("InBattle:%v->%v", gym.InBattle, v)) + } gym.InBattle = v //Do not set to dirty, as don't trigger an update gym.internalDirty = true @@ -356,196 +360,199 @@ func (gym *Gym) SetInBattle(v null.Int) { func (gym *Gym) SetRaidPokemonMove1(v null.Int) { if gym.RaidPokemonMove1 != v { - gym.RaidPokemonMove1 = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonMove1") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove1:%v->%v", gym.RaidPokemonMove1, v)) } + gym.RaidPokemonMove1 = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonMove2(v null.Int) { if gym.RaidPokemonMove2 != v { - gym.RaidPokemonMove2 = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonMove2") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove2:%v->%v", gym.RaidPokemonMove2, v)) } + gym.RaidPokemonMove2 = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonForm(v null.Int) { if gym.RaidPokemonForm != v { - gym.RaidPokemonForm = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonForm") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonForm:%v->%v", gym.RaidPokemonForm, v)) } + gym.RaidPokemonForm = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { if gym.RaidPokemonAlignment != v { - gym.RaidPokemonAlignment = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonAlignment") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonAlignment:%v->%v", gym.RaidPokemonAlignment, v)) } + gym.RaidPokemonAlignment = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonCp(v null.Int) { if gym.RaidPokemonCp != v { - gym.RaidPokemonCp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonCp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCp:%v->%v", gym.RaidPokemonCp, v)) } + gym.RaidPokemonCp = v + gym.dirty = true } } func (gym *Gym) SetRaidIsExclusive(v null.Int) { if gym.RaidIsExclusive != v { - gym.RaidIsExclusive = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidIsExclusive") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidIsExclusive:%v->%v", gym.RaidIsExclusive, v)) } + gym.RaidIsExclusive = v + gym.dirty = true } } func (gym *Gym) SetCellId(v null.Int) { if gym.CellId != v { - gym.CellId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "CellId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("CellId:%v->%v", gym.CellId, v)) } + gym.CellId = v + gym.dirty = true } } func (gym *Gym) SetDeleted(v bool) { if gym.Deleted != v { - gym.Deleted = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Deleted") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Deleted:%t->%t", gym.Deleted, v)) } + gym.Deleted = v + gym.dirty = true } } func (gym *Gym) SetTotalCp(v null.Int) { if gym.TotalCp != v { - gym.TotalCp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "TotalCp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("TotalCp:%v->%v", gym.TotalCp, v)) } + gym.TotalCp = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonGender(v null.Int) { if gym.RaidPokemonGender != v { - gym.RaidPokemonGender = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonGender") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonGender:%v->%v", gym.RaidPokemonGender, v)) } + gym.RaidPokemonGender = v + gym.dirty = true } } func (gym *Gym) SetSponsorId(v null.Int) { if gym.SponsorId != v { - gym.SponsorId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "SponsorId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("SponsorId:%v->%v", gym.SponsorId, v)) } + gym.SponsorId = v + gym.dirty = true } } func (gym *Gym) SetPartnerId(v null.String) { if gym.PartnerId != v { - gym.PartnerId = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "PartnerId") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PartnerId:%v->%v", gym.PartnerId, v)) } + gym.PartnerId = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonCostume(v null.Int) { if gym.RaidPokemonCostume != v { - gym.RaidPokemonCostume = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonCostume") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCostume:%v->%v", gym.RaidPokemonCostume, v)) } + gym.RaidPokemonCostume = v + gym.dirty = true } } func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { if gym.RaidPokemonEvolution != v { - gym.RaidPokemonEvolution = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "RaidPokemonEvolution") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonEvolution:%v->%v", gym.RaidPokemonEvolution, v)) } + gym.RaidPokemonEvolution = v + gym.dirty = true } } func (gym *Gym) SetArScanEligible(v null.Int) { if gym.ArScanEligible != v { - gym.ArScanEligible = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "ArScanEligible") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("ArScanEligible:%v->%v", gym.ArScanEligible, v)) } + gym.ArScanEligible = v + gym.dirty = true } } func (gym *Gym) SetPowerUpLevel(v null.Int) { if gym.PowerUpLevel != v { - gym.PowerUpLevel = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "PowerUpLevel") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpLevel:%v->%v", gym.PowerUpLevel, v)) } + gym.PowerUpLevel = v + gym.dirty = true } } func (gym *Gym) SetPowerUpPoints(v null.Int) { if gym.PowerUpPoints != v { - gym.PowerUpPoints = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "PowerUpPoints") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpPoints:%v->%v", gym.PowerUpPoints, v)) } + gym.PowerUpPoints = v + gym.dirty = true } } func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { if gym.PowerUpEndTimestamp != v { - gym.PowerUpEndTimestamp = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "PowerUpEndTimestamp") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%v->%v", gym.PowerUpEndTimestamp, v)) } + gym.PowerUpEndTimestamp = v + gym.dirty = true } } func (gym *Gym) SetDescription(v null.String) { if gym.Description != v { - gym.Description = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Description") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Description:%v->%v", gym.Description, v)) } + gym.Description = v + gym.dirty = true } } func (gym *Gym) SetDefenders(v null.String) { if gym.Defenders != v { + if dbDebugEnabled { + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Defenders:%v->%v", gym.Defenders, v)) + } gym.Defenders = v //Do not set to dirty, as don't trigger an update gym.internalDirty = true @@ -554,20 +561,20 @@ func (gym *Gym) SetDefenders(v null.String) { func (gym *Gym) SetRsvps(v null.String) { if gym.Rsvps != v { - gym.Rsvps = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Rsvps") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Rsvps:%v->%v", gym.Rsvps, v)) } + gym.Rsvps = v + gym.dirty = true } } func (gym *Gym) SetUpdated(v int64) { if gym.Updated != v { - gym.Updated = v - gym.dirty = true if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, "Updated") + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Updated:%d->%d", gym.Updated, v)) } + gym.Updated = v + gym.dirty = true } } diff --git a/decoder/incident.go b/decoder/incident.go index 707ed5ee..dd85d474 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -27,8 +28,9 @@ type Incident struct { Slot3PokemonId null.Int `db:"slot_3_pokemon_id"` Slot3Form null.Int `db:"slot_3_form"` - dirty bool `db:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-"` // Track which fields changed (only when dbDebugEnabled) oldValues IncidentOldValues `db:"-"` // Old values for webhook comparison } @@ -118,6 +120,9 @@ func (incident *Incident) snapshotOldValues() { func (incident *Incident) SetId(v string) { if incident.Id != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Id:%s->%s", incident.Id, v)) + } incident.Id = v incident.dirty = true } @@ -125,6 +130,9 @@ func (incident *Incident) SetId(v string) { func (incident *Incident) SetPokestopId(v string) { if incident.PokestopId != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("PokestopId:%s->%s", incident.PokestopId, v)) + } incident.PokestopId = v incident.dirty = true } @@ -132,6 +140,9 @@ func (incident *Incident) SetPokestopId(v string) { func (incident *Incident) SetStartTime(v int64) { if incident.StartTime != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("StartTime:%d->%d", incident.StartTime, v)) + } incident.StartTime = v incident.dirty = true } @@ -139,6 +150,9 @@ func (incident *Incident) SetStartTime(v int64) { func (incident *Incident) SetExpirationTime(v int64) { if incident.ExpirationTime != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("ExpirationTime:%d->%d", incident.ExpirationTime, v)) + } incident.ExpirationTime = v incident.dirty = true } @@ -146,6 +160,9 @@ func (incident *Incident) SetExpirationTime(v int64) { func (incident *Incident) SetDisplayType(v int16) { if incident.DisplayType != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("DisplayType:%d->%d", incident.DisplayType, v)) + } incident.DisplayType = v incident.dirty = true } @@ -153,6 +170,9 @@ func (incident *Incident) SetDisplayType(v int16) { func (incident *Incident) SetStyle(v int16) { if incident.Style != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Style:%d->%d", incident.Style, v)) + } incident.Style = v incident.dirty = true } @@ -160,6 +180,9 @@ func (incident *Incident) SetStyle(v int16) { func (incident *Incident) SetCharacter(v int16) { if incident.Character != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Character:%d->%d", incident.Character, v)) + } incident.Character = v incident.dirty = true } @@ -167,6 +190,9 @@ func (incident *Incident) SetCharacter(v int16) { func (incident *Incident) SetConfirmed(v bool) { if incident.Confirmed != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Confirmed:%t->%t", incident.Confirmed, v)) + } incident.Confirmed = v incident.dirty = true } @@ -174,6 +200,9 @@ func (incident *Incident) SetConfirmed(v bool) { func (incident *Incident) SetSlot1PokemonId(v null.Int) { if incident.Slot1PokemonId != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1PokemonId:%v->%v", incident.Slot1PokemonId, v)) + } incident.Slot1PokemonId = v incident.dirty = true } @@ -181,6 +210,9 @@ func (incident *Incident) SetSlot1PokemonId(v null.Int) { func (incident *Incident) SetSlot1Form(v null.Int) { if incident.Slot1Form != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1Form:%v->%v", incident.Slot1Form, v)) + } incident.Slot1Form = v incident.dirty = true } @@ -188,6 +220,9 @@ func (incident *Incident) SetSlot1Form(v null.Int) { func (incident *Incident) SetSlot2PokemonId(v null.Int) { if incident.Slot2PokemonId != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2PokemonId:%v->%v", incident.Slot2PokemonId, v)) + } incident.Slot2PokemonId = v incident.dirty = true } @@ -195,6 +230,9 @@ func (incident *Incident) SetSlot2PokemonId(v null.Int) { func (incident *Incident) SetSlot2Form(v null.Int) { if incident.Slot2Form != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2Form:%v->%v", incident.Slot2Form, v)) + } incident.Slot2Form = v incident.dirty = true } @@ -202,6 +240,9 @@ func (incident *Incident) SetSlot2Form(v null.Int) { func (incident *Incident) SetSlot3PokemonId(v null.Int) { if incident.Slot3PokemonId != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3PokemonId:%v->%v", incident.Slot3PokemonId, v)) + } incident.Slot3PokemonId = v incident.dirty = true } @@ -209,6 +250,9 @@ func (incident *Incident) SetSlot3PokemonId(v null.Int) { func (incident *Incident) SetSlot3Form(v null.Int) { if incident.Slot3Form != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3Form:%v->%v", incident.Slot3Form, v)) + } incident.Slot3Form = v incident.dirty = true } @@ -216,6 +260,9 @@ func (incident *Incident) SetSlot3Form(v null.Int) { func (incident *Incident) SetUpdated(v int64) { if incident.Updated != v { + if dbDebugEnabled { + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Updated:%d->%d", incident.Updated, v)) + } incident.Updated = v incident.dirty = true } diff --git a/decoder/incident_state.go b/decoder/incident_state.go index c9da5ba7..641bc06b 100644 --- a/decoder/incident_state.go +++ b/decoder/incident_state.go @@ -114,6 +114,9 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident incident.SetUpdated(time.Now().Unix()) if incident.IsNewRecord() { + if dbDebugEnabled { + dbDebugLog("INSERT", "Incident", incident.Id, incident.changedFields) + } res, err := db.GeneralDb.NamedExec("INSERT INTO incident (id, pokestop_id, start, expiration, display_type, style, `character`, updated, confirmed, slot_1_pokemon_id, slot_1_form, slot_2_pokemon_id, slot_2_form, slot_3_pokemon_id, slot_3_form) "+ "VALUES (:id, :pokestop_id, :start, :expiration, :display_type, :style, :character, :updated, :confirmed, :slot_1_pokemon_id, :slot_1_form, :slot_2_pokemon_id, :slot_2_form, :slot_3_pokemon_id, :slot_3_form)", incident) @@ -124,6 +127,9 @@ func saveIncidentRecord(ctx context.Context, db db.DbDetails, incident *Incident statsCollector.IncDbQuery("insert incident", err) _, _ = res, err } else { + if dbDebugEnabled { + dbDebugLog("UPDATE", "Incident", incident.Id, incident.changedFields) + } res, err := db.GeneralDb.NamedExec("UPDATE incident SET "+ "start = :start, "+ "expiration = :expiration, "+ diff --git a/decoder/player.go b/decoder/player.go index c24054e3..433885bb 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -2,6 +2,7 @@ package decoder import ( "database/sql" + "fmt" "reflect" "strconv" "time" @@ -103,8 +104,9 @@ type Player struct { CaughtDark null.Int `db:"caught_dark"` CaughtFairy null.Int `db:"caught_fairy"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) } // IsDirty returns true if any field has been modified @@ -131,6 +133,9 @@ func (p *Player) setFieldDirty() { func (p *Player) SetFriendshipId(v null.String) { if p.FriendshipId != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("FriendshipId:%v->%v", p.FriendshipId, v)) + } p.FriendshipId = v p.dirty = true } @@ -138,6 +143,9 @@ func (p *Player) SetFriendshipId(v null.String) { func (p *Player) SetFriendCode(v null.String) { if p.FriendCode != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("FriendCode:%v->%v", p.FriendCode, v)) + } p.FriendCode = v p.dirty = true } @@ -145,6 +153,9 @@ func (p *Player) SetFriendCode(v null.String) { func (p *Player) SetTeam(v null.Int) { if p.Team != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Team:%v->%v", p.Team, v)) + } p.Team = v p.dirty = true } @@ -152,6 +163,9 @@ func (p *Player) SetTeam(v null.Int) { func (p *Player) SetLevel(v null.Int) { if p.Level != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Level:%v->%v", p.Level, v)) + } p.Level = v p.dirty = true } @@ -159,6 +173,9 @@ func (p *Player) SetLevel(v null.Int) { func (p *Player) SetXp(v null.Int) { if p.Xp != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Xp:%v->%v", p.Xp, v)) + } p.Xp = v p.dirty = true } @@ -166,6 +183,9 @@ func (p *Player) SetXp(v null.Int) { func (p *Player) SetBattlesWon(v null.Int) { if p.BattlesWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("BattlesWon:%v->%v", p.BattlesWon, v)) + } p.BattlesWon = v p.dirty = true } @@ -173,6 +193,9 @@ func (p *Player) SetBattlesWon(v null.Int) { func (p *Player) SetKmWalked(v null.Float) { if !nullFloatAlmostEqual(p.KmWalked, v, 0.001) { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("KmWalked:%v->%v", p.KmWalked, v)) + } p.KmWalked = v p.dirty = true } @@ -180,6 +203,9 @@ func (p *Player) SetKmWalked(v null.Float) { func (p *Player) SetCaughtPokemon(v null.Int) { if p.CaughtPokemon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPokemon:%v->%v", p.CaughtPokemon, v)) + } p.CaughtPokemon = v p.dirty = true } @@ -187,6 +213,9 @@ func (p *Player) SetCaughtPokemon(v null.Int) { func (p *Player) SetGblRank(v null.Int) { if p.GblRank != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GblRank:%v->%v", p.GblRank, v)) + } p.GblRank = v p.dirty = true } @@ -194,6 +223,9 @@ func (p *Player) SetGblRank(v null.Int) { func (p *Player) SetGblRating(v null.Int) { if p.GblRating != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GblRating:%v->%v", p.GblRating, v)) + } p.GblRating = v p.dirty = true } @@ -201,6 +233,9 @@ func (p *Player) SetGblRating(v null.Int) { func (p *Player) SetEventBadges(v null.String) { if p.EventBadges != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("EventBadges:%v->%v", p.EventBadges, v)) + } p.EventBadges = v p.dirty = true } @@ -208,426 +243,708 @@ func (p *Player) SetEventBadges(v null.String) { func (p *Player) SetStopsSpun(v null.Int) { if p.StopsSpun != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("StopsSpun:%v->%v", p.StopsSpun, v)) + } p.StopsSpun = v p.dirty = true } } + func (p *Player) SetEvolved(v null.Int) { if p.Evolved != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Evolved:%v->%v", p.Evolved, v)) + } p.Evolved = v p.dirty = true } } + func (p *Player) SetHatched(v null.Int) { if p.Hatched != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Hatched:%v->%v", p.Hatched, v)) + } p.Hatched = v p.dirty = true } } + func (p *Player) SetQuests(v null.Int) { if p.Quests != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Quests:%v->%v", p.Quests, v)) + } p.Quests = v p.dirty = true } } + func (p *Player) SetTrades(v null.Int) { if p.Trades != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Trades:%v->%v", p.Trades, v)) + } p.Trades = v p.dirty = true } } + func (p *Player) SetPhotobombs(v null.Int) { if p.Photobombs != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Photobombs:%v->%v", p.Photobombs, v)) + } p.Photobombs = v p.dirty = true } } + func (p *Player) SetPurified(v null.Int) { if p.Purified != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Purified:%v->%v", p.Purified, v)) + } p.Purified = v p.dirty = true } } + func (p *Player) SetGruntsDefeated(v null.Int) { if p.GruntsDefeated != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GruntsDefeated:%v->%v", p.GruntsDefeated, v)) + } p.GruntsDefeated = v p.dirty = true } } + func (p *Player) SetGymBattlesWon(v null.Int) { if p.GymBattlesWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GymBattlesWon:%v->%v", p.GymBattlesWon, v)) + } p.GymBattlesWon = v p.dirty = true } } + func (p *Player) SetNormalRaidsWon(v null.Int) { if p.NormalRaidsWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("NormalRaidsWon:%v->%v", p.NormalRaidsWon, v)) + } p.NormalRaidsWon = v p.dirty = true } } + func (p *Player) SetLegendaryRaidsWon(v null.Int) { if p.LegendaryRaidsWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LegendaryRaidsWon:%v->%v", p.LegendaryRaidsWon, v)) + } p.LegendaryRaidsWon = v p.dirty = true } } + func (p *Player) SetTrainingsWon(v null.Int) { if p.TrainingsWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TrainingsWon:%v->%v", p.TrainingsWon, v)) + } p.TrainingsWon = v p.dirty = true } } + func (p *Player) SetBerriesFed(v null.Int) { if p.BerriesFed != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("BerriesFed:%v->%v", p.BerriesFed, v)) + } p.BerriesFed = v p.dirty = true } } + func (p *Player) SetHoursDefended(v null.Int) { if p.HoursDefended != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("HoursDefended:%v->%v", p.HoursDefended, v)) + } p.HoursDefended = v p.dirty = true } } + func (p *Player) SetBestFriends(v null.Int) { if p.BestFriends != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("BestFriends:%v->%v", p.BestFriends, v)) + } p.BestFriends = v p.dirty = true } } + func (p *Player) SetBestBuddies(v null.Int) { if p.BestBuddies != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("BestBuddies:%v->%v", p.BestBuddies, v)) + } p.BestBuddies = v p.dirty = true } } + func (p *Player) SetGiovanniDefeated(v null.Int) { if p.GiovanniDefeated != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("GiovanniDefeated:%v->%v", p.GiovanniDefeated, v)) + } p.GiovanniDefeated = v p.dirty = true } } + func (p *Player) SetMegaEvos(v null.Int) { if p.MegaEvos != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("MegaEvos:%v->%v", p.MegaEvos, v)) + } p.MegaEvos = v p.dirty = true } } + func (p *Player) SetCollectionsDone(v null.Int) { if p.CollectionsDone != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CollectionsDone:%v->%v", p.CollectionsDone, v)) + } p.CollectionsDone = v p.dirty = true } } + func (p *Player) SetUniqueStopsSpun(v null.Int) { if p.UniqueStopsSpun != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueStopsSpun:%v->%v", p.UniqueStopsSpun, v)) + } p.UniqueStopsSpun = v p.dirty = true } } + func (p *Player) SetUniqueMegaEvos(v null.Int) { if p.UniqueMegaEvos != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueMegaEvos:%v->%v", p.UniqueMegaEvos, v)) + } p.UniqueMegaEvos = v p.dirty = true } } + func (p *Player) SetUniqueRaidBosses(v null.Int) { if p.UniqueRaidBosses != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueRaidBosses:%v->%v", p.UniqueRaidBosses, v)) + } p.UniqueRaidBosses = v p.dirty = true } } + func (p *Player) SetUniqueUnown(v null.Int) { if p.UniqueUnown != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueUnown:%v->%v", p.UniqueUnown, v)) + } p.UniqueUnown = v p.dirty = true } } + func (p *Player) SetSevenDayStreaks(v null.Int) { if p.SevenDayStreaks != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("SevenDayStreaks:%v->%v", p.SevenDayStreaks, v)) + } p.SevenDayStreaks = v p.dirty = true } } + func (p *Player) SetTradeKm(v null.Int) { if p.TradeKm != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TradeKm:%v->%v", p.TradeKm, v)) + } p.TradeKm = v p.dirty = true } } + func (p *Player) SetRaidsWithFriends(v null.Int) { if p.RaidsWithFriends != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("RaidsWithFriends:%v->%v", p.RaidsWithFriends, v)) + } p.RaidsWithFriends = v p.dirty = true } } + func (p *Player) SetCaughtAtLure(v null.Int) { if p.CaughtAtLure != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtAtLure:%v->%v", p.CaughtAtLure, v)) + } p.CaughtAtLure = v p.dirty = true } } + func (p *Player) SetWayfarerAgreements(v null.Int) { if p.WayfarerAgreements != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("WayfarerAgreements:%v->%v", p.WayfarerAgreements, v)) + } p.WayfarerAgreements = v p.dirty = true } } + func (p *Player) SetTrainersReferred(v null.Int) { if p.TrainersReferred != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TrainersReferred:%v->%v", p.TrainersReferred, v)) + } p.TrainersReferred = v p.dirty = true } } + func (p *Player) SetRaidAchievements(v null.Int) { if p.RaidAchievements != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("RaidAchievements:%v->%v", p.RaidAchievements, v)) + } p.RaidAchievements = v p.dirty = true } } + func (p *Player) SetXlKarps(v null.Int) { if p.XlKarps != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("XlKarps:%v->%v", p.XlKarps, v)) + } p.XlKarps = v p.dirty = true } } + func (p *Player) SetXsRats(v null.Int) { if p.XsRats != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("XsRats:%v->%v", p.XsRats, v)) + } p.XsRats = v p.dirty = true } } + func (p *Player) SetPikachuCaught(v null.Int) { if p.PikachuCaught != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("PikachuCaught:%v->%v", p.PikachuCaught, v)) + } p.PikachuCaught = v p.dirty = true } } + func (p *Player) SetLeagueGreatWon(v null.Int) { if p.LeagueGreatWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueGreatWon:%v->%v", p.LeagueGreatWon, v)) + } p.LeagueGreatWon = v p.dirty = true } } + func (p *Player) SetLeagueUltraWon(v null.Int) { if p.LeagueUltraWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueUltraWon:%v->%v", p.LeagueUltraWon, v)) + } p.LeagueUltraWon = v p.dirty = true } } + func (p *Player) SetLeagueMasterWon(v null.Int) { if p.LeagueMasterWon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueMasterWon:%v->%v", p.LeagueMasterWon, v)) + } p.LeagueMasterWon = v p.dirty = true } } + func (p *Player) SetTinyPokemonCaught(v null.Int) { if p.TinyPokemonCaught != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TinyPokemonCaught:%v->%v", p.TinyPokemonCaught, v)) + } p.TinyPokemonCaught = v p.dirty = true } } + func (p *Player) SetJumboPokemonCaught(v null.Int) { if p.JumboPokemonCaught != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("JumboPokemonCaught:%v->%v", p.JumboPokemonCaught, v)) + } p.JumboPokemonCaught = v p.dirty = true } } + func (p *Player) SetVivillon(v null.Int) { if p.Vivillon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("Vivillon:%v->%v", p.Vivillon, v)) + } p.Vivillon = v p.dirty = true } } + func (p *Player) SetMaxSizeFirstPlace(v null.Int) { if p.MaxSizeFirstPlace != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("MaxSizeFirstPlace:%v->%v", p.MaxSizeFirstPlace, v)) + } p.MaxSizeFirstPlace = v p.dirty = true } } + func (p *Player) SetTotalRoutePlay(v null.Int) { if p.TotalRoutePlay != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("TotalRoutePlay:%v->%v", p.TotalRoutePlay, v)) + } p.TotalRoutePlay = v p.dirty = true } } + func (p *Player) SetPartiesCompleted(v null.Int) { if p.PartiesCompleted != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("PartiesCompleted:%v->%v", p.PartiesCompleted, v)) + } p.PartiesCompleted = v p.dirty = true } } + func (p *Player) SetEventCheckIns(v null.Int) { if p.EventCheckIns != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("EventCheckIns:%v->%v", p.EventCheckIns, v)) + } p.EventCheckIns = v p.dirty = true } } func (p *Player) SetDexGen1(v null.Int) { if p.DexGen1 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen1:%v->%v", p.DexGen1, v)) + } p.DexGen1 = v p.dirty = true } } + func (p *Player) SetDexGen2(v null.Int) { if p.DexGen2 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen2:%v->%v", p.DexGen2, v)) + } p.DexGen2 = v p.dirty = true } } + func (p *Player) SetDexGen3(v null.Int) { if p.DexGen3 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen3:%v->%v", p.DexGen3, v)) + } p.DexGen3 = v p.dirty = true } } + func (p *Player) SetDexGen4(v null.Int) { if p.DexGen4 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen4:%v->%v", p.DexGen4, v)) + } p.DexGen4 = v p.dirty = true } } + func (p *Player) SetDexGen5(v null.Int) { if p.DexGen5 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen5:%v->%v", p.DexGen5, v)) + } p.DexGen5 = v p.dirty = true } } + func (p *Player) SetDexGen6(v null.Int) { if p.DexGen6 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen6:%v->%v", p.DexGen6, v)) + } p.DexGen6 = v p.dirty = true } } + func (p *Player) SetDexGen7(v null.Int) { if p.DexGen7 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen7:%v->%v", p.DexGen7, v)) + } p.DexGen7 = v p.dirty = true } } + func (p *Player) SetDexGen8(v null.Int) { if p.DexGen8 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8:%v->%v", p.DexGen8, v)) + } p.DexGen8 = v p.dirty = true } } + func (p *Player) SetDexGen8A(v null.Int) { if p.DexGen8A != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8A:%v->%v", p.DexGen8A, v)) + } p.DexGen8A = v p.dirty = true } } + func (p *Player) SetDexGen9(v null.Int) { if p.DexGen9 != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen9:%v->%v", p.DexGen9, v)) + } p.DexGen9 = v p.dirty = true } } + func (p *Player) SetCaughtNormal(v null.Int) { if p.CaughtNormal != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtNormal:%v->%v", p.CaughtNormal, v)) + } p.CaughtNormal = v p.dirty = true } } + func (p *Player) SetCaughtFighting(v null.Int) { if p.CaughtFighting != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFighting:%v->%v", p.CaughtFighting, v)) + } p.CaughtFighting = v p.dirty = true } } + func (p *Player) SetCaughtFlying(v null.Int) { if p.CaughtFlying != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFlying:%v->%v", p.CaughtFlying, v)) + } p.CaughtFlying = v p.dirty = true } } + func (p *Player) SetCaughtPoison(v null.Int) { if p.CaughtPoison != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPoison:%v->%v", p.CaughtPoison, v)) + } p.CaughtPoison = v p.dirty = true } } + func (p *Player) SetCaughtGround(v null.Int) { if p.CaughtGround != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGround:%v->%v", p.CaughtGround, v)) + } p.CaughtGround = v p.dirty = true } } + func (p *Player) SetCaughtRock(v null.Int) { if p.CaughtRock != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtRock:%v->%v", p.CaughtRock, v)) + } p.CaughtRock = v p.dirty = true } } + func (p *Player) SetCaughtBug(v null.Int) { if p.CaughtBug != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtBug:%v->%v", p.CaughtBug, v)) + } p.CaughtBug = v p.dirty = true } } + func (p *Player) SetCaughtGhost(v null.Int) { if p.CaughtGhost != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGhost:%v->%v", p.CaughtGhost, v)) + } p.CaughtGhost = v p.dirty = true } } + func (p *Player) SetCaughtSteel(v null.Int) { if p.CaughtSteel != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtSteel:%v->%v", p.CaughtSteel, v)) + } p.CaughtSteel = v p.dirty = true } } + func (p *Player) SetCaughtFire(v null.Int) { if p.CaughtFire != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFire:%v->%v", p.CaughtFire, v)) + } p.CaughtFire = v p.dirty = true } } + func (p *Player) SetCaughtWater(v null.Int) { if p.CaughtWater != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtWater:%v->%v", p.CaughtWater, v)) + } p.CaughtWater = v p.dirty = true } } + func (p *Player) SetCaughtGrass(v null.Int) { if p.CaughtGrass != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGrass:%v->%v", p.CaughtGrass, v)) + } p.CaughtGrass = v p.dirty = true } } + func (p *Player) SetCaughtElectric(v null.Int) { if p.CaughtElectric != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtElectric:%v->%v", p.CaughtElectric, v)) + } p.CaughtElectric = v p.dirty = true } } + func (p *Player) SetCaughtPsychic(v null.Int) { if p.CaughtPsychic != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPsychic:%v->%v", p.CaughtPsychic, v)) + } p.CaughtPsychic = v p.dirty = true } } + func (p *Player) SetCaughtIce(v null.Int) { if p.CaughtIce != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtIce:%v->%v", p.CaughtIce, v)) + } p.CaughtIce = v p.dirty = true } } + func (p *Player) SetCaughtDragon(v null.Int) { if p.CaughtDragon != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDragon:%v->%v", p.CaughtDragon, v)) + } p.CaughtDragon = v p.dirty = true } } + func (p *Player) SetCaughtDark(v null.Int) { if p.CaughtDark != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDark:%v->%v", p.CaughtDark, v)) + } p.CaughtDark = v p.dirty = true } } + func (p *Player) SetCaughtFairy(v null.Int) { if p.CaughtFairy != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFairy:%v->%v", p.CaughtFairy, v)) + } p.CaughtFairy = v p.dirty = true } @@ -635,6 +952,9 @@ func (p *Player) SetCaughtFairy(v null.Int) { func (p *Player) SetLastSeen(v int64) { if p.LastSeen != v { + if dbDebugEnabled { + p.changedFields = append(p.changedFields, fmt.Sprintf("LastSeen:%d->%d", p.LastSeen, v)) + } p.LastSeen = v p.dirty = true } @@ -791,6 +1111,14 @@ func savePlayerRecord(db db.DbDetails, player *Player) { player.SetLastSeen(time.Now().Unix()) + if dbDebugEnabled { + if player.IsNewRecord() { + dbDebugLog("INSERT", "Player", player.Name, player.changedFields) + } else { + dbDebugLog("UPDATE", "Player", player.Name, player.changedFields) + } + } + if player.IsNewRecord() { _, err := db.GeneralDb.NamedExec( ` diff --git a/decoder/pokemon.go b/decoder/pokemon.go index 81e0f0f8..d20ba93b 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "golbat/grpc" @@ -168,131 +169,131 @@ func (pokemon *Pokemon) Unlock() { func (pokemon *Pokemon) SetPokestopId(v null.String) { if pokemon.PokestopId != v { - pokemon.PokestopId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "PokestopId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("PokestopId:%v->%v", pokemon.PokestopId, v)) } + pokemon.PokestopId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetSpawnId(v null.Int) { if pokemon.SpawnId != v { - pokemon.SpawnId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "SpawnId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SpawnId:%v->%v", pokemon.SpawnId, v)) } + pokemon.SpawnId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetLat(v float64) { if !floatAlmostEqual(pokemon.Lat, v, floatTolerance) { - pokemon.Lat = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Lat") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Lat:%f->%f", pokemon.Lat, v)) } + pokemon.Lat = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetLon(v float64) { if !floatAlmostEqual(pokemon.Lon, v, floatTolerance) { - pokemon.Lon = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Lon") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Lon:%f->%f", pokemon.Lon, v)) } + pokemon.Lon = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetPokemonId(v int16) { if pokemon.PokemonId != v { - pokemon.PokemonId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "PokemonId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("PokemonId:%d->%d", pokemon.PokemonId, v)) } + pokemon.PokemonId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetForm(v null.Int) { if pokemon.Form != v { - pokemon.Form = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Form") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Form:%v->%v", pokemon.Form, v)) } + pokemon.Form = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCostume(v null.Int) { if pokemon.Costume != v { - pokemon.Costume = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Costume") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Costume:%v->%v", pokemon.Costume, v)) } + pokemon.Costume = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetGender(v null.Int) { if pokemon.Gender != v { - pokemon.Gender = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Gender") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Gender:%v->%v", pokemon.Gender, v)) } + pokemon.Gender = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetWeather(v null.Int) { if pokemon.Weather != v { - pokemon.Weather = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Weather") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weather:%v->%v", pokemon.Weather, v)) } + pokemon.Weather = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetIsStrong(v null.Bool) { if pokemon.IsStrong != v { - pokemon.IsStrong = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "IsStrong") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsStrong:%v->%v", pokemon.IsStrong, v)) } + pokemon.IsStrong = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetExpireTimestamp(v null.Int) { if pokemon.ExpireTimestamp != v { - pokemon.ExpireTimestamp = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "ExpireTimestamp") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("ExpireTimestamp:%v->%v", pokemon.ExpireTimestamp, v)) } + pokemon.ExpireTimestamp = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetExpireTimestampVerified(v bool) { if pokemon.ExpireTimestampVerified != v { - pokemon.ExpireTimestampVerified = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "ExpireTimestampVerified") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("ExpireTimestampVerified:%t->%t", pokemon.ExpireTimestampVerified, v)) } + pokemon.ExpireTimestampVerified = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetSeenType(v null.String) { if pokemon.SeenType != v { - pokemon.SeenType = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "SeenType") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SeenType:%v->%v", pokemon.SeenType, v)) } + pokemon.SeenType = v + pokemon.dirty = true } } @@ -305,180 +306,180 @@ func (pokemon *Pokemon) SetUsername(v null.String) { func (pokemon *Pokemon) SetCellId(v null.Int) { if pokemon.CellId != v { - pokemon.CellId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "CellId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("CellId:%v->%v", pokemon.CellId, v)) } + pokemon.CellId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetIsEvent(v int8) { if pokemon.IsEvent != v { - pokemon.IsEvent = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "IsEvent") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsEvent:%d->%d", pokemon.IsEvent, v)) } + pokemon.IsEvent = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetShiny(v null.Bool) { if pokemon.Shiny != v { - pokemon.Shiny = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Shiny") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Shiny:%v->%v", pokemon.Shiny, v)) } + pokemon.Shiny = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCp(v null.Int) { if pokemon.Cp != v { - pokemon.Cp = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Cp") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Cp:%v->%v", pokemon.Cp, v)) } + pokemon.Cp = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetLevel(v null.Int) { if pokemon.Level != v { - pokemon.Level = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Level") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Level:%v->%v", pokemon.Level, v)) } + pokemon.Level = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetMove1(v null.Int) { if pokemon.Move1 != v { - pokemon.Move1 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Move1") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move1:%v->%v", pokemon.Move1, v)) } + pokemon.Move1 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetMove2(v null.Int) { if pokemon.Move2 != v { - pokemon.Move2 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Move2") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move2:%v->%v", pokemon.Move2, v)) } + pokemon.Move2 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetHeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Height, v, floatTolerance) { - pokemon.Height = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Height") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Height:%v->%v", pokemon.Height, v)) } + pokemon.Height = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetWeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Weight, v, floatTolerance) { - pokemon.Weight = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Weight") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weight:%v->%v", pokemon.Weight, v)) } + pokemon.Weight = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetSize(v null.Int) { if pokemon.Size != v { - pokemon.Size = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Size") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Size:%v->%v", pokemon.Size, v)) } + pokemon.Size = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetIsDitto(v bool) { if pokemon.IsDitto != v { - pokemon.IsDitto = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "IsDitto") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsDitto:%t->%t", pokemon.IsDitto, v)) } + pokemon.IsDitto = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { if pokemon.DisplayPokemonId != v { - pokemon.DisplayPokemonId = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "DisplayPokemonId") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("DisplayPokemonId:%v->%v", pokemon.DisplayPokemonId, v)) } + pokemon.DisplayPokemonId = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetPvp(v null.String) { if pokemon.Pvp != v { - pokemon.Pvp = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Pvp") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Pvp:%v->%v", pokemon.Pvp, v)) } + pokemon.Pvp = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCapture1(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture1, v, floatTolerance) { - pokemon.Capture1 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Capture1") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture1:%v->%v", pokemon.Capture1, v)) } + pokemon.Capture1 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCapture2(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture2, v, floatTolerance) { - pokemon.Capture2 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Capture2") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture2:%v->%v", pokemon.Capture2, v)) } + pokemon.Capture2 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetCapture3(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture3, v, floatTolerance) { - pokemon.Capture3 = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Capture3") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture3:%v->%v", pokemon.Capture3, v)) } + pokemon.Capture3 = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetUpdated(v null.Int) { if pokemon.Updated != v { - pokemon.Updated = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Updated") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Updated:%v->%v", pokemon.Updated, v)) } + pokemon.Updated = v + pokemon.dirty = true } } func (pokemon *Pokemon) SetChanged(v int64) { if pokemon.Changed != v { - pokemon.Changed = v - pokemon.dirty = true if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, "Changed") + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Changed:%d->%d", pokemon.Changed, v)) } + pokemon.Changed = v + pokemon.dirty = true } } diff --git a/decoder/pokemon_state.go b/decoder/pokemon_state.go index 286dcf44..dcfb2840 100644 --- a/decoder/pokemon_state.go +++ b/decoder/pokemon_state.go @@ -339,6 +339,10 @@ func savePokemonRecordAsAtTime(ctx context.Context, db db.DbDetails, pokemon *Po rows, rowsErr := res.RowsAffected() log.Debugf("Updating pokemon [%d] after update res = %d %v", pokemon.Id, rows, rowsErr) } + } else { + if dbDebugEnabled { + dbDebugLog("MEMORY", "Pokemon", strconv.FormatUint(pokemon.Id, 10), pokemon.changedFields) + } } // Update pokemon rtree diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 3289cfe4..8d97c8a1 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -159,430 +160,430 @@ func (p *Pokestop) Unlock() { func (p *Pokestop) SetId(v string) { if p.Id != v { - p.Id = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Id") + p.changedFields = append(p.changedFields, fmt.Sprintf("Id:%s->%s", p.Id, v)) } + p.Id = v + p.dirty = true } } func (p *Pokestop) SetLat(v float64) { if !floatAlmostEqual(p.Lat, v, floatTolerance) { - p.Lat = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Lat") + p.changedFields = append(p.changedFields, fmt.Sprintf("Lat:%f->%f", p.Lat, v)) } + p.Lat = v + p.dirty = true } } func (p *Pokestop) SetLon(v float64) { if !floatAlmostEqual(p.Lon, v, floatTolerance) { - p.Lon = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Lon") + p.changedFields = append(p.changedFields, fmt.Sprintf("Lon:%f->%f", p.Lon, v)) } + p.Lon = v + p.dirty = true } } func (p *Pokestop) SetName(v null.String) { if p.Name != v { - p.Name = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Name") + p.changedFields = append(p.changedFields, fmt.Sprintf("Name:%v->%v", p.Name, v)) } + p.Name = v + p.dirty = true } } func (p *Pokestop) SetUrl(v null.String) { if p.Url != v { - p.Url = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Url") + p.changedFields = append(p.changedFields, fmt.Sprintf("Url:%v->%v", p.Url, v)) } + p.Url = v + p.dirty = true } } func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { if p.LureExpireTimestamp != v { - p.LureExpireTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "LureExpireTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("LureExpireTimestamp:%v->%v", p.LureExpireTimestamp, v)) } + p.LureExpireTimestamp = v + p.dirty = true } } func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { if p.LastModifiedTimestamp != v { - p.LastModifiedTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "LastModifiedTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("LastModifiedTimestamp:%v->%v", p.LastModifiedTimestamp, v)) } + p.LastModifiedTimestamp = v + p.dirty = true } } func (p *Pokestop) SetEnabled(v null.Bool) { if p.Enabled != v { - p.Enabled = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Enabled") + p.changedFields = append(p.changedFields, fmt.Sprintf("Enabled:%v->%v", p.Enabled, v)) } + p.Enabled = v + p.dirty = true } } func (p *Pokestop) SetQuestType(v null.Int) { if p.QuestType != v { - p.QuestType = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestType") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestType:%v->%v", p.QuestType, v)) } + p.QuestType = v + p.dirty = true } } func (p *Pokestop) SetQuestTimestamp(v null.Int) { if p.QuestTimestamp != v { - p.QuestTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTimestamp:%v->%v", p.QuestTimestamp, v)) } + p.QuestTimestamp = v + p.dirty = true } } func (p *Pokestop) SetQuestTarget(v null.Int) { if p.QuestTarget != v { - p.QuestTarget = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestTarget") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTarget:%v->%v", p.QuestTarget, v)) } + p.QuestTarget = v + p.dirty = true } } func (p *Pokestop) SetQuestConditions(v null.String) { if p.QuestConditions != v { - p.QuestConditions = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestConditions") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestConditions:%v->%v", p.QuestConditions, v)) } + p.QuestConditions = v + p.dirty = true } } func (p *Pokestop) SetQuestRewards(v null.String) { if p.QuestRewards != v { - p.QuestRewards = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestRewards") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestRewards:%v->%v", p.QuestRewards, v)) } + p.QuestRewards = v + p.dirty = true } } func (p *Pokestop) SetQuestTemplate(v null.String) { if p.QuestTemplate != v { - p.QuestTemplate = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestTemplate") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTemplate:%v->%v", p.QuestTemplate, v)) } + p.QuestTemplate = v + p.dirty = true } } func (p *Pokestop) SetQuestTitle(v null.String) { if p.QuestTitle != v { - p.QuestTitle = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestTitle") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTitle:%v->%v", p.QuestTitle, v)) } + p.QuestTitle = v + p.dirty = true } } func (p *Pokestop) SetQuestExpiry(v null.Int) { if p.QuestExpiry != v { - p.QuestExpiry = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "QuestExpiry") + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestExpiry:%v->%v", p.QuestExpiry, v)) } + p.QuestExpiry = v + p.dirty = true } } func (p *Pokestop) SetCellId(v null.Int) { if p.CellId != v { - p.CellId = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "CellId") + p.changedFields = append(p.changedFields, fmt.Sprintf("CellId:%v->%v", p.CellId, v)) } + p.CellId = v + p.dirty = true } } func (p *Pokestop) SetDeleted(v bool) { if p.Deleted != v { - p.Deleted = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Deleted") + p.changedFields = append(p.changedFields, fmt.Sprintf("Deleted:%t->%t", p.Deleted, v)) } + p.Deleted = v + p.dirty = true } } func (p *Pokestop) SetLureId(v int16) { if p.LureId != v { - p.LureId = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "LureId") + p.changedFields = append(p.changedFields, fmt.Sprintf("LureId:%d->%d", p.LureId, v)) } + p.LureId = v + p.dirty = true } } func (p *Pokestop) SetFirstSeenTimestamp(v int16) { if p.FirstSeenTimestamp != v { - p.FirstSeenTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "FirstSeenTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("FirstSeenTimestamp:%d->%d", p.FirstSeenTimestamp, v)) } + p.FirstSeenTimestamp = v + p.dirty = true } } func (p *Pokestop) SetSponsorId(v null.Int) { if p.SponsorId != v { - p.SponsorId = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "SponsorId") + p.changedFields = append(p.changedFields, fmt.Sprintf("SponsorId:%v->%v", p.SponsorId, v)) } + p.SponsorId = v + p.dirty = true } } func (p *Pokestop) SetPartnerId(v null.String) { if p.PartnerId != v { - p.PartnerId = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "PartnerId") + p.changedFields = append(p.changedFields, fmt.Sprintf("PartnerId:%v->%v", p.PartnerId, v)) } + p.PartnerId = v + p.dirty = true } } func (p *Pokestop) SetArScanEligible(v null.Int) { if p.ArScanEligible != v { - p.ArScanEligible = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ArScanEligible") + p.changedFields = append(p.changedFields, fmt.Sprintf("ArScanEligible:%v->%v", p.ArScanEligible, v)) } + p.ArScanEligible = v + p.dirty = true } } func (p *Pokestop) SetPowerUpLevel(v null.Int) { if p.PowerUpLevel != v { - p.PowerUpLevel = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "PowerUpLevel") + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpLevel:%v->%v", p.PowerUpLevel, v)) } + p.PowerUpLevel = v + p.dirty = true } } func (p *Pokestop) SetPowerUpPoints(v null.Int) { if p.PowerUpPoints != v { - p.PowerUpPoints = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "PowerUpPoints") + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpPoints:%v->%v", p.PowerUpPoints, v)) } + p.PowerUpPoints = v + p.dirty = true } } func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { if p.PowerUpEndTimestamp != v { - p.PowerUpEndTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "PowerUpEndTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%v->%v", p.PowerUpEndTimestamp, v)) } + p.PowerUpEndTimestamp = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestType(v null.Int) { if p.AlternativeQuestType != v { - p.AlternativeQuestType = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestType") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestType:%v->%v", p.AlternativeQuestType, v)) } + p.AlternativeQuestType = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { if p.AlternativeQuestTimestamp != v { - p.AlternativeQuestTimestamp = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestTimestamp") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTimestamp:%v->%v", p.AlternativeQuestTimestamp, v)) } + p.AlternativeQuestTimestamp = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { if p.AlternativeQuestTarget != v { - p.AlternativeQuestTarget = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestTarget") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTarget:%v->%v", p.AlternativeQuestTarget, v)) } + p.AlternativeQuestTarget = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { if p.AlternativeQuestConditions != v { - p.AlternativeQuestConditions = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestConditions") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestConditions:%v->%v", p.AlternativeQuestConditions, v)) } + p.AlternativeQuestConditions = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { if p.AlternativeQuestRewards != v { - p.AlternativeQuestRewards = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestRewards") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestRewards:%v->%v", p.AlternativeQuestRewards, v)) } + p.AlternativeQuestRewards = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { if p.AlternativeQuestTemplate != v { - p.AlternativeQuestTemplate = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestTemplate") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTemplate:%v->%v", p.AlternativeQuestTemplate, v)) } + p.AlternativeQuestTemplate = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { if p.AlternativeQuestTitle != v { - p.AlternativeQuestTitle = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestTitle") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTitle:%v->%v", p.AlternativeQuestTitle, v)) } + p.AlternativeQuestTitle = v + p.dirty = true } } func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { if p.AlternativeQuestExpiry != v { - p.AlternativeQuestExpiry = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "AlternativeQuestExpiry") + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestExpiry:%v->%v", p.AlternativeQuestExpiry, v)) } + p.AlternativeQuestExpiry = v + p.dirty = true } } func (p *Pokestop) SetDescription(v null.String) { if p.Description != v { - p.Description = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Description") + p.changedFields = append(p.changedFields, fmt.Sprintf("Description:%v->%v", p.Description, v)) } + p.Description = v + p.dirty = true } } func (p *Pokestop) SetShowcaseFocus(v null.String) { if p.ShowcaseFocus != v { - p.ShowcaseFocus = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcaseFocus") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseFocus:%v->%v", p.ShowcaseFocus, v)) } + p.ShowcaseFocus = v + p.dirty = true } } func (p *Pokestop) SetShowcasePokemon(v null.Int) { if p.ShowcasePokemon != v { - p.ShowcasePokemon = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcasePokemon") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemon:%v->%v", p.ShowcasePokemon, v)) } + p.ShowcasePokemon = v + p.dirty = true } } func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { if p.ShowcasePokemonForm != v { - p.ShowcasePokemonForm = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcasePokemonForm") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonForm:%v->%v", p.ShowcasePokemonForm, v)) } + p.ShowcasePokemonForm = v + p.dirty = true } } func (p *Pokestop) SetShowcasePokemonType(v null.Int) { if p.ShowcasePokemonType != v { - p.ShowcasePokemonType = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcasePokemonType") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonType:%v->%v", p.ShowcasePokemonType, v)) } + p.ShowcasePokemonType = v + p.dirty = true } } func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { if p.ShowcaseRankingStandard != v { - p.ShowcaseRankingStandard = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcaseRankingStandard") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankingStandard:%v->%v", p.ShowcaseRankingStandard, v)) } + p.ShowcaseRankingStandard = v + p.dirty = true } } func (p *Pokestop) SetShowcaseExpiry(v null.Int) { if p.ShowcaseExpiry != v { - p.ShowcaseExpiry = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcaseExpiry") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseExpiry:%v->%v", p.ShowcaseExpiry, v)) } + p.ShowcaseExpiry = v + p.dirty = true } } func (p *Pokestop) SetShowcaseRankings(v null.String) { if p.ShowcaseRankings != v { - p.ShowcaseRankings = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "ShowcaseRankings") + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankings:%v->%v", p.ShowcaseRankings, v)) } + p.ShowcaseRankings = v + p.dirty = true } } func (p *Pokestop) SetUpdated(v int64) { if p.Updated != v { - p.Updated = v - p.dirty = true if dbDebugEnabled { - p.changedFields = append(p.changedFields, "Updated") + p.changedFields = append(p.changedFields, fmt.Sprintf("Updated:%d->%d", p.Updated, v)) } + p.Updated = v + p.dirty = true } } diff --git a/decoder/routes.go b/decoder/routes.go index 2a6aaf75..0fa7ee6a 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -83,210 +84,210 @@ func (r *Route) snapshotOldValues() { func (r *Route) SetName(v string) { if r.Name != v { - r.Name = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Name") + r.changedFields = append(r.changedFields, fmt.Sprintf("Name:%s->%s", r.Name, v)) } + r.Name = v + r.dirty = true } } func (r *Route) SetShortcode(v string) { if r.Shortcode != v { - r.Shortcode = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Shortcode") + r.changedFields = append(r.changedFields, fmt.Sprintf("Shortcode:%s->%s", r.Shortcode, v)) } + r.Shortcode = v + r.dirty = true } } func (r *Route) SetDescription(v string) { if r.Description != v { - r.Description = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Description") + r.changedFields = append(r.changedFields, fmt.Sprintf("Description:%s->%s", r.Description, v)) } + r.Description = v + r.dirty = true } } func (r *Route) SetDistanceMeters(v int64) { if r.DistanceMeters != v { - r.DistanceMeters = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "DistanceMeters") + r.changedFields = append(r.changedFields, fmt.Sprintf("DistanceMeters:%d->%d", r.DistanceMeters, v)) } + r.DistanceMeters = v + r.dirty = true } } func (r *Route) SetDurationSeconds(v int64) { if r.DurationSeconds != v { - r.DurationSeconds = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "DurationSeconds") + r.changedFields = append(r.changedFields, fmt.Sprintf("DurationSeconds:%d->%d", r.DurationSeconds, v)) } + r.DurationSeconds = v + r.dirty = true } } func (r *Route) SetEndFortId(v string) { if r.EndFortId != v { - r.EndFortId = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "EndFortId") + r.changedFields = append(r.changedFields, fmt.Sprintf("EndFortId:%s->%s", r.EndFortId, v)) } + r.EndFortId = v + r.dirty = true } } func (r *Route) SetEndImage(v string) { if r.EndImage != v { - r.EndImage = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "EndImage") + r.changedFields = append(r.changedFields, fmt.Sprintf("EndImage:%s->%s", r.EndImage, v)) } + r.EndImage = v + r.dirty = true } } func (r *Route) SetEndLat(v float64) { if !floatAlmostEqual(r.EndLat, v, floatTolerance) { - r.EndLat = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "EndLat") + r.changedFields = append(r.changedFields, fmt.Sprintf("EndLat:%f->%f", r.EndLat, v)) } + r.EndLat = v + r.dirty = true } } func (r *Route) SetEndLon(v float64) { if !floatAlmostEqual(r.EndLon, v, floatTolerance) { - r.EndLon = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "EndLon") + r.changedFields = append(r.changedFields, fmt.Sprintf("EndLon:%f->%f", r.EndLon, v)) } + r.EndLon = v + r.dirty = true } } func (r *Route) SetImage(v string) { if r.Image != v { - r.Image = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Image") + r.changedFields = append(r.changedFields, fmt.Sprintf("Image:%s->%s", r.Image, v)) } + r.Image = v + r.dirty = true } } func (r *Route) SetImageBorderColor(v string) { if r.ImageBorderColor != v { - r.ImageBorderColor = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "ImageBorderColor") + r.changedFields = append(r.changedFields, fmt.Sprintf("ImageBorderColor:%s->%s", r.ImageBorderColor, v)) } + r.ImageBorderColor = v + r.dirty = true } } func (r *Route) SetReversible(v bool) { if r.Reversible != v { - r.Reversible = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Reversible") + r.changedFields = append(r.changedFields, fmt.Sprintf("Reversible:%t->%t", r.Reversible, v)) } + r.Reversible = v + r.dirty = true } } func (r *Route) SetStartFortId(v string) { if r.StartFortId != v { - r.StartFortId = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "StartFortId") + r.changedFields = append(r.changedFields, fmt.Sprintf("StartFortId:%s->%s", r.StartFortId, v)) } + r.StartFortId = v + r.dirty = true } } func (r *Route) SetStartImage(v string) { if r.StartImage != v { - r.StartImage = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "StartImage") + r.changedFields = append(r.changedFields, fmt.Sprintf("StartImage:%s->%s", r.StartImage, v)) } + r.StartImage = v + r.dirty = true } } func (r *Route) SetStartLat(v float64) { if !floatAlmostEqual(r.StartLat, v, floatTolerance) { - r.StartLat = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "StartLat") + r.changedFields = append(r.changedFields, fmt.Sprintf("StartLat:%f->%f", r.StartLat, v)) } + r.StartLat = v + r.dirty = true } } func (r *Route) SetStartLon(v float64) { if !floatAlmostEqual(r.StartLon, v, floatTolerance) { - r.StartLon = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "StartLon") + r.changedFields = append(r.changedFields, fmt.Sprintf("StartLon:%f->%f", r.StartLon, v)) } + r.StartLon = v + r.dirty = true } } func (r *Route) SetTags(v null.String) { if r.Tags != v { - r.Tags = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Tags") + r.changedFields = append(r.changedFields, fmt.Sprintf("Tags:%v->%v", r.Tags, v)) } + r.Tags = v + r.dirty = true } } func (r *Route) SetType(v int8) { if r.Type != v { - r.Type = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Type") + r.changedFields = append(r.changedFields, fmt.Sprintf("Type:%d->%d", r.Type, v)) } + r.Type = v + r.dirty = true } } func (r *Route) SetVersion(v int64) { if r.Version != v { - r.Version = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Version") + r.changedFields = append(r.changedFields, fmt.Sprintf("Version:%d->%d", r.Version, v)) } + r.Version = v + r.dirty = true } } func (r *Route) SetWaypoints(v string) { if r.Waypoints != v { - r.Waypoints = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Waypoints") + r.changedFields = append(r.changedFields, fmt.Sprintf("Waypoints:%s->%s", r.Waypoints, v)) } + r.Waypoints = v + r.dirty = true } } func (r *Route) SetUpdated(v int64) { if r.Updated != v { - r.Updated = v - r.dirty = true if dbDebugEnabled { - r.changedFields = append(r.changedFields, "Updated") + r.changedFields = append(r.changedFields, fmt.Sprintf("Updated:%d->%d", r.Updated, v)) } + r.Updated = v + r.dirty = true } } diff --git a/decoder/s2cell.go b/decoder/s2cell.go index 736b302f..60513492 100644 --- a/decoder/s2cell.go +++ b/decoder/s2cell.go @@ -69,7 +69,7 @@ func saveS2CellRecords(ctx context.Context, db db.DbDetails, cellIds []uint64) { for _, s2cell := range outputCellIds { updatedCells = append(updatedCells, strconv.FormatUint(s2cell.Id, 10)) } - log.Debugf("[DB_S2CELL] Updated cells: %s", strings.Join(updatedCells, ",")) + log.Debugf("[DB_UPDATE] S2Cell Updated cells: %s", strings.Join(updatedCells, ",")) } // run bulk query diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 9a3a4a8a..9171a852 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "fmt" "strconv" "sync" "time" @@ -28,8 +29,9 @@ type Spawnpoint struct { LastSeen int64 `db:"last_seen"` DespawnSec null.Int `db:"despawn_sec"` - dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving - newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + dirty bool `db:"-" json:"-"` // Not persisted - tracks if object needs saving + newRecord bool `db:"-" json:"-"` // Not persisted - tracks if this is a new record + changedFields []string `db:"-" json:"-"` // Track which fields changed (only when dbDebugEnabled) } //CREATE TABLE `spawnpoint` ( @@ -74,6 +76,9 @@ func (s *Spawnpoint) Unlock() { func (s *Spawnpoint) SetLat(v float64) { if !floatAlmostEqual(s.Lat, v, floatTolerance) { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("Lat:%f->%f", s.Lat, v)) + } s.Lat = v s.dirty = true } @@ -81,6 +86,9 @@ func (s *Spawnpoint) SetLat(v float64) { func (s *Spawnpoint) SetLon(v float64) { if !floatAlmostEqual(s.Lon, v, floatTolerance) { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("Lon:%f->%f", s.Lon, v)) + } s.Lon = v s.dirty = true } @@ -90,6 +98,9 @@ func (s *Spawnpoint) SetLon(v float64) { func (s *Spawnpoint) SetDespawnSec(v null.Int) { // Handle validity changes if (s.DespawnSec.Valid && !v.Valid) || (!s.DespawnSec.Valid && v.Valid) { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%v->%v", s.DespawnSec, v)) + } s.DespawnSec = v s.dirty = true return @@ -114,6 +125,9 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { // Allow 2-second tolerance for despawn time if Abs(oldVal-newVal) > 2 { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%v->%v", s.DespawnSec, v)) + } s.DespawnSec = v s.dirty = true } @@ -121,6 +135,9 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { func (s *Spawnpoint) SetUpdated(v int64) { if s.Updated != v { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("Updated:%d->%d", s.Updated, v)) + } s.Updated = v s.dirty = true } @@ -128,6 +145,9 @@ func (s *Spawnpoint) SetUpdated(v int64) { func (s *Spawnpoint) SetLastSeen(v int64) { if s.LastSeen != v { + if dbDebugEnabled { + s.changedFields = append(s.changedFields, fmt.Sprintf("LastSeen:%d->%d", s.LastSeen, v)) + } s.LastSeen = v s.dirty = true } @@ -266,6 +286,14 @@ func spawnpointUpdate(ctx context.Context, db db.DbDetails, spawnpoint *Spawnpoi spawnpoint.SetUpdated(time.Now().Unix()) // ensure future updates are set correctly spawnpoint.SetLastSeen(time.Now().Unix()) // ensure future updates are set correctly + if dbDebugEnabled { + if spawnpoint.IsNewRecord() { + dbDebugLog("INSERT", "Spawnpoint", strconv.FormatInt(spawnpoint.Id, 10), spawnpoint.changedFields) + } else { + dbDebugLog("UPDATE", "Spawnpoint", strconv.FormatInt(spawnpoint.Id, 10), spawnpoint.changedFields) + } + } + _, err := db.GeneralDb.NamedExecContext(ctx, "INSERT INTO spawnpoint (id, lat, lon, updated, last_seen, despawn_sec)"+ "VALUES (:id, :lat, :lon, :updated, :last_seen, :despawn_sec)"+ "ON DUPLICATE KEY UPDATE "+ diff --git a/decoder/station.go b/decoder/station.go index d46bd2c2..8eb1e307 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -102,270 +103,270 @@ func (station *Station) snapshotOldValues() { func (station *Station) SetId(v string) { if station.Id != v { - station.Id = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Id") + station.changedFields = append(station.changedFields, fmt.Sprintf("Id:%s->%s", station.Id, v)) } + station.Id = v + station.dirty = true } } func (station *Station) SetLat(v float64) { if !floatAlmostEqual(station.Lat, v, floatTolerance) { - station.Lat = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Lat") + station.changedFields = append(station.changedFields, fmt.Sprintf("Lat:%f->%f", station.Lat, v)) } + station.Lat = v + station.dirty = true } } func (station *Station) SetLon(v float64) { if !floatAlmostEqual(station.Lon, v, floatTolerance) { - station.Lon = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Lon") + station.changedFields = append(station.changedFields, fmt.Sprintf("Lon:%f->%f", station.Lon, v)) } + station.Lon = v + station.dirty = true } } func (station *Station) SetName(v string) { if station.Name != v { - station.Name = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Name") + station.changedFields = append(station.changedFields, fmt.Sprintf("Name:%s->%s", station.Name, v)) } + station.Name = v + station.dirty = true } } func (station *Station) SetCellId(v int64) { if station.CellId != v { - station.CellId = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "CellId") + station.changedFields = append(station.changedFields, fmt.Sprintf("CellId:%d->%d", station.CellId, v)) } + station.CellId = v + station.dirty = true } } func (station *Station) SetStartTime(v int64) { if station.StartTime != v { - station.StartTime = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "StartTime") + station.changedFields = append(station.changedFields, fmt.Sprintf("StartTime:%d->%d", station.StartTime, v)) } + station.StartTime = v + station.dirty = true } } func (station *Station) SetEndTime(v int64) { if station.EndTime != v { - station.EndTime = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "EndTime") + station.changedFields = append(station.changedFields, fmt.Sprintf("EndTime:%d->%d", station.EndTime, v)) } + station.EndTime = v + station.dirty = true } } func (station *Station) SetCooldownComplete(v int64) { if station.CooldownComplete != v { - station.CooldownComplete = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "CooldownComplete") + station.changedFields = append(station.changedFields, fmt.Sprintf("CooldownComplete:%d->%d", station.CooldownComplete, v)) } + station.CooldownComplete = v + station.dirty = true } } func (station *Station) SetIsBattleAvailable(v bool) { if station.IsBattleAvailable != v { - station.IsBattleAvailable = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "IsBattleAvailable") + station.changedFields = append(station.changedFields, fmt.Sprintf("IsBattleAvailable:%t->%t", station.IsBattleAvailable, v)) } + station.IsBattleAvailable = v + station.dirty = true } } func (station *Station) SetIsInactive(v bool) { if station.IsInactive != v { - station.IsInactive = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "IsInactive") + station.changedFields = append(station.changedFields, fmt.Sprintf("IsInactive:%t->%t", station.IsInactive, v)) } + station.IsInactive = v + station.dirty = true } } func (station *Station) SetBattleLevel(v null.Int) { if station.BattleLevel != v { - station.BattleLevel = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattleLevel") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleLevel:%v->%v", station.BattleLevel, v)) } + station.BattleLevel = v + station.dirty = true } } func (station *Station) SetBattleStart(v null.Int) { if station.BattleStart != v { - station.BattleStart = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattleStart") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleStart:%v->%v", station.BattleStart, v)) } + station.BattleStart = v + station.dirty = true } } func (station *Station) SetBattleEnd(v null.Int) { if station.BattleEnd != v { - station.BattleEnd = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattleEnd") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleEnd:%v->%v", station.BattleEnd, v)) } + station.BattleEnd = v + station.dirty = true } } func (station *Station) SetBattlePokemonId(v null.Int) { if station.BattlePokemonId != v { - station.BattlePokemonId = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonId") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonId:%v->%v", station.BattlePokemonId, v)) } + station.BattlePokemonId = v + station.dirty = true } } func (station *Station) SetBattlePokemonForm(v null.Int) { if station.BattlePokemonForm != v { - station.BattlePokemonForm = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonForm") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonForm:%v->%v", station.BattlePokemonForm, v)) } + station.BattlePokemonForm = v + station.dirty = true } } func (station *Station) SetBattlePokemonCostume(v null.Int) { if station.BattlePokemonCostume != v { - station.BattlePokemonCostume = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonCostume") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCostume:%v->%v", station.BattlePokemonCostume, v)) } + station.BattlePokemonCostume = v + station.dirty = true } } func (station *Station) SetBattlePokemonGender(v null.Int) { if station.BattlePokemonGender != v { - station.BattlePokemonGender = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonGender") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonGender:%v->%v", station.BattlePokemonGender, v)) } + station.BattlePokemonGender = v + station.dirty = true } } func (station *Station) SetBattlePokemonAlignment(v null.Int) { if station.BattlePokemonAlignment != v { - station.BattlePokemonAlignment = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonAlignment") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonAlignment:%v->%v", station.BattlePokemonAlignment, v)) } + station.BattlePokemonAlignment = v + station.dirty = true } } func (station *Station) SetBattlePokemonBreadMode(v null.Int) { if station.BattlePokemonBreadMode != v { - station.BattlePokemonBreadMode = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonBreadMode") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonBreadMode:%v->%v", station.BattlePokemonBreadMode, v)) } + station.BattlePokemonBreadMode = v + station.dirty = true } } func (station *Station) SetBattlePokemonMove1(v null.Int) { if station.BattlePokemonMove1 != v { - station.BattlePokemonMove1 = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonMove1") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove1:%v->%v", station.BattlePokemonMove1, v)) } + station.BattlePokemonMove1 = v + station.dirty = true } } func (station *Station) SetBattlePokemonMove2(v null.Int) { if station.BattlePokemonMove2 != v { - station.BattlePokemonMove2 = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonMove2") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove2:%v->%v", station.BattlePokemonMove2, v)) } + station.BattlePokemonMove2 = v + station.dirty = true } } func (station *Station) SetBattlePokemonStamina(v null.Int) { if station.BattlePokemonStamina != v { - station.BattlePokemonStamina = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonStamina") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonStamina:%v->%v", station.BattlePokemonStamina, v)) } + station.BattlePokemonStamina = v + station.dirty = true } } func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { if !nullFloatAlmostEqual(station.BattlePokemonCpMultiplier, v, floatTolerance) { - station.BattlePokemonCpMultiplier = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "BattlePokemonCpMultiplier") + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCpMultiplier:%v->%v", station.BattlePokemonCpMultiplier, v)) } + station.BattlePokemonCpMultiplier = v + station.dirty = true } } func (station *Station) SetTotalStationedPokemon(v null.Int) { if station.TotalStationedPokemon != v { - station.TotalStationedPokemon = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "TotalStationedPokemon") + station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedPokemon:%v->%v", station.TotalStationedPokemon, v)) } + station.TotalStationedPokemon = v + station.dirty = true } } func (station *Station) SetTotalStationedGmax(v null.Int) { if station.TotalStationedGmax != v { - station.TotalStationedGmax = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "TotalStationedGmax") + station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedGmax:%v->%v", station.TotalStationedGmax, v)) } + station.TotalStationedGmax = v + station.dirty = true } } func (station *Station) SetStationedPokemon(v null.String) { if station.StationedPokemon != v { - station.StationedPokemon = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "StationedPokemon") + station.changedFields = append(station.changedFields, fmt.Sprintf("StationedPokemon:%v->%v", station.StationedPokemon, v)) } + station.StationedPokemon = v + station.dirty = true } } func (station *Station) SetUpdated(v int64) { if station.Updated != v { - station.Updated = v - station.dirty = true if dbDebugEnabled { - station.changedFields = append(station.changedFields, "Updated") + station.changedFields = append(station.changedFields, fmt.Sprintf("Updated:%d->%d", station.Updated, v)) } + station.Updated = v + station.dirty = true } } diff --git a/decoder/tappable.go b/decoder/tappable.go index 94ab1332..98573571 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "sync" "github.com/guregu/null/v6" @@ -58,110 +59,110 @@ func (ta *Tappable) Unlock() { func (ta *Tappable) SetLat(v float64) { if !floatAlmostEqual(ta.Lat, v, floatTolerance) { - ta.Lat = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Lat") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Lat:%f->%f", ta.Lat, v)) } + ta.Lat = v + ta.dirty = true } } func (ta *Tappable) SetLon(v float64) { if !floatAlmostEqual(ta.Lon, v, floatTolerance) { - ta.Lon = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Lon") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Lon:%f->%f", ta.Lon, v)) } + ta.Lon = v + ta.dirty = true } } func (ta *Tappable) SetFortId(v null.String) { if ta.FortId != v { - ta.FortId = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "FortId") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("FortId:%v->%v", ta.FortId, v)) } + ta.FortId = v + ta.dirty = true } } func (ta *Tappable) SetSpawnId(v null.Int) { if ta.SpawnId != v { - ta.SpawnId = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "SpawnId") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("SpawnId:%v->%v", ta.SpawnId, v)) } + ta.SpawnId = v + ta.dirty = true } } func (ta *Tappable) SetType(v string) { if ta.Type != v { - ta.Type = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Type") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Type:%s->%s", ta.Type, v)) } + ta.Type = v + ta.dirty = true } } func (ta *Tappable) SetEncounter(v null.Int) { if ta.Encounter != v { - ta.Encounter = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Encounter") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Encounter:%v->%v", ta.Encounter, v)) } + ta.Encounter = v + ta.dirty = true } } func (ta *Tappable) SetItemId(v null.Int) { if ta.ItemId != v { - ta.ItemId = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "ItemId") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ItemId:%v->%v", ta.ItemId, v)) } + ta.ItemId = v + ta.dirty = true } } func (ta *Tappable) SetCount(v null.Int) { if ta.Count != v { - ta.Count = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Count") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Count:%v->%v", ta.Count, v)) } + ta.Count = v + ta.dirty = true } } func (ta *Tappable) SetExpireTimestamp(v null.Int) { if ta.ExpireTimestamp != v { - ta.ExpireTimestamp = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "ExpireTimestamp") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ExpireTimestamp:%v->%v", ta.ExpireTimestamp, v)) } + ta.ExpireTimestamp = v + ta.dirty = true } } func (ta *Tappable) SetExpireTimestampVerified(v bool) { if ta.ExpireTimestampVerified != v { - ta.ExpireTimestampVerified = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "ExpireTimestampVerified") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ExpireTimestampVerified:%t->%t", ta.ExpireTimestampVerified, v)) } + ta.ExpireTimestampVerified = v + ta.dirty = true } } func (ta *Tappable) SetUpdated(v int64) { if ta.Updated != v { - ta.Updated = v - ta.dirty = true if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, "Updated") + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Updated:%d->%d", ta.Updated, v)) } + ta.Updated = v + ta.dirty = true } } From ba42ef9a82c21481864cb2b3296ae0961def1241 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 22:20:07 +0000 Subject: [PATCH 32/35] Ability to turn off nearby cell pokemon --- config/config.go | 1 + decoder/gmo_decode.go | 24 +++++++++++++----------- decoder/scanarea.go | 16 +++++++++++++++- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/config/config.go b/config/config.go index ab2423d4..6eacd7da 100644 --- a/config/config.go +++ b/config/config.go @@ -134,6 +134,7 @@ type scanRule struct { ProcessPokemon *bool `koanf:"pokemon"` ProcessWilds *bool `koanf:"wild_pokemon"` ProcessNearby *bool `koanf:"nearby_pokemon"` + ProcessNearbyCell *bool `koanf:"nearby_cell_pokemon"` ProcessWeather *bool `koanf:"weather"` ProcessCells *bool `koanf:"cells"` ProcessPokestops *bool `koanf:"pokestops"` diff --git a/decoder/gmo_decode.go b/decoder/gmo_decode.go index 95784faf..8f2a7f54 100644 --- a/decoder/gmo_decode.go +++ b/decoder/gmo_decode.go @@ -145,19 +145,21 @@ func UpdatePokemonBatch(ctx context.Context, db db.DbDetails, scanParameters Sca for _, nearby := range nearbyPokemonList { encounterId := nearby.Data.EncounterId - pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) - if err != nil { - log.Printf("getOrCreatePokemonRecord: %s", err) - continue - } + if nearby.Data.FortId != "" || scanParameters.ProcessNearbyCell { + pokemon, unlock, err := getOrCreatePokemonRecord(ctx, db, encounterId) + if err != nil { + log.Printf("getOrCreatePokemonRecord: %s", err) + continue + } - updateTime := nearby.Timestamp / 1000 - if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { - pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) - savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) - } + updateTime := nearby.Timestamp / 1000 + if pokemon.isNewRecord() || pokemon.nearbySignificantUpdate(nearby.Data, updateTime) { + pokemon.updateFromNearby(ctx, db, nearby.Data, int64(nearby.Cell), weatherLookup, nearby.Timestamp, username) + savePokemonRecordAsAtTime(ctx, db, pokemon, false, true, true, nearby.Timestamp/1000) + } - unlock() + unlock() + } } } diff --git a/decoder/scanarea.go b/decoder/scanarea.go index 9e7831ae..7b52636f 100644 --- a/decoder/scanarea.go +++ b/decoder/scanarea.go @@ -1,15 +1,17 @@ package decoder import ( + "strings" + "golbat/config" "golbat/geo" - "strings" ) type ScanParameters struct { ProcessPokemon bool ProcessWild bool ProcessNearby bool + ProcessNearbyCell bool ProcessWeather bool ProcessPokestops bool ProcessGyms bool @@ -56,6 +58,16 @@ func FindScanConfiguration(scanContext string, lat, lon float64) ScanParameters return *value } + defaultTrueFirst := func(value *bool, value2 *bool) bool { + if value != nil { + return *value + } + if value2 != nil { + return *value2 + } + return true + } + defaultFromWeatherConfig := func(value *bool, weatherDefault bool) bool { if value == nil { return weatherDefault @@ -67,6 +79,7 @@ func FindScanConfiguration(scanContext string, lat, lon float64) ScanParameters ProcessPokemon: defaultTrue(rule.ProcessPokemon), ProcessWild: defaultTrue(rule.ProcessWilds), ProcessNearby: defaultTrue(rule.ProcessNearby), + ProcessNearbyCell: defaultTrueFirst(rule.ProcessNearbyCell, rule.ProcessNearby), ProcessCells: defaultTrue(rule.ProcessCells), ProcessWeather: defaultTrue(rule.ProcessWeather), ProcessPokestops: defaultTrue(rule.ProcessPokestops), @@ -82,6 +95,7 @@ func FindScanConfiguration(scanContext string, lat, lon float64) ScanParameters ProcessPokemon: true, ProcessWild: true, ProcessNearby: true, + ProcessNearbyCell: true, ProcessCells: true, ProcessWeather: true, ProcessGyms: true, From dba646a78462cf01d9637925d1ac2d11cdf25fa0 Mon Sep 17 00:00:00 2001 From: James Berry Date: Sun, 1 Feb 2026 22:37:47 +0000 Subject: [PATCH 33/35] API documentation --- api.md | 760 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 760 insertions(+) create mode 100644 api.md diff --git a/api.md b/api.md new file mode 100644 index 00000000..953f71c5 --- /dev/null +++ b/api.md @@ -0,0 +1,760 @@ +# Golbat API Documentation + +Golbat provides both HTTP REST and gRPC APIs for querying Pokemon GO data. + +## Table of Contents + +- [Authentication](#authentication) +- [Health Check](#health-check) +- [Raw Data Ingestion](#raw-data-ingestion) +- [Pokemon Endpoints](#pokemon-endpoints) +- [Pokestop Endpoints](#pokestop-endpoints) +- [Gym Endpoints](#gym-endpoints) +- [Quest Endpoints](#quest-endpoints) +- [Tappable Endpoints](#tappable-endpoints) +- [Device Endpoints](#device-endpoints) +- [Debug Endpoints](#debug-endpoints) +- [gRPC API](#grpc-api) +- [Data Structures](#data-structures) + +--- + +## Authentication + +### API Authentication + +All `/api/*` endpoints require authentication via the `X-Golbat-Secret` header. + +``` +X-Golbat-Secret: your_api_secret +``` + +The secret is configured via `api_secret` in the configuration file. + +### Raw Endpoint Authentication + +The `/raw` endpoint optionally supports Bearer token authentication: + +``` +Authorization: Bearer your_raw_bearer_token +``` + +This is only enforced if `raw_bearer` is configured. + +--- + +## Health Check + +### GET /health + +Unrestricted health check endpoint for monitoring. + +**Authentication:** Not required + +**Response:** +```json +{ + "status": "ok" +} +``` + +### GET /api/health + +Authenticated health check endpoint. + +**Authentication:** Required + +**Response:** +```json +{ + "status": "ok" +} +``` + +--- + +## Raw Data Ingestion + +### POST /raw + +Accept raw protobuf data from scanning clients. + +**Authentication:** Bearer token (optional, if configured) + +**Request Body:** +```json +{ + "uuid": "device_uuid", + "username": "account_name", + "trainerlvl": 30, + "scan_context": "context_string", + "lat_target": 40.7128, + "lon_target": -74.0060, + "timestamp_ms": 1234567890, + "have_ar": true, + "contents": [ + { + "payload": "base64_encoded_proto", + "type": 1, + "request": "optional_request_proto" + } + ] +} +``` + +**Response:** HTTP 201 Created (async processing) + +**Notes:** +- Multiple provider formats supported (Pogodroid, standard format) +- Processing timeout: 5s normal, 30s if `extended_timeout` enabled +- Content can use `data` or `payload` for the proto data +- Content can use `method` or `type` for the method number + +--- + +## Pokemon Endpoints + +### GET /api/pokemon/id/:pokemon_id + +Retrieve a single pokemon by encounter ID. + +**Authentication:** Required + +**Parameters:** +| Name | Type | Location | Description | +|------|------|----------|-------------| +| pokemon_id | uint64 | path | Pokemon encounter ID | + +**Response:** [ApiPokemonResult](#apipokemonresult) + +**Status Codes:** +- 200: Pokemon found +- 404: Pokemon not found + +--- + +### GET /api/pokemon/available + +List all available pokemon species with counts. + +**Authentication:** Required + +**Response:** +```json +[ + { + "id": 1, + "form": 0, + "count": 42 + } +] +``` + +--- + +### POST /api/pokemon/scan + +Query pokemon in a geographic area with filters (v1 - legacy). + +**Authentication:** Required + +**Request Body:** +```json +{ + "min": {"lat": 40.7, "lon": -74.0}, + "max": {"lat": 40.8, "lon": -73.9}, + "center": {"lat": 40.75, "lon": -73.95}, + "limit": 500, + "global": { + "iv": [0, 100], + "atk_iv": [0, 15], + "def_iv": [0, 15], + "sta_iv": [0, 15], + "level": [1, 50], + "cp": [0, 3000], + "gender": 1, + "additional": { + "include_everything": false, + "include_hundoiv": true, + "include_zeroiv": false, + "include_xxs": true, + "include_xxl": false + }, + "pvp": { + "little": [1, 100], + "great": [1, 100], + "ultra": [1, 100] + } + }, + "filters": { + "1-0": {} + } +} +``` + +**Response:** Array of [ApiPokemonResult](#apipokemonresult) + +--- + +### POST /api/pokemon/v2/scan + +Query pokemon with DNF (Disjunctive Normal Form) filters - more efficient filtering. + +**Authentication:** Required + +**Request Body:** +```json +{ + "min": {"lat": 40.7, "lon": -74.0}, + "max": {"lat": 40.8, "lon": -73.9}, + "limit": 500, + "filters": [ + { + "pokemon": [{"id": 1, "form": 0}], + "iv": {"min": 90, "max": 100}, + "atk_iv": {"min": 10, "max": 15}, + "def_iv": {"min": 10, "max": 15}, + "sta_iv": {"min": 10, "max": 15}, + "level": {"min": 30, "max": 50}, + "cp": {"min": 2000, "max": 3000}, + "gender": {"min": 0, "max": 2}, + "size": {"min": 0, "max": 5}, + "pvp_little": {"min": 1, "max": 100}, + "pvp_great": {"min": 1, "max": 100}, + "pvp_ultra": {"min": 1, "max": 100} + } + ] +} +``` + +**Response:** Array of [ApiPokemonResult](#apipokemonresult) + +--- + +### POST /api/pokemon/v3/scan + +Query pokemon with advanced DNF filters, returns metadata about scan. + +**Authentication:** Required + +**Request Body:** Same as v2, with gender as array + +**Response:** +```json +{ + "pokemon": [], + "examined": 1000, + "skipped": 50, + "total": 1050 +} +``` + +--- + +### POST /api/pokemon/search + +Advanced search using center point and distance. + +**Authentication:** Required + +**Request Body:** +```json +{ + "min": {"lat": 40.7, "lon": -74.0}, + "max": {"lat": 40.8, "lon": -73.9}, + "center": {"lat": 40.75, "lon": -73.95}, + "limit": 500, + "searchIds": [1, 4, 7] +} +``` + +**Response:** Array of [ApiPokemonResult](#apipokemonresult) + +**Status Codes:** +- 200: Success +- 400: Bad Request (validation failed) + +--- + +## Pokestop Endpoints + +### GET /api/pokestop/id/:fort_id + +Retrieve a single pokestop by fort ID. + +**Authentication:** Required + +**Parameters:** +| Name | Type | Location | Description | +|------|------|----------|-------------| +| fort_id | string | path | Pokestop fort ID | + +**Response:** [ApiPokestopResult](#apipokestopresult) + +**Status Codes:** +- 200: Pokestop found +- 404: Pokestop not found + +--- + +### POST /api/pokestop-positions + +Get coordinates of all pokestops within a geofence. + +**Authentication:** Required + +**Request Body:** GeoJSON Feature, Geometry, or Golbat Geofence format +```json +{ + "fence": [ + {"lat": 40.7, "lon": -74.0}, + {"lat": 40.8, "lon": -74.0}, + {"lat": 40.8, "lon": -73.9}, + {"lat": 40.7, "lon": -73.9} + ] +} +``` + +**Response:** +```json +[ + { + "id": "fort_id", + "latitude": 40.7128, + "longitude": -74.0060 + } +] +``` + +--- + +## Gym Endpoints + +### GET /api/gym/id/:gym_id + +Retrieve a single gym by gym ID. + +**Authentication:** Required + +**Parameters:** +| Name | Type | Location | Description | +|------|------|----------|-------------| +| gym_id | string | path | Gym ID | + +**Response:** [ApiGymResult](#apigymresult) + +**Status Codes:** +- 200: Gym found +- 404: Gym not found + +--- + +### POST /api/gym/query + +Get multiple gyms by IDs. + +**Authentication:** Required + +**Request Body:** +```json +{ + "ids": ["gym_id1", "gym_id2"] +} +``` +Or as an array: +```json +["gym_id1", "gym_id2"] +``` + +**Response:** Array of [ApiGymResult](#apigymresult) + +**Limits:** +- Maximum 500 IDs per request +- Duplicates are filtered + +**Status Codes:** +- 200: Success +- 413: Request Entity Too Large (exceeds 500 IDs) + +--- + +### POST /api/gym/search + +Advanced gym search with filters. + +**Authentication:** Required + +**Request Body:** +```json +{ + "filters": [ + { + "name": "central park", + "description": "playground", + "location_distance": { + "location": {"lat": 40.7829, "lon": -73.9654}, + "distance": 500 + }, + "bbox": { + "min_lon": -74.0, + "min_lat": 40.7, + "max_lon": -73.9, + "max_lat": 40.8 + } + } + ], + "limit": 100 +} +``` + +**Response:** Array of [ApiGymResult](#apigymresult) + +**Limits:** +- Default limit: 500 +- Max limit: 10,000 +- Max distance: 500,000 meters + +**Status Codes:** +- 200: Success +- 400: Bad Request (invalid filters) +- 504: Gateway Timeout + +--- + +## Quest Endpoints + +### POST /api/quest-status + +Get quest statistics for a geofence area. + +**Authentication:** Required + +**Request Body:** GeoJSON Feature, Geometry, or Golbat Geofence format + +**Response:** +```json +{ + "ar_quests": 50, + "no_ar_quests": 100, + "total": 200 +} +``` + +--- + +### POST /api/clear-quests +### DELETE /api/clear-quests + +Clear all quests within a geofence area. + +**Authentication:** Required + +**Request Body:** GeoJSON Feature, Geometry, or Golbat Geofence format + +**Response:** +```json +{ + "status": "ok" +} +``` + +--- + +### POST /api/reload-geojson +### GET /api/reload-geojson + +Reload geofence boundaries and clear stats. + +**Authentication:** Required + +**Response:** +```json +{ + "status": "ok" +} +``` + +--- + +## Tappable Endpoints + +### GET /api/tappable/id/:tappable_id + +Retrieve a tappable (invasions, research, etc.). + +**Authentication:** Required + +**Parameters:** +| Name | Type | Location | Description | +|------|------|----------|-------------| +| tappable_id | uint64 | path | Tappable ID | + +**Response:** [ApiTappableResult](#apitappableresult) + +**Status Codes:** +- 200: Tappable found +- 400: Invalid ID +- 404: Tappable not found + +--- + +## Device Endpoints + +### GET /api/devices/all + +Get information about all connected/known devices. + +**Authentication:** Required + +**Response:** +```json +{ + "devices": [ + { + "uuid": "device_uuid", + "lat": 40.7128, + "lon": -74.0060, + "last_scan": 1234567890 + } + ] +} +``` + +--- + +## Debug Endpoints + +These endpoints are only available if `tuning.profile_routes` is enabled in configuration. + +**Authentication:** Required + +| Endpoint | Description | +|----------|-------------| +| GET /debug/pprof/cmdline | Command line arguments | +| GET /debug/pprof/heap | Heap memory profile | +| GET /debug/pprof/block | Block profile | +| GET /debug/pprof/mutex | Mutex profile | +| GET /debug/pprof/trace | Execution trace | +| GET /debug/pprof/profile | CPU profile | +| GET /debug/pprof/symbol | Symbol lookup | + +--- + +## gRPC API + +Golbat also provides a gRPC API running on a separate port (configured via `grpc_port`). + +### Authentication + +Use the `authorization` metadata header with the API secret. + +### Pokemon Service + +```protobuf +service Pokemon { + rpc Search(PokemonScanRequest) returns (PokemonScanResponse); + rpc SearchV3(PokemonScanRequestV3) returns (PokemonScanResponseV3); +} +``` + +The gRPC endpoints mirror the HTTP v2/v3 scan endpoints. + +--- + +## Data Structures + +### Location + +```json +{ + "lat": 40.7128, + "lon": -74.0060 +} +``` + +### Bounding Box (Bbox) + +```json +{ + "min_lon": -74.0, + "min_lat": 40.7, + "max_lon": -73.9, + "max_lat": 40.8 +} +``` + +### ApiPokemonResult + +```json +{ + "id": "encounter_id", + "pokestop_id": "fort_id_or_null", + "spawn_id": 123456789, + "lat": 40.7128, + "lon": -74.0060, + "weight": 5.5, + "size": 2, + "height": 0.8, + "expire_timestamp": 1234567890, + "updated": 1234567800, + "pokemon_id": 1, + "move_1": 100, + "move_2": 200, + "gender": 1, + "cp": 500, + "atk_iv": 15, + "def_iv": 15, + "sta_iv": 15, + "iv": 100.0, + "form": 0, + "level": 30, + "weather": 1, + "costume": 0, + "first_seen_timestamp": 1234567000, + "changed": 1234567800, + "cell_id": 123456789, + "expire_timestamp_verified": true, + "display_pokemon_id": 1, + "is_ditto": false, + "seen_type": "encounter", + "shiny": false, + "username": "trainer_name", + "capture_1": 0.5, + "capture_2": 0.6, + "capture_3": 0.7, + "pvp": {}, + "is_event": 0 +} +``` + +### ApiPokestopResult + +```json +{ + "id": "fort_id", + "lat": 40.7128, + "lon": -74.0060, + "name": "Pokestop Name", + "url": "image_url", + "lure_expire_timestamp": 1234567890, + "last_modified_timestamp": 1234567800, + "updated": 1234567800, + "enabled": true, + "quest_type": 1, + "quest_timestamp": 1234567800, + "quest_target": 3, + "quest_conditions": "json_conditions", + "quest_rewards": "json_rewards", + "quest_template": "template_string", + "quest_title": "Quest Title", + "quest_expiry": 1234667800, + "cell_id": 123456789, + "deleted": false, + "lure_id": 501, + "first_seen_timestamp": 1234567000, + "sponsor_id": 1, + "partner_id": "partner_code", + "ar_scan_eligible": 1, + "power_up_level": 1, + "power_up_points": 100, + "power_up_end_timestamp": 1234567890, + "alternative_quest_type": null, + "alternative_quest_timestamp": null, + "alternative_quest_target": null, + "alternative_quest_conditions": null, + "alternative_quest_rewards": null, + "alternative_quest_template": null, + "alternative_quest_title": null, + "alternative_quest_expiry": null, + "description": "Pokestop description", + "showcase_focus": "focus_pokemon", + "showcase_pokemon_id": 1, + "showcase_pokemon_form_id": 0, + "showcase_pokemon_type_id": 1, + "showcase_ranking_standard": 1, + "showcase_expiry": 1234567890, + "showcase_rankings": "json_rankings" +} +``` + +### ApiGymResult + +```json +{ + "id": "gym_id", + "lat": 40.7128, + "lon": -74.0060, + "name": "Gym Name", + "url": "image_url", + "last_modified_timestamp": 1234567800, + "raid_end_timestamp": 1234567890, + "raid_spawn_timestamp": 1234567800, + "raid_battle_timestamp": 1234567850, + "updated": 1234567800, + "raid_pokemon_id": 1, + "guarding_pokemon_id": 25, + "guarding_pokemon_display": "display_string", + "available_slots": 3, + "team_id": 1, + "raid_level": 3, + "enabled": 1, + "ex_raid_eligible": 1, + "in_battle": 0, + "raid_pokemon_move_1": 100, + "raid_pokemon_move_2": 200, + "raid_pokemon_form": 0, + "raid_pokemon_alignment": 1, + "raid_pokemon_cp": 30000, + "raid_is_exclusive": 0, + "cell_id": 123456789, + "deleted": false, + "total_cp": 150000, + "first_seen_timestamp": 1234567000, + "raid_pokemon_gender": 1, + "sponsor_id": 1, + "partner_id": "partner_code", + "raid_pokemon_costume": 0, + "raid_pokemon_evolution": 0, + "ar_scan_eligible": 1, + "power_up_level": 1, + "power_up_points": 100, + "power_up_end_timestamp": 1234567890, + "description": "Gym description", + "defenders": "json_defenders", + "rsvps": "json_rsvps" +} +``` + +### ApiTappableResult + +```json +{ + "id": 1234567890, + "lat": 40.7128, + "lon": -74.0060, + "fort_id": "gym_or_pokestop_id", + "spawn_id": 987654321, + "type": "invasion", + "pokemon_id": 1, + "item_id": 1, + "count": 1, + "expire_timestamp": 1234567890, + "expire_timestamp_verified": true, + "updated": 1234567800 +} +``` + +--- + +## Configuration Reference + +| Key | Description | +|-----|-------------| +| `api_secret` | API authentication token (header: `X-Golbat-Secret`) | +| `raw_bearer` | Bearer token for raw endpoint (header: `Authorization: Bearer`) | +| `port` | HTTP server port | +| `grpc_port` | gRPC server port | +| `tuning.extended_timeout` | Enable 30s timeout for raw processing | +| `tuning.profile_routes` | Enable pprof debug endpoints | +| `tuning.max_pokemon_results` | Max pokemon returned per query | +| `tuning.max_pokemon_distance` | Max distance between min/max points in searches | From c02ade86e81b8025724e44f05ca90d9074f86b6e Mon Sep 17 00:00:00 2001 From: James Berry Date: Mon, 2 Feb 2026 08:21:41 +0000 Subject: [PATCH 34/35] Change null debug output so it is more readable --- decoder/gym.go | 70 +++++++++--------- decoder/incident.go | 12 ++-- decoder/main.go | 15 ++++ decoder/player.go | 164 +++++++++++++++++++++--------------------- decoder/pokemon.go | 48 ++++++------- decoder/pokestop.go | 72 +++++++++---------- decoder/routes.go | 2 +- decoder/spawnpoint.go | 4 +- decoder/station.go | 32 ++++----- decoder/tappable.go | 12 ++-- 10 files changed, 223 insertions(+), 208 deletions(-) diff --git a/decoder/gym.go b/decoder/gym.go index d906a0c0..902cb15b 100644 --- a/decoder/gym.go +++ b/decoder/gym.go @@ -210,7 +210,7 @@ func (gym *Gym) SetLon(v float64) { func (gym *Gym) SetName(v null.String) { if gym.Name != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Name:%v->%v", gym.Name, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Name:%s->%s", FormatNull(gym.Name), FormatNull(v))) } gym.Name = v gym.dirty = true @@ -220,7 +220,7 @@ func (gym *Gym) SetName(v null.String) { func (gym *Gym) SetUrl(v null.String) { if gym.Url != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Url:%v->%v", gym.Url, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Url:%s->%s", FormatNull(gym.Url), FormatNull(v))) } gym.Url = v gym.dirty = true @@ -230,7 +230,7 @@ func (gym *Gym) SetUrl(v null.String) { func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { if gym.LastModifiedTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("LastModifiedTimestamp:%v->%v", gym.LastModifiedTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("LastModifiedTimestamp:%s->%s", FormatNull(gym.LastModifiedTimestamp), FormatNull(v))) } gym.LastModifiedTimestamp = v gym.dirty = true @@ -240,7 +240,7 @@ func (gym *Gym) SetLastModifiedTimestamp(v null.Int) { func (gym *Gym) SetRaidEndTimestamp(v null.Int) { if gym.RaidEndTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidEndTimestamp:%v->%v", gym.RaidEndTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidEndTimestamp:%s->%s", FormatNull(gym.RaidEndTimestamp), FormatNull(v))) } gym.RaidEndTimestamp = v gym.dirty = true @@ -250,7 +250,7 @@ func (gym *Gym) SetRaidEndTimestamp(v null.Int) { func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { if gym.RaidSpawnTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidSpawnTimestamp:%v->%v", gym.RaidSpawnTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidSpawnTimestamp:%s->%s", FormatNull(gym.RaidSpawnTimestamp), FormatNull(v))) } gym.RaidSpawnTimestamp = v gym.dirty = true @@ -260,7 +260,7 @@ func (gym *Gym) SetRaidSpawnTimestamp(v null.Int) { func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { if gym.RaidBattleTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidBattleTimestamp:%v->%v", gym.RaidBattleTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidBattleTimestamp:%s->%s", FormatNull(gym.RaidBattleTimestamp), FormatNull(v))) } gym.RaidBattleTimestamp = v gym.dirty = true @@ -270,7 +270,7 @@ func (gym *Gym) SetRaidBattleTimestamp(v null.Int) { func (gym *Gym) SetRaidPokemonId(v null.Int) { if gym.RaidPokemonId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonId:%v->%v", gym.RaidPokemonId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonId:%s->%s", FormatNull(gym.RaidPokemonId), FormatNull(v))) } gym.RaidPokemonId = v gym.dirty = true @@ -280,7 +280,7 @@ func (gym *Gym) SetRaidPokemonId(v null.Int) { func (gym *Gym) SetGuardingPokemonId(v null.Int) { if gym.GuardingPokemonId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonId:%v->%v", gym.GuardingPokemonId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonId:%s->%s", FormatNull(gym.GuardingPokemonId), FormatNull(v))) } gym.GuardingPokemonId = v gym.dirty = true @@ -290,7 +290,7 @@ func (gym *Gym) SetGuardingPokemonId(v null.Int) { func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { if gym.GuardingPokemonDisplay != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonDisplay:%v->%v", gym.GuardingPokemonDisplay, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("GuardingPokemonDisplay:%s->%s", FormatNull(gym.GuardingPokemonDisplay), FormatNull(v))) } gym.GuardingPokemonDisplay = v gym.dirty = true @@ -300,7 +300,7 @@ func (gym *Gym) SetGuardingPokemonDisplay(v null.String) { func (gym *Gym) SetAvailableSlots(v null.Int) { if gym.AvailableSlots != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("AvailableSlots:%v->%v", gym.AvailableSlots, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("AvailableSlots:%s->%s", FormatNull(gym.AvailableSlots), FormatNull(v))) } gym.AvailableSlots = v gym.dirty = true @@ -310,7 +310,7 @@ func (gym *Gym) SetAvailableSlots(v null.Int) { func (gym *Gym) SetTeamId(v null.Int) { if gym.TeamId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("TeamId:%v->%v", gym.TeamId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("TeamId:%s->%s", FormatNull(gym.TeamId), FormatNull(v))) } gym.TeamId = v gym.dirty = true @@ -320,7 +320,7 @@ func (gym *Gym) SetTeamId(v null.Int) { func (gym *Gym) SetRaidLevel(v null.Int) { if gym.RaidLevel != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidLevel:%v->%v", gym.RaidLevel, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidLevel:%s->%s", FormatNull(gym.RaidLevel), FormatNull(v))) } gym.RaidLevel = v gym.dirty = true @@ -330,7 +330,7 @@ func (gym *Gym) SetRaidLevel(v null.Int) { func (gym *Gym) SetEnabled(v null.Int) { if gym.Enabled != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Enabled:%v->%v", gym.Enabled, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Enabled:%s->%s", FormatNull(gym.Enabled), FormatNull(v))) } gym.Enabled = v gym.dirty = true @@ -340,7 +340,7 @@ func (gym *Gym) SetEnabled(v null.Int) { func (gym *Gym) SetExRaidEligible(v null.Int) { if gym.ExRaidEligible != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("ExRaidEligible:%v->%v", gym.ExRaidEligible, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("ExRaidEligible:%s->%s", FormatNull(gym.ExRaidEligible), FormatNull(v))) } gym.ExRaidEligible = v gym.dirty = true @@ -350,7 +350,7 @@ func (gym *Gym) SetExRaidEligible(v null.Int) { func (gym *Gym) SetInBattle(v null.Int) { if gym.InBattle != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("InBattle:%v->%v", gym.InBattle, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("InBattle:%s->%s", FormatNull(gym.InBattle), FormatNull(v))) } gym.InBattle = v //Do not set to dirty, as don't trigger an update @@ -361,7 +361,7 @@ func (gym *Gym) SetInBattle(v null.Int) { func (gym *Gym) SetRaidPokemonMove1(v null.Int) { if gym.RaidPokemonMove1 != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove1:%v->%v", gym.RaidPokemonMove1, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove1:%s->%s", FormatNull(gym.RaidPokemonMove1), FormatNull(v))) } gym.RaidPokemonMove1 = v gym.dirty = true @@ -371,7 +371,7 @@ func (gym *Gym) SetRaidPokemonMove1(v null.Int) { func (gym *Gym) SetRaidPokemonMove2(v null.Int) { if gym.RaidPokemonMove2 != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove2:%v->%v", gym.RaidPokemonMove2, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonMove2:%s->%s", FormatNull(gym.RaidPokemonMove2), FormatNull(v))) } gym.RaidPokemonMove2 = v gym.dirty = true @@ -381,7 +381,7 @@ func (gym *Gym) SetRaidPokemonMove2(v null.Int) { func (gym *Gym) SetRaidPokemonForm(v null.Int) { if gym.RaidPokemonForm != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonForm:%v->%v", gym.RaidPokemonForm, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonForm:%s->%s", FormatNull(gym.RaidPokemonForm), FormatNull(v))) } gym.RaidPokemonForm = v gym.dirty = true @@ -391,7 +391,7 @@ func (gym *Gym) SetRaidPokemonForm(v null.Int) { func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { if gym.RaidPokemonAlignment != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonAlignment:%v->%v", gym.RaidPokemonAlignment, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonAlignment:%s->%s", FormatNull(gym.RaidPokemonAlignment), FormatNull(v))) } gym.RaidPokemonAlignment = v gym.dirty = true @@ -401,7 +401,7 @@ func (gym *Gym) SetRaidPokemonAlignment(v null.Int) { func (gym *Gym) SetRaidPokemonCp(v null.Int) { if gym.RaidPokemonCp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCp:%v->%v", gym.RaidPokemonCp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCp:%s->%s", FormatNull(gym.RaidPokemonCp), FormatNull(v))) } gym.RaidPokemonCp = v gym.dirty = true @@ -411,7 +411,7 @@ func (gym *Gym) SetRaidPokemonCp(v null.Int) { func (gym *Gym) SetRaidIsExclusive(v null.Int) { if gym.RaidIsExclusive != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidIsExclusive:%v->%v", gym.RaidIsExclusive, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidIsExclusive:%s->%s", FormatNull(gym.RaidIsExclusive), FormatNull(v))) } gym.RaidIsExclusive = v gym.dirty = true @@ -421,7 +421,7 @@ func (gym *Gym) SetRaidIsExclusive(v null.Int) { func (gym *Gym) SetCellId(v null.Int) { if gym.CellId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("CellId:%v->%v", gym.CellId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("CellId:%s->%s", FormatNull(gym.CellId), FormatNull(v))) } gym.CellId = v gym.dirty = true @@ -441,7 +441,7 @@ func (gym *Gym) SetDeleted(v bool) { func (gym *Gym) SetTotalCp(v null.Int) { if gym.TotalCp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("TotalCp:%v->%v", gym.TotalCp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("TotalCp:%s->%s", FormatNull(gym.TotalCp), FormatNull(v))) } gym.TotalCp = v gym.dirty = true @@ -451,7 +451,7 @@ func (gym *Gym) SetTotalCp(v null.Int) { func (gym *Gym) SetRaidPokemonGender(v null.Int) { if gym.RaidPokemonGender != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonGender:%v->%v", gym.RaidPokemonGender, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonGender:%s->%s", FormatNull(gym.RaidPokemonGender), FormatNull(v))) } gym.RaidPokemonGender = v gym.dirty = true @@ -461,7 +461,7 @@ func (gym *Gym) SetRaidPokemonGender(v null.Int) { func (gym *Gym) SetSponsorId(v null.Int) { if gym.SponsorId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("SponsorId:%v->%v", gym.SponsorId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("SponsorId:%s->%s", FormatNull(gym.SponsorId), FormatNull(v))) } gym.SponsorId = v gym.dirty = true @@ -471,7 +471,7 @@ func (gym *Gym) SetSponsorId(v null.Int) { func (gym *Gym) SetPartnerId(v null.String) { if gym.PartnerId != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("PartnerId:%v->%v", gym.PartnerId, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PartnerId:%s->%s", FormatNull(gym.PartnerId), FormatNull(v))) } gym.PartnerId = v gym.dirty = true @@ -481,7 +481,7 @@ func (gym *Gym) SetPartnerId(v null.String) { func (gym *Gym) SetRaidPokemonCostume(v null.Int) { if gym.RaidPokemonCostume != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCostume:%v->%v", gym.RaidPokemonCostume, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonCostume:%s->%s", FormatNull(gym.RaidPokemonCostume), FormatNull(v))) } gym.RaidPokemonCostume = v gym.dirty = true @@ -491,7 +491,7 @@ func (gym *Gym) SetRaidPokemonCostume(v null.Int) { func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { if gym.RaidPokemonEvolution != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonEvolution:%v->%v", gym.RaidPokemonEvolution, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("RaidPokemonEvolution:%s->%s", FormatNull(gym.RaidPokemonEvolution), FormatNull(v))) } gym.RaidPokemonEvolution = v gym.dirty = true @@ -501,7 +501,7 @@ func (gym *Gym) SetRaidPokemonEvolution(v null.Int) { func (gym *Gym) SetArScanEligible(v null.Int) { if gym.ArScanEligible != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("ArScanEligible:%v->%v", gym.ArScanEligible, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("ArScanEligible:%s->%s", FormatNull(gym.ArScanEligible), FormatNull(v))) } gym.ArScanEligible = v gym.dirty = true @@ -511,7 +511,7 @@ func (gym *Gym) SetArScanEligible(v null.Int) { func (gym *Gym) SetPowerUpLevel(v null.Int) { if gym.PowerUpLevel != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpLevel:%v->%v", gym.PowerUpLevel, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpLevel:%s->%s", FormatNull(gym.PowerUpLevel), FormatNull(v))) } gym.PowerUpLevel = v gym.dirty = true @@ -521,7 +521,7 @@ func (gym *Gym) SetPowerUpLevel(v null.Int) { func (gym *Gym) SetPowerUpPoints(v null.Int) { if gym.PowerUpPoints != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpPoints:%v->%v", gym.PowerUpPoints, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpPoints:%s->%s", FormatNull(gym.PowerUpPoints), FormatNull(v))) } gym.PowerUpPoints = v gym.dirty = true @@ -531,7 +531,7 @@ func (gym *Gym) SetPowerUpPoints(v null.Int) { func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { if gym.PowerUpEndTimestamp != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%v->%v", gym.PowerUpEndTimestamp, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%s->%s", FormatNull(gym.PowerUpEndTimestamp), FormatNull(v))) } gym.PowerUpEndTimestamp = v gym.dirty = true @@ -541,7 +541,7 @@ func (gym *Gym) SetPowerUpEndTimestamp(v null.Int) { func (gym *Gym) SetDescription(v null.String) { if gym.Description != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Description:%v->%v", gym.Description, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Description:%s->%s", FormatNull(gym.Description), FormatNull(v))) } gym.Description = v gym.dirty = true @@ -551,7 +551,7 @@ func (gym *Gym) SetDescription(v null.String) { func (gym *Gym) SetDefenders(v null.String) { if gym.Defenders != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Defenders:%v->%v", gym.Defenders, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Defenders:%s->%s", FormatNull(gym.Defenders), FormatNull(v))) } gym.Defenders = v //Do not set to dirty, as don't trigger an update @@ -562,7 +562,7 @@ func (gym *Gym) SetDefenders(v null.String) { func (gym *Gym) SetRsvps(v null.String) { if gym.Rsvps != v { if dbDebugEnabled { - gym.changedFields = append(gym.changedFields, fmt.Sprintf("Rsvps:%v->%v", gym.Rsvps, v)) + gym.changedFields = append(gym.changedFields, fmt.Sprintf("Rsvps:%s->%s", FormatNull(gym.Rsvps), FormatNull(v))) } gym.Rsvps = v gym.dirty = true diff --git a/decoder/incident.go b/decoder/incident.go index dd85d474..c30e135d 100644 --- a/decoder/incident.go +++ b/decoder/incident.go @@ -201,7 +201,7 @@ func (incident *Incident) SetConfirmed(v bool) { func (incident *Incident) SetSlot1PokemonId(v null.Int) { if incident.Slot1PokemonId != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1PokemonId:%v->%v", incident.Slot1PokemonId, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1PokemonId:%s->%s", FormatNull(incident.Slot1PokemonId), FormatNull(v))) } incident.Slot1PokemonId = v incident.dirty = true @@ -211,7 +211,7 @@ func (incident *Incident) SetSlot1PokemonId(v null.Int) { func (incident *Incident) SetSlot1Form(v null.Int) { if incident.Slot1Form != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1Form:%v->%v", incident.Slot1Form, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot1Form:%s->%s", FormatNull(incident.Slot1Form), FormatNull(v))) } incident.Slot1Form = v incident.dirty = true @@ -221,7 +221,7 @@ func (incident *Incident) SetSlot1Form(v null.Int) { func (incident *Incident) SetSlot2PokemonId(v null.Int) { if incident.Slot2PokemonId != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2PokemonId:%v->%v", incident.Slot2PokemonId, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2PokemonId:%s->%s", FormatNull(incident.Slot2PokemonId), FormatNull(v))) } incident.Slot2PokemonId = v incident.dirty = true @@ -231,7 +231,7 @@ func (incident *Incident) SetSlot2PokemonId(v null.Int) { func (incident *Incident) SetSlot2Form(v null.Int) { if incident.Slot2Form != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2Form:%v->%v", incident.Slot2Form, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot2Form:%s->%s", FormatNull(incident.Slot2Form), FormatNull(v))) } incident.Slot2Form = v incident.dirty = true @@ -241,7 +241,7 @@ func (incident *Incident) SetSlot2Form(v null.Int) { func (incident *Incident) SetSlot3PokemonId(v null.Int) { if incident.Slot3PokemonId != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3PokemonId:%v->%v", incident.Slot3PokemonId, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3PokemonId:%s->%s", FormatNull(incident.Slot3PokemonId), FormatNull(v))) } incident.Slot3PokemonId = v incident.dirty = true @@ -251,7 +251,7 @@ func (incident *Incident) SetSlot3PokemonId(v null.Int) { func (incident *Incident) SetSlot3Form(v null.Int) { if incident.Slot3Form != v { if dbDebugEnabled { - incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3Form:%v->%v", incident.Slot3Form, v)) + incident.changedFields = append(incident.changedFields, fmt.Sprintf("Slot3Form:%s->%s", FormatNull(incident.Slot3Form), FormatNull(v))) } incident.Slot3Form = v incident.dirty = true diff --git a/decoder/main.go b/decoder/main.go index b1e2b589..3fcb043a 100644 --- a/decoder/main.go +++ b/decoder/main.go @@ -1,6 +1,7 @@ package decoder import ( + "fmt" "math" "runtime" "time" @@ -246,6 +247,20 @@ func nullFloatAlmostEqual(a, b null.Float, tolerance float64) bool { } } +// Ptrable is an interface for any type that has a Ptr() method returning *T +// specifically these are the null objects +type Ptrable[T any] interface { + Ptr() *T +} + +// FormatNull returns "NULL" if the nullable value is not valid, otherwise formats the value +func FormatNull[T any](n Ptrable[T]) string { + if ptr := n.Ptr(); ptr != nil { + return fmt.Sprintf("%v", *ptr) + } + return "NULL" +} + func SetWebhooksSender(whSender webhooksSenderInterface) { webhooksSender = whSender } diff --git a/decoder/player.go b/decoder/player.go index 433885bb..0bd20785 100644 --- a/decoder/player.go +++ b/decoder/player.go @@ -134,7 +134,7 @@ func (p *Player) setFieldDirty() { func (p *Player) SetFriendshipId(v null.String) { if p.FriendshipId != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("FriendshipId:%v->%v", p.FriendshipId, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("FriendshipId:%s->%s", FormatNull(p.FriendshipId), FormatNull(v))) } p.FriendshipId = v p.dirty = true @@ -144,7 +144,7 @@ func (p *Player) SetFriendshipId(v null.String) { func (p *Player) SetFriendCode(v null.String) { if p.FriendCode != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("FriendCode:%v->%v", p.FriendCode, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("FriendCode:%s->%s", FormatNull(p.FriendCode), FormatNull(v))) } p.FriendCode = v p.dirty = true @@ -154,7 +154,7 @@ func (p *Player) SetFriendCode(v null.String) { func (p *Player) SetTeam(v null.Int) { if p.Team != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Team:%v->%v", p.Team, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Team:%s->%s", FormatNull(p.Team), FormatNull(v))) } p.Team = v p.dirty = true @@ -164,7 +164,7 @@ func (p *Player) SetTeam(v null.Int) { func (p *Player) SetLevel(v null.Int) { if p.Level != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Level:%v->%v", p.Level, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Level:%s->%s", FormatNull(p.Level), FormatNull(v))) } p.Level = v p.dirty = true @@ -174,7 +174,7 @@ func (p *Player) SetLevel(v null.Int) { func (p *Player) SetXp(v null.Int) { if p.Xp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Xp:%v->%v", p.Xp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Xp:%s->%s", FormatNull(p.Xp), FormatNull(v))) } p.Xp = v p.dirty = true @@ -184,7 +184,7 @@ func (p *Player) SetXp(v null.Int) { func (p *Player) SetBattlesWon(v null.Int) { if p.BattlesWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("BattlesWon:%v->%v", p.BattlesWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("BattlesWon:%s->%s", FormatNull(p.BattlesWon), FormatNull(v))) } p.BattlesWon = v p.dirty = true @@ -194,7 +194,7 @@ func (p *Player) SetBattlesWon(v null.Int) { func (p *Player) SetKmWalked(v null.Float) { if !nullFloatAlmostEqual(p.KmWalked, v, 0.001) { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("KmWalked:%v->%v", p.KmWalked, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("KmWalked:%s->%s", FormatNull(p.KmWalked), FormatNull(v))) } p.KmWalked = v p.dirty = true @@ -204,7 +204,7 @@ func (p *Player) SetKmWalked(v null.Float) { func (p *Player) SetCaughtPokemon(v null.Int) { if p.CaughtPokemon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPokemon:%v->%v", p.CaughtPokemon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPokemon:%s->%s", FormatNull(p.CaughtPokemon), FormatNull(v))) } p.CaughtPokemon = v p.dirty = true @@ -214,7 +214,7 @@ func (p *Player) SetCaughtPokemon(v null.Int) { func (p *Player) SetGblRank(v null.Int) { if p.GblRank != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GblRank:%v->%v", p.GblRank, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GblRank:%s->%s", FormatNull(p.GblRank), FormatNull(v))) } p.GblRank = v p.dirty = true @@ -224,7 +224,7 @@ func (p *Player) SetGblRank(v null.Int) { func (p *Player) SetGblRating(v null.Int) { if p.GblRating != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GblRating:%v->%v", p.GblRating, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GblRating:%s->%s", FormatNull(p.GblRating), FormatNull(v))) } p.GblRating = v p.dirty = true @@ -234,7 +234,7 @@ func (p *Player) SetGblRating(v null.Int) { func (p *Player) SetEventBadges(v null.String) { if p.EventBadges != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("EventBadges:%v->%v", p.EventBadges, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("EventBadges:%s->%s", FormatNull(p.EventBadges), FormatNull(v))) } p.EventBadges = v p.dirty = true @@ -244,7 +244,7 @@ func (p *Player) SetEventBadges(v null.String) { func (p *Player) SetStopsSpun(v null.Int) { if p.StopsSpun != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("StopsSpun:%v->%v", p.StopsSpun, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("StopsSpun:%s->%s", FormatNull(p.StopsSpun), FormatNull(v))) } p.StopsSpun = v p.dirty = true @@ -254,7 +254,7 @@ func (p *Player) SetStopsSpun(v null.Int) { func (p *Player) SetEvolved(v null.Int) { if p.Evolved != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Evolved:%v->%v", p.Evolved, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Evolved:%s->%s", FormatNull(p.Evolved), FormatNull(v))) } p.Evolved = v p.dirty = true @@ -264,7 +264,7 @@ func (p *Player) SetEvolved(v null.Int) { func (p *Player) SetHatched(v null.Int) { if p.Hatched != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Hatched:%v->%v", p.Hatched, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Hatched:%s->%s", FormatNull(p.Hatched), FormatNull(v))) } p.Hatched = v p.dirty = true @@ -274,7 +274,7 @@ func (p *Player) SetHatched(v null.Int) { func (p *Player) SetQuests(v null.Int) { if p.Quests != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Quests:%v->%v", p.Quests, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Quests:%s->%s", FormatNull(p.Quests), FormatNull(v))) } p.Quests = v p.dirty = true @@ -284,7 +284,7 @@ func (p *Player) SetQuests(v null.Int) { func (p *Player) SetTrades(v null.Int) { if p.Trades != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Trades:%v->%v", p.Trades, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Trades:%s->%s", FormatNull(p.Trades), FormatNull(v))) } p.Trades = v p.dirty = true @@ -294,7 +294,7 @@ func (p *Player) SetTrades(v null.Int) { func (p *Player) SetPhotobombs(v null.Int) { if p.Photobombs != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Photobombs:%v->%v", p.Photobombs, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Photobombs:%s->%s", FormatNull(p.Photobombs), FormatNull(v))) } p.Photobombs = v p.dirty = true @@ -304,7 +304,7 @@ func (p *Player) SetPhotobombs(v null.Int) { func (p *Player) SetPurified(v null.Int) { if p.Purified != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Purified:%v->%v", p.Purified, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Purified:%s->%s", FormatNull(p.Purified), FormatNull(v))) } p.Purified = v p.dirty = true @@ -314,7 +314,7 @@ func (p *Player) SetPurified(v null.Int) { func (p *Player) SetGruntsDefeated(v null.Int) { if p.GruntsDefeated != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GruntsDefeated:%v->%v", p.GruntsDefeated, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GruntsDefeated:%s->%s", FormatNull(p.GruntsDefeated), FormatNull(v))) } p.GruntsDefeated = v p.dirty = true @@ -324,7 +324,7 @@ func (p *Player) SetGruntsDefeated(v null.Int) { func (p *Player) SetGymBattlesWon(v null.Int) { if p.GymBattlesWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GymBattlesWon:%v->%v", p.GymBattlesWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GymBattlesWon:%s->%s", FormatNull(p.GymBattlesWon), FormatNull(v))) } p.GymBattlesWon = v p.dirty = true @@ -334,7 +334,7 @@ func (p *Player) SetGymBattlesWon(v null.Int) { func (p *Player) SetNormalRaidsWon(v null.Int) { if p.NormalRaidsWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("NormalRaidsWon:%v->%v", p.NormalRaidsWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("NormalRaidsWon:%s->%s", FormatNull(p.NormalRaidsWon), FormatNull(v))) } p.NormalRaidsWon = v p.dirty = true @@ -344,7 +344,7 @@ func (p *Player) SetNormalRaidsWon(v null.Int) { func (p *Player) SetLegendaryRaidsWon(v null.Int) { if p.LegendaryRaidsWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LegendaryRaidsWon:%v->%v", p.LegendaryRaidsWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LegendaryRaidsWon:%s->%s", FormatNull(p.LegendaryRaidsWon), FormatNull(v))) } p.LegendaryRaidsWon = v p.dirty = true @@ -354,7 +354,7 @@ func (p *Player) SetLegendaryRaidsWon(v null.Int) { func (p *Player) SetTrainingsWon(v null.Int) { if p.TrainingsWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TrainingsWon:%v->%v", p.TrainingsWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TrainingsWon:%s->%s", FormatNull(p.TrainingsWon), FormatNull(v))) } p.TrainingsWon = v p.dirty = true @@ -364,7 +364,7 @@ func (p *Player) SetTrainingsWon(v null.Int) { func (p *Player) SetBerriesFed(v null.Int) { if p.BerriesFed != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("BerriesFed:%v->%v", p.BerriesFed, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("BerriesFed:%s->%s", FormatNull(p.BerriesFed), FormatNull(v))) } p.BerriesFed = v p.dirty = true @@ -374,7 +374,7 @@ func (p *Player) SetBerriesFed(v null.Int) { func (p *Player) SetHoursDefended(v null.Int) { if p.HoursDefended != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("HoursDefended:%v->%v", p.HoursDefended, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("HoursDefended:%s->%s", FormatNull(p.HoursDefended), FormatNull(v))) } p.HoursDefended = v p.dirty = true @@ -384,7 +384,7 @@ func (p *Player) SetHoursDefended(v null.Int) { func (p *Player) SetBestFriends(v null.Int) { if p.BestFriends != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("BestFriends:%v->%v", p.BestFriends, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("BestFriends:%s->%s", FormatNull(p.BestFriends), FormatNull(v))) } p.BestFriends = v p.dirty = true @@ -394,7 +394,7 @@ func (p *Player) SetBestFriends(v null.Int) { func (p *Player) SetBestBuddies(v null.Int) { if p.BestBuddies != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("BestBuddies:%v->%v", p.BestBuddies, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("BestBuddies:%s->%s", FormatNull(p.BestBuddies), FormatNull(v))) } p.BestBuddies = v p.dirty = true @@ -404,7 +404,7 @@ func (p *Player) SetBestBuddies(v null.Int) { func (p *Player) SetGiovanniDefeated(v null.Int) { if p.GiovanniDefeated != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("GiovanniDefeated:%v->%v", p.GiovanniDefeated, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("GiovanniDefeated:%s->%s", FormatNull(p.GiovanniDefeated), FormatNull(v))) } p.GiovanniDefeated = v p.dirty = true @@ -414,7 +414,7 @@ func (p *Player) SetGiovanniDefeated(v null.Int) { func (p *Player) SetMegaEvos(v null.Int) { if p.MegaEvos != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("MegaEvos:%v->%v", p.MegaEvos, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("MegaEvos:%s->%s", FormatNull(p.MegaEvos), FormatNull(v))) } p.MegaEvos = v p.dirty = true @@ -424,7 +424,7 @@ func (p *Player) SetMegaEvos(v null.Int) { func (p *Player) SetCollectionsDone(v null.Int) { if p.CollectionsDone != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CollectionsDone:%v->%v", p.CollectionsDone, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CollectionsDone:%s->%s", FormatNull(p.CollectionsDone), FormatNull(v))) } p.CollectionsDone = v p.dirty = true @@ -434,7 +434,7 @@ func (p *Player) SetCollectionsDone(v null.Int) { func (p *Player) SetUniqueStopsSpun(v null.Int) { if p.UniqueStopsSpun != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueStopsSpun:%v->%v", p.UniqueStopsSpun, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueStopsSpun:%s->%s", FormatNull(p.UniqueStopsSpun), FormatNull(v))) } p.UniqueStopsSpun = v p.dirty = true @@ -444,7 +444,7 @@ func (p *Player) SetUniqueStopsSpun(v null.Int) { func (p *Player) SetUniqueMegaEvos(v null.Int) { if p.UniqueMegaEvos != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueMegaEvos:%v->%v", p.UniqueMegaEvos, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueMegaEvos:%s->%s", FormatNull(p.UniqueMegaEvos), FormatNull(v))) } p.UniqueMegaEvos = v p.dirty = true @@ -454,7 +454,7 @@ func (p *Player) SetUniqueMegaEvos(v null.Int) { func (p *Player) SetUniqueRaidBosses(v null.Int) { if p.UniqueRaidBosses != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueRaidBosses:%v->%v", p.UniqueRaidBosses, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueRaidBosses:%s->%s", FormatNull(p.UniqueRaidBosses), FormatNull(v))) } p.UniqueRaidBosses = v p.dirty = true @@ -464,7 +464,7 @@ func (p *Player) SetUniqueRaidBosses(v null.Int) { func (p *Player) SetUniqueUnown(v null.Int) { if p.UniqueUnown != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueUnown:%v->%v", p.UniqueUnown, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("UniqueUnown:%s->%s", FormatNull(p.UniqueUnown), FormatNull(v))) } p.UniqueUnown = v p.dirty = true @@ -474,7 +474,7 @@ func (p *Player) SetUniqueUnown(v null.Int) { func (p *Player) SetSevenDayStreaks(v null.Int) { if p.SevenDayStreaks != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("SevenDayStreaks:%v->%v", p.SevenDayStreaks, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("SevenDayStreaks:%s->%s", FormatNull(p.SevenDayStreaks), FormatNull(v))) } p.SevenDayStreaks = v p.dirty = true @@ -484,7 +484,7 @@ func (p *Player) SetSevenDayStreaks(v null.Int) { func (p *Player) SetTradeKm(v null.Int) { if p.TradeKm != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TradeKm:%v->%v", p.TradeKm, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TradeKm:%s->%s", FormatNull(p.TradeKm), FormatNull(v))) } p.TradeKm = v p.dirty = true @@ -494,7 +494,7 @@ func (p *Player) SetTradeKm(v null.Int) { func (p *Player) SetRaidsWithFriends(v null.Int) { if p.RaidsWithFriends != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("RaidsWithFriends:%v->%v", p.RaidsWithFriends, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("RaidsWithFriends:%s->%s", FormatNull(p.RaidsWithFriends), FormatNull(v))) } p.RaidsWithFriends = v p.dirty = true @@ -504,7 +504,7 @@ func (p *Player) SetRaidsWithFriends(v null.Int) { func (p *Player) SetCaughtAtLure(v null.Int) { if p.CaughtAtLure != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtAtLure:%v->%v", p.CaughtAtLure, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtAtLure:%s->%s", FormatNull(p.CaughtAtLure), FormatNull(v))) } p.CaughtAtLure = v p.dirty = true @@ -514,7 +514,7 @@ func (p *Player) SetCaughtAtLure(v null.Int) { func (p *Player) SetWayfarerAgreements(v null.Int) { if p.WayfarerAgreements != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("WayfarerAgreements:%v->%v", p.WayfarerAgreements, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("WayfarerAgreements:%s->%s", FormatNull(p.WayfarerAgreements), FormatNull(v))) } p.WayfarerAgreements = v p.dirty = true @@ -524,7 +524,7 @@ func (p *Player) SetWayfarerAgreements(v null.Int) { func (p *Player) SetTrainersReferred(v null.Int) { if p.TrainersReferred != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TrainersReferred:%v->%v", p.TrainersReferred, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TrainersReferred:%s->%s", FormatNull(p.TrainersReferred), FormatNull(v))) } p.TrainersReferred = v p.dirty = true @@ -534,7 +534,7 @@ func (p *Player) SetTrainersReferred(v null.Int) { func (p *Player) SetRaidAchievements(v null.Int) { if p.RaidAchievements != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("RaidAchievements:%v->%v", p.RaidAchievements, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("RaidAchievements:%s->%s", FormatNull(p.RaidAchievements), FormatNull(v))) } p.RaidAchievements = v p.dirty = true @@ -544,7 +544,7 @@ func (p *Player) SetRaidAchievements(v null.Int) { func (p *Player) SetXlKarps(v null.Int) { if p.XlKarps != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("XlKarps:%v->%v", p.XlKarps, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("XlKarps:%s->%s", FormatNull(p.XlKarps), FormatNull(v))) } p.XlKarps = v p.dirty = true @@ -554,7 +554,7 @@ func (p *Player) SetXlKarps(v null.Int) { func (p *Player) SetXsRats(v null.Int) { if p.XsRats != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("XsRats:%v->%v", p.XsRats, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("XsRats:%s->%s", FormatNull(p.XsRats), FormatNull(v))) } p.XsRats = v p.dirty = true @@ -564,7 +564,7 @@ func (p *Player) SetXsRats(v null.Int) { func (p *Player) SetPikachuCaught(v null.Int) { if p.PikachuCaught != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PikachuCaught:%v->%v", p.PikachuCaught, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PikachuCaught:%s->%s", FormatNull(p.PikachuCaught), FormatNull(v))) } p.PikachuCaught = v p.dirty = true @@ -574,7 +574,7 @@ func (p *Player) SetPikachuCaught(v null.Int) { func (p *Player) SetLeagueGreatWon(v null.Int) { if p.LeagueGreatWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueGreatWon:%v->%v", p.LeagueGreatWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueGreatWon:%s->%s", FormatNull(p.LeagueGreatWon), FormatNull(v))) } p.LeagueGreatWon = v p.dirty = true @@ -584,7 +584,7 @@ func (p *Player) SetLeagueGreatWon(v null.Int) { func (p *Player) SetLeagueUltraWon(v null.Int) { if p.LeagueUltraWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueUltraWon:%v->%v", p.LeagueUltraWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueUltraWon:%s->%s", FormatNull(p.LeagueUltraWon), FormatNull(v))) } p.LeagueUltraWon = v p.dirty = true @@ -594,7 +594,7 @@ func (p *Player) SetLeagueUltraWon(v null.Int) { func (p *Player) SetLeagueMasterWon(v null.Int) { if p.LeagueMasterWon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueMasterWon:%v->%v", p.LeagueMasterWon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LeagueMasterWon:%s->%s", FormatNull(p.LeagueMasterWon), FormatNull(v))) } p.LeagueMasterWon = v p.dirty = true @@ -604,7 +604,7 @@ func (p *Player) SetLeagueMasterWon(v null.Int) { func (p *Player) SetTinyPokemonCaught(v null.Int) { if p.TinyPokemonCaught != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TinyPokemonCaught:%v->%v", p.TinyPokemonCaught, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TinyPokemonCaught:%s->%s", FormatNull(p.TinyPokemonCaught), FormatNull(v))) } p.TinyPokemonCaught = v p.dirty = true @@ -614,7 +614,7 @@ func (p *Player) SetTinyPokemonCaught(v null.Int) { func (p *Player) SetJumboPokemonCaught(v null.Int) { if p.JumboPokemonCaught != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("JumboPokemonCaught:%v->%v", p.JumboPokemonCaught, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("JumboPokemonCaught:%s->%s", FormatNull(p.JumboPokemonCaught), FormatNull(v))) } p.JumboPokemonCaught = v p.dirty = true @@ -624,7 +624,7 @@ func (p *Player) SetJumboPokemonCaught(v null.Int) { func (p *Player) SetVivillon(v null.Int) { if p.Vivillon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Vivillon:%v->%v", p.Vivillon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Vivillon:%s->%s", FormatNull(p.Vivillon), FormatNull(v))) } p.Vivillon = v p.dirty = true @@ -634,7 +634,7 @@ func (p *Player) SetVivillon(v null.Int) { func (p *Player) SetMaxSizeFirstPlace(v null.Int) { if p.MaxSizeFirstPlace != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("MaxSizeFirstPlace:%v->%v", p.MaxSizeFirstPlace, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("MaxSizeFirstPlace:%s->%s", FormatNull(p.MaxSizeFirstPlace), FormatNull(v))) } p.MaxSizeFirstPlace = v p.dirty = true @@ -644,7 +644,7 @@ func (p *Player) SetMaxSizeFirstPlace(v null.Int) { func (p *Player) SetTotalRoutePlay(v null.Int) { if p.TotalRoutePlay != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("TotalRoutePlay:%v->%v", p.TotalRoutePlay, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("TotalRoutePlay:%s->%s", FormatNull(p.TotalRoutePlay), FormatNull(v))) } p.TotalRoutePlay = v p.dirty = true @@ -654,7 +654,7 @@ func (p *Player) SetTotalRoutePlay(v null.Int) { func (p *Player) SetPartiesCompleted(v null.Int) { if p.PartiesCompleted != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PartiesCompleted:%v->%v", p.PartiesCompleted, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PartiesCompleted:%s->%s", FormatNull(p.PartiesCompleted), FormatNull(v))) } p.PartiesCompleted = v p.dirty = true @@ -664,7 +664,7 @@ func (p *Player) SetPartiesCompleted(v null.Int) { func (p *Player) SetEventCheckIns(v null.Int) { if p.EventCheckIns != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("EventCheckIns:%v->%v", p.EventCheckIns, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("EventCheckIns:%s->%s", FormatNull(p.EventCheckIns), FormatNull(v))) } p.EventCheckIns = v p.dirty = true @@ -673,7 +673,7 @@ func (p *Player) SetEventCheckIns(v null.Int) { func (p *Player) SetDexGen1(v null.Int) { if p.DexGen1 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen1:%v->%v", p.DexGen1, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen1:%s->%s", FormatNull(p.DexGen1), FormatNull(v))) } p.DexGen1 = v p.dirty = true @@ -683,7 +683,7 @@ func (p *Player) SetDexGen1(v null.Int) { func (p *Player) SetDexGen2(v null.Int) { if p.DexGen2 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen2:%v->%v", p.DexGen2, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen2:%s->%s", FormatNull(p.DexGen2), FormatNull(v))) } p.DexGen2 = v p.dirty = true @@ -693,7 +693,7 @@ func (p *Player) SetDexGen2(v null.Int) { func (p *Player) SetDexGen3(v null.Int) { if p.DexGen3 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen3:%v->%v", p.DexGen3, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen3:%s->%s", FormatNull(p.DexGen3), FormatNull(v))) } p.DexGen3 = v p.dirty = true @@ -703,7 +703,7 @@ func (p *Player) SetDexGen3(v null.Int) { func (p *Player) SetDexGen4(v null.Int) { if p.DexGen4 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen4:%v->%v", p.DexGen4, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen4:%s->%s", FormatNull(p.DexGen4), FormatNull(v))) } p.DexGen4 = v p.dirty = true @@ -713,7 +713,7 @@ func (p *Player) SetDexGen4(v null.Int) { func (p *Player) SetDexGen5(v null.Int) { if p.DexGen5 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen5:%v->%v", p.DexGen5, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen5:%s->%s", FormatNull(p.DexGen5), FormatNull(v))) } p.DexGen5 = v p.dirty = true @@ -723,7 +723,7 @@ func (p *Player) SetDexGen5(v null.Int) { func (p *Player) SetDexGen6(v null.Int) { if p.DexGen6 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen6:%v->%v", p.DexGen6, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen6:%s->%s", FormatNull(p.DexGen6), FormatNull(v))) } p.DexGen6 = v p.dirty = true @@ -733,7 +733,7 @@ func (p *Player) SetDexGen6(v null.Int) { func (p *Player) SetDexGen7(v null.Int) { if p.DexGen7 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen7:%v->%v", p.DexGen7, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen7:%s->%s", FormatNull(p.DexGen7), FormatNull(v))) } p.DexGen7 = v p.dirty = true @@ -743,7 +743,7 @@ func (p *Player) SetDexGen7(v null.Int) { func (p *Player) SetDexGen8(v null.Int) { if p.DexGen8 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8:%v->%v", p.DexGen8, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8:%s->%s", FormatNull(p.DexGen8), FormatNull(v))) } p.DexGen8 = v p.dirty = true @@ -753,7 +753,7 @@ func (p *Player) SetDexGen8(v null.Int) { func (p *Player) SetDexGen8A(v null.Int) { if p.DexGen8A != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8A:%v->%v", p.DexGen8A, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen8A:%s->%s", FormatNull(p.DexGen8A), FormatNull(v))) } p.DexGen8A = v p.dirty = true @@ -763,7 +763,7 @@ func (p *Player) SetDexGen8A(v null.Int) { func (p *Player) SetDexGen9(v null.Int) { if p.DexGen9 != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen9:%v->%v", p.DexGen9, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("DexGen9:%s->%s", FormatNull(p.DexGen9), FormatNull(v))) } p.DexGen9 = v p.dirty = true @@ -773,7 +773,7 @@ func (p *Player) SetDexGen9(v null.Int) { func (p *Player) SetCaughtNormal(v null.Int) { if p.CaughtNormal != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtNormal:%v->%v", p.CaughtNormal, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtNormal:%s->%s", FormatNull(p.CaughtNormal), FormatNull(v))) } p.CaughtNormal = v p.dirty = true @@ -783,7 +783,7 @@ func (p *Player) SetCaughtNormal(v null.Int) { func (p *Player) SetCaughtFighting(v null.Int) { if p.CaughtFighting != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFighting:%v->%v", p.CaughtFighting, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFighting:%s->%s", FormatNull(p.CaughtFighting), FormatNull(v))) } p.CaughtFighting = v p.dirty = true @@ -793,7 +793,7 @@ func (p *Player) SetCaughtFighting(v null.Int) { func (p *Player) SetCaughtFlying(v null.Int) { if p.CaughtFlying != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFlying:%v->%v", p.CaughtFlying, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFlying:%s->%s", FormatNull(p.CaughtFlying), FormatNull(v))) } p.CaughtFlying = v p.dirty = true @@ -803,7 +803,7 @@ func (p *Player) SetCaughtFlying(v null.Int) { func (p *Player) SetCaughtPoison(v null.Int) { if p.CaughtPoison != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPoison:%v->%v", p.CaughtPoison, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPoison:%s->%s", FormatNull(p.CaughtPoison), FormatNull(v))) } p.CaughtPoison = v p.dirty = true @@ -813,7 +813,7 @@ func (p *Player) SetCaughtPoison(v null.Int) { func (p *Player) SetCaughtGround(v null.Int) { if p.CaughtGround != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGround:%v->%v", p.CaughtGround, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGround:%s->%s", FormatNull(p.CaughtGround), FormatNull(v))) } p.CaughtGround = v p.dirty = true @@ -823,7 +823,7 @@ func (p *Player) SetCaughtGround(v null.Int) { func (p *Player) SetCaughtRock(v null.Int) { if p.CaughtRock != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtRock:%v->%v", p.CaughtRock, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtRock:%s->%s", FormatNull(p.CaughtRock), FormatNull(v))) } p.CaughtRock = v p.dirty = true @@ -833,7 +833,7 @@ func (p *Player) SetCaughtRock(v null.Int) { func (p *Player) SetCaughtBug(v null.Int) { if p.CaughtBug != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtBug:%v->%v", p.CaughtBug, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtBug:%s->%s", FormatNull(p.CaughtBug), FormatNull(v))) } p.CaughtBug = v p.dirty = true @@ -843,7 +843,7 @@ func (p *Player) SetCaughtBug(v null.Int) { func (p *Player) SetCaughtGhost(v null.Int) { if p.CaughtGhost != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGhost:%v->%v", p.CaughtGhost, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGhost:%s->%s", FormatNull(p.CaughtGhost), FormatNull(v))) } p.CaughtGhost = v p.dirty = true @@ -853,7 +853,7 @@ func (p *Player) SetCaughtGhost(v null.Int) { func (p *Player) SetCaughtSteel(v null.Int) { if p.CaughtSteel != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtSteel:%v->%v", p.CaughtSteel, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtSteel:%s->%s", FormatNull(p.CaughtSteel), FormatNull(v))) } p.CaughtSteel = v p.dirty = true @@ -863,7 +863,7 @@ func (p *Player) SetCaughtSteel(v null.Int) { func (p *Player) SetCaughtFire(v null.Int) { if p.CaughtFire != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFire:%v->%v", p.CaughtFire, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFire:%s->%s", FormatNull(p.CaughtFire), FormatNull(v))) } p.CaughtFire = v p.dirty = true @@ -873,7 +873,7 @@ func (p *Player) SetCaughtFire(v null.Int) { func (p *Player) SetCaughtWater(v null.Int) { if p.CaughtWater != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtWater:%v->%v", p.CaughtWater, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtWater:%s->%s", FormatNull(p.CaughtWater), FormatNull(v))) } p.CaughtWater = v p.dirty = true @@ -883,7 +883,7 @@ func (p *Player) SetCaughtWater(v null.Int) { func (p *Player) SetCaughtGrass(v null.Int) { if p.CaughtGrass != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGrass:%v->%v", p.CaughtGrass, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtGrass:%s->%s", FormatNull(p.CaughtGrass), FormatNull(v))) } p.CaughtGrass = v p.dirty = true @@ -893,7 +893,7 @@ func (p *Player) SetCaughtGrass(v null.Int) { func (p *Player) SetCaughtElectric(v null.Int) { if p.CaughtElectric != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtElectric:%v->%v", p.CaughtElectric, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtElectric:%s->%s", FormatNull(p.CaughtElectric), FormatNull(v))) } p.CaughtElectric = v p.dirty = true @@ -903,7 +903,7 @@ func (p *Player) SetCaughtElectric(v null.Int) { func (p *Player) SetCaughtPsychic(v null.Int) { if p.CaughtPsychic != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPsychic:%v->%v", p.CaughtPsychic, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtPsychic:%s->%s", FormatNull(p.CaughtPsychic), FormatNull(v))) } p.CaughtPsychic = v p.dirty = true @@ -913,7 +913,7 @@ func (p *Player) SetCaughtPsychic(v null.Int) { func (p *Player) SetCaughtIce(v null.Int) { if p.CaughtIce != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtIce:%v->%v", p.CaughtIce, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtIce:%s->%s", FormatNull(p.CaughtIce), FormatNull(v))) } p.CaughtIce = v p.dirty = true @@ -923,7 +923,7 @@ func (p *Player) SetCaughtIce(v null.Int) { func (p *Player) SetCaughtDragon(v null.Int) { if p.CaughtDragon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDragon:%v->%v", p.CaughtDragon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDragon:%s->%s", FormatNull(p.CaughtDragon), FormatNull(v))) } p.CaughtDragon = v p.dirty = true @@ -933,7 +933,7 @@ func (p *Player) SetCaughtDragon(v null.Int) { func (p *Player) SetCaughtDark(v null.Int) { if p.CaughtDark != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDark:%v->%v", p.CaughtDark, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtDark:%s->%s", FormatNull(p.CaughtDark), FormatNull(v))) } p.CaughtDark = v p.dirty = true @@ -943,7 +943,7 @@ func (p *Player) SetCaughtDark(v null.Int) { func (p *Player) SetCaughtFairy(v null.Int) { if p.CaughtFairy != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFairy:%v->%v", p.CaughtFairy, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CaughtFairy:%s->%s", FormatNull(p.CaughtFairy), FormatNull(v))) } p.CaughtFairy = v p.dirty = true diff --git a/decoder/pokemon.go b/decoder/pokemon.go index d20ba93b..56cf62b1 100644 --- a/decoder/pokemon.go +++ b/decoder/pokemon.go @@ -170,7 +170,7 @@ func (pokemon *Pokemon) Unlock() { func (pokemon *Pokemon) SetPokestopId(v null.String) { if pokemon.PokestopId != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("PokestopId:%v->%v", pokemon.PokestopId, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("PokestopId:%s->%s", FormatNull(pokemon.PokestopId), FormatNull(v))) } pokemon.PokestopId = v pokemon.dirty = true @@ -180,7 +180,7 @@ func (pokemon *Pokemon) SetPokestopId(v null.String) { func (pokemon *Pokemon) SetSpawnId(v null.Int) { if pokemon.SpawnId != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SpawnId:%v->%v", pokemon.SpawnId, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SpawnId:%s->%s", FormatNull(pokemon.SpawnId), FormatNull(v))) } pokemon.SpawnId = v pokemon.dirty = true @@ -220,7 +220,7 @@ func (pokemon *Pokemon) SetPokemonId(v int16) { func (pokemon *Pokemon) SetForm(v null.Int) { if pokemon.Form != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Form:%v->%v", pokemon.Form, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Form:%s->%s", FormatNull(pokemon.Form), FormatNull(v))) } pokemon.Form = v pokemon.dirty = true @@ -230,7 +230,7 @@ func (pokemon *Pokemon) SetForm(v null.Int) { func (pokemon *Pokemon) SetCostume(v null.Int) { if pokemon.Costume != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Costume:%v->%v", pokemon.Costume, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Costume:%s->%s", FormatNull(pokemon.Costume), FormatNull(v))) } pokemon.Costume = v pokemon.dirty = true @@ -240,7 +240,7 @@ func (pokemon *Pokemon) SetCostume(v null.Int) { func (pokemon *Pokemon) SetGender(v null.Int) { if pokemon.Gender != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Gender:%v->%v", pokemon.Gender, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Gender:%s->%s", FormatNull(pokemon.Gender), FormatNull(v))) } pokemon.Gender = v pokemon.dirty = true @@ -250,7 +250,7 @@ func (pokemon *Pokemon) SetGender(v null.Int) { func (pokemon *Pokemon) SetWeather(v null.Int) { if pokemon.Weather != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weather:%v->%v", pokemon.Weather, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weather:%s->%s", FormatNull(pokemon.Weather), FormatNull(v))) } pokemon.Weather = v pokemon.dirty = true @@ -260,7 +260,7 @@ func (pokemon *Pokemon) SetWeather(v null.Int) { func (pokemon *Pokemon) SetIsStrong(v null.Bool) { if pokemon.IsStrong != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsStrong:%v->%v", pokemon.IsStrong, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("IsStrong:%s->%s", FormatNull(pokemon.IsStrong), FormatNull(v))) } pokemon.IsStrong = v pokemon.dirty = true @@ -270,7 +270,7 @@ func (pokemon *Pokemon) SetIsStrong(v null.Bool) { func (pokemon *Pokemon) SetExpireTimestamp(v null.Int) { if pokemon.ExpireTimestamp != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("ExpireTimestamp:%v->%v", pokemon.ExpireTimestamp, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("ExpireTimestamp:%s->%s", FormatNull(pokemon.ExpireTimestamp), FormatNull(v))) } pokemon.ExpireTimestamp = v pokemon.dirty = true @@ -290,7 +290,7 @@ func (pokemon *Pokemon) SetExpireTimestampVerified(v bool) { func (pokemon *Pokemon) SetSeenType(v null.String) { if pokemon.SeenType != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SeenType:%v->%v", pokemon.SeenType, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("SeenType:%s->%s", FormatNull(pokemon.SeenType), FormatNull(v))) } pokemon.SeenType = v pokemon.dirty = true @@ -307,7 +307,7 @@ func (pokemon *Pokemon) SetUsername(v null.String) { func (pokemon *Pokemon) SetCellId(v null.Int) { if pokemon.CellId != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("CellId:%v->%v", pokemon.CellId, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("CellId:%s->%s", FormatNull(pokemon.CellId), FormatNull(v))) } pokemon.CellId = v pokemon.dirty = true @@ -327,7 +327,7 @@ func (pokemon *Pokemon) SetIsEvent(v int8) { func (pokemon *Pokemon) SetShiny(v null.Bool) { if pokemon.Shiny != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Shiny:%v->%v", pokemon.Shiny, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Shiny:%s->%s", FormatNull(pokemon.Shiny), FormatNull(v))) } pokemon.Shiny = v pokemon.dirty = true @@ -337,7 +337,7 @@ func (pokemon *Pokemon) SetShiny(v null.Bool) { func (pokemon *Pokemon) SetCp(v null.Int) { if pokemon.Cp != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Cp:%v->%v", pokemon.Cp, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Cp:%s->%s", FormatNull(pokemon.Cp), FormatNull(v))) } pokemon.Cp = v pokemon.dirty = true @@ -347,7 +347,7 @@ func (pokemon *Pokemon) SetCp(v null.Int) { func (pokemon *Pokemon) SetLevel(v null.Int) { if pokemon.Level != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Level:%v->%v", pokemon.Level, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Level:%s->%s", FormatNull(pokemon.Level), FormatNull(v))) } pokemon.Level = v pokemon.dirty = true @@ -357,7 +357,7 @@ func (pokemon *Pokemon) SetLevel(v null.Int) { func (pokemon *Pokemon) SetMove1(v null.Int) { if pokemon.Move1 != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move1:%v->%v", pokemon.Move1, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move1:%s->%s", FormatNull(pokemon.Move1), FormatNull(v))) } pokemon.Move1 = v pokemon.dirty = true @@ -367,7 +367,7 @@ func (pokemon *Pokemon) SetMove1(v null.Int) { func (pokemon *Pokemon) SetMove2(v null.Int) { if pokemon.Move2 != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move2:%v->%v", pokemon.Move2, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Move2:%s->%s", FormatNull(pokemon.Move2), FormatNull(v))) } pokemon.Move2 = v pokemon.dirty = true @@ -377,7 +377,7 @@ func (pokemon *Pokemon) SetMove2(v null.Int) { func (pokemon *Pokemon) SetHeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Height, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Height:%v->%v", pokemon.Height, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Height:%s->%s", FormatNull(pokemon.Height), FormatNull(v))) } pokemon.Height = v pokemon.dirty = true @@ -387,7 +387,7 @@ func (pokemon *Pokemon) SetHeight(v null.Float) { func (pokemon *Pokemon) SetWeight(v null.Float) { if !nullFloatAlmostEqual(pokemon.Weight, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weight:%v->%v", pokemon.Weight, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Weight:%s->%s", FormatNull(pokemon.Weight), FormatNull(v))) } pokemon.Weight = v pokemon.dirty = true @@ -397,7 +397,7 @@ func (pokemon *Pokemon) SetWeight(v null.Float) { func (pokemon *Pokemon) SetSize(v null.Int) { if pokemon.Size != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Size:%v->%v", pokemon.Size, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Size:%s->%s", FormatNull(pokemon.Size), FormatNull(v))) } pokemon.Size = v pokemon.dirty = true @@ -417,7 +417,7 @@ func (pokemon *Pokemon) SetIsDitto(v bool) { func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { if pokemon.DisplayPokemonId != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("DisplayPokemonId:%v->%v", pokemon.DisplayPokemonId, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("DisplayPokemonId:%s->%s", FormatNull(pokemon.DisplayPokemonId), FormatNull(v))) } pokemon.DisplayPokemonId = v pokemon.dirty = true @@ -427,7 +427,7 @@ func (pokemon *Pokemon) SetDisplayPokemonId(v null.Int) { func (pokemon *Pokemon) SetPvp(v null.String) { if pokemon.Pvp != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Pvp:%v->%v", pokemon.Pvp, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Pvp:%s->%s", FormatNull(pokemon.Pvp), FormatNull(v))) } pokemon.Pvp = v pokemon.dirty = true @@ -437,7 +437,7 @@ func (pokemon *Pokemon) SetPvp(v null.String) { func (pokemon *Pokemon) SetCapture1(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture1, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture1:%v->%v", pokemon.Capture1, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture1:%s->%s", FormatNull(pokemon.Capture1), FormatNull(v))) } pokemon.Capture1 = v pokemon.dirty = true @@ -447,7 +447,7 @@ func (pokemon *Pokemon) SetCapture1(v null.Float) { func (pokemon *Pokemon) SetCapture2(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture2, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture2:%v->%v", pokemon.Capture2, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture2:%s->%s", FormatNull(pokemon.Capture2), FormatNull(v))) } pokemon.Capture2 = v pokemon.dirty = true @@ -457,7 +457,7 @@ func (pokemon *Pokemon) SetCapture2(v null.Float) { func (pokemon *Pokemon) SetCapture3(v null.Float) { if !nullFloatAlmostEqual(pokemon.Capture3, v, floatTolerance) { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture3:%v->%v", pokemon.Capture3, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Capture3:%s->%s", FormatNull(pokemon.Capture3), FormatNull(v))) } pokemon.Capture3 = v pokemon.dirty = true @@ -467,7 +467,7 @@ func (pokemon *Pokemon) SetCapture3(v null.Float) { func (pokemon *Pokemon) SetUpdated(v null.Int) { if pokemon.Updated != v { if dbDebugEnabled { - pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Updated:%v->%v", pokemon.Updated, v)) + pokemon.changedFields = append(pokemon.changedFields, fmt.Sprintf("Updated:%s->%s", FormatNull(pokemon.Updated), FormatNull(v))) } pokemon.Updated = v pokemon.dirty = true diff --git a/decoder/pokestop.go b/decoder/pokestop.go index 8d97c8a1..20d771b7 100644 --- a/decoder/pokestop.go +++ b/decoder/pokestop.go @@ -191,7 +191,7 @@ func (p *Pokestop) SetLon(v float64) { func (p *Pokestop) SetName(v null.String) { if p.Name != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Name:%v->%v", p.Name, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Name:%s->%s", FormatNull(p.Name), FormatNull(v))) } p.Name = v p.dirty = true @@ -201,7 +201,7 @@ func (p *Pokestop) SetName(v null.String) { func (p *Pokestop) SetUrl(v null.String) { if p.Url != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Url:%v->%v", p.Url, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Url:%s->%s", FormatNull(p.Url), FormatNull(v))) } p.Url = v p.dirty = true @@ -211,7 +211,7 @@ func (p *Pokestop) SetUrl(v null.String) { func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { if p.LureExpireTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LureExpireTimestamp:%v->%v", p.LureExpireTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LureExpireTimestamp:%s->%s", FormatNull(p.LureExpireTimestamp), FormatNull(v))) } p.LureExpireTimestamp = v p.dirty = true @@ -221,7 +221,7 @@ func (p *Pokestop) SetLureExpireTimestamp(v null.Int) { func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { if p.LastModifiedTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("LastModifiedTimestamp:%v->%v", p.LastModifiedTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("LastModifiedTimestamp:%s->%s", FormatNull(p.LastModifiedTimestamp), FormatNull(v))) } p.LastModifiedTimestamp = v p.dirty = true @@ -231,7 +231,7 @@ func (p *Pokestop) SetLastModifiedTimestamp(v null.Int) { func (p *Pokestop) SetEnabled(v null.Bool) { if p.Enabled != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Enabled:%v->%v", p.Enabled, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Enabled:%s->%s", FormatNull(p.Enabled), FormatNull(v))) } p.Enabled = v p.dirty = true @@ -241,7 +241,7 @@ func (p *Pokestop) SetEnabled(v null.Bool) { func (p *Pokestop) SetQuestType(v null.Int) { if p.QuestType != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestType:%v->%v", p.QuestType, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestType:%s->%s", FormatNull(p.QuestType), FormatNull(v))) } p.QuestType = v p.dirty = true @@ -251,7 +251,7 @@ func (p *Pokestop) SetQuestType(v null.Int) { func (p *Pokestop) SetQuestTimestamp(v null.Int) { if p.QuestTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTimestamp:%v->%v", p.QuestTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTimestamp:%s->%s", FormatNull(p.QuestTimestamp), FormatNull(v))) } p.QuestTimestamp = v p.dirty = true @@ -261,7 +261,7 @@ func (p *Pokestop) SetQuestTimestamp(v null.Int) { func (p *Pokestop) SetQuestTarget(v null.Int) { if p.QuestTarget != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTarget:%v->%v", p.QuestTarget, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTarget:%s->%s", FormatNull(p.QuestTarget), FormatNull(v))) } p.QuestTarget = v p.dirty = true @@ -271,7 +271,7 @@ func (p *Pokestop) SetQuestTarget(v null.Int) { func (p *Pokestop) SetQuestConditions(v null.String) { if p.QuestConditions != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestConditions:%v->%v", p.QuestConditions, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestConditions:%s->%s", FormatNull(p.QuestConditions), FormatNull(v))) } p.QuestConditions = v p.dirty = true @@ -281,7 +281,7 @@ func (p *Pokestop) SetQuestConditions(v null.String) { func (p *Pokestop) SetQuestRewards(v null.String) { if p.QuestRewards != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestRewards:%v->%v", p.QuestRewards, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestRewards:%s->%s", FormatNull(p.QuestRewards), FormatNull(v))) } p.QuestRewards = v p.dirty = true @@ -291,7 +291,7 @@ func (p *Pokestop) SetQuestRewards(v null.String) { func (p *Pokestop) SetQuestTemplate(v null.String) { if p.QuestTemplate != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTemplate:%v->%v", p.QuestTemplate, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTemplate:%s->%s", FormatNull(p.QuestTemplate), FormatNull(v))) } p.QuestTemplate = v p.dirty = true @@ -301,7 +301,7 @@ func (p *Pokestop) SetQuestTemplate(v null.String) { func (p *Pokestop) SetQuestTitle(v null.String) { if p.QuestTitle != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTitle:%v->%v", p.QuestTitle, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestTitle:%s->%s", FormatNull(p.QuestTitle), FormatNull(v))) } p.QuestTitle = v p.dirty = true @@ -311,7 +311,7 @@ func (p *Pokestop) SetQuestTitle(v null.String) { func (p *Pokestop) SetQuestExpiry(v null.Int) { if p.QuestExpiry != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("QuestExpiry:%v->%v", p.QuestExpiry, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("QuestExpiry:%s->%s", FormatNull(p.QuestExpiry), FormatNull(v))) } p.QuestExpiry = v p.dirty = true @@ -321,7 +321,7 @@ func (p *Pokestop) SetQuestExpiry(v null.Int) { func (p *Pokestop) SetCellId(v null.Int) { if p.CellId != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("CellId:%v->%v", p.CellId, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("CellId:%s->%s", FormatNull(p.CellId), FormatNull(v))) } p.CellId = v p.dirty = true @@ -361,7 +361,7 @@ func (p *Pokestop) SetFirstSeenTimestamp(v int16) { func (p *Pokestop) SetSponsorId(v null.Int) { if p.SponsorId != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("SponsorId:%v->%v", p.SponsorId, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("SponsorId:%s->%s", FormatNull(p.SponsorId), FormatNull(v))) } p.SponsorId = v p.dirty = true @@ -371,7 +371,7 @@ func (p *Pokestop) SetSponsorId(v null.Int) { func (p *Pokestop) SetPartnerId(v null.String) { if p.PartnerId != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PartnerId:%v->%v", p.PartnerId, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PartnerId:%s->%s", FormatNull(p.PartnerId), FormatNull(v))) } p.PartnerId = v p.dirty = true @@ -381,7 +381,7 @@ func (p *Pokestop) SetPartnerId(v null.String) { func (p *Pokestop) SetArScanEligible(v null.Int) { if p.ArScanEligible != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ArScanEligible:%v->%v", p.ArScanEligible, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ArScanEligible:%s->%s", FormatNull(p.ArScanEligible), FormatNull(v))) } p.ArScanEligible = v p.dirty = true @@ -391,7 +391,7 @@ func (p *Pokestop) SetArScanEligible(v null.Int) { func (p *Pokestop) SetPowerUpLevel(v null.Int) { if p.PowerUpLevel != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpLevel:%v->%v", p.PowerUpLevel, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpLevel:%s->%s", FormatNull(p.PowerUpLevel), FormatNull(v))) } p.PowerUpLevel = v p.dirty = true @@ -401,7 +401,7 @@ func (p *Pokestop) SetPowerUpLevel(v null.Int) { func (p *Pokestop) SetPowerUpPoints(v null.Int) { if p.PowerUpPoints != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpPoints:%v->%v", p.PowerUpPoints, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpPoints:%s->%s", FormatNull(p.PowerUpPoints), FormatNull(v))) } p.PowerUpPoints = v p.dirty = true @@ -411,7 +411,7 @@ func (p *Pokestop) SetPowerUpPoints(v null.Int) { func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { if p.PowerUpEndTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%v->%v", p.PowerUpEndTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("PowerUpEndTimestamp:%s->%s", FormatNull(p.PowerUpEndTimestamp), FormatNull(v))) } p.PowerUpEndTimestamp = v p.dirty = true @@ -421,7 +421,7 @@ func (p *Pokestop) SetPowerUpEndTimestamp(v null.Int) { func (p *Pokestop) SetAlternativeQuestType(v null.Int) { if p.AlternativeQuestType != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestType:%v->%v", p.AlternativeQuestType, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestType:%s->%s", FormatNull(p.AlternativeQuestType), FormatNull(v))) } p.AlternativeQuestType = v p.dirty = true @@ -431,7 +431,7 @@ func (p *Pokestop) SetAlternativeQuestType(v null.Int) { func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { if p.AlternativeQuestTimestamp != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTimestamp:%v->%v", p.AlternativeQuestTimestamp, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTimestamp:%s->%s", FormatNull(p.AlternativeQuestTimestamp), FormatNull(v))) } p.AlternativeQuestTimestamp = v p.dirty = true @@ -441,7 +441,7 @@ func (p *Pokestop) SetAlternativeQuestTimestamp(v null.Int) { func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { if p.AlternativeQuestTarget != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTarget:%v->%v", p.AlternativeQuestTarget, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTarget:%s->%s", FormatNull(p.AlternativeQuestTarget), FormatNull(v))) } p.AlternativeQuestTarget = v p.dirty = true @@ -451,7 +451,7 @@ func (p *Pokestop) SetAlternativeQuestTarget(v null.Int) { func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { if p.AlternativeQuestConditions != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestConditions:%v->%v", p.AlternativeQuestConditions, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestConditions:%s->%s", FormatNull(p.AlternativeQuestConditions), FormatNull(v))) } p.AlternativeQuestConditions = v p.dirty = true @@ -461,7 +461,7 @@ func (p *Pokestop) SetAlternativeQuestConditions(v null.String) { func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { if p.AlternativeQuestRewards != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestRewards:%v->%v", p.AlternativeQuestRewards, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestRewards:%s->%s", FormatNull(p.AlternativeQuestRewards), FormatNull(v))) } p.AlternativeQuestRewards = v p.dirty = true @@ -471,7 +471,7 @@ func (p *Pokestop) SetAlternativeQuestRewards(v null.String) { func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { if p.AlternativeQuestTemplate != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTemplate:%v->%v", p.AlternativeQuestTemplate, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTemplate:%s->%s", FormatNull(p.AlternativeQuestTemplate), FormatNull(v))) } p.AlternativeQuestTemplate = v p.dirty = true @@ -481,7 +481,7 @@ func (p *Pokestop) SetAlternativeQuestTemplate(v null.String) { func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { if p.AlternativeQuestTitle != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTitle:%v->%v", p.AlternativeQuestTitle, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestTitle:%s->%s", FormatNull(p.AlternativeQuestTitle), FormatNull(v))) } p.AlternativeQuestTitle = v p.dirty = true @@ -491,7 +491,7 @@ func (p *Pokestop) SetAlternativeQuestTitle(v null.String) { func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { if p.AlternativeQuestExpiry != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestExpiry:%v->%v", p.AlternativeQuestExpiry, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("AlternativeQuestExpiry:%s->%s", FormatNull(p.AlternativeQuestExpiry), FormatNull(v))) } p.AlternativeQuestExpiry = v p.dirty = true @@ -501,7 +501,7 @@ func (p *Pokestop) SetAlternativeQuestExpiry(v null.Int) { func (p *Pokestop) SetDescription(v null.String) { if p.Description != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("Description:%v->%v", p.Description, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("Description:%s->%s", FormatNull(p.Description), FormatNull(v))) } p.Description = v p.dirty = true @@ -511,7 +511,7 @@ func (p *Pokestop) SetDescription(v null.String) { func (p *Pokestop) SetShowcaseFocus(v null.String) { if p.ShowcaseFocus != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseFocus:%v->%v", p.ShowcaseFocus, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseFocus:%s->%s", FormatNull(p.ShowcaseFocus), FormatNull(v))) } p.ShowcaseFocus = v p.dirty = true @@ -521,7 +521,7 @@ func (p *Pokestop) SetShowcaseFocus(v null.String) { func (p *Pokestop) SetShowcasePokemon(v null.Int) { if p.ShowcasePokemon != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemon:%v->%v", p.ShowcasePokemon, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemon:%s->%s", FormatNull(p.ShowcasePokemon), FormatNull(v))) } p.ShowcasePokemon = v p.dirty = true @@ -531,7 +531,7 @@ func (p *Pokestop) SetShowcasePokemon(v null.Int) { func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { if p.ShowcasePokemonForm != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonForm:%v->%v", p.ShowcasePokemonForm, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonForm:%s->%s", FormatNull(p.ShowcasePokemonForm), FormatNull(v))) } p.ShowcasePokemonForm = v p.dirty = true @@ -541,7 +541,7 @@ func (p *Pokestop) SetShowcasePokemonForm(v null.Int) { func (p *Pokestop) SetShowcasePokemonType(v null.Int) { if p.ShowcasePokemonType != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonType:%v->%v", p.ShowcasePokemonType, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcasePokemonType:%s->%s", FormatNull(p.ShowcasePokemonType), FormatNull(v))) } p.ShowcasePokemonType = v p.dirty = true @@ -551,7 +551,7 @@ func (p *Pokestop) SetShowcasePokemonType(v null.Int) { func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { if p.ShowcaseRankingStandard != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankingStandard:%v->%v", p.ShowcaseRankingStandard, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankingStandard:%s->%s", FormatNull(p.ShowcaseRankingStandard), FormatNull(v))) } p.ShowcaseRankingStandard = v p.dirty = true @@ -561,7 +561,7 @@ func (p *Pokestop) SetShowcaseRankingStandard(v null.Int) { func (p *Pokestop) SetShowcaseExpiry(v null.Int) { if p.ShowcaseExpiry != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseExpiry:%v->%v", p.ShowcaseExpiry, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseExpiry:%s->%s", FormatNull(p.ShowcaseExpiry), FormatNull(v))) } p.ShowcaseExpiry = v p.dirty = true @@ -571,7 +571,7 @@ func (p *Pokestop) SetShowcaseExpiry(v null.Int) { func (p *Pokestop) SetShowcaseRankings(v null.String) { if p.ShowcaseRankings != v { if dbDebugEnabled { - p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankings:%v->%v", p.ShowcaseRankings, v)) + p.changedFields = append(p.changedFields, fmt.Sprintf("ShowcaseRankings:%s->%s", FormatNull(p.ShowcaseRankings), FormatNull(v))) } p.ShowcaseRankings = v p.dirty = true diff --git a/decoder/routes.go b/decoder/routes.go index 0fa7ee6a..567a7adb 100644 --- a/decoder/routes.go +++ b/decoder/routes.go @@ -245,7 +245,7 @@ func (r *Route) SetStartLon(v float64) { func (r *Route) SetTags(v null.String) { if r.Tags != v { if dbDebugEnabled { - r.changedFields = append(r.changedFields, fmt.Sprintf("Tags:%v->%v", r.Tags, v)) + r.changedFields = append(r.changedFields, fmt.Sprintf("Tags:%s->%s", FormatNull(r.Tags), FormatNull(v))) } r.Tags = v r.dirty = true diff --git a/decoder/spawnpoint.go b/decoder/spawnpoint.go index 9171a852..0f5f0f3b 100644 --- a/decoder/spawnpoint.go +++ b/decoder/spawnpoint.go @@ -99,7 +99,7 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { // Handle validity changes if (s.DespawnSec.Valid && !v.Valid) || (!s.DespawnSec.Valid && v.Valid) { if dbDebugEnabled { - s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%v->%v", s.DespawnSec, v)) + s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%s->%s", FormatNull(s.DespawnSec), FormatNull(v))) } s.DespawnSec = v s.dirty = true @@ -126,7 +126,7 @@ func (s *Spawnpoint) SetDespawnSec(v null.Int) { // Allow 2-second tolerance for despawn time if Abs(oldVal-newVal) > 2 { if dbDebugEnabled { - s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%v->%v", s.DespawnSec, v)) + s.changedFields = append(s.changedFields, fmt.Sprintf("DespawnSec:%s->%s", FormatNull(s.DespawnSec), FormatNull(v))) } s.DespawnSec = v s.dirty = true diff --git a/decoder/station.go b/decoder/station.go index 8eb1e307..8a7b7bcd 100644 --- a/decoder/station.go +++ b/decoder/station.go @@ -204,7 +204,7 @@ func (station *Station) SetIsInactive(v bool) { func (station *Station) SetBattleLevel(v null.Int) { if station.BattleLevel != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattleLevel:%v->%v", station.BattleLevel, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleLevel:%s->%s", FormatNull(station.BattleLevel), FormatNull(v))) } station.BattleLevel = v station.dirty = true @@ -214,7 +214,7 @@ func (station *Station) SetBattleLevel(v null.Int) { func (station *Station) SetBattleStart(v null.Int) { if station.BattleStart != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattleStart:%v->%v", station.BattleStart, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleStart:%s->%s", FormatNull(station.BattleStart), FormatNull(v))) } station.BattleStart = v station.dirty = true @@ -224,7 +224,7 @@ func (station *Station) SetBattleStart(v null.Int) { func (station *Station) SetBattleEnd(v null.Int) { if station.BattleEnd != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattleEnd:%v->%v", station.BattleEnd, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattleEnd:%s->%s", FormatNull(station.BattleEnd), FormatNull(v))) } station.BattleEnd = v station.dirty = true @@ -234,7 +234,7 @@ func (station *Station) SetBattleEnd(v null.Int) { func (station *Station) SetBattlePokemonId(v null.Int) { if station.BattlePokemonId != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonId:%v->%v", station.BattlePokemonId, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonId:%s->%s", FormatNull(station.BattlePokemonId), FormatNull(v))) } station.BattlePokemonId = v station.dirty = true @@ -244,7 +244,7 @@ func (station *Station) SetBattlePokemonId(v null.Int) { func (station *Station) SetBattlePokemonForm(v null.Int) { if station.BattlePokemonForm != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonForm:%v->%v", station.BattlePokemonForm, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonForm:%s->%s", FormatNull(station.BattlePokemonForm), FormatNull(v))) } station.BattlePokemonForm = v station.dirty = true @@ -254,7 +254,7 @@ func (station *Station) SetBattlePokemonForm(v null.Int) { func (station *Station) SetBattlePokemonCostume(v null.Int) { if station.BattlePokemonCostume != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCostume:%v->%v", station.BattlePokemonCostume, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCostume:%s->%s", FormatNull(station.BattlePokemonCostume), FormatNull(v))) } station.BattlePokemonCostume = v station.dirty = true @@ -264,7 +264,7 @@ func (station *Station) SetBattlePokemonCostume(v null.Int) { func (station *Station) SetBattlePokemonGender(v null.Int) { if station.BattlePokemonGender != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonGender:%v->%v", station.BattlePokemonGender, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonGender:%s->%s", FormatNull(station.BattlePokemonGender), FormatNull(v))) } station.BattlePokemonGender = v station.dirty = true @@ -274,7 +274,7 @@ func (station *Station) SetBattlePokemonGender(v null.Int) { func (station *Station) SetBattlePokemonAlignment(v null.Int) { if station.BattlePokemonAlignment != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonAlignment:%v->%v", station.BattlePokemonAlignment, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonAlignment:%s->%s", FormatNull(station.BattlePokemonAlignment), FormatNull(v))) } station.BattlePokemonAlignment = v station.dirty = true @@ -284,7 +284,7 @@ func (station *Station) SetBattlePokemonAlignment(v null.Int) { func (station *Station) SetBattlePokemonBreadMode(v null.Int) { if station.BattlePokemonBreadMode != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonBreadMode:%v->%v", station.BattlePokemonBreadMode, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonBreadMode:%s->%s", FormatNull(station.BattlePokemonBreadMode), FormatNull(v))) } station.BattlePokemonBreadMode = v station.dirty = true @@ -294,7 +294,7 @@ func (station *Station) SetBattlePokemonBreadMode(v null.Int) { func (station *Station) SetBattlePokemonMove1(v null.Int) { if station.BattlePokemonMove1 != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove1:%v->%v", station.BattlePokemonMove1, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove1:%s->%s", FormatNull(station.BattlePokemonMove1), FormatNull(v))) } station.BattlePokemonMove1 = v station.dirty = true @@ -304,7 +304,7 @@ func (station *Station) SetBattlePokemonMove1(v null.Int) { func (station *Station) SetBattlePokemonMove2(v null.Int) { if station.BattlePokemonMove2 != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove2:%v->%v", station.BattlePokemonMove2, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonMove2:%s->%s", FormatNull(station.BattlePokemonMove2), FormatNull(v))) } station.BattlePokemonMove2 = v station.dirty = true @@ -314,7 +314,7 @@ func (station *Station) SetBattlePokemonMove2(v null.Int) { func (station *Station) SetBattlePokemonStamina(v null.Int) { if station.BattlePokemonStamina != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonStamina:%v->%v", station.BattlePokemonStamina, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonStamina:%s->%s", FormatNull(station.BattlePokemonStamina), FormatNull(v))) } station.BattlePokemonStamina = v station.dirty = true @@ -324,7 +324,7 @@ func (station *Station) SetBattlePokemonStamina(v null.Int) { func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { if !nullFloatAlmostEqual(station.BattlePokemonCpMultiplier, v, floatTolerance) { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCpMultiplier:%v->%v", station.BattlePokemonCpMultiplier, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("BattlePokemonCpMultiplier:%s->%s", FormatNull(station.BattlePokemonCpMultiplier), FormatNull(v))) } station.BattlePokemonCpMultiplier = v station.dirty = true @@ -334,7 +334,7 @@ func (station *Station) SetBattlePokemonCpMultiplier(v null.Float) { func (station *Station) SetTotalStationedPokemon(v null.Int) { if station.TotalStationedPokemon != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedPokemon:%v->%v", station.TotalStationedPokemon, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedPokemon:%s->%s", FormatNull(station.TotalStationedPokemon), FormatNull(v))) } station.TotalStationedPokemon = v station.dirty = true @@ -344,7 +344,7 @@ func (station *Station) SetTotalStationedPokemon(v null.Int) { func (station *Station) SetTotalStationedGmax(v null.Int) { if station.TotalStationedGmax != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedGmax:%v->%v", station.TotalStationedGmax, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("TotalStationedGmax:%s->%s", FormatNull(station.TotalStationedGmax), FormatNull(v))) } station.TotalStationedGmax = v station.dirty = true @@ -354,7 +354,7 @@ func (station *Station) SetTotalStationedGmax(v null.Int) { func (station *Station) SetStationedPokemon(v null.String) { if station.StationedPokemon != v { if dbDebugEnabled { - station.changedFields = append(station.changedFields, fmt.Sprintf("StationedPokemon:%v->%v", station.StationedPokemon, v)) + station.changedFields = append(station.changedFields, fmt.Sprintf("StationedPokemon:%s->%s", FormatNull(station.StationedPokemon), FormatNull(v))) } station.StationedPokemon = v station.dirty = true diff --git a/decoder/tappable.go b/decoder/tappable.go index 98573571..20ff851d 100644 --- a/decoder/tappable.go +++ b/decoder/tappable.go @@ -80,7 +80,7 @@ func (ta *Tappable) SetLon(v float64) { func (ta *Tappable) SetFortId(v null.String) { if ta.FortId != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("FortId:%v->%v", ta.FortId, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("FortId:%s->%s", FormatNull(ta.FortId), FormatNull(v))) } ta.FortId = v ta.dirty = true @@ -90,7 +90,7 @@ func (ta *Tappable) SetFortId(v null.String) { func (ta *Tappable) SetSpawnId(v null.Int) { if ta.SpawnId != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("SpawnId:%v->%v", ta.SpawnId, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("SpawnId:%s->%s", FormatNull(ta.SpawnId), FormatNull(v))) } ta.SpawnId = v ta.dirty = true @@ -110,7 +110,7 @@ func (ta *Tappable) SetType(v string) { func (ta *Tappable) SetEncounter(v null.Int) { if ta.Encounter != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("Encounter:%v->%v", ta.Encounter, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Encounter:%s->%s", FormatNull(ta.Encounter), FormatNull(v))) } ta.Encounter = v ta.dirty = true @@ -120,7 +120,7 @@ func (ta *Tappable) SetEncounter(v null.Int) { func (ta *Tappable) SetItemId(v null.Int) { if ta.ItemId != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("ItemId:%v->%v", ta.ItemId, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ItemId:%s->%s", FormatNull(ta.ItemId), FormatNull(v))) } ta.ItemId = v ta.dirty = true @@ -130,7 +130,7 @@ func (ta *Tappable) SetItemId(v null.Int) { func (ta *Tappable) SetCount(v null.Int) { if ta.Count != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("Count:%v->%v", ta.Count, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("Count:%s->%s", FormatNull(ta.Count), FormatNull(v))) } ta.Count = v ta.dirty = true @@ -140,7 +140,7 @@ func (ta *Tappable) SetCount(v null.Int) { func (ta *Tappable) SetExpireTimestamp(v null.Int) { if ta.ExpireTimestamp != v { if dbDebugEnabled { - ta.changedFields = append(ta.changedFields, fmt.Sprintf("ExpireTimestamp:%v->%v", ta.ExpireTimestamp, v)) + ta.changedFields = append(ta.changedFields, fmt.Sprintf("ExpireTimestamp:%s->%s", FormatNull(ta.ExpireTimestamp), FormatNull(v))) } ta.ExpireTimestamp = v ta.dirty = true From 0382c1d906a4ddf74a7c309c3428ff5644bd7932 Mon Sep 17 00:00:00 2001 From: James Berry Date: Tue, 3 Feb 2026 17:07:16 +0000 Subject: [PATCH 35/35] fix location of reduce_updates --- config.toml.example | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/config.toml.example b/config.toml.example index 2c018fcb..81b9e971 100644 --- a/config.toml.example +++ b/config.toml.example @@ -3,12 +3,6 @@ port = 9001 # Listening port for golbat raw_bearer = "" # Raw bearer (password) required api_secret = "golbat" # Golbat secret required on api calls (blank for none) -# When enabled, reduce_updates will make fort update debounce windows much longer -# to reduce database churn. Specifically, gym/pokestop/station debounce will be -# extended from 15 minutes (900s) to 12 hours (43200s) and spawnpoint last_seen -# will be updated every 12 hours instead of the default 6 hours. -reduce_updates = false - pokemon_memory_only = false # Use in-memory storage for pokemon only [koji] @@ -25,6 +19,14 @@ stats_days = 7 # Remove entries from "pokemon_stats", "pokemon_ device_hours = 24 # Remove devices from in memory after not seen for x hours forts_stale_threshold = 3600 # Seconds before a fort is considered stale (default: 1 hour) +[tuning] +# When enabled, reduce_updates will make fort update debounce windows much longer +# to reduce database churn. Specifically, gym/pokestop/station debounce will be +# extended from 15 minutes (900s) to 12 hours (43200s) and spawnpoint last_seen +# will be updated every 12 hours instead of the default 6 hours. +reduce_updates = false + + [logging] debug = false save_logs = true