Skip to content

feat: invasion_lineup_stats #311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Mygod wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat: invasion_lineup_stats #311

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion config.toml.example
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ incidents = true # Remove incidents after expiry
quests = true # Remove quests after expiry
tappables = true # Remove tappables after expiry
stats = true # Enable/Disable stats history
stats_days = 7 # Remove entries from "pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats", "invasion_stats", "quest_stats", "raid_stats" after x days
stats_days = 7 # Remove entries from "pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats", "invasion_stats", "invasion_lineup_stats", "quest_stats", "raid_stats" after x days
device_hours = 24 # Remove devices from in memory after not seen for x hours

[logging]
Expand Down
178 changes: 164 additions & 14 deletions decoder/stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
null "gopkg.in/guregu/null.v4"

"golbat/encounter_cache"
"golbat/geo"
Expand Down Expand Up @@ -65,7 +66,8 @@ type areaRaidCountDetail struct {
}

type areaInvasionCountDetail struct {
count [maxInvasionCharacter + 1]int
count [maxInvasionCharacter + 1]int
lineup map[invasionLineupKey]int
}

type areaQuestCountDetail struct {
Expand All @@ -74,6 +76,19 @@ type areaQuestCountDetail struct {
itemDetails [maxItemNo + 1]map[int]int // for each itemId[amount] keep a count
}

type invasionLineupKey struct {
Character int
Slot uint8
PokemonID int
FormID int
}

func newAreaInvasionCountDetail() *areaInvasionCountDetail {
return &areaInvasionCountDetail{
lineup: make(map[invasionLineupKey]int),
}
}

// a cache indexed by encounterId (Pokemon.Id)
var encounterCache *encounter_cache.EncounterCache

Expand Down Expand Up @@ -509,24 +524,112 @@ func updateIncidentStats(old *Incident, new *Incident, areas []geo.AreaName) {
for i := 0; i < len(areas); i++ {
area := areas[i]

// Check if StartTime has changed, then we can assume a new Incident has appeared.
if old == nil || old.StartTime != new.StartTime {
if new.Character == 0 {
continue
}

if !locked {
incidentStatsLock.Lock()
locked = true
}
needGeneralCount := old == nil || old.StartTime != new.StartTime

invasionStats := invasionCount[area]
if invasionStats == nil {
invasionStats = &areaInvasionCountDetail{}
invasionCount[area] = invasionStats
shouldCountSlot := func(newValid bool, newID int64, newFormValid bool, newForm int64, oldValid bool, oldID int64, oldFormValid bool, oldForm int64) bool {
if !newValid {
return false
}
if newID <= 0 {
return false
}
if !oldValid {
return true
}
if oldID != newID {
return true
}
if oldFormValid != newFormValid {
return true
}
if newFormValid && oldForm != newForm {
return true
}
return false
}

lineupChanged := false

var oldSlot1Valid, oldSlot1FormValid bool
var oldSlot1ID, oldSlot1Form int64
if old != nil {
oldSlot1Valid = old.Slot1PokemonId.Valid
oldSlot1ID = old.Slot1PokemonId.ValueOrZero()
oldSlot1FormValid = old.Slot1Form.Valid
oldSlot1Form = old.Slot1Form.ValueOrZero()
}
lineupChanged = lineupChanged || shouldCountSlot(new.Slot1PokemonId.Valid, new.Slot1PokemonId.ValueOrZero(), new.Slot1Form.Valid, new.Slot1Form.ValueOrZero(),
oldSlot1Valid, oldSlot1ID, oldSlot1FormValid, oldSlot1Form)

var oldSlot2Valid, oldSlot2FormValid bool
var oldSlot2ID, oldSlot2Form int64
if old != nil {
oldSlot2Valid = old.Slot2PokemonId.Valid
oldSlot2ID = old.Slot2PokemonId.ValueOrZero()
oldSlot2FormValid = old.Slot2Form.Valid
oldSlot2Form = old.Slot2Form.ValueOrZero()
}
lineupChanged = lineupChanged || shouldCountSlot(new.Slot2PokemonId.Valid, new.Slot2PokemonId.ValueOrZero(), new.Slot2Form.Valid, new.Slot2Form.ValueOrZero(),
oldSlot2Valid, oldSlot2ID, oldSlot2FormValid, oldSlot2Form)

var oldSlot3Valid, oldSlot3FormValid bool
var oldSlot3ID, oldSlot3Form int64
if old != nil {
oldSlot3Valid = old.Slot3PokemonId.Valid
oldSlot3ID = old.Slot3PokemonId.ValueOrZero()
oldSlot3FormValid = old.Slot3Form.Valid
oldSlot3Form = old.Slot3Form.ValueOrZero()
}
lineupChanged = lineupChanged || shouldCountSlot(new.Slot3PokemonId.Valid, new.Slot3PokemonId.ValueOrZero(), new.Slot3Form.Valid, new.Slot3Form.ValueOrZero(),
oldSlot3Valid, oldSlot3ID, oldSlot3FormValid, oldSlot3Form)

if !needGeneralCount && !lineupChanged {
continue
}

if !locked {
incidentStatsLock.Lock()
locked = true
}

invasionStats := invasionCount[area]
if invasionStats == nil {
invasionStats = newAreaInvasionCountDetail()
invasionCount[area] = invasionStats
} else if invasionStats.lineup == nil {
invasionStats.lineup = make(map[invasionLineupKey]int)
}

if needGeneralCount {
invasionStats.count[new.Character]++
}

// Exclude Kecleon, Showcases and other UNSET characters for invasionStats.
if new.Character != 0 {
invasionStats.count[new.Character]++
if lineupChanged {
addSlot := func(slot uint8, pokemonID null.Int, formID null.Int) {
if !pokemonID.Valid {
return
}
pid := pokemonID.ValueOrZero()
if pid <= 0 {
return
}

key := invasionLineupKey{
Character: int(new.Character),
Slot: slot,
PokemonID: int(pid),
FormID: int(formID.ValueOrZero()),
}
invasionStats.lineup[key]++
}

addSlot(1, new.Slot1PokemonId, new.Slot1Form)
addSlot(2, new.Slot2PokemonId, new.Slot2Form)
addSlot(3, new.Slot3PokemonId, new.Slot3Form)
}
}

Expand Down Expand Up @@ -927,6 +1030,17 @@ type invasionStatsDbRow struct {
Count int `db:"count"`
}

type invasionLineupStatsDbRow struct {
Date string `db:"date"`
Area string `db:"area"`
Fence string `db:"fence"`
Character int `db:"character"`
Slot int `db:"slot"`
PokemonID int `db:"pokemon_id"`
FormID int `db:"form_id"`
Count int `db:"count"`
}

func logInvasionStats(statsDb *sqlx.DB) {
incidentStatsLock.Lock()
log.Infof("STATS: Write invasion stats")
Expand All @@ -937,6 +1051,7 @@ func logInvasionStats(statsDb *sqlx.DB) {

go func() {
var rows []invasionStatsDbRow
var lineupRows []invasionLineupStatsDbRow

t := time.Now().In(time.Local)
midnightString := t.Format("2006-01-02")
Expand All @@ -957,6 +1072,24 @@ func logInvasionStats(statsDb *sqlx.DB) {
addRows(&rows, character, count)
}
}

if stats.lineup != nil {
for key, count := range stats.lineup {
if count <= 0 {
continue
}
lineupRows = append(lineupRows, invasionLineupStatsDbRow{
Date: midnightString,
Area: area.Parent,
Fence: area.Name,
Character: key.Character,
Slot: int(key.Slot),
PokemonID: key.PokemonID,
FormID: key.FormID,
Count: count,
})
}
}
}

for i := 0; i < len(rows); i += batchInsertSize {
Expand All @@ -975,6 +1108,23 @@ func logInvasionStats(statsDb *sqlx.DB) {
log.Errorf("Error inserting invasion_stats: %v", err)
}
}

for i := 0; i < len(lineupRows); i += batchInsertSize {
end := i + batchInsertSize
if end > len(lineupRows) {
end = len(lineupRows)
}

batchRows := lineupRows[i:end]
_, err := statsDb.NamedExec(
"INSERT INTO invasion_lineup_stats "+
"(date, area, fence, `character`, slot, pokemon_id, form_id, `count`)"+
" VALUES (:date, :area, :fence, :character, :slot, :pokemon_id, :form_id, :count)"+
" ON DUPLICATE KEY UPDATE `count` = `count` + VALUES(`count`);", batchRows)
if err != nil {
log.Errorf("Error inserting invasion_lineup_stats: %v", err)
}
}
}()
}

Expand Down
11 changes: 11 additions & 0 deletions sql/50_invasion_lineup_stats.up.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
CREATE TABLE IF NOT EXISTS `invasion_lineup_stats` (
`date` date NOT NULL,
`area` varchar(255) NOT NULL DEFAULT '',
`fence` varchar(255) NOT NULL DEFAULT '',
`character` smallint unsigned NOT NULL,
`slot` tinyint unsigned NOT NULL,
`pokemon_id` smallint unsigned NOT NULL,
`form_id` smallint unsigned NOT NULL DEFAULT 0,
`count` int NOT NULL,
PRIMARY KEY (`date`, `area`, `fence`, `character`, `slot`, `pokemon_id`, `form_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
20 changes: 20 additions & 0 deletions sql/rdm/rdmdb.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,26 @@ CREATE TABLE `invasion_stats` (
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `invasion_lineup_stats`
--

DROP TABLE IF EXISTS `invasion_lineup_stats`;
/*!40101 SET @saved_cs_client = @@character_set_client */;
/*!50503 SET character_set_client = utf8mb4 */;
CREATE TABLE `invasion_lineup_stats` (
`date` date NOT NULL,
`area` varchar(255) NOT NULL DEFAULT '',
`fence` varchar(255) NOT NULL DEFAULT '',
`character` smallint unsigned NOT NULL,
`slot` tinyint unsigned NOT NULL,
`pokemon_id` smallint unsigned NOT NULL,
`form_id` smallint unsigned NOT NULL DEFAULT '0',
`count` int NOT NULL,
PRIMARY KEY (`date`,`area`,`fence`,`character`,`slot`,`pokemon_id`,`form_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
-- Table structure for table `metadata`
--
Expand Down
2 changes: 1 addition & 1 deletion stats.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func StartStatsExpiry(db *sqlx.DB) {
log.Infof("DB - Cleanup of pokemon_area_stats table took %s (%d rows)", elapsed, rows)
}

tables := []string{"pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats", "invasion_stats", "quest_stats", "raid_stats"}
tables := []string{"pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats", "invasion_stats", "invasion_lineup_stats", "quest_stats", "raid_stats"}

for _, table := range tables {
start = time.Now()
Expand Down