diff --git a/config.toml.example b/config.toml.example index 868b0efc..fd4e28b5 100644 --- a/config.toml.example +++ b/config.toml.example @@ -15,7 +15,7 @@ incidents = true # Remove incidents after expiry quests = true # Remove quests after expiry tappables = true # Remove tappables after expiry stats = true # Enable/Disable stats history -stats_days = 7 # Remove entries from "pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats", "invasion_stats", "quest_stats", "raid_stats" after x days +stats_days = 7 # Remove entries from "pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats", "invasion_stats", "invasion_lineup_stats", "quest_stats", "raid_stats" after x days device_hours = 24 # Remove devices from in memory after not seen for x hours [logging] diff --git a/decoder/stats.go b/decoder/stats.go index 86ea7956..34868ca0 100644 --- a/decoder/stats.go +++ b/decoder/stats.go @@ -11,6 +11,7 @@ import ( "github.com/jmoiron/sqlx" log "github.com/sirupsen/logrus" + null "gopkg.in/guregu/null.v4" "golbat/encounter_cache" "golbat/geo" @@ -65,7 +66,8 @@ type areaRaidCountDetail struct { } type areaInvasionCountDetail struct { - count [maxInvasionCharacter + 1]int + count [maxInvasionCharacter + 1]int + lineup map[invasionLineupKey]int } type areaQuestCountDetail struct { @@ -74,6 +76,19 @@ type areaQuestCountDetail struct { itemDetails [maxItemNo + 1]map[int]int // for each itemId[amount] keep a count } +type invasionLineupKey struct { + Character int + Slot uint8 + PokemonID int + FormID int +} + +func newAreaInvasionCountDetail() *areaInvasionCountDetail { + return &areaInvasionCountDetail{ + lineup: make(map[invasionLineupKey]int), + } +} + // a cache indexed by encounterId (Pokemon.Id) var encounterCache *encounter_cache.EncounterCache @@ -509,24 +524,112 @@ func updateIncidentStats(old *Incident, new *Incident, areas []geo.AreaName) { for i := 0; i < len(areas); i++ { area := areas[i] - // Check if StartTime has changed, then we can assume a new Incident has appeared. - if old == nil || old.StartTime != new.StartTime { + if new.Character == 0 { + continue + } - if !locked { - incidentStatsLock.Lock() - locked = true - } + needGeneralCount := old == nil || old.StartTime != new.StartTime - invasionStats := invasionCount[area] - if invasionStats == nil { - invasionStats = &areaInvasionCountDetail{} - invasionCount[area] = invasionStats + shouldCountSlot := func(newValid bool, newID int64, newFormValid bool, newForm int64, oldValid bool, oldID int64, oldFormValid bool, oldForm int64) bool { + if !newValid { + return false + } + if newID <= 0 { + return false + } + if !oldValid { + return true + } + if oldID != newID { + return true } + if oldFormValid != newFormValid { + return true + } + if newFormValid && oldForm != newForm { + return true + } + return false + } + + lineupChanged := false + + var oldSlot1Valid, oldSlot1FormValid bool + var oldSlot1ID, oldSlot1Form int64 + if old != nil { + oldSlot1Valid = old.Slot1PokemonId.Valid + oldSlot1ID = old.Slot1PokemonId.ValueOrZero() + oldSlot1FormValid = old.Slot1Form.Valid + oldSlot1Form = old.Slot1Form.ValueOrZero() + } + lineupChanged = lineupChanged || shouldCountSlot(new.Slot1PokemonId.Valid, new.Slot1PokemonId.ValueOrZero(), new.Slot1Form.Valid, new.Slot1Form.ValueOrZero(), + oldSlot1Valid, oldSlot1ID, oldSlot1FormValid, oldSlot1Form) + + var oldSlot2Valid, oldSlot2FormValid bool + var oldSlot2ID, oldSlot2Form int64 + if old != nil { + oldSlot2Valid = old.Slot2PokemonId.Valid + oldSlot2ID = old.Slot2PokemonId.ValueOrZero() + oldSlot2FormValid = old.Slot2Form.Valid + oldSlot2Form = old.Slot2Form.ValueOrZero() + } + lineupChanged = lineupChanged || shouldCountSlot(new.Slot2PokemonId.Valid, new.Slot2PokemonId.ValueOrZero(), new.Slot2Form.Valid, new.Slot2Form.ValueOrZero(), + oldSlot2Valid, oldSlot2ID, oldSlot2FormValid, oldSlot2Form) + + var oldSlot3Valid, oldSlot3FormValid bool + var oldSlot3ID, oldSlot3Form int64 + if old != nil { + oldSlot3Valid = old.Slot3PokemonId.Valid + oldSlot3ID = old.Slot3PokemonId.ValueOrZero() + oldSlot3FormValid = old.Slot3Form.Valid + oldSlot3Form = old.Slot3Form.ValueOrZero() + } + lineupChanged = lineupChanged || shouldCountSlot(new.Slot3PokemonId.Valid, new.Slot3PokemonId.ValueOrZero(), new.Slot3Form.Valid, new.Slot3Form.ValueOrZero(), + oldSlot3Valid, oldSlot3ID, oldSlot3FormValid, oldSlot3Form) + + if !needGeneralCount && !lineupChanged { + continue + } + + if !locked { + incidentStatsLock.Lock() + locked = true + } + + invasionStats := invasionCount[area] + if invasionStats == nil { + invasionStats = newAreaInvasionCountDetail() + invasionCount[area] = invasionStats + } else if invasionStats.lineup == nil { + invasionStats.lineup = make(map[invasionLineupKey]int) + } + + if needGeneralCount { + invasionStats.count[new.Character]++ + } - // Exclude Kecleon, Showcases and other UNSET characters for invasionStats. - if new.Character != 0 { - invasionStats.count[new.Character]++ + if lineupChanged { + addSlot := func(slot uint8, pokemonID null.Int, formID null.Int) { + if !pokemonID.Valid { + return + } + pid := pokemonID.ValueOrZero() + if pid <= 0 { + return + } + + key := invasionLineupKey{ + Character: int(new.Character), + Slot: slot, + PokemonID: int(pid), + FormID: int(formID.ValueOrZero()), + } + invasionStats.lineup[key]++ } + + addSlot(1, new.Slot1PokemonId, new.Slot1Form) + addSlot(2, new.Slot2PokemonId, new.Slot2Form) + addSlot(3, new.Slot3PokemonId, new.Slot3Form) } } @@ -927,6 +1030,17 @@ type invasionStatsDbRow struct { Count int `db:"count"` } +type invasionLineupStatsDbRow struct { + Date string `db:"date"` + Area string `db:"area"` + Fence string `db:"fence"` + Character int `db:"character"` + Slot int `db:"slot"` + PokemonID int `db:"pokemon_id"` + FormID int `db:"form_id"` + Count int `db:"count"` +} + func logInvasionStats(statsDb *sqlx.DB) { incidentStatsLock.Lock() log.Infof("STATS: Write invasion stats") @@ -937,6 +1051,7 @@ func logInvasionStats(statsDb *sqlx.DB) { go func() { var rows []invasionStatsDbRow + var lineupRows []invasionLineupStatsDbRow t := time.Now().In(time.Local) midnightString := t.Format("2006-01-02") @@ -957,6 +1072,24 @@ func logInvasionStats(statsDb *sqlx.DB) { addRows(&rows, character, count) } } + + if stats.lineup != nil { + for key, count := range stats.lineup { + if count <= 0 { + continue + } + lineupRows = append(lineupRows, invasionLineupStatsDbRow{ + Date: midnightString, + Area: area.Parent, + Fence: area.Name, + Character: key.Character, + Slot: int(key.Slot), + PokemonID: key.PokemonID, + FormID: key.FormID, + Count: count, + }) + } + } } for i := 0; i < len(rows); i += batchInsertSize { @@ -975,6 +1108,23 @@ func logInvasionStats(statsDb *sqlx.DB) { log.Errorf("Error inserting invasion_stats: %v", err) } } + + for i := 0; i < len(lineupRows); i += batchInsertSize { + end := i + batchInsertSize + if end > len(lineupRows) { + end = len(lineupRows) + } + + batchRows := lineupRows[i:end] + _, err := statsDb.NamedExec( + "INSERT INTO invasion_lineup_stats "+ + "(date, area, fence, `character`, slot, pokemon_id, form_id, `count`)"+ + " VALUES (:date, :area, :fence, :character, :slot, :pokemon_id, :form_id, :count)"+ + " ON DUPLICATE KEY UPDATE `count` = `count` + VALUES(`count`);", batchRows) + if err != nil { + log.Errorf("Error inserting invasion_lineup_stats: %v", err) + } + } }() } diff --git a/sql/50_invasion_lineup_stats.up.sql b/sql/50_invasion_lineup_stats.up.sql new file mode 100644 index 00000000..a8fdef5d --- /dev/null +++ b/sql/50_invasion_lineup_stats.up.sql @@ -0,0 +1,11 @@ +CREATE TABLE IF NOT EXISTS `invasion_lineup_stats` ( + `date` date NOT NULL, + `area` varchar(255) NOT NULL DEFAULT '', + `fence` varchar(255) NOT NULL DEFAULT '', + `character` smallint unsigned NOT NULL, + `slot` tinyint unsigned NOT NULL, + `pokemon_id` smallint unsigned NOT NULL, + `form_id` smallint unsigned NOT NULL DEFAULT 0, + `count` int NOT NULL, + PRIMARY KEY (`date`, `area`, `fence`, `character`, `slot`, `pokemon_id`, `form_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; diff --git a/sql/rdm/rdmdb.sql b/sql/rdm/rdmdb.sql index 9b6b9637..aee9bbd5 100644 --- a/sql/rdm/rdmdb.sql +++ b/sql/rdm/rdmdb.sql @@ -386,6 +386,26 @@ CREATE TABLE `invasion_stats` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; /*!40101 SET character_set_client = @saved_cs_client */; +-- +-- Table structure for table `invasion_lineup_stats` +-- + +DROP TABLE IF EXISTS `invasion_lineup_stats`; +/*!40101 SET @saved_cs_client = @@character_set_client */; +/*!50503 SET character_set_client = utf8mb4 */; +CREATE TABLE `invasion_lineup_stats` ( + `date` date NOT NULL, + `area` varchar(255) NOT NULL DEFAULT '', + `fence` varchar(255) NOT NULL DEFAULT '', + `character` smallint unsigned NOT NULL, + `slot` tinyint unsigned NOT NULL, + `pokemon_id` smallint unsigned NOT NULL, + `form_id` smallint unsigned NOT NULL DEFAULT '0', + `count` int NOT NULL, + PRIMARY KEY (`date`,`area`,`fence`,`character`,`slot`,`pokemon_id`,`form_id`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci; +/*!40101 SET character_set_client = @saved_cs_client */; + -- -- Table structure for table `metadata` -- diff --git a/stats.go b/stats.go index 1cf549f0..d8bc4c31 100644 --- a/stats.go +++ b/stats.go @@ -148,7 +148,7 @@ func StartStatsExpiry(db *sqlx.DB) { log.Infof("DB - Cleanup of pokemon_area_stats table took %s (%d rows)", elapsed, rows) } - tables := []string{"pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats", "invasion_stats", "quest_stats", "raid_stats"} + tables := []string{"pokemon_stats", "pokemon_shiny_stats", "pokemon_iv_stats", "pokemon_hundo_stats", "pokemon_nundo_stats", "invasion_stats", "invasion_lineup_stats", "quest_stats", "raid_stats"} for _, table := range tables { start = time.Now()