Add fleet statistics

.github/workflows/docker-publish.yml

@@ -50,15 +50,16 @@ jobs:
             type=sha,format=short
             type=ref,event=branch
             type=ref,event=pr
-            latest
+            type=raw,value=pr-${{ github.head_ref || github.ref_name }},enable=${{ github.event_name == 'pull_request' }}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
 
       # Build and push Docker image
       - name: Build and push Docker image
         id: build-and-push
         uses: docker/build-push-action@v4
         with:
           context: ./go-rewrite
-          push: ${{ github.event_name != 'pull_request' }}
+          push: true
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.gitignore

@@ -1,6 +1,7 @@
-*.html
+
 *.json
 bmwtools-server
+go-rewrite/data/bmwtools.db
 go-rewrite/bmwtools-server
 go-rewrite/traefik/*
 go-rewrite/pkg/data/real_data_test.go

README.md

@@ -1 +1,31 @@
-Most of this is just generated with CoPilot as I was very lazy with this :D. go-rewrite in `go-rewrite`
+# BMW Tools
+
+This repository contains tools for analyzing BMW electric vehicle charging data from the My BMW app.
+
+## Overview
+
+BMW Tools allows you to analyze your charging sessions, track battery health, visualize charging locations, and gain insights into your EV's performance over time.
+
+## Project Structure
+
+- **Python Version**: Original implementation in the root directory
+- **Go Version**: Improved implementation in the `go-rewrite` folder with better performance and features
+
+## Getting Started
+
+For the latest version with improved performance and battery health tracking, see the Go implementation:
+
+```
+cd go-rewrite
+```
+
+Refer to the [Go Version README](/go-rewrite/README.md) for detailed instructions on installation and usage.
+
+## Features
+
+- Upload and analyze BMW CarData JSON files
+- Interactive dashboard with visualizations
+- Track battery health over time
+- Analyze charging efficiency and power consumption
+- View charging locations on a map
+- Compare charging sessions and providers

go-rewrite/BATTERY-HEALTH.md

@@ -0,0 +1,73 @@
+# BMW CarData Battery Health Tracking
+
+This extension to the BMW CarData Tools adds persistent storage and fleet-wide battery health tracking, allowing you to:
+
+1. Store all uploaded charging data in SQLite for persistence between service restarts
+2. Track battery degradation over time across the fleet, similar to TeslaLogger
+3. Filter data by different BMW models
+4. Prevent duplicate uploads with content hashing
+5. Preserve privacy through FIN (vehicle ID) hashing
+
+## New Features
+
+### Database Storage
+
+All charging data is now stored in an SQLite database located in the `./data` directory, making it persistent between application restarts. The original JSON files are never stored - only the processed, anonymized data is saved.
+
+### Battery Health Tracking
+
+The system now tracks battery health (estimated capacity) over time, calculated from charging sessions with significant SOC changes. This gives you insights into battery degradation patterns across your fleet.
+
+### Model Filtering
+
+You can now filter battery health data by different BMW models, allowing you to compare degradation patterns across different vehicle types.
+
+### Privacy Protection
+
+Vehicle identification numbers (FINs) are securely hashed before storage to maintain privacy while still allowing tracking of individual vehicles over time. No personal data or location information is stored in a way that could identify users.
+
+## Using the Battery Health Dashboard
+
+1. Navigate to `/battery` in your browser
+2. View fleet-wide battery health trends
+3. Use the model filter to focus on specific BMW models
+4. Analyze both individual data points and the monthly trend line
+
+## Technical Implementation
+
+- SQLite database for persistent storage
+- Content hashing to prevent duplicate uploads
+- One-way hashing for vehicle identifiers
+- Responsive visualization using Plotly.js
+
+## Database Schema
+
+The system uses the following tables:
+
+- `uploads`: Tracks uploaded files with content hashes to prevent duplicates
+- `vehicles`: Stores hashed vehicle identifiers and models
+- `sessions`: Stores anonymized charging session data
+- `battery_health`: Tracks battery capacity estimates over time with mileage information
+
+**Note:** If you're upgrading from a previous version, you may need to run the database migration script to add the mileage column. See [MIGRATION.md](MIGRATION.md) for details.
+
+## Building with Database Support
+
+```bash
+# Install SQLite dependencies and build
+make setup-db
+make build
+
+# Or do everything at once
+make all
+```
+
+## Docker Deployment
+
+The Docker configuration includes a persistent volume for the database:
+
+```bash
+docker-compose up -d
+```
+
+This will mount a `./data` directory to store the database file.

go-rewrite/Dockerfile

@@ -1,6 +1,10 @@
 # Build stage
 FROM golang:1.22-alpine AS builder
 
+# Install build dependencies including gcc and required libraries
+RUN apk update && apk add --no-cache gcc g++ make musl-dev libc-dev sqlite-dev && \
+    gcc --version
+
 WORKDIR /app
 
 # Copy go mod and sum files
@@ -13,11 +17,14 @@ RUN go mod download
 COPY . .
 
 # Build the application
-RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o bmwtools-server ./cmd/server/main.go
+RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o bmwtools-server ./cmd/server/main.go
 
 # Final stage
 FROM alpine:latest
 
+# Add runtime dependencies if any CGO-compiled libraries are needed
+RUN apk add --no-cache sqlite-libs
+
 WORKDIR /app
 
 # Copy the binary from builder

go-rewrite/Makefile

@@ -8,10 +8,10 @@ GO_MODULE=github.com/awlx/bmwtools
 # Go build flags
 LDFLAGS=-ldflags "-s -w"
 
-.PHONY: all build clean run docker-build docker-run docker-stop test lint vet fmt help
+.PHONY: all build clean run docker-build docker-run docker-stop test lint vet fmt help setup-db
 
 # Default target
-all: clean build
+all: clean setup-db build
 
 # Build the application
 build:
@@ -75,6 +75,11 @@ compose-down:
 	@echo "Stopping Docker Compose services..."
 	docker-compose down
 
+# Setup database dependencies
+setup-db:
+	@echo "Setting up database dependencies..."
+	go get github.com/mattn/go-sqlite3
+
 # Help
 help:
 	@echo "BMW CarData Dashboard Go Implementation"
@@ -87,6 +92,7 @@ help:
 	@echo "  make lint           - Run linter"
 	@echo "  make vet            - Run go vet"
 	@echo "  make fmt            - Format code"
+	@echo "  make setup-db       - Set up database dependencies"
 	@echo "  make docker-build   - Build Docker image"
 	@echo "  make docker-run     - Run in Docker container"
 	@echo "  make docker-stop    - Stop Docker container"

go-rewrite/README.md

@@ -67,6 +67,7 @@ The application will be available at https://bmwtools.localhost (or your configu
 #### Customizing the Domain
 
 Edit the `docker-compose.yaml` file and update the domain in the Traefik labels:
+
 ```yaml
 labels:
   - "traefik.http.routers.bmwtools-server.rule=Host(`your-domain.com`)"
@@ -116,6 +117,11 @@ The Go implementation offers several advantages over the original Python version
 - **Deployment**: Single binary deployment
 - **Security**: Strong type system and memory safety
 
+## Database Migration
+
+If you're upgrading from a previous version, you may need to migrate your database schema.
+See [MIGRATION.md](MIGRATION.md) for instructions on updating your database.
+
 ## Disclaimer
 
 This application stores all uploaded data in memory. If you refresh, your session is lost.

go-rewrite/TEST_README.md

@@ -0,0 +1,77 @@
+# Session Isolation Test Scripts
+
+These scripts are designed to test the session isolation capabilities of the BMW Tools application to ensure that user data doesn't leak between sessions. The scripts include extensive debugging capabilities to help diagnose any issues with cookie handling or session management.
+
+## Basic Session Isolation Test
+
+The `test_session_isolation.sh` script performs a simple test by uploading two different JSON files in parallel and verifying that each session only has access to its own data.
+
+### Usage:
+
+```bash
+./test_session_isolation.sh <json_file1> <json_file2>
+```
+
+Example:
+```bash
+./test_session_isolation.sh BMW-CarData-Ladehistorie_*.json BMW-CarData-Ladehistorie_*.json
+```
+
+By default, the script connects to a server running at `http://localhost:8080`. You can change this by setting the `SERVER_URL` environment variable:
+
+```bash
+SERVER_URL=http://your-server-url:8080 ./test_session_isolation.sh file1.json file2.json
+```
+
+## Comprehensive Session Isolation Test
+
+The `test_comprehensive_isolation.sh` script performs a more thorough test by checking isolation across multiple API endpoints:
+- `/api/sessions` - Verifies session data isolation
+- `/api/stats` - Verifies statistics data isolation
+- `/api/map` - Verifies map data isolation
+- `/api/providers` - Verifies provider data isolation
+
+### Usage:
+
+```bash
+./test_comprehensive_isolation.sh <json_file1> <json_file2>
+```
+
+Example:
+```bash
+./test_comprehensive_isolation.sh BMW-CarData-Ladehistorie_1.json BMW-CarData-Ladehistorie_2.json
+```
+
+## Test Results
+
+Both scripts create a temporary directory to store test results. The path to this directory is displayed at the end of the test run. You can inspect the JSON responses from each endpoint to verify that data isolation is working correctly.
+
+### Debugging Information
+
+The test scripts generate extensive debugging information to help diagnose any issues with cookie handling or session management:
+
+- Verbose curl output with request/response headers
+- Cookie file contents
+- Raw session IDs extracted from cookies
+- Raw API responses for inspection
+- Detailed error messages when tests fail
+
+All this information is saved in the temporary output directory, which is displayed at the end of the test run.
+
+## Testing With Delays
+
+The scripts include deliberate delays between requests to ensure that concurrent sessions are properly isolated even under load. This simulates real-world scenarios where multiple users might be interacting with the system simultaneously.
+
+## Troubleshooting
+
+If the tests fail with "Session count mismatch" errors, check the following:
+
+1. **Cookie Handling**: Make sure the server is setting the `session_id` cookie properly. Inspect the verbose curl output in the debug files.
+
+2. **Server Configuration**: Verify that the server is running on the expected port (default: 8080). You can change this using the `SERVER_URL` environment variable.
+
+3. **Session Timeout**: If sessions are expiring too quickly, it could cause tests to fail. Check the session expiration time in the server code.
+
+4. **Raw Responses**: Examine the raw API responses saved in the output directory to see what data the server is actually returning.
+
+5. **Server Logs**: Check the server logs for any errors or warnings related to session handling.

go-rewrite/cmd/server/main.go

@@ -5,11 +5,14 @@ import (
 	"log"
 	"net/http"
 	"os"
+	"path/filepath"
 
 	"github.com/awlx/bmwtools/pkg/api"
 	"github.com/awlx/bmwtools/pkg/data"
+	"github.com/awlx/bmwtools/pkg/database"
 	"github.com/gin-contrib/cors"
 	"github.com/gin-gonic/gin"
+	_ "github.com/mattn/go-sqlite3"
 )
 
 func main() {
@@ -26,8 +29,22 @@ func main() {
 	// Initialize the data manager
 	dataManager := data.NewManager()
 
+	// Initialize the database manager
+	// Create data directory if it doesn't exist
+	dataDir := filepath.Join(".", "data")
+	if err := os.MkdirAll(dataDir, 0755); err != nil {
+		log.Fatalf("Failed to create data directory: %v", err)
+	}
+
+	dbPath := filepath.Join(dataDir, "bmwtools.db")
+	dbManager, err := database.New(dbPath)
+	if err != nil {
+		log.Fatalf("Failed to initialize database: %v", err)
+	}
+	defer dbManager.Close()
+
 	// Create the API handler
-	apiHandler := api.NewHandler(dataManager)
+	apiHandler := api.NewHandler(dataManager, dbManager)
 
 	// Set up routes
 	// API routes
@@ -39,10 +56,22 @@ func main() {
 	r.GET("/api/map", apiHandler.GetMapData)
 	r.GET("/api/grouped-providers", apiHandler.GetGroupedProviders)
 	r.GET("/api/version", apiHandler.GetVersion)
+	r.GET("/api/anonymous-stats", apiHandler.GetAnonymousStats)
+	r.GET("/api/battery-health", apiHandler.GetBatteryHealth)
+
+	// Create a custom static file handler with cache control headers
+	staticHandler := func(c *gin.Context) {
+		c.Header("Cache-Control", "no-cache, no-store, must-revalidate")
+		c.Header("Pragma", "no-cache")
+		c.Header("Expires", "0")
+		c.Next()
+	}
 
-	// Static file serving for the frontend
-	r.StaticFS("/static", http.Dir("./static"))
+	// Static file serving for the frontend with cache control
+	r.Group("/static").Use(staticHandler).StaticFS("", http.Dir("./static"))
 	r.StaticFile("/", "./static/index.html")
+	r.StaticFile("/stats", "./static/stats.html")
+	r.StaticFile("/battery", "./static/battery.html")
 
 	// Add a catch-all route for SPA
 	r.NoRoute(func(c *gin.Context) {

go-rewrite/docker-compose.yaml

@@ -7,6 +7,8 @@ services:
     restart: always
     environment:
       - PORT=8050
+    volumes:
+      - ./data:/app/data
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.bmwtools-server.rule=Host(`bmwtools.localhost`)"

go-rewrite/go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/kr/text v0.2.0 // indirect
 	github.com/leodido/go-urn v1.3.0 // indirect
 	github.com/mattn/go-isatty v0.0.19 // indirect
+	github.com/mattn/go-sqlite3 v1.14.28 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/pelletier/go-toml/v2 v2.1.0 // indirect

go-rewrite/go.sum

@@ -48,6 +48,8 @@ github.com/leodido/go-urn v1.3.0 h1:jX8FDLfW4ThVXctBNZ+3cIWnCSnrACDV73r76dy0aQQ=
 github.com/leodido/go-urn v1.3.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
 github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
+github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=