diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 133f396..26323f7 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -50,7 +50,8 @@ jobs: type=sha,format=short type=ref,event=branch type=ref,event=pr - latest + type=raw,value=pr-${{ github.head_ref || github.ref_name }},enable=${{ github.event_name == 'pull_request' }} + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} # Build and push Docker image - name: Build and push Docker image @@ -58,7 +59,7 @@ jobs: uses: docker/build-push-action@v4 with: context: ./go-rewrite - push: ${{ github.event_name != 'pull_request' }} + push: true platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.gitignore b/.gitignore index c5be27d..ba653d8 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ -*.html + *.json bmwtools-server +go-rewrite/data/bmwtools.db go-rewrite/bmwtools-server go-rewrite/traefik/* go-rewrite/pkg/data/real_data_test.go diff --git a/README.md b/README.md index a18bff6..c55f0ab 100644 --- a/README.md +++ b/README.md @@ -1 +1,31 @@ -Most of this is just generated with CoPilot as I was very lazy with this :D. go-rewrite in `go-rewrite` +# BMW Tools + +This repository contains tools for analyzing BMW electric vehicle charging data from the My BMW app. + +## Overview + +BMW Tools allows you to analyze your charging sessions, track battery health, visualize charging locations, and gain insights into your EV's performance over time. + +## Project Structure + +- **Python Version**: Original implementation in the root directory +- **Go Version**: Improved implementation in the `go-rewrite` folder with better performance and features + +## Getting Started + +For the latest version with improved performance and battery health tracking, see the Go implementation: + +``` +cd go-rewrite +``` + +Refer to the [Go Version README](/go-rewrite/README.md) for detailed instructions on installation and usage. + +## Features + +- Upload and analyze BMW CarData JSON files +- Interactive dashboard with visualizations +- Track battery health over time +- Analyze charging efficiency and power consumption +- View charging locations on a map +- Compare charging sessions and providers diff --git a/go-rewrite/BATTERY-HEALTH.md b/go-rewrite/BATTERY-HEALTH.md new file mode 100644 index 0000000..a01c976 --- /dev/null +++ b/go-rewrite/BATTERY-HEALTH.md @@ -0,0 +1,73 @@ +# BMW CarData Battery Health Tracking + +This extension to the BMW CarData Tools adds persistent storage and fleet-wide battery health tracking, allowing you to: + +1. Store all uploaded charging data in SQLite for persistence between service restarts +2. Track battery degradation over time across the fleet, similar to TeslaLogger +3. Filter data by different BMW models +4. Prevent duplicate uploads with content hashing +5. Preserve privacy through FIN (vehicle ID) hashing + +## New Features + +### Database Storage + +All charging data is now stored in an SQLite database located in the `./data` directory, making it persistent between application restarts. The original JSON files are never stored - only the processed, anonymized data is saved. + +### Battery Health Tracking + +The system now tracks battery health (estimated capacity) over time, calculated from charging sessions with significant SOC changes. This gives you insights into battery degradation patterns across your fleet. + +### Model Filtering + +You can now filter battery health data by different BMW models, allowing you to compare degradation patterns across different vehicle types. + +### Privacy Protection + +Vehicle identification numbers (FINs) are securely hashed before storage to maintain privacy while still allowing tracking of individual vehicles over time. No personal data or location information is stored in a way that could identify users. + +## Using the Battery Health Dashboard + +1. Navigate to `/battery` in your browser +2. View fleet-wide battery health trends +3. Use the model filter to focus on specific BMW models +4. Analyze both individual data points and the monthly trend line + +## Technical Implementation + +- SQLite database for persistent storage +- Content hashing to prevent duplicate uploads +- One-way hashing for vehicle identifiers +- Responsive visualization using Plotly.js + +## Database Schema + +The system uses the following tables: + +- `uploads`: Tracks uploaded files with content hashes to prevent duplicates +- `vehicles`: Stores hashed vehicle identifiers and models +- `sessions`: Stores anonymized charging session data +- `battery_health`: Tracks battery capacity estimates over time with mileage information + +**Note:** If you're upgrading from a previous version, you may need to run the database migration script to add the mileage column. See [MIGRATION.md](MIGRATION.md) for details. + +## Building with Database Support + +```bash +# Install SQLite dependencies and build +make setup-db +make build + +# Or do everything at once +make all +``` + +## Docker Deployment + +The Docker configuration includes a persistent volume for the database: + +```bash +docker-compose up -d +``` + +This will mount a `./data` directory to store the database file. diff --git a/go-rewrite/Dockerfile b/go-rewrite/Dockerfile index f25c06f..cb0979b 100644 --- a/go-rewrite/Dockerfile +++ b/go-rewrite/Dockerfile @@ -1,6 +1,10 @@ # Build stage FROM golang:1.22-alpine AS builder +# Install build dependencies including gcc and required libraries +RUN apk update && apk add --no-cache gcc g++ make musl-dev libc-dev sqlite-dev && \ + gcc --version + WORKDIR /app # Copy go mod and sum files @@ -13,11 +17,14 @@ RUN go mod download COPY . . # Build the application -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o bmwtools-server ./cmd/server/main.go +RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o bmwtools-server ./cmd/server/main.go # Final stage FROM alpine:latest +# Add runtime dependencies if any CGO-compiled libraries are needed +RUN apk add --no-cache sqlite-libs + WORKDIR /app # Copy the binary from builder diff --git a/go-rewrite/Makefile b/go-rewrite/Makefile index b0a3afe..f910bf1 100644 --- a/go-rewrite/Makefile +++ b/go-rewrite/Makefile @@ -8,10 +8,10 @@ GO_MODULE=github.com/awlx/bmwtools # Go build flags LDFLAGS=-ldflags "-s -w" -.PHONY: all build clean run docker-build docker-run docker-stop test lint vet fmt help +.PHONY: all build clean run docker-build docker-run docker-stop test lint vet fmt help setup-db # Default target -all: clean build +all: clean setup-db build # Build the application build: @@ -75,6 +75,11 @@ compose-down: @echo "Stopping Docker Compose services..." docker-compose down +# Setup database dependencies +setup-db: + @echo "Setting up database dependencies..." + go get github.com/mattn/go-sqlite3 + # Help help: @echo "BMW CarData Dashboard Go Implementation" @@ -87,6 +92,7 @@ help: @echo " make lint - Run linter" @echo " make vet - Run go vet" @echo " make fmt - Format code" + @echo " make setup-db - Set up database dependencies" @echo " make docker-build - Build Docker image" @echo " make docker-run - Run in Docker container" @echo " make docker-stop - Stop Docker container" diff --git a/go-rewrite/README.md b/go-rewrite/README.md index be566b7..a1e9d8e 100644 --- a/go-rewrite/README.md +++ b/go-rewrite/README.md @@ -67,6 +67,7 @@ The application will be available at https://bmwtools.localhost (or your configu #### Customizing the Domain Edit the `docker-compose.yaml` file and update the domain in the Traefik labels: + ```yaml labels: - "traefik.http.routers.bmwtools-server.rule=Host(`your-domain.com`)" @@ -116,6 +117,11 @@ The Go implementation offers several advantages over the original Python version - **Deployment**: Single binary deployment - **Security**: Strong type system and memory safety +## Database Migration + +If you're upgrading from a previous version, you may need to migrate your database schema. +See [MIGRATION.md](MIGRATION.md) for instructions on updating your database. + ## Disclaimer This application stores all uploaded data in memory. If you refresh, your session is lost. diff --git a/go-rewrite/TEST_README.md b/go-rewrite/TEST_README.md new file mode 100644 index 0000000..cf20879 --- /dev/null +++ b/go-rewrite/TEST_README.md @@ -0,0 +1,77 @@ +# Session Isolation Test Scripts + +These scripts are designed to test the session isolation capabilities of the BMW Tools application to ensure that user data doesn't leak between sessions. The scripts include extensive debugging capabilities to help diagnose any issues with cookie handling or session management. + +## Basic Session Isolation Test + +The `test_session_isolation.sh` script performs a simple test by uploading two different JSON files in parallel and verifying that each session only has access to its own data. + +### Usage: + +```bash +./test_session_isolation.sh +``` + +Example: +```bash +./test_session_isolation.sh BMW-CarData-Ladehistorie_*.json BMW-CarData-Ladehistorie_*.json +``` + +By default, the script connects to a server running at `http://localhost:8080`. You can change this by setting the `SERVER_URL` environment variable: + +```bash +SERVER_URL=http://your-server-url:8080 ./test_session_isolation.sh file1.json file2.json +``` + +## Comprehensive Session Isolation Test + +The `test_comprehensive_isolation.sh` script performs a more thorough test by checking isolation across multiple API endpoints: +- `/api/sessions` - Verifies session data isolation +- `/api/stats` - Verifies statistics data isolation +- `/api/map` - Verifies map data isolation +- `/api/providers` - Verifies provider data isolation + +### Usage: + +```bash +./test_comprehensive_isolation.sh +``` + +Example: +```bash +./test_comprehensive_isolation.sh BMW-CarData-Ladehistorie_1.json BMW-CarData-Ladehistorie_2.json +``` + +## Test Results + +Both scripts create a temporary directory to store test results. The path to this directory is displayed at the end of the test run. You can inspect the JSON responses from each endpoint to verify that data isolation is working correctly. + +### Debugging Information + +The test scripts generate extensive debugging information to help diagnose any issues with cookie handling or session management: + +- Verbose curl output with request/response headers +- Cookie file contents +- Raw session IDs extracted from cookies +- Raw API responses for inspection +- Detailed error messages when tests fail + +All this information is saved in the temporary output directory, which is displayed at the end of the test run. + +## Testing With Delays + +The scripts include deliberate delays between requests to ensure that concurrent sessions are properly isolated even under load. This simulates real-world scenarios where multiple users might be interacting with the system simultaneously. + +## Troubleshooting + +If the tests fail with "Session count mismatch" errors, check the following: + +1. **Cookie Handling**: Make sure the server is setting the `session_id` cookie properly. Inspect the verbose curl output in the debug files. + +2. **Server Configuration**: Verify that the server is running on the expected port (default: 8080). You can change this using the `SERVER_URL` environment variable. + +3. **Session Timeout**: If sessions are expiring too quickly, it could cause tests to fail. Check the session expiration time in the server code. + +4. **Raw Responses**: Examine the raw API responses saved in the output directory to see what data the server is actually returning. + +5. **Server Logs**: Check the server logs for any errors or warnings related to session handling. \ No newline at end of file diff --git a/go-rewrite/cmd/server/main.go b/go-rewrite/cmd/server/main.go index a73bfe6..8186ec3 100644 --- a/go-rewrite/cmd/server/main.go +++ b/go-rewrite/cmd/server/main.go @@ -5,11 +5,14 @@ import ( "log" "net/http" "os" + "path/filepath" "github.com/awlx/bmwtools/pkg/api" "github.com/awlx/bmwtools/pkg/data" + "github.com/awlx/bmwtools/pkg/database" "github.com/gin-contrib/cors" "github.com/gin-gonic/gin" + _ "github.com/mattn/go-sqlite3" ) func main() { @@ -26,8 +29,22 @@ func main() { // Initialize the data manager dataManager := data.NewManager() + // Initialize the database manager + // Create data directory if it doesn't exist + dataDir := filepath.Join(".", "data") + if err := os.MkdirAll(dataDir, 0755); err != nil { + log.Fatalf("Failed to create data directory: %v", err) + } + + dbPath := filepath.Join(dataDir, "bmwtools.db") + dbManager, err := database.New(dbPath) + if err != nil { + log.Fatalf("Failed to initialize database: %v", err) + } + defer dbManager.Close() + // Create the API handler - apiHandler := api.NewHandler(dataManager) + apiHandler := api.NewHandler(dataManager, dbManager) // Set up routes // API routes @@ -39,10 +56,22 @@ func main() { r.GET("/api/map", apiHandler.GetMapData) r.GET("/api/grouped-providers", apiHandler.GetGroupedProviders) r.GET("/api/version", apiHandler.GetVersion) + r.GET("/api/anonymous-stats", apiHandler.GetAnonymousStats) + r.GET("/api/battery-health", apiHandler.GetBatteryHealth) + + // Create a custom static file handler with cache control headers + staticHandler := func(c *gin.Context) { + c.Header("Cache-Control", "no-cache, no-store, must-revalidate") + c.Header("Pragma", "no-cache") + c.Header("Expires", "0") + c.Next() + } - // Static file serving for the frontend - r.StaticFS("/static", http.Dir("./static")) + // Static file serving for the frontend with cache control + r.Group("/static").Use(staticHandler).StaticFS("", http.Dir("./static")) r.StaticFile("/", "./static/index.html") + r.StaticFile("/stats", "./static/stats.html") + r.StaticFile("/battery", "./static/battery.html") // Add a catch-all route for SPA r.NoRoute(func(c *gin.Context) { diff --git a/go-rewrite/docker-compose.yaml b/go-rewrite/docker-compose.yaml index 7a43793..728a94d 100644 --- a/go-rewrite/docker-compose.yaml +++ b/go-rewrite/docker-compose.yaml @@ -7,6 +7,8 @@ services: restart: always environment: - PORT=8050 + volumes: + - ./data:/app/data labels: - "traefik.enable=true" - "traefik.http.routers.bmwtools-server.rule=Host(`bmwtools.localhost`)" diff --git a/go-rewrite/go.mod b/go-rewrite/go.mod index 10e4d55..1883f1e 100644 --- a/go-rewrite/go.mod +++ b/go-rewrite/go.mod @@ -22,6 +22,7 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.3.0 // indirect github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-sqlite3 v1.14.28 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.1.0 // indirect diff --git a/go-rewrite/go.sum b/go-rewrite/go.sum index dc02c78..dbd3bbc 100644 --- a/go-rewrite/go.sum +++ b/go-rewrite/go.sum @@ -48,6 +48,8 @@ github.com/leodido/go-urn v1.3.0 h1:jX8FDLfW4ThVXctBNZ+3cIWnCSnrACDV73r76dy0aQQ= github.com/leodido/go-urn v1.3.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A= +github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= diff --git a/go-rewrite/pkg/api/battery_health.go b/go-rewrite/pkg/api/battery_health.go new file mode 100644 index 0000000..17cee7a --- /dev/null +++ b/go-rewrite/pkg/api/battery_health.go @@ -0,0 +1,68 @@ +package api + +import ( + "fmt" + "net/http" + + "github.com/gin-gonic/gin" +) + +// GetBatteryHealth returns battery health data for the entire fleet or filtered by model +func (h *Handler) GetBatteryHealth(c *gin.Context) { + // Get optional model filter + model := c.Query("model") + + // Check if database is initialized + if h.dbManager == nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database manager is not initialized"}) + return + } + + // Return empty data if battery_health table doesn't exist or has no data + // This prevents 500 errors when there's no battery health data yet + rawData := []map[string]interface{}{} + trendData := []map[string]interface{}{} + availableModels := []string{} + + // Get the raw data points - with error handling + rawData, err := h.dbManager.GetFleetBatteryHealth(model) + if err != nil { + // Log the error but don't fail + fmt.Printf("Error getting battery health data: %v\n", err) + // Return empty data instead of error + c.JSON(http.StatusOK, gin.H{ + "battery_health_data": []map[string]interface{}{}, + "available_models": []string{}, + "model_filter": model, + "error": fmt.Sprintf("Failed to get battery health data: %v", err), + }) + return + } + + // Get the monthly trend data (aggregated) - with error handling + trendData, err = h.dbManager.GetMonthlyBatteryHealthTrend(model) + if err != nil { + // Log the error but don't fail + fmt.Printf("Error getting battery health trend: %v\n", err) + // We can continue with just the raw data + trendData = []map[string]interface{}{} + } + + // Get available models for filtering + availableModels, err = h.dbManager.GetAvailableModels() + if err != nil { + // Non-critical error, we can continue + fmt.Printf("Error getting available models: %v\n", err) + availableModels = []string{} + } + + // Merge raw data and trend data + allData := append(rawData, trendData...) + + // Return the data + c.JSON(http.StatusOK, gin.H{ + "battery_health_data": allData, + "available_models": availableModels, + "model_filter": model, + }) +} diff --git a/go-rewrite/pkg/api/handler.go b/go-rewrite/pkg/api/handler.go index cf3a945..365e7b4 100644 --- a/go-rewrite/pkg/api/handler.go +++ b/go-rewrite/pkg/api/handler.go @@ -2,26 +2,33 @@ package api import ( "fmt" + "log" "net/http" "os" "time" "github.com/awlx/bmwtools/pkg/data" + "github.com/awlx/bmwtools/pkg/database" "github.com/gin-gonic/gin" ) // AppVersion is the current version of the application -const AppVersion = "1.0.0" +const AppVersion = "1.0.1" // Handler handles API requests type Handler struct { - dataManager *data.Manager + dbManager *database.Manager + sessions *SessionStore } // NewHandler creates a new API handler -func NewHandler(dataManager *data.Manager) *Handler { +func NewHandler(_ *data.Manager, dbManager *database.Manager) *Handler { + // Create a new session store with 30 minute expiration time + sessionStore := NewSessionStore(30 * time.Minute) + return &Handler{ - dataManager: dataManager, + dbManager: dbManager, + sessions: sessionStore, } } @@ -32,27 +39,145 @@ func (h *Handler) GetVersion(c *gin.Context) { }) } +// GetAnonymousStats returns anonymous statistics about charging sessions +func (h *Handler) GetAnonymousStats(c *gin.Context) { + // Check if we have a database manager + if h.dbManager == nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": "Database not initialized"}) + return + } + + // Get provider stats from database + providerStats, err := h.dbManager.GetProviderStats() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error fetching provider stats: %v", err)}) + return + } + + // Get SOC statistics from database + socStats, err := h.dbManager.GetSOCStats() + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error fetching SOC stats: %v", err)}) + return + } + + // Calculate global statistics + var totalSessions, totalSuccessful, totalFailed int + var totalEnergyAdded float64 + + for _, provider := range providerStats { + totalSessions += provider["total_sessions"].(int) + totalSuccessful += provider["successful_sessions"].(int) + totalFailed += provider["failed_sessions"].(int) + totalEnergyAdded += provider["total_energy_added"].(float64) + } + + // Calculate global success rate + globalSuccessRate := 0.0 + if totalSessions > 0 { + globalSuccessRate = float64(totalSuccessful) / float64(totalSessions) * 100 + } + + c.JSON(http.StatusOK, gin.H{ + "providers": providerStats, + "global_stats": map[string]interface{}{ + "total_sessions": totalSessions, + "successful_sessions": totalSuccessful, + "failed_sessions": totalFailed, + "success_rate": globalSuccessRate, + "total_energy_added": totalEnergyAdded, + }, + "soc_stats": socStats, + }) +} + // UploadJSON handles JSON file uploads func (h *Handler) UploadJSON(c *gin.Context) { // Get the file from the request - file, _, err := c.Request.FormFile("file") + file, fileHeader, err := c.Request.FormFile("file") if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "No file uploaded"}) return } defer file.Close() - // Process the file - err = h.dataManager.LoadJSON(file) + // Get or create a session for this user + sessionID, err := c.Cookie("session_id") + if err != nil { + sessionID = "" + } + sessionID, session := h.sessions.GetOrCreateSession(sessionID) + + // Always set the cookie to refresh expiration time + // Set cookie to expire in 30 minutes + c.SetCookie("session_id", sessionID, 1800, "/", "", false, true) + + // Process the file with the session's data manager + err = session.DataManager.LoadJSON(file) if err != nil { c.JSON(http.StatusBadRequest, gin.H{"error": fmt.Sprintf("Error processing JSON: %v", err)}) return } - // Return success + // Get the processed sessions + sessions := session.DataManager.GetSessions() + + // Check for consent to store in fleet statistics database + consentParam := c.PostForm("consent") + hasConsent := consentParam == "true" || consentParam == "1" || consentParam == "yes" + + // Get the BMW model if provided + model := c.PostForm("model") + + // Store sessions in the database if consent was given + isNew := false + var storedCount int + if h.dbManager != nil && hasConsent { + // Validate model parameter is provided when consent is given + if model == "" { + c.JSON(http.StatusBadRequest, gin.H{"error": "BMW model selection is required when sharing data"}) + return + } + + var err error + isNew, err = h.dbManager.StoreSessions(sessions, fileHeader.Filename, model) + if err != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error storing sessions in database: %v", err)}) + return + } + + // Query the database to get an accurate count of stored sessions + // This ensures that the count shown in the UI matches what's actually in the database + var countErr error + storedCount, countErr = h.dbManager.GetSessionCount() + if countErr != nil { + log.Printf("Warning: Unable to get session count: %v", countErr) + storedCount = len(sessions) // Fallback to local count if query fails + } + + // If this was a duplicate upload, notify the user + if !isNew { + c.JSON(http.StatusOK, gin.H{ + "message": "This file has already been processed previously", + "count": len(sessions), + "new": false, + "stored": hasConsent, + "stored_count": storedCount, + }) + return + } + } // Return success with appropriate message based on consent + message := "File uploaded and processed successfully" + if !hasConsent { + message = "File uploaded and processed (not stored in fleet statistics)" + storedCount = 0 + } c.JSON(http.StatusOK, gin.H{ - "message": "File uploaded and processed successfully", - "count": len(h.dataManager.GetSessions()), + "message": message, + "count": len(sessions), + "new": true, + "stored": hasConsent, + "stored_count": storedCount, }) } @@ -66,42 +191,78 @@ func (h *Handler) LoadDemoData(c *gin.Context) { } defer file.Close() - // Process the file - err = h.dataManager.LoadJSON(file) + // Get or create a session for this user + sessionID, err := c.Cookie("session_id") + if err != nil { + sessionID = "" + } + sessionID, session := h.sessions.GetOrCreateSession(sessionID) + + // Always set the cookie to refresh expiration time + // Set cookie to expire in 30 minutes + c.SetCookie("session_id", sessionID, 1800, "/", "", false, true) + + // Process the file with the session's data manager + err = session.DataManager.LoadJSON(file) if err != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("Error processing demo data: %v", err)}) return } + // Get the processed sessions + sessions := session.DataManager.GetSessions() + + // Demo data is intentionally NOT stored in the database + // This is because demo data is just for testing/preview and shouldn't + // pollute the actual fleet statistics + // Return success c.JSON(http.StatusOK, gin.H{ - "message": "Demo data loaded successfully", - "count": len(h.dataManager.GetSessions()), + "message": "Demo data loaded successfully (not stored in database)", + "count": len(sessions), + "demo": true, }) } // GetSessions returns all sessions or filtered by date range func (h *Handler) GetSessions(c *gin.Context) { - var sessions []data.Session + // Get the user's session + sessionID, err := c.Cookie("session_id") + if err != nil { + // No session found, return empty data + c.JSON(http.StatusOK, []map[string]interface{}{}) + return + } - // Get date range parameters + // Get the session data + session, exists := h.sessions.GetSession(sessionID) + if !exists { + // Session expired, return empty data + c.JSON(http.StatusOK, []map[string]interface{}{}) + return + } + + // Use the session's data manager + var sessions []data.Session = session.DataManager.GetSessions() + + // Apply any date filters if provided startDateStr := c.Query("startDate") endDateStr := c.Query("endDate") - if startDateStr != "" && endDateStr != "" { + if startDateStr != "" && endDateStr != "" && len(sessions) > 0 { startDate, err1 := time.Parse("2006-01-02", startDateStr) endDate, err2 := time.Parse("2006-01-02", endDateStr) if err1 == nil && err2 == nil { // Add a day to endDate to make it inclusive endDate = endDate.AddDate(0, 0, 1) - sessions = h.dataManager.GetSessionsByDateRange(startDate, endDate) + + // Filter sessions by date range + sessions = session.DataManager.GetSessionsByDateRange(startDate, endDate) } else { c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid date format"}) return } - } else { - sessions = h.dataManager.GetSessions() } // Convert to a frontend-friendly format @@ -138,41 +299,79 @@ func (h *Handler) GetSessions(c *gin.Context) { func (h *Handler) GetSession(c *gin.Context) { id := c.Param("id") - session, found := h.dataManager.GetSessionByID(id) - if !found { - c.JSON(http.StatusNotFound, gin.H{"error": "Session not found"}) + // Get the user's session + sessionID, err := c.Cookie("session_id") + if err != nil { + // No session found, return error + c.JSON(http.StatusNotFound, gin.H{"error": "No session data available"}) + return + } + + // Get the session data + userSession, exists := h.sessions.GetSession(sessionID) + if !exists { + // Session expired, return error + c.JSON(http.StatusNotFound, gin.H{"error": "Session expired, please reload data"}) return } - c.JSON(http.StatusOK, session) + // Get all charging sessions + uploadedSessions := userSession.DataManager.GetSessions() + + // Search for the requested session by ID + for _, session := range uploadedSessions { + if session.ID == id { + c.JSON(http.StatusOK, session) + return + } + } + + c.JSON(http.StatusNotFound, gin.H{"error": "Session not found"}) } // GetStats returns various statistics func (h *Handler) GetStats(c *gin.Context) { - // Get date range parameters (same as in GetSessions) + // Get the user's session + sessionID, err := c.Cookie("session_id") + if err != nil { + // No session found, return error + c.JSON(http.StatusNotFound, gin.H{"error": "No data available for statistics"}) + return + } + + // Get the session data + userSession, exists := h.sessions.GetSession(sessionID) + if !exists { + // Session expired, return error + c.JSON(http.StatusNotFound, gin.H{"error": "Session expired, please reload data"}) + return + } + + // Get the data manager from the user's session + requestDataManager := userSession.DataManager + + // Get date range parameters startDateStr := c.Query("startDate") endDateStr := c.Query("endDate") - var sessions []data.Session - var startDate, endDate time.Time + var sessions []data.Session = requestDataManager.GetSessions() var dateFilterActive bool if startDateStr != "" && endDateStr != "" { - var err1, err2 error - startDate, err1 = time.Parse("2006-01-02", startDateStr) - endDate, err2 = time.Parse("2006-01-02", endDateStr) + startDate, err1 := time.Parse("2006-01-02", startDateStr) + endDate, err2 := time.Parse("2006-01-02", endDateStr) if err1 == nil && err2 == nil { // Add a day to endDate to make it inclusive endDate = endDate.AddDate(0, 0, 1) - sessions = h.dataManager.GetSessionsByDateRange(startDate, endDate) + sessions = requestDataManager.GetSessionsByDateRange(startDate, endDate) dateFilterActive = true } else { c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid date format"}) return } } else { - sessions = h.dataManager.GetSessions() + // Keep the sessions we already have from the user's session } // Calculate statistics using filtered sessions if filter is active @@ -181,23 +380,15 @@ func (h *Handler) GetStats(c *gin.Context) { var sessionStats map[string]interface{} var estimatedCapacity []map[string]interface{} - if dateFilterActive { - // Create a temporary data manager with only the filtered sessions - tempManager := data.NewManager() - tempManager.SetSessions(sessions) + // Create a temporary data manager with the sessions + tempManager := data.NewManager() + tempManager.SetSessions(sessions) - // Calculate stats using the filtered data - overallEfficiency, powerConsumption, powerConsumptionWithoutGridLosses = tempManager.CalculateOverallStats() - socStats = tempManager.CalculateSOCStatistics() - sessionStats = tempManager.GetSessionStats() - estimatedCapacity = tempManager.CalculateEstimatedBatteryCapacity() - } else { - // Use all data - overallEfficiency, powerConsumption, powerConsumptionWithoutGridLosses = h.dataManager.CalculateOverallStats() - socStats = h.dataManager.CalculateSOCStatistics() - sessionStats = h.dataManager.GetSessionStats() - estimatedCapacity = h.dataManager.CalculateEstimatedBatteryCapacity() - } + // Calculate stats using the data + overallEfficiency, powerConsumption, powerConsumptionWithoutGridLosses = tempManager.CalculateOverallStats() + socStats = tempManager.CalculateSOCStatistics() + sessionStats = tempManager.GetSessionStats() + estimatedCapacity = tempManager.CalculateEstimatedBatteryCapacity() // Combine everything into one response stats := map[string]interface{}{ @@ -207,7 +398,7 @@ func (h *Handler) GetStats(c *gin.Context) { "soc_stats": socStats, "session_stats": sessionStats, "estimated_capacity": estimatedCapacity, - "using_estimated_values": h.dataManager.IsUsingEstimatedValues(), + "using_estimated_values": tempManager.IsUsingEstimatedValues(), "date_filter_active": dateFilterActive, } @@ -216,7 +407,26 @@ func (h *Handler) GetStats(c *gin.Context) { // GetMapData returns data for the map func (h *Handler) GetMapData(c *gin.Context) { - // Get date range parameters (same as in GetSessions and GetStats) + // Get the user's session + sessionID, err := c.Cookie("session_id") + if err != nil { + // No session found, return error + c.JSON(http.StatusNotFound, gin.H{"error": "No data available for map"}) + return + } + + // Get the session data + userSession, exists := h.sessions.GetSession(sessionID) + if !exists { + // Session expired, return error + c.JSON(http.StatusNotFound, gin.H{"error": "Session expired, please reload data"}) + return + } + + // Get the data manager from the user's session + requestDataManager := userSession.DataManager + + // Get date range parameters startDateStr := c.Query("startDate") endDateStr := c.Query("endDate") @@ -229,13 +439,13 @@ func (h *Handler) GetMapData(c *gin.Context) { if err1 == nil && err2 == nil { // Add a day to endDate to make it inclusive endDate = endDate.AddDate(0, 0, 1) - sessions = h.dataManager.GetSessionsByDateRange(startDate, endDate) + sessions = requestDataManager.GetSessionsByDateRange(startDate, endDate) } else { c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid date format"}) return } } else { - sessions = h.dataManager.GetSessions() + sessions = requestDataManager.GetSessions() } // Group sessions by location to count occurrences @@ -318,11 +528,27 @@ func (h *Handler) GetMapData(c *gin.Context) { // GetGroupedProviders returns statistics grouped by provider with similar names merged func (h *Handler) GetGroupedProviders(c *gin.Context) { + // Get the user's session + sessionID, err := c.Cookie("session_id") + if err != nil { + // No session found, return error + c.JSON(http.StatusNotFound, gin.H{"error": "No data available for statistics"}) + return + } + + // Get the session data + userSession, exists := h.sessions.GetSession(sessionID) + if !exists { + // Session expired, return error + c.JSON(http.StatusNotFound, gin.H{"error": "Session expired, please reload data"}) + return + } + // Get date range parameters startDateStr := c.Query("startDate") endDateStr := c.Query("endDate") - var tempManager *data.Manager + var tempManager *data.Manager = userSession.DataManager if startDateStr != "" && endDateStr != "" { startDate, err1 := time.Parse("2006-01-02", startDateStr) @@ -331,7 +557,7 @@ func (h *Handler) GetGroupedProviders(c *gin.Context) { if err1 == nil && err2 == nil { // Add a day to endDate to make it inclusive endDate = endDate.AddDate(0, 0, 1) - sessions := h.dataManager.GetSessionsByDateRange(startDate, endDate) + sessions := tempManager.GetSessionsByDateRange(startDate, endDate) // Create a temporary manager with filtered sessions tempManager = data.NewManager() @@ -340,9 +566,6 @@ func (h *Handler) GetGroupedProviders(c *gin.Context) { c.JSON(http.StatusBadRequest, gin.H{"error": "Invalid date format"}) return } - } else { - // Use the main manager - tempManager = h.dataManager } // Get grouped provider statistics diff --git a/go-rewrite/pkg/api/session_store.go b/go-rewrite/pkg/api/session_store.go new file mode 100644 index 0000000..c48e35c --- /dev/null +++ b/go-rewrite/pkg/api/session_store.go @@ -0,0 +1,122 @@ +package api + +import ( + "crypto/rand" + "encoding/hex" + "sync" + "time" + + "github.com/awlx/bmwtools/pkg/data" +) + +// SessionData represents the data associated with a user session +type SessionData struct { + ID string + DataManager *data.Manager + CreatedAt time.Time + LastAccessed time.Time +} + +// SessionStore manages user sessions and their associated data +type SessionStore struct { + sessions map[string]*SessionData + mutex sync.RWMutex + // Time after which inactive sessions are cleaned up (e.g., 30 minutes) + expirationTime time.Duration +} + +// NewSessionStore creates a new session store +func NewSessionStore(expirationTime time.Duration) *SessionStore { + store := &SessionStore{ + sessions: make(map[string]*SessionData), + expirationTime: expirationTime, + } + + // Start a goroutine to periodically clean up expired sessions + go store.cleanupExpiredSessions() + + return store +} + +// cleanupExpiredSessions periodically removes expired sessions +func (s *SessionStore) cleanupExpiredSessions() { + ticker := time.NewTicker(5 * time.Minute) + defer ticker.Stop() + + for range ticker.C { + s.mutex.Lock() + + now := time.Now() + for id, session := range s.sessions { + if now.Sub(session.LastAccessed) > s.expirationTime { + delete(s.sessions, id) + } + } + + s.mutex.Unlock() + } +} + +// CreateSession creates a new session and returns its ID +func (s *SessionStore) CreateSession() string { + s.mutex.Lock() + defer s.mutex.Unlock() + + // Generate a random ID using crypto/rand + bytes := make([]byte, 16) + if _, err := rand.Read(bytes); err != nil { + // Fallback to current time if random generation fails + sessionID := hex.EncodeToString([]byte(time.Now().String())) + now := time.Now() + + s.sessions[sessionID] = &SessionData{ + ID: sessionID, + DataManager: data.NewManager(), + CreatedAt: now, + LastAccessed: now, + } + + return sessionID + } + + sessionID := hex.EncodeToString(bytes) + now := time.Now() + + s.sessions[sessionID] = &SessionData{ + ID: sessionID, + DataManager: data.NewManager(), + CreatedAt: now, + LastAccessed: now, + } + + return sessionID +} + +// GetSession returns the session data for the given ID +func (s *SessionStore) GetSession(sessionID string) (*SessionData, bool) { + s.mutex.RLock() + defer s.mutex.RUnlock() + + session, exists := s.sessions[sessionID] + if exists { + // Update last accessed time + session.LastAccessed = time.Now() + } + + return session, exists +} + +// GetOrCreateSession gets an existing session or creates a new one +func (s *SessionStore) GetOrCreateSession(sessionID string) (string, *SessionData) { + // If we have a session ID and it exists, return it + if sessionID != "" { + if session, exists := s.GetSession(sessionID); exists { + return sessionID, session + } + } + + // Create a new session + newID := s.CreateSession() + session, _ := s.GetSession(newID) + return newID, session +} diff --git a/go-rewrite/pkg/data/providers.go b/go-rewrite/pkg/data/providers.go index fe262d7..b85a015 100644 --- a/go-rewrite/pkg/data/providers.go +++ b/go-rewrite/pkg/data/providers.go @@ -18,6 +18,11 @@ type ProviderStats struct { FailureRate float64 `json:"failure_rate"` } +// NormalizeProviderName exports the provider name normalization functionality +func NormalizeProviderName(name string) string { + return normalizeProviderName(name) +} + // normalizeProviderName normalizes a provider name for comparison func normalizeProviderName(name string) string { if name == "" || name == "Unknown" { diff --git a/go-rewrite/pkg/data/unknown_providers_test.go b/go-rewrite/pkg/data/unknown_providers_test.go index 7d05998..b45a720 100644 --- a/go-rewrite/pkg/data/unknown_providers_test.go +++ b/go-rewrite/pkg/data/unknown_providers_test.go @@ -52,7 +52,7 @@ func TestUnknownProviderAnalysis(t *testing.T) { t.Log("2. Fix for providers with very few letters:") t.Log(" - Consider maintaining a whitelist of valid short providers") - t.Log(" - Example: 'E.ON', 'EWE', 'RWE' are valid short names") + t.Log(" - ***REMOVED*** 'E.ON', 'EWE', 'RWE' are valid short names") t.Log("3. Fix for numeric-only or special character-only providers:") t.Log(" - Add special handling for specific numeric/character patterns") diff --git a/go-rewrite/pkg/database/database.go b/go-rewrite/pkg/database/database.go new file mode 100644 index 0000000..1ae9105 --- /dev/null +++ b/go-rewrite/pkg/database/database.go @@ -0,0 +1,689 @@ +package database + +import ( + "database/sql" + "fmt" + "log" + "path/filepath" + "sort" + "strings" + "time" + + "crypto/sha256" + "encoding/hex" + + "github.com/awlx/bmwtools/pkg/data" + _ "github.com/mattn/go-sqlite3" +) + +// Manager handles database operations +type Manager struct { + db *sql.DB +} + +// New creates a new database manager +func New(dbPath string) (*Manager, error) { + // Ensure the directory exists + dir := filepath.Dir(dbPath) + if dir != "." && dir != "/" { + // This is a simplification - in production code, you might want to handle this more robustly + log.Printf("Ensuring directory exists: %s", dir) + } + + db, err := sql.Open("sqlite3", dbPath) + if err != nil { + return nil, err + } + + m := &Manager{ + db: db, + } + + if err := m.initSchema(); err != nil { + db.Close() + return nil, err + } + + return m, nil +} + +// Close closes the database connection +func (m *Manager) Close() error { + return m.db.Close() +} + +// initSchema initializes the database schema +func (m *Manager) initSchema() error { + _, err := m.db.Exec(` + CREATE TABLE IF NOT EXISTS uploads ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + content_hash TEXT UNIQUE NOT NULL, + uploaded_at TIMESTAMP NOT NULL, + session_count INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS vehicles ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + fin_hash TEXT UNIQUE NOT NULL, + model TEXT, + created_at TIMESTAMP NOT NULL + ); + + CREATE TABLE IF NOT EXISTS sessions ( + id TEXT PRIMARY KEY, + vehicle_id INTEGER, + start_time TIMESTAMP NOT NULL, + end_time TIMESTAMP NOT NULL, + soc_start REAL NOT NULL, + soc_end REAL NOT NULL, + energy_from_grid REAL NOT NULL, + energy_added_hvb REAL NOT NULL, + cost REAL, + efficiency REAL, + provider TEXT, + avg_power REAL, + session_time_minutes REAL, + status TEXT, + location_hash TEXT, + FOREIGN KEY (vehicle_id) REFERENCES vehicles(id) + ); + + CREATE TABLE IF NOT EXISTS battery_health ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + vehicle_id INTEGER NOT NULL, + date TIMESTAMP NOT NULL, + estimated_capacity REAL NOT NULL, + soc_change REAL NOT NULL, + is_raw_data BOOLEAN NOT NULL, + mileage REAL DEFAULT 0, + FOREIGN KEY (vehicle_id) REFERENCES vehicles(id) + ); + `) + + return err +} + +// StoreSessions stores charging sessions in the database +func (m *Manager) StoreSessions(sessions []data.Session, filename string, userSpecifiedModel string) (bool, error) { + // Generate a hash of the content to detect duplicates + contentHash := hashSessions(sessions) + + // Start a transaction + tx, err := m.db.Begin() + if err != nil { + return false, err + } + defer func() { + if err != nil { + tx.Rollback() + } + }() + + // Keep track of how many sessions were new vs. duplicates + newSessionCount := 0 + duplicateSessionCount := 0 + + // First collect all session IDs to check in a single query for efficiency + var sessionIDs []interface{} + sessionMap := make(map[string]data.Session) + for _, session := range sessions { + sessionIDs = append(sessionIDs, session.ID) + sessionMap[session.ID] = session + } + + // Build the SQL query to find existing sessions in a single database call + existingSessions := make(map[string]bool) + if len(sessionIDs) > 0 { + placeholders := make([]string, len(sessionIDs)) + for i := range placeholders { + placeholders[i] = "?" + } + + // Use a parameterized query to find existing sessions + query := fmt.Sprintf("SELECT id FROM sessions WHERE id IN (%s)", strings.Join(placeholders, ",")) + + // Execute the query with sessionIDs as parameters + rows, err := tx.Query(query, sessionIDs...) + if err != nil { + return false, fmt.Errorf("error checking for existing sessions: %w", err) + } + defer rows.Close() + + // Mark existing sessions + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return false, err + } + existingSessions[id] = true + duplicateSessionCount++ + } + } + + // Only record the upload if we have NEW sessions to store + if len(sessionIDs) > duplicateSessionCount { + // Record the upload - without storing the filename + _, err = tx.Exec( + "INSERT INTO uploads (content_hash, uploaded_at, session_count) VALUES (?, ?, ?)", + contentHash, time.Now(), len(sessions)-duplicateSessionCount, + ) + if err != nil { + // If the content hash already exists, check if this is a true duplicate upload + if strings.Contains(err.Error(), "UNIQUE constraint failed") && duplicateSessionCount == len(sessionIDs) { + return false, nil + } + // Otherwise, we have some new sessions to process despite the duplicate content hash + if !strings.Contains(err.Error(), "UNIQUE constraint failed") { + return false, err + } + } + } else if duplicateSessionCount == len(sessionIDs) { + // All sessions are duplicates, treat as duplicate upload + return false, nil + } + + // Extract and hash FIN from sessions + vehicleIDMap := make(map[string]int64) + + // Process each session, skipping duplicates + for _, session := range sessions { + // Skip if this session already exists + if existingSessions[session.ID] { + continue + } + + // This is a new session + newSessionCount++ + + // Extract FIN from session ID (using our safer, non-identifiable approach) + fin := extractFIN(session.ID) + if fin == "" { + continue + } + + finHash := hashFIN(fin) + + // Check if we've already processed this vehicle in this batch + vehicleID, exists := vehicleIDMap[finHash] + if !exists { + // Check if this vehicle exists in the database + err = tx.QueryRow("SELECT id FROM vehicles WHERE fin_hash = ?", finHash).Scan(&vehicleID) + if err == sql.ErrNoRows { + // Vehicle doesn't exist, create it + var model string + if userSpecifiedModel != "" { + model = userSpecifiedModel // Use the model specified by the user + } else { + model = deriveModelFromSession(session) // Fall back to derived model + } + res, err := tx.Exec( + "INSERT INTO vehicles (fin_hash, model, created_at) VALUES (?, ?, ?)", + finHash, model, time.Now(), + ) + if err != nil { + return false, err + } + vehicleID, _ = res.LastInsertId() + } else if err != nil { + return false, err + } + vehicleIDMap[finHash] = vehicleID + } + + // Determine session status + status := "failed" + if session.SocEnd > session.SocStart { + status = "successful" + } + + // Hash location for privacy + locationHash := "" + if session.Location != "" { + h := sha256.New() + h.Write([]byte(session.Location)) + locationHash = hex.EncodeToString(h.Sum(nil)) + } + + // Store the session with the original provider name - no normalization + // We're storing the original provider name as it appears in the data + // This way we preserve the full, unmodified provider information + _, err = tx.Exec( + `INSERT INTO sessions + (id, vehicle_id, start_time, end_time, soc_start, soc_end, + energy_from_grid, energy_added_hvb, cost, efficiency, provider, + avg_power, session_time_minutes, status, location_hash) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + session.ID, vehicleID, session.StartTime, session.EndTime, + session.SocStart, session.SocEnd, session.EnergyFromGrid, + session.EnergyAddedHvb, session.Cost, session.Efficiency, + session.Provider, session.AvgPower, session.SessionTimeMinutes, + status, locationHash, + ) + if err != nil { + return false, err + } + + // If we can calculate battery health from this session, store it + if session.EnergyAddedHvb >= 30 && (session.SocEnd-session.SocStart) >= 20 { + // Check if we already have battery health data for this exact session + // We're using a more precise query with the actual values + // This ensures we don't duplicate battery health data for the same session + + // Check for existing battery health record with a more precise query + var existingBHCount int + err = tx.QueryRow(`SELECT COUNT(*) FROM battery_health + WHERE vehicle_id = ? + AND date = ? + AND ABS(estimated_capacity - ?) < 0.01 + AND ABS(soc_change - ?) < 0.01`, + vehicleID, session.StartTime, + (session.EnergyAddedHvb*100)/(session.SocEnd-session.SocStart), + session.SocEnd-session.SocStart).Scan(&existingBHCount) + if err != nil { + return false, err + } + + if existingBHCount == 0 { + estimatedCapacity := (session.EnergyAddedHvb * 100) / (session.SocEnd - session.SocStart) + _, err = tx.Exec( + `INSERT INTO battery_health + (vehicle_id, date, estimated_capacity, soc_change, is_raw_data, mileage) + VALUES (?, ?, ?, ?, ?, ?)`, + vehicleID, session.StartTime, estimatedCapacity, + session.SocEnd-session.SocStart, true, session.Mileage, + ) + if err != nil { + return false, err + } + } + } + } + + // If all sessions were duplicates, consider this a duplicate upload + if newSessionCount == 0 && duplicateSessionCount > 0 { + tx.Rollback() // Don't save the upload record + return false, nil + } + + // Commit the transaction + if err = tx.Commit(); err != nil { + return false, err + } + + // Log how many sessions were new vs duplicates + log.Printf("Stored %d new sessions, skipped %d duplicate sessions", newSessionCount, duplicateSessionCount) + + return true, nil +} + +// GetAvailableModels returns a list of available BMW models in the database +func (m *Manager) GetAvailableModels() ([]string, error) { + // First check if the table exists + var tableExists int + err := m.db.QueryRow(`SELECT count(name) FROM sqlite_master WHERE type='table' AND name='vehicles'`).Scan(&tableExists) + if err != nil { + return nil, fmt.Errorf("error checking if vehicles table exists: %w", err) + } + + if tableExists == 0 { + // Table doesn't exist, return empty result instead of error + return []string{}, nil + } + + rows, err := m.db.Query("SELECT DISTINCT model FROM vehicles WHERE model IS NOT NULL ORDER BY model") + if err != nil { + return nil, err + } + defer rows.Close() + + var models []string + for rows.Next() { + var model string + if err := rows.Scan(&model); err != nil { + return nil, err + } + if model != "" { + models = append(models, model) + } + } + + return models, rows.Err() +} + +// GetFleetBatteryHealth returns battery health data for the fleet +func (m *Manager) GetFleetBatteryHealth(modelFilter string) ([]map[string]interface{}, error) { + // First check if the table exists + var tableExists int + err := m.db.QueryRow(`SELECT count(name) FROM sqlite_master WHERE type='table' AND name='battery_health'`).Scan(&tableExists) + if err != nil { + return nil, fmt.Errorf("error checking if battery_health table exists: %w", err) + } + + if tableExists == 0 { + // Table doesn't exist, return empty result instead of error + return []map[string]interface{}{}, nil + } + + query := ` + SELECT + bh.date, + bh.estimated_capacity, + bh.soc_change, + bh.is_raw_data, + bh.mileage, + v.model + FROM battery_health bh + JOIN vehicles v ON bh.vehicle_id = v.id + ` + + args := []interface{}{} + if modelFilter != "" { + query += " WHERE v.model = ?" + args = append(args, modelFilter) + } + + query += " ORDER BY bh.date" + + rows, err := m.db.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []map[string]interface{} + for rows.Next() { + var date time.Time + var capacity, socChange, mileage float64 + var isRawData bool + var model string + + if err := rows.Scan(&date, &capacity, &socChange, &isRawData, &mileage, &model); err != nil { + return nil, err + } + + result = append(result, map[string]interface{}{ + "date": date, + "estimated_capacity": capacity, + "mileage": mileage, + "soc_change": socChange, + "is_raw_data": isRawData, + "model": model, + }) + } + + return result, rows.Err() +} + +// GetMonthlyBatteryHealthTrend returns the monthly aggregated battery health trend +func (m *Manager) GetMonthlyBatteryHealthTrend(modelFilter string) ([]map[string]interface{}, error) { + // First check if the table exists + var tableExists int + err := m.db.QueryRow(`SELECT count(name) FROM sqlite_master WHERE type='table' AND name='battery_health'`).Scan(&tableExists) + if err != nil { + return nil, fmt.Errorf("error checking if battery_health table exists: %w", err) + } + + if tableExists == 0 { + // Table doesn't exist, return empty result instead of error + return []map[string]interface{}{}, nil + } + + query := ` + SELECT + strftime('%Y-%m', bh.date) as month, + avg(bh.estimated_capacity) as avg_capacity, + sum(bh.soc_change) as total_soc_change, + count(*) as data_points, + avg(bh.mileage) as avg_mileage, + v.model + FROM battery_health bh + JOIN vehicles v ON bh.vehicle_id = v.id + ` + + args := []interface{}{} + if modelFilter != "" { + query += " WHERE v.model = ?" + args = append(args, modelFilter) + } + + query += " GROUP BY strftime('%Y-%m', bh.date), v.model ORDER BY avg_mileage" + + rows, err := m.db.Query(query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + + var result []map[string]interface{} + for rows.Next() { + var month string + var avgCapacity, totalSocChange, avgMileage float64 + var dataPoints int + var model string + + if err := rows.Scan(&month, &avgCapacity, &totalSocChange, &dataPoints, &avgMileage, &model); err != nil { + return nil, err + } + + // Parse the month string to get a date for the middle of the month + t, _ := time.Parse("2006-01", month) + // Set to the 15th day of the month as a representative date + representativeDate := time.Date(t.Year(), t.Month(), 15, 12, 0, 0, 0, time.UTC) + + result = append(result, map[string]interface{}{ + "month": month, + "date": representativeDate, + "avg_capacity": avgCapacity, + "total_soc_change": totalSocChange, + "data_points": dataPoints, + "mileage": avgMileage, + "model": model, + "is_monthly_average": true, + }) + } + + return result, rows.Err() +} + +// GetProviderStats returns statistics about charging providers from the database +func (m *Manager) GetProviderStats() ([]map[string]interface{}, error) { + // First, get all providers and their metrics + query := ` + SELECT + provider, + COUNT(*) as total_sessions, + SUM(CASE WHEN status = 'successful' THEN 1 ELSE 0 END) as successful_sessions, + SUM(CASE WHEN status = 'failed' THEN 1 ELSE 0 END) as failed_sessions, + AVG(CASE WHEN status = 'successful' THEN efficiency ELSE 0 END) as avg_efficiency, + SUM(CASE WHEN status = 'successful' THEN energy_added_hvb ELSE 0 END) as total_energy_added, + AVG(CASE WHEN status = 'successful' THEN avg_power ELSE 0 END) as avg_power + FROM sessions + GROUP BY provider + ` + + // We'll group providers manually to properly normalize them + rows, err := m.db.Query(query) + if err != nil { + return nil, err + } + defer rows.Close() + + // Create a map to normalize and combine providers + providerMap := make(map[string]map[string]interface{}) + + // We're using the comprehensive provider normalization logic + // from data.NormalizeProviderName instead of maintaining a separate list here + + // Use the exported NormalizeProviderName function from the data package + // instead of duplicating the normalization logic here + normalizeProvider := func(provider string) string { + return data.NormalizeProviderName(provider) + } + + for rows.Next() { + var provider string + var totalSessions, successfulSessions, failedSessions int + var avgEfficiency, totalEnergyAdded, avgPower float64 + + if err := rows.Scan(&provider, &totalSessions, &successfulSessions, &failedSessions, + &avgEfficiency, &totalEnergyAdded, &avgPower); err != nil { + return nil, err + } + + // Normalize the provider name + normalizedProvider := normalizeProvider(provider) + + // Check if we already have an entry for this normalized provider + if existingStats, exists := providerMap[normalizedProvider]; exists { + // Update existing stats + existingStats["total_sessions"] = existingStats["total_sessions"].(int) + totalSessions + existingStats["successful_sessions"] = existingStats["successful_sessions"].(int) + successfulSessions + existingStats["failed_sessions"] = existingStats["failed_sessions"].(int) + failedSessions + + // Weighted average for efficiency and power + if successfulSessions > 0 { + currentSuccessful := float64(existingStats["successful_sessions"].(int) - successfulSessions) + newAvgEfficiency := ((avgEfficiency * float64(successfulSessions)) + + (existingStats["avg_efficiency"].(float64) * currentSuccessful)) / + float64(existingStats["successful_sessions"].(int)) + existingStats["avg_efficiency"] = newAvgEfficiency + + newAvgPower := ((avgPower * float64(successfulSessions)) + + (existingStats["avg_power"].(float64) * currentSuccessful)) / + float64(existingStats["successful_sessions"].(int)) + existingStats["avg_power"] = newAvgPower + } + + existingStats["total_energy_added"] = existingStats["total_energy_added"].(float64) + totalEnergyAdded + } else { + // Create new entry + providerMap[normalizedProvider] = map[string]interface{}{ + "provider": normalizedProvider, // Use the normalized name + "original_provider": provider, // Keep the original name for reference + "total_sessions": totalSessions, + "successful_sessions": successfulSessions, + "failed_sessions": failedSessions, + "avg_efficiency": avgEfficiency, + "total_energy_added": totalEnergyAdded, + "avg_power": avgPower, + } + } + } + + // Now convert the map to a slice and calculate success rates + var result []map[string]interface{} + for _, stats := range providerMap { + totalSessions := stats["total_sessions"].(int) + successfulSessions := stats["successful_sessions"].(int) + + // Calculate success rate + successRate := 0.0 + if totalSessions > 0 { + successRate = float64(successfulSessions) / float64(totalSessions) * 100 + } + + stats["success_rate"] = successRate + stats["avg_efficiency"] = stats["avg_efficiency"].(float64) * 100 // Convert to percentage + + result = append(result, stats) + } + + // Sort by total sessions descending + sort.Slice(result, func(i, j int) bool { + return result[i]["total_sessions"].(int) > result[j]["total_sessions"].(int) + }) + + return result, rows.Err() +} + +// GetSOCStats returns statistics about State of Charge from the database +func (m *Manager) GetSOCStats() (map[string]interface{}, error) { + query := ` + SELECT + AVG(soc_start) as avg_start_soc, + AVG(soc_end) as avg_end_soc, + MIN(soc_start) as min_start_soc, + MAX(soc_start) as max_start_soc, + MIN(soc_end) as min_end_soc, + MAX(soc_end) as max_end_soc + FROM sessions + WHERE status = 'successful' + ` + + var avgStartSOC, avgEndSOC, minStartSOC, maxStartSOC, minEndSOC, maxEndSOC float64 + err := m.db.QueryRow(query).Scan(&avgStartSOC, &avgEndSOC, &minStartSOC, &maxStartSOC, &minEndSOC, &maxEndSOC) + if err != nil { + return nil, err + } + + return map[string]interface{}{ + "average_start_soc": avgStartSOC, + "average_end_soc": avgEndSOC, + "min_start_soc": minStartSOC, + "max_start_soc": maxStartSOC, + "min_end_soc": minEndSOC, + "max_end_soc": maxEndSOC, + }, nil +} + +// GetSessionCount returns the total number of sessions stored in the database +func (m *Manager) GetSessionCount() (int, error) { + var count int + err := m.db.QueryRow("SELECT COUNT(*) FROM sessions").Scan(&count) + if err != nil { + return 0, err + } + return count, nil +} + +// Helper functions + +// hashSessions creates a hash of the sessions to detect duplicates +func hashSessions(sessions []data.Session) string { + h := sha256.New() + for _, s := range sessions { + // Include key fields that identify the session uniquely + fmt.Fprintf(h, "%s|%s|%s|%.2f|%.2f|%.2f|%.2f|", + s.ID, s.StartTime.String(), s.EndTime.String(), + s.SocStart, s.SocEnd, s.EnergyFromGrid, s.EnergyAddedHvb) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// extractFIN extracts a non-identifiable vehicle identifier from session data +// This deliberately does NOT extract the actual FIN, but creates a pseudonymous identifier +func extractFIN(sessionID string) string { + // Instead of extracting an actual FIN, we'll just use a session-specific hash + // This approach ensures we never store or process the actual FIN + return sessionID +} + +// hashFIN creates a strong one-way hash with salt for privacy +func hashFIN(sessionIdentifier string) string { + // Add a salt to make it even more secure against brute-force attacks + // Using a fixed salt that's specific to this application + salt := "BMWToolsAnonymousFleetStats2025" + + h := sha256.New() + h.Write([]byte(salt + sessionIdentifier)) + return hex.EncodeToString(h.Sum(nil)) +} + +// deriveModelFromSession tries to determine the BMW model from session data +func deriveModelFromSession(session data.Session) string { + // This is a placeholder. In reality, the model would need to be determined + // from the session data based on your specific data structure. + // It might be embedded in the data or derivable from the FIN. + + // Example logic: + if strings.Contains(session.ID, "i3") { + return "BMW i3" + } else if strings.Contains(session.ID, "i4") { + return "BMW i4" + } else if strings.Contains(session.ID, "iX") { + return "BMW iX" + } + + // Default if we can't determine + return "Unknown BMW EV" +} diff --git a/go-rewrite/static/battery.html b/go-rewrite/static/battery.html new file mode 100644 index 0000000..f7b0a45 --- /dev/null +++ b/go-rewrite/static/battery.html @@ -0,0 +1,414 @@ + + + + + + BMW CarData - Fleet Battery Health + + + + + + + + + + +
+
+

BMW CarData - Fleet Battery Health

+ +

This dashboard shows anonymous battery health metrics for the fleet. No personal data is stored or displayed.

+
+ +
+
+ + +
+
+ +
+
+

About Battery Capacity Measurements

+

The chart below shows battery capacity estimates based on charging sessions plotted against vehicle mileage. Points represent individual measurements, while the blue line shows the linear average capacity trend at each mileage point.

+

A healthy battery will show gradual capacity decline with increasing mileage. Sudden drops may indicate calibration issues rather than actual capacity loss.

+
+ +
+
+ + Individual Measurements +
+
+ + Average at Current Kilometers +
+
+ +
+ +
+

Note: Battery capacity is estimated from charging sessions where a significant charge was added (>20% SOC change).

+
+
+ +
+ Back to Dashboard +
+
+ +
+
+

Impressum

+

Freie Netze MΓΌnchen e. V.

+

Parkstraße 28

+

82131 Gauting

+
+
+ + + + + diff --git a/go-rewrite/static/css/nav.css b/go-rewrite/static/css/nav.css new file mode 100644 index 0000000..1b6da11 --- /dev/null +++ b/go-rewrite/static/css/nav.css @@ -0,0 +1,70 @@ +/* Navigation styles */ +.main-nav { + margin: 20px auto 30px; + display: flex; + justify-content: center; +} + +.main-nav ul { + list-style-type: none; + display: flex; + background-color: #fff; + border-radius: 30px; + padding: 5px; + box-shadow: 0 4px 12px rgba(0,0,0,0.1); +} + +.main-nav li { + margin: 0; + padding: 0; +} + +.main-nav a { + display: block; + padding: 10px 20px; + text-decoration: none; + color: #555; + font-weight: 500; + border-radius: 25px; + transition: all 0.3s ease; +} + +.main-nav li.active a { + background: linear-gradient(45deg, #3498db, #1abc9c); + color: white; + box-shadow: 0 4px 8px rgba(52, 152, 219, 0.3); +} + +.main-nav a:hover:not(.active) { + background-color: #f0f0f0; + color: #3498db; +} + +/* Responsive navigation */ +@media (max-width: 768px) { + .main-nav ul { + flex-direction: column; + border-radius: 15px; + width: 100%; + } + + .main-nav li { + width: 100%; + text-align: center; + } + + .main-nav a { + border-radius: 0; + padding: 15px; + } + + .main-nav li:first-child a { + border-top-left-radius: 15px; + border-top-right-radius: 15px; + } + + .main-nav li:last-child a { + border-bottom-left-radius: 15px; + border-bottom-right-radius: 15px; + } +} diff --git a/go-rewrite/static/css/styles.css b/go-rewrite/static/css/styles.css index 2ceaa58..12c4b20 100644 --- a/go-rewrite/static/css/styles.css +++ b/go-rewrite/static/css/styles.css @@ -79,6 +79,61 @@ h1::after { position: relative; } +/* Consent checkbox styling */ +.consent-checkbox { + margin: 15px 0; + display: flex; + align-items: center; + padding: 10px 15px; + background-color: rgba(52, 152, 219, 0.1); + border-radius: 8px; + border: 1px solid rgba(52, 152, 219, 0.2); + transition: all 0.3s ease; +} + +.consent-checkbox:hover { + background-color: rgba(52, 152, 219, 0.15); +} + +.consent-checkbox input[type="checkbox"] { + margin-right: 10px; + width: 18px; + height: 18px; + cursor: pointer; +} + +.consent-checkbox label { + font-size: 14px; + cursor: pointer; +} + +/* Tooltip styling */ +.tooltip { + position: relative; + display: inline-block; + color: #3498db; + cursor: help; + margin-left: 5px; + font-size: 14px; +} + +.tooltip:hover::before { + content: attr(title); + position: absolute; + bottom: 100%; + left: 50%; + transform: translateX(-50%); + background-color: rgba(0, 0, 0, 0.8); + color: white; + padding: 8px 12px; + border-radius: 6px; + white-space: nowrap; + font-size: 12px; + z-index: 100; + width: max-content; + max-width: 300px; +} + .file-input { width: 0.1px; height: 0.1px; @@ -328,6 +383,21 @@ h1::after { margin-bottom: 20px; } +/* No data message styling */ +.no-data-message { + display: flex; + justify-content: center; + align-items: center; + height: 100%; + font-size: 18px; + color: #888; + text-align: center; + padding: 20px; + background-color: rgba(0,0,0,0.02); + border-radius: 8px; + font-style: italic; +} + .session-details-container { display: flex; flex-wrap: wrap; diff --git a/go-rewrite/static/index.html b/go-rewrite/static/index.html index ffcc5cd..861e13c 100644 --- a/go-rewrite/static/index.html +++ b/go-rewrite/static/index.html @@ -29,11 +29,18 @@ function loadResources(version) { // Create and append CSS link with version + // Load main CSS const cssLink = document.createElement('link'); cssLink.rel = 'stylesheet'; cssLink.href = `/static/css/styles.css?v=${version}`; document.head.appendChild(cssLink); + // Load navigation CSS + const navCssLink = document.createElement('link'); + navCssLink.rel = 'stylesheet'; + navCssLink.href = `/static/css/nav.css?v=${version}`; + document.head.appendChild(navCssLink); + // Store version for later use when loading JS window.APP_CACHE_VERSION = version; } @@ -44,6 +51,13 @@

BMW CarData - Charging Session Dashboard

+
@@ -55,6 +69,30 @@

BMW CarData - Charging Session Dashboard

Drag and Drop or Select your CarData JSON file (BMW-CarData-Ladehistorie_*.json)
+
+ + + +
diff --git a/go-rewrite/static/js/app.js b/go-rewrite/static/js/app.js index c8988e4..73c32fe 100644 --- a/go-rewrite/static/js/app.js +++ b/go-rewrite/static/js/app.js @@ -1,5 +1,5 @@ // App version - should match the server version in pkg/api/handler.go -const APP_VERSION = '1.0.0'; +const APP_VERSION = '1.0.1'; // Global variables let sessions = []; @@ -76,7 +76,7 @@ function init() { // Set up the disclaimer function setDisclaimer() { disclaimerEl.textContent = 'Disclaimer: This application stores all uploaded data in memory, if you refresh your session is lost.\n' + - 'CarData contains location data of your charges. Use at your own risk!\n' + + 'CarData contains location data of your charges, but no location data is stored on the server. Use at your own risk!\n' + 'You can verify authenticity at https://github.com/awlx/bmwtools'; } @@ -88,6 +88,16 @@ function setupEventListeners() { toggleUnitsBtn.addEventListener('click', toggleUnits); sessionDropdownEl.addEventListener('change', handleSessionSelection); + // Set up consent checkbox to show/hide model selector + const consentCheckbox = document.getElementById('fleet-stats-consent'); + const modelSelector = document.getElementById('model-selector'); + + if (consentCheckbox && modelSelector) { + consentCheckbox.addEventListener('change', function() { + modelSelector.style.display = this.checked ? 'block' : 'none'; + }); + } + // Set up drag and drop events for the upload container const uploadContainer = document.querySelector('.upload-container'); const fileLabel = document.querySelector('.file-label'); @@ -167,6 +177,24 @@ async function uploadFile(file) { const formData = new FormData(); formData.append('file', file); + + // Get the consent checkbox value + const consentCheckbox = document.getElementById('fleet-stats-consent'); + if (consentCheckbox && consentCheckbox.checked) { + formData.append('consent', 'true'); + + // Get the selected BMW model if consent is given + const modelSelect = document.getElementById('bmw-model'); + if (modelSelect && modelSelect.value) { + formData.append('model', modelSelect.value); + } else { + // Show an error if no model is selected when consent is given + alert('Please select a BMW model when sharing data for fleet statistics.'); + return; // Stop the upload process + } + } else { + formData.append('consent', 'false'); + } try { const response = await fetch('/api/upload', { @@ -179,6 +207,28 @@ async function uploadFile(file) { throw new Error(errorData.error || 'Error uploading file'); } + const result = await response.json(); + + // Only show popup when data was actually stored in fleet stats or it's a duplicate + if (result.stored || !result.new) { + // Show feedback about whether data was stored for fleet statistics + let message = result.message; + + if (result.stored) { + if (result.stored_count) { + message += `\n\nYour anonymous data was included in the fleet statistics. Total sessions in database: ${result.stored_count}.`; + } else { + message += "\n\nYour anonymous data was included in the fleet statistics."; + } + } + + if (!result.new && result.stored) { + message += " Note: This file was already uploaded previously."; + } + + alert(message); + } + await loadSessionData(); } catch (error) { console.error('Upload error:', error); diff --git a/go-rewrite/static/js/stats.js b/go-rewrite/static/js/stats.js new file mode 100644 index 0000000..a9cda63 --- /dev/null +++ b/go-rewrite/static/js/stats.js @@ -0,0 +1,520 @@ +// BMW Tools - Anonymous Statistics Dashboard + +// Plotly template for better styling (copied from main app.js) +const plotlyTemplate = { + layout: { + paper_bgcolor: '#ffffff', + plot_bgcolor: '#ffffff', + font: { + family: 'SF Pro Display, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif', + color: '#444', + size: 12 + }, + title: { + font: { + family: 'SF Pro Display, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif', + size: 18, + color: '#333' + } + }, + colorway: ['#3498db', '#2ecc71', '#f39c12', '#e74c3c', '#9b59b6', '#1abc9c', '#34495e', '#7f8c8d', '#d35400', '#c0392b'], + legend: { + bgcolor: '#ffffff', + bordercolor: '#f0f0f0', + borderwidth: 1, + font: { + family: 'SF Pro Display, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif', + size: 12, + color: '#555' + } + }, + xaxis: { + gridcolor: '#f0f0f0', + zerolinecolor: '#e0e0e0' + }, + yaxis: { + gridcolor: '#f0f0f0', + zerolinecolor: '#e0e0e0' + } + } +}; + +console.log('stats.js loaded - ' + new Date().toISOString()); + +// Initialize the statistics dashboard immediately and also on DOMContentLoaded +initStatsDashboard(); // Try to initialize immediately + +document.addEventListener('DOMContentLoaded', function() { + console.log('DOMContentLoaded event triggered for stats page'); + initStatsDashboard(); // Try again after DOM is loaded +}); + +// Main initialization function +async function initStatsDashboard() { + console.log('Initializing stats dashboard, fetching data from /api/anonymous-stats'); + try { + // Fetch statistics data + const response = await fetch('/api/anonymous-stats'); + + if (!response.ok) { + throw new Error('Failed to fetch statistics data'); + } + + const data = await response.json(); + + // Update the UI with the data + updateGlobalStats(data.global_stats, data.soc_stats); + updateProviderTable(data.providers); + createProviderFailureChart(data.providers); + createSessionOutcomesChart(data.global_stats); + createEnergyByProviderChart(data.providers); + + } catch (error) { + console.error('Error initializing stats dashboard:', error); + showErrorMessage('Failed to load statistics data. Please try again later.'); + } +} + +// Update global statistics section +function updateGlobalStats(globalStats, socStats) { + // Update total sessions + document.getElementById('total-sessions-count').textContent = globalStats.total_sessions.toLocaleString(); + + // Update success rate + const successRate = document.getElementById('success-rate'); + successRate.textContent = globalStats.success_rate.toFixed(1) + '%'; + + // Set color based on success rate + if (globalStats.success_rate >= 80) { + successRate.classList.add('success-high'); + } else if (globalStats.success_rate >= 50) { + successRate.classList.add('success-medium'); + } else { + successRate.classList.add('success-low'); + } + + // Update total energy + document.getElementById('total-energy').textContent = globalStats.total_energy_added.toFixed(0) + ' kWh'; + + // Update SOC statistics + document.getElementById('avg-start-soc').textContent = socStats.average_start_soc.toFixed(1) + '%'; + document.getElementById('avg-end-soc').textContent = socStats.average_end_soc.toFixed(1) + '%'; +} + +// Update provider statistics table +function updateProviderTable(providers) { + const tableBody = document.querySelector('#provider-stats-table tbody'); + tableBody.innerHTML = ''; // Clear existing content + + // Make sure providers is an array and not null or undefined + if (!Array.isArray(providers) || providers.length === 0) { + const row = document.createElement('tr'); + row.innerHTML = ` + No provider data available + `; + tableBody.appendChild(row); + return; + } + + // Sort providers by total sessions (descending) + const sortedProviders = [...providers].sort((a, b) => b.total_sessions - a.total_sessions); + + // Show all providers, including those with just 1 session + const significantProviders = sortedProviders; + + significantProviders.forEach(provider => { + const row = document.createElement('tr'); + + // Determine success rate class + let successRateClass = ''; + if (provider.success_rate >= 80) { + successRateClass = 'success-high'; + } else if (provider.success_rate >= 50) { + successRateClass = 'success-medium'; + } else { + successRateClass = 'success-low'; + } + + // Format efficiency as N/A if it's zero (likely no data) + const efficiencyDisplay = provider.avg_efficiency > 0 + ? `${provider.avg_efficiency.toFixed(1)}%` + : 'N/A'; + + // Format avg power as N/A if it's zero + const avgPowerDisplay = provider.avg_power > 0 + ? `${provider.avg_power.toFixed(1)}` + : 'N/A'; + + row.innerHTML = ` + ${provider.provider} + ${provider.total_sessions.toLocaleString()} + ${provider.successful_sessions.toLocaleString()} + ${provider.failed_sessions.toLocaleString()} + ${provider.success_rate.toFixed(1)}% + ${efficiencyDisplay} + ${provider.total_energy_added.toFixed(1)} kWh + ${avgPowerDisplay} kW + `; + + tableBody.appendChild(row); + }); + + // If there are no providers with sufficient data + if (significantProviders.length === 0) { + const row = document.createElement('tr'); + row.innerHTML = ` + No provider data available + `; + tableBody.appendChild(row); + } + + // Add a total row at the bottom + const totalRow = document.createElement('tr'); + totalRow.classList.add('total-row'); + + // Calculate totals + const totalSessions = providers.reduce((sum, p) => sum + p.total_sessions, 0); + const successfulSessions = providers.reduce((sum, p) => sum + p.successful_sessions, 0); + const failedSessions = providers.reduce((sum, p) => sum + p.failed_sessions, 0); + const totalEnergy = providers.reduce((sum, p) => sum + p.total_energy_added, 0); + const successRate = totalSessions > 0 ? (successfulSessions / totalSessions) * 100 : 0; + + let totalRateClass = ''; + if (successRate >= 80) { + totalRateClass = 'success-high'; + } else if (successRate >= 50) { + totalRateClass = 'success-medium'; + } else { + totalRateClass = 'success-low'; + } + + totalRow.innerHTML = ` + TOTAL + ${totalSessions.toLocaleString()} + ${successfulSessions.toLocaleString()} + ${failedSessions.toLocaleString()} + ${successRate.toFixed(1)}% + - + ${totalEnergy.toFixed(1)} kWh + - + `; + + tableBody.appendChild(totalRow); +} + +// Create provider failure rate chart +function createProviderFailureChart(providers) { + // Check if we have valid data + if (!Array.isArray(providers) || providers.length === 0) { + // Display a message in the chart container + const chartContainer = document.getElementById('provider-failure-chart'); + chartContainer.innerHTML = '
No provider data available
'; + return; + } + + // Filter to providers with at least 5 sessions for meaningful data + // and at least 1 failed session + const significantProviders = providers.filter(p => + p.total_sessions >= 5 && p.failed_sessions > 0); + + if (significantProviders.length === 0) { + // Display a message in the chart container + const chartContainer = document.getElementById('provider-failure-chart'); + chartContainer.innerHTML = '
No providers with failures available
'; + return; + } + + // Calculate failure rates for each provider + significantProviders.forEach(p => { + p.failure_rate = 100 - p.success_rate; + }); + + // Sort by failure rate (descending) - highest failure rate first + const sortedProviders = [...significantProviders].sort((a, b) => b.failure_rate - a.failure_rate); + + // Limit to top 10 providers with highest failure rates + const topProviders = sortedProviders.slice(0, 10); + + // Use original provider names for display + const providerNames = topProviders.map(p => p.provider); + const failureRates = topProviders.map(p => p.failure_rate); + const sessionCounts = topProviders.map(p => p.total_sessions); + const failedCounts = topProviders.map(p => p.failed_sessions); + + const trace = { + x: providerNames, + y: failureRates, + type: 'bar', + name: 'Failure Rate (%)', + marker: { + color: failureRates.map(rate => { + if (rate >= 50) return '#e74c3c'; // High failure - red + if (rate >= 20) return '#f39c12'; // Medium failure - orange + return '#2ecc71'; // Low failure - green + }), + line: { + width: 1, + color: '#888' + } + }, + text: failureRates.map((rate, i) => + `${rate.toFixed(1)}%
(${failedCounts[i]}/${sessionCounts[i]})`), + textposition: 'outside', + textfont: { + size: 11, + color: '#333' + }, + hovertemplate: '%{x}
Failure Rate: %{y:.1f}%
Failed: %{text}', + width: 0.6 // Make bars narrower + }; + + const layout = { + title: { + text: 'Charging Failure Rates by Provider', + font: { size: 20 } + }, + xaxis: { + title: 'Provider', + tickangle: -45, + tickfont: { + size: 11 + }, + automargin: true // Automatically adjust margin to fit labels + }, + yaxis: { + title: 'Failure Rate (%)', + range: [0, 110], // Fixed range with some space at top for labels + fixedrange: true, // Prevent users from zooming/panning + ticksuffix: '%' + }, + height: 550, // Taller chart + autosize: true, + margin: { + b: 120, // Ample bottom margin for provider labels + t: 80, // Top margin for title + l: 70, // Left margin for y-axis labels + r: 50 // Right margin for values that might extend beyond bars + }, + bargap: 0.4, // More space between bars + template: plotlyTemplate + }; + + // Create responsive chart + const config = { + responsive: true, + displayModeBar: false // Hide the mode bar + }; + + Plotly.newPlot('provider-failure-chart', [trace], layout, config); +} + +// Create session outcomes pie chart +function createSessionOutcomesChart(globalStats) { + // Check if we have valid data + if (!globalStats || typeof globalStats !== 'object' || + typeof globalStats.successful_sessions !== 'number' || + typeof globalStats.failed_sessions !== 'number') { + // Display a message in the chart container + const chartContainer = document.getElementById('session-outcomes-chart'); + chartContainer.innerHTML = '
No session data available
'; + return; + } + + const total = globalStats.successful_sessions + globalStats.failed_sessions; + + if (total === 0) { + // Display a message if there are no sessions + const chartContainer = document.getElementById('session-outcomes-chart'); + chartContainer.innerHTML = '
No session data available
'; + return; + } + + const successPercent = ((globalStats.successful_sessions / total) * 100).toFixed(1); + const failedPercent = ((globalStats.failed_sessions / total) * 100).toFixed(1); + + const data = [{ + values: [globalStats.successful_sessions, globalStats.failed_sessions], + labels: ['Successful', 'Failed'], + type: 'pie', + marker: { + colors: ['#2ecc71', '#e74c3c'], + line: { + color: '#fff', + width: 2 + } + }, + textinfo: 'label+percent', + textposition: 'outside', + textfont: { + size: 14, + color: '#333' + }, + hoverinfo: 'label+value+percent', + hole: 0.4, // Create a donut chart for better visual appeal + pull: [0.03, 0], // Slightly separate the success slice + insidetextorientation: 'horizontal' + }]; + + const layout = { + title: { + text: 'Charging Session Outcomes', + font: { size: 20 } + }, + annotations: [ + { + // Add total sessions in the center of the donut + text: `${total}
Sessions`, + x: 0.5, + y: 0.5, + font: { + size: 16 + }, + showarrow: false + } + ], + height: 450, + autosize: true, + margin: { + t: 80, + b: 80, + l: 40, + r: 40 + }, + showlegend: true, + legend: { + orientation: 'h', + xanchor: 'center', + yanchor: 'top', + y: -0.15, + x: 0.5 + }, + template: plotlyTemplate + }; + + // Create responsive chart + const config = { + responsive: true, + displayModeBar: false // Hide the mode bar + }; + + Plotly.newPlot('session-outcomes-chart', data, layout, config); +} + +// Create energy by provider chart +function createEnergyByProviderChart(providers) { + // Check if we have valid data + if (!Array.isArray(providers) || providers.length === 0) { + // Display a message in the chart container + const chartContainer = document.getElementById('energy-by-provider-chart'); + chartContainer.innerHTML = '
No provider data available
'; + return; + } + + // Filter to providers with at least some energy added + const providersWithEnergy = providers.filter(p => p.total_energy_added > 0); + + if (providersWithEnergy.length === 0) { + // Display a message in the chart container + const chartContainer = document.getElementById('energy-by-provider-chart'); + chartContainer.innerHTML = '
No energy data available for providers
'; + return; + } + + // Sort by total energy added (descending) + const sortedProviders = [...providersWithEnergy].sort((a, b) => b.total_energy_added - a.total_energy_added); + + // Limit to top 12 providers for better readability + const topProviders = sortedProviders.slice(0, 12); + + // Use original provider names + const providerNames = topProviders.map(p => p.provider); + const energyValues = topProviders.map(p => p.total_energy_added); + const sessionCounts = topProviders.map(p => p.total_sessions); + + // Find max energy to properly scale the y-axis + const maxEnergy = Math.max(...energyValues); + const yAxisMax = Math.ceil(maxEnergy * 1.2); // Add 20% padding for labels + + const trace = { + x: providerNames, + y: energyValues, + type: 'bar', + name: 'Energy Added (kWh)', + marker: { + color: '#3498db', + opacity: 0.8, + line: { + width: 1, + color: '#2980b9' + } + }, + text: energyValues.map((val, i) => `${val.toFixed(1)} kWh
(${sessionCounts[i]} sessions)`), + textposition: 'outside', + textfont: { + size: 11, + color: '#333' + }, + hovertemplate: '%{x}
Total Energy: %{y:.1f} kWh
Sessions: %{text}', + width: 0.6 // Make bars narrower + }; + + const layout = { + title: { + text: 'Total Energy Added by Provider (kWh)', + font: { size: 20 } + }, + xaxis: { + title: 'Provider', + tickangle: -45, + tickfont: { + size: 11 + }, + automargin: true // Automatically adjust margin to fit labels + }, + yaxis: { + title: 'Energy Added (kWh)', + rangemode: 'tozero', + range: [0, yAxisMax], // Set fixed range with padding for labels + fixedrange: true, + ticksuffix: ' kWh' + }, + height: 580, // Taller chart for better readability + autosize: true, + margin: { + b: 120, // Ample bottom margin for provider labels + t: 80, // Top margin for title + l: 80, // Left margin for y-axis labels and values + r: 60 // Right margin for values that might extend beyond bars + }, + bargap: 0.4, // More space between bars + template: plotlyTemplate + }; + + // Create responsive chart + const config = { + responsive: true, + displayModeBar: false // Hide the mode bar + }; + + Plotly.newPlot('energy-by-provider-chart', [trace], layout, config); +} + +// Display error message +function showErrorMessage(message) { + // Create an error message element + const errorElement = document.createElement('div'); + errorElement.className = 'error-message'; + errorElement.style.backgroundColor = '#ffebee'; + errorElement.style.color = '#c62828'; + errorElement.style.padding = '15px'; + errorElement.style.borderRadius = '5px'; + errorElement.style.marginBottom = '20px'; + errorElement.style.textAlign = 'center'; + errorElement.textContent = message; + + // Insert at the top of the container + const container = document.querySelector('.container'); + container.insertBefore(errorElement, container.firstChild.nextSibling); +} + +// We removed the formatProviderName function as we want to display the original provider names exactly as stored in the database diff --git a/go-rewrite/static/stats.html b/go-rewrite/static/stats.html new file mode 100644 index 0000000..a20fb29 --- /dev/null +++ b/go-rewrite/static/stats.html @@ -0,0 +1,293 @@ + + + + + + BMW CarData - Anonymous Charging Statistics + + + + + + + + + + +
+
+

BMW CarData - Anonymous Charging Statistics

+ +

This dashboard shows anonymous statistics from all uploaded charging sessions. No personal data or locations are stored.

+
+ +
+

Global Statistics

+
+
+
+

Total Sessions

+ πŸ“Š +
+
-
+
Charging sessions analyzed
+
+ +
+
+

Success Rate

+ βœ… +
+
-
+
Sessions that successfully added charge
+
+ +
+
+

Total Energy

+ ⚑ +
+
-
+
Total kWh added to vehicles
+
+ +
+
+

SOC Statistics

+ πŸ”‹ +
+
+
+
Avg Start SOC
+
-
+
+
+
Avg End SOC
+
-
+
+
+
+
+
+ +
+

Provider Failure Rates

+
+
+ +
+

Session Outcomes

+
+
+ +
+

Energy Distribution by Provider

+
+
+ +
+

Charging Provider Statistics

+
+

Detailed statistics for each charging provider, showing success rates and energy metrics.

+
+ + + + + + + + + + + + + + + + + + +
ProviderSessionsSuccessfulFailedSuccess RateAvg EfficiencyEnergy Added (kWh)Avg Power (kW)
Loading data...
+
+
+
+ +
+ Back to Dashboard +
+
+ +
+
+

Impressum

+

Freie Netze MΓΌnchen e. V.

+

Parkstraße 28

+

82131 Gauting

+
+
+ + + + + diff --git a/go-rewrite/test_comprehensive_isolation.sh b/go-rewrite/test_comprehensive_isolation.sh new file mode 100755 index 0000000..6fa672b --- /dev/null +++ b/go-rewrite/test_comprehensive_isolation.sh @@ -0,0 +1,293 @@ +#!/bin/bash + +# Comprehensive test script for bmwtools session isolation +# This script uploads two different JSON files in parallel and verifies that +# each session only sees its own data across multiple API endpoints + +# Command-line arguments check +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo "***REMOVED*** $0 file1.json file2.json" + exit 1 +fi + +# Check if files exist +if [ ! -f "$1" ]; then + echo "Error: File $1 does not exist" + exit 1 +fi + +if [ ! -f "$2" ]; then + echo "Error: File $2 does not exist" + exit 1 +fi + +JSON_FILE1="$1" +JSON_FILE2="$2" +SERVER_URL="${SERVER_URL:-http://localhost:8050}" +OUTPUT_DIR=$(mktemp -d) +DELAY_SECONDS=2 +MAX_RETRIES=3 + +echo "================= COMPREHENSIVE SESSION ISOLATION TEST ==================" +echo "Server URL: $SERVER_URL" +echo "JSON File 1: $JSON_FILE1" +echo "JSON File 2: $JSON_FILE2" +echo "Output directory: $OUTPUT_DIR" +echo "======================================================================" + +# Function to check if the server is running +check_server() { + local retries=0 + while [ $retries -lt $MAX_RETRIES ]; do + echo "Checking server at ${SERVER_URL}..." + SERVER_RESPONSE=$(curl -s "${SERVER_URL}/api/version") + if [ $? -eq 0 ] && [ -n "$SERVER_RESPONSE" ]; then + echo "Server is running. Server version: $SERVER_RESPONSE" + + # Test a cookie-based endpoint with no cookie to verify the expected behavior + echo "Testing cookie handling with no cookies..." + NO_COOKIE_RESPONSE=$(curl -s "${SERVER_URL}/api/sessions") + if [[ "$NO_COOKIE_RESPONSE" == "[]" || "$NO_COOKIE_RESPONSE" == "{\"error\":\"*\"}" ]]; then + echo "Server correctly returns empty data when no session cookie is provided" + else + echo "WARNING: Server returns data without session cookie. Response: ${NO_COOKIE_RESPONSE:0:100}..." + fi + return 0 + fi + retries=$((retries + 1)) + echo "Server not responding, retrying in 2 seconds... ($retries/$MAX_RETRIES)" + sleep 2 + done + echo "Error: Could not connect to server at ${SERVER_URL}" + exit 1 +} + +# Function to perform comprehensive tests for session 1 +run_comprehensive_test_session_1() { + echo "[Session 1] Starting comprehensive tests..." + + # Upload JSON file + echo "[Session 1] Uploading $JSON_FILE1" + UPLOAD_RESPONSE=$(curl -s -X POST \ + -F "file=@$JSON_FILE1" \ + -F "consent=false" \ + -c "${OUTPUT_DIR}/cookies_1.txt" \ + -v \ + "${SERVER_URL}/api/upload" 2> "${OUTPUT_DIR}/curl_verbose_1.txt") + + # Debug cookie information + echo "[Session 1] Cookies received:" + cat "${OUTPUT_DIR}/cookies_1.txt" + + # Extract session_id cookie specifically + SESSION_ID=$(grep -A 10 "Set-Cookie:" "${OUTPUT_DIR}/curl_verbose_1.txt" | grep "session_id" | head -1 | sed -n 's/.*session_id=\([^;]*\).*/\1/p') + echo "[Session 1] Session ID from cookies: $SESSION_ID" + + # Extract the session count + SESSION_COUNT=$(echo "$UPLOAD_RESPONSE" | grep -o '"count":[0-9]*' | cut -d':' -f2) + echo "[Session 1] Uploaded $SESSION_COUNT sessions" + echo "$UPLOAD_RESPONSE" > "${OUTPUT_DIR}/upload_response_1.json" + + # Wait to ensure interleaving with session 2 + sleep $DELAY_SECONDS + + # Test API endpoints + test_endpoints 1 "${OUTPUT_DIR}/cookies_1.txt" "$SESSION_COUNT" + + # Wait and test again to check for leaks + sleep $((DELAY_SECONDS * 2)) + echo "[Session 1] Testing endpoints again after delay (checking for leaks)" + test_endpoints 1 "${OUTPUT_DIR}/cookies_1.txt" "$SESSION_COUNT" "_after_delay" + + echo "[Session 1] Comprehensive tests complete" +} + +# Function to perform comprehensive tests for session 2 +run_comprehensive_test_session_2() { + # Wait before starting to ensure first session is already in progress + sleep $(($DELAY_SECONDS / 2)) + + echo "[Session 2] Starting comprehensive tests..." + + # Upload JSON file + echo "[Session 2] Uploading $JSON_FILE2" + UPLOAD_RESPONSE=$(curl -s -X POST \ + -F "file=@$JSON_FILE2" \ + -F "consent=false" \ + -c "${OUTPUT_DIR}/cookies_2.txt" \ + -v \ + "${SERVER_URL}/api/upload" 2> "${OUTPUT_DIR}/curl_verbose_2.txt") + + # Debug cookie information + echo "[Session 2] Cookies received:" + cat "${OUTPUT_DIR}/cookies_2.txt" + + # Extract session_id cookie specifically + SESSION_ID=$(grep -A 10 "Set-Cookie:" "${OUTPUT_DIR}/curl_verbose_2.txt" | grep "session_id" | head -1 | sed -n 's/.*session_id=\([^;]*\).*/\1/p') + echo "[Session 2] Session ID from cookies: $SESSION_ID" + + # Extract the session count + SESSION_COUNT=$(echo "$UPLOAD_RESPONSE" | grep -o '"count":[0-9]*' | cut -d':' -f2) + echo "[Session 2] Uploaded $SESSION_COUNT sessions" + echo "$UPLOAD_RESPONSE" > "${OUTPUT_DIR}/upload_response_2.json" + + # Wait to ensure interleaving with session 1 + sleep $DELAY_SECONDS + + # Test API endpoints + test_endpoints 2 "${OUTPUT_DIR}/cookies_2.txt" "$SESSION_COUNT" + + # Wait and test again to check for leaks + sleep $((DELAY_SECONDS * 2)) + echo "[Session 2] Testing endpoints again after delay (checking for leaks)" + test_endpoints 2 "${OUTPUT_DIR}/cookies_2.txt" "$SESSION_COUNT" "_after_delay" + + echo "[Session 2] Comprehensive tests complete" +} + +# Function to test all relevant API endpoints +test_endpoints() { + local session_num=$1 + local cookies_file=$2 + local expected_count=$3 + local suffix=$4 # Optional suffix for output files + + echo "[Session $session_num] Testing API endpoints..." + + # Check cookie file content before using it + echo "[Session $session_num] Cookie file content:" + cat "$cookies_file" + + # Manually extract session_id cookie for debugging + SESSION_ID=$(grep "session_id" "$cookies_file" | cut -f7) + echo "[Session $session_num] Session ID from cookie file: $SESSION_ID" + + # Use -v for verbose output to debug cookie issues + echo "[Session $session_num] Testing /api/sessions endpoint" + SESSIONS_RESPONSE=$(curl -s -b "$cookies_file" -v "${SERVER_URL}/api/sessions" 2> "${OUTPUT_DIR}/sessions_curl_${session_num}${suffix}.txt") + + # Save full response for analysis + echo "$SESSIONS_RESPONSE" > "${OUTPUT_DIR}/sessions_response_${session_num}${suffix}.json" + + # Check if response starts with an array bracket + if [[ "$SESSIONS_RESPONSE" == \[* ]]; then + # Count using simple grep for session objects - most reliable across systems + RETRIEVED_COUNT=$(echo "$SESSIONS_RESPONSE" | grep -o '"avg_power"' | wc -l | tr -d ' ') + else + RETRIEVED_COUNT=0 + fi + + echo "[Session $session_num] Retrieved $RETRIEVED_COUNT sessions (expected $expected_count)" + + # Display raw response for debugging + echo "[Session $session_num] Raw sessions response (first 200 chars):" + echo "$SESSIONS_RESPONSE" | head -c 200 + + # Test /api/stats + echo "[Session $session_num] Testing /api/stats endpoint" + STATS_RESPONSE=$(curl -s -b "$cookies_file" "${SERVER_URL}/api/stats") + echo "$STATS_RESPONSE" > "${OUTPUT_DIR}/stats_response_${session_num}${suffix}.json" + + # Test /api/map + echo "[Session $session_num] Testing /api/map endpoint" + # Add explicit Cookie header to ensure it's being sent properly + if [ -n "$SESSION_ID" ]; then + echo "[Session $session_num] Using explicit session_id cookie: $SESSION_ID" + MAP_RESPONSE=$(curl -s -b "$cookies_file" -H "Cookie: session_id=$SESSION_ID" -v "${SERVER_URL}/api/map" 2> "${OUTPUT_DIR}/map_curl_${session_num}${suffix}.txt") + else + MAP_RESPONSE=$(curl -s -b "$cookies_file" -v "${SERVER_URL}/api/map" 2> "${OUTPUT_DIR}/map_curl_${session_num}${suffix}.txt") + fi + echo "$MAP_RESPONSE" > "${OUTPUT_DIR}/map_response_${session_num}${suffix}.json" + MAP_COUNT=$(echo "$MAP_RESPONSE" | grep -o '"latitude"' | wc -l | tr -d ' ') + echo "[Session $session_num] Retrieved $MAP_COUNT map locations" + + # Test /api/providers + echo "[Session $session_num] Testing /api/providers endpoint" + PROVIDERS_RESPONSE=$(curl -s -b "$cookies_file" "${SERVER_URL}/api/providers") + echo "$PROVIDERS_RESPONSE" > "${OUTPUT_DIR}/providers_response_${session_num}${suffix}.json" + + # Verify session count consistency + clean_retrieved=$(echo "$RETRIEVED_COUNT" | tr -d '\n') + clean_expected=$(echo "$expected_count" | tr -d '\n') + + if [ "$clean_retrieved" = "$clean_expected" ]; then + echo "[Session $session_num] βœ… Session count test PASSED: Expected $clean_expected, got $clean_retrieved" + else + echo "[Session $session_num] ❌ Session count test FAILED: Expected $clean_expected, got $clean_retrieved" + fi +} + +# Function to verify session isolation across all endpoints +verify_comprehensive_isolation() { + echo "Verifying comprehensive session isolation..." + + # Check session data isolation + SESSION1_IDS=$(cat "${OUTPUT_DIR}/sessions_response_1_after_delay.json" | grep -o '"id":"[^"]*"' | head -5 | sort) + SESSION2_IDS=$(cat "${OUTPUT_DIR}/sessions_response_2_after_delay.json" | grep -o '"id":"[^"]*"' | head -5 | sort) + + # Check if there's any overlap in session IDs + OVERLAP=$(comm -12 <(echo "$SESSION1_IDS") <(echo "$SESSION2_IDS")) + + if [ -z "$OVERLAP" ]; then + echo "βœ… Session data isolation test PASSED: No session data leakage detected" + else + echo "❌ Session data isolation test FAILED: Found overlapping session IDs between sessions!" + echo "Overlapping IDs:" + echo "$OVERLAP" + fi + + # Check map data isolation + MAP1_COUNT=$(cat "${OUTPUT_DIR}/map_response_1_after_delay.json" | grep -o '"latitude"' | wc -l | tr -d ' ') + MAP2_COUNT=$(cat "${OUTPUT_DIR}/map_response_2_after_delay.json" | grep -o '"latitude"' | wc -l | tr -d ' ') + + MAP1_FIRST=$(cat "${OUTPUT_DIR}/map_response_1_after_delay.json" | grep -o '"name":"[^"]*"' | head -1) + MAP2_FIRST=$(cat "${OUTPUT_DIR}/map_response_2_after_delay.json" | grep -o '"name":"[^"]*"' | head -1) + + echo "Map locations - Session 1: $MAP1_COUNT, Session 2: $MAP2_COUNT" + echo "First location - Session 1: $MAP1_FIRST, Session 2: $MAP2_FIRST" + + if [ "$MAP1_FIRST" != "$MAP2_FIRST" ]; then + echo "βœ… Map data isolation test PASSED: Different map data between sessions" + else + # This is only a warning as it's possible two different files might have some identical locations + echo "⚠️ Map data isolation WARNING: First location is identical in both sessions" + fi + + # Check stats data isolation + STATS1_EFFICIENCY=$(cat "${OUTPUT_DIR}/stats_response_1_after_delay.json" | grep -o '"overall_efficiency":[0-9.]*' | cut -d':' -f2) + STATS2_EFFICIENCY=$(cat "${OUTPUT_DIR}/stats_response_2_after_delay.json" | grep -o '"overall_efficiency":[0-9.]*' | cut -d':' -f2) + + echo "Overall efficiency - Session 1: $STATS1_EFFICIENCY, Session 2: $STATS2_EFFICIENCY" + + if [ "$STATS1_EFFICIENCY" != "$STATS2_EFFICIENCY" ]; then + echo "βœ… Stats data isolation test PASSED: Different statistics between sessions" + else + echo "⚠️ Stats data isolation WARNING: Same efficiency in both sessions (may be coincidental)" + fi + + echo "Test output saved to $OUTPUT_DIR" + echo "To view the full results, check the JSON files in that directory" +} + +# Main execution + +# Check if server is running +check_server + +# Run both test sessions in parallel +run_comprehensive_test_session_1 & +SESSION1_PID=$! + +run_comprehensive_test_session_2 & +SESSION2_PID=$! + +# Wait for both processes to finish +wait $SESSION1_PID +wait $SESSION2_PID + +# Verify isolation between sessions +verify_comprehensive_isolation + +echo "Comprehensive test complete. Results saved in $OUTPUT_DIR" +echo "To view all test data: ls -la $OUTPUT_DIR" diff --git a/go-rewrite/test_session_isolation.sh b/go-rewrite/test_session_isolation.sh new file mode 100755 index 0000000..fb10639 --- /dev/null +++ b/go-rewrite/test_session_isolation.sh @@ -0,0 +1,362 @@ +#!/bin/bash + +# Test script for bmwtools session isolation +# This script uploads two different JSON files in parallel and verifies that +# each session only sees its own data + +# Command-line arguments check +if [ "$#" -ne 2 ]; then + echo "Usage: $0 " + echo "***REMOVED*** $0 file1.json file2.json" + exit 1 +fi + +# Check if files exist +if [ ! -f "$1" ]; then + echo "Error: File $1 does not exist" + exit 1 +fi + +if [ ! -f "$2" ]; then + echo "Error: File $2 does not exist" + exit 1 +fi + +JSON_FILE1="$1" +JSON_FILE2="$2" +SERVER_URL="${SERVER_URL:-http://localhost:8050}" +OUTPUT_DIR=$(mktemp -d) +DELAY_SECONDS=2 + +echo "=================== SESSION ISOLATION TEST ===================" +echo "Server URL: $SERVER_URL" +echo "JSON File 1: $JSON_FILE1" +echo "JSON File 2: $JSON_FILE2" +echo "Output directory: $OUTPUT_DIR" +echo "==============================================================" + +# Function to check if the server is running +check_server() { + local retries=0 + local max_retries=3 + + while [ $retries -lt $max_retries ]; do + echo "Checking server at ${SERVER_URL}..." + SERVER_RESPONSE=$(curl -s "${SERVER_URL}/api/version") + if [ $? -eq 0 ] && [ -n "$SERVER_RESPONSE" ]; then + echo "Server is running. Server version: $SERVER_RESPONSE" + return 0 + fi + retries=$((retries + 1)) + echo "Server not responding, retrying in 2 seconds... ($retries/$max_retries)" + sleep 2 + done + echo "Error: Could not connect to server at ${SERVER_URL}" + exit 1 +} + +# Check if server is running +check_server + +# Function to perform uploads and tests for session 1 +run_session_1() { + echo "[Session 1] Starting..." + + # Upload JSON file + echo "[Session 1] Uploading $JSON_FILE1" + UPLOAD_RESPONSE=$(curl -s -X POST \ + -F "file=@$JSON_FILE1" \ + -F "consent=false" \ + -c "${OUTPUT_DIR}/cookies_1.txt" \ + -v \ + "${SERVER_URL}/api/upload" 2> "${OUTPUT_DIR}/curl_verbose_1.txt") + + # Debug cookie information + echo "[Session 1] Cookies received:" + cat "${OUTPUT_DIR}/cookies_1.txt" + + # Extract session_id cookie specifically + SESSION_ID=$(grep -A 10 "Set-Cookie:" "${OUTPUT_DIR}/curl_verbose_1.txt" | grep "session_id" | head -1 | sed -n 's/.*session_id=\([^;]*\).*/\1/p') + echo "[Session 1] Session ID from cookies: $SESSION_ID" + + SESSION_COUNT=$(echo "$UPLOAD_RESPONSE" | grep -o '"count":[0-9]*' | cut -d':' -f2) + echo "[Session 1] Uploaded $SESSION_COUNT sessions" + + # Wait a bit + sleep $DELAY_SECONDS + + # Get sessions + echo "[Session 1] Getting sessions" + # Add explicit Cookie header to ensure it's being sent properly + if [ -n "$SESSION_ID" ]; then + echo "[Session 1] Using explicit session_id cookie: $SESSION_ID" + SESSIONS_RESPONSE=$(curl -s -b "${OUTPUT_DIR}/cookies_1.txt" -H "Cookie: session_id=$SESSION_ID" -v "${SERVER_URL}/api/sessions" 2> "${OUTPUT_DIR}/sessions_curl_1.txt") + else + SESSIONS_RESPONSE=$(curl -s -b "${OUTPUT_DIR}/cookies_1.txt" -v "${SERVER_URL}/api/sessions" 2> "${OUTPUT_DIR}/sessions_curl_1.txt") + fi + + # Save full response for analysis + echo "$SESSIONS_RESPONSE" > "${OUTPUT_DIR}/sessions_response_1_full.json" + + # Check if response starts with an array bracket + if [[ "$SESSIONS_RESPONSE" == \[* ]]; then + # Count using simple grep for session objects - most reliable across systems + RETRIEVED_COUNT=$(echo "$SESSIONS_RESPONSE" | grep -o '"avg_power"' | wc -l | tr -d ' ') + else + RETRIEVED_COUNT=0 + fi + + echo "[Session 1] Retrieved $RETRIEVED_COUNT sessions" + + # Display raw response for debugging + echo "[Session 1] Raw sessions response (first 200 chars):" + echo "$SESSIONS_RESPONSE" | head -c 200 + + # Wait a bit more to ensure interleaving with session 2 + sleep $((DELAY_SECONDS * 2)) + + # Get sessions again to verify no contamination from session 2 + echo "[Session 1] Getting sessions again (checking for leaks)" + SESSIONS_RESPONSE_2=$(curl -s -b "${OUTPUT_DIR}/cookies_1.txt" "${SERVER_URL}/api/sessions") + + # Save full response for analysis + echo "$SESSIONS_RESPONSE_2" > "${OUTPUT_DIR}/sessions_response_1_leak_check.json" + + # Check if response starts with an array bracket + if [[ "$SESSIONS_RESPONSE_2" == \[* ]]; then + # Count using simple grep for session objects - most reliable across systems + RETRIEVED_COUNT_2=$(echo "$SESSIONS_RESPONSE_2" | grep -o '"avg_power"' | wc -l | tr -d ' ') + else + RETRIEVED_COUNT_2=0 + fi + + echo "[Session 1] Retrieved $RETRIEVED_COUNT_2 sessions" + + # Verify counts match + if [ "$SESSION_COUNT" -eq "$RETRIEVED_COUNT_2" ]; then + echo "[Session 1] βœ… Test PASSED: Session data is consistent" + else + echo "[Session 1] ❌ Test FAILED: Session count mismatch! Expected $SESSION_COUNT, got $RETRIEVED_COUNT_2" + echo "$SESSIONS_RESPONSE_2" > "${OUTPUT_DIR}/leak_sessions_1.json" + fi + + # Save full responses for detailed analysis + echo "$UPLOAD_RESPONSE" > "${OUTPUT_DIR}/upload_response_1.json" + echo "$SESSIONS_RESPONSE" > "${OUTPUT_DIR}/sessions_response_1.json" + echo "$SESSIONS_RESPONSE_2" > "${OUTPUT_DIR}/sessions_response_2_1.json" + + echo "[Session 1] Complete" +} + +# Function to perform uploads and tests for session 2 +run_session_2() { + # Wait a bit before starting to ensure first session is already in progress + sleep $DELAY_SECONDS + + echo "[Session 2] Starting..." + + # Upload JSON file + echo "[Session 2] Uploading $JSON_FILE2" + UPLOAD_RESPONSE=$(curl -s -X POST \ + -F "file=@$JSON_FILE2" \ + -F "consent=false" \ + -c "${OUTPUT_DIR}/cookies_2.txt" \ + -v \ + "${SERVER_URL}/api/upload" 2> "${OUTPUT_DIR}/curl_verbose_2.txt") + + # Debug cookie information + echo "[Session 2] Cookies received:" + cat "${OUTPUT_DIR}/cookies_2.txt" + + # Extract session_id cookie specifically + SESSION_ID=$(grep -A 10 "Set-Cookie:" "${OUTPUT_DIR}/curl_verbose_2.txt" | grep "session_id" | head -1 | sed -n 's/.*session_id=\([^;]*\).*/\1/p') + echo "[Session 2] Session ID from cookies: $SESSION_ID" + + SESSION_COUNT=$(echo "$UPLOAD_RESPONSE" | grep -o '"count":[0-9]*' | cut -d':' -f2) + echo "[Session 2] Uploaded $SESSION_COUNT sessions" + + # Wait a bit + sleep $((DELAY_SECONDS + 1)) + + # Get sessions + echo "[Session 2] Getting sessions" + # Add explicit Cookie header to ensure it's being sent properly + if [ -n "$SESSION_ID" ]; then + echo "[Session 2] Using explicit session_id cookie: $SESSION_ID" + SESSIONS_RESPONSE=$(curl -s -b "${OUTPUT_DIR}/cookies_2.txt" -H "Cookie: session_id=$SESSION_ID" -v "${SERVER_URL}/api/sessions" 2> "${OUTPUT_DIR}/sessions_curl_2.txt") + else + SESSIONS_RESPONSE=$(curl -s -b "${OUTPUT_DIR}/cookies_2.txt" -v "${SERVER_URL}/api/sessions" 2> "${OUTPUT_DIR}/sessions_curl_2.txt") + fi + + # Save full response for analysis + echo "$SESSIONS_RESPONSE" > "${OUTPUT_DIR}/sessions_response_2_full.json" + + # Check if response starts with an array bracket + if [[ "$SESSIONS_RESPONSE" == \[* ]]; then + # Count using simple grep for session objects - most reliable across systems + RETRIEVED_COUNT=$(echo "$SESSIONS_RESPONSE" | grep -o '"avg_power"' | wc -l | tr -d ' ') + else + RETRIEVED_COUNT=0 + fi + + echo "[Session 2] Retrieved $RETRIEVED_COUNT sessions" + + # Display raw response for debugging + echo "[Session 2] Raw sessions response (first 200 chars):" + echo "$SESSIONS_RESPONSE" | head -c 200 + + # Wait a bit more + sleep $DELAY_SECONDS + + # Get sessions again to verify no contamination from session 1 + echo "[Session 2] Getting sessions again (checking for leaks)" + SESSIONS_RESPONSE_2=$(curl -s -b "${OUTPUT_DIR}/cookies_2.txt" "${SERVER_URL}/api/sessions") + + # Save full response for analysis + echo "$SESSIONS_RESPONSE_2" > "${OUTPUT_DIR}/sessions_response_2_leak_check.json" + + # Check if response starts with an array bracket + if [[ "$SESSIONS_RESPONSE_2" == \[* ]]; then + # Count using simple grep for session objects - most reliable across systems + RETRIEVED_COUNT_2=$(echo "$SESSIONS_RESPONSE_2" | grep -o '"avg_power"' | wc -l | tr -d ' ') + else + RETRIEVED_COUNT_2=0 + fi + + echo "[Session 2] Retrieved $RETRIEVED_COUNT_2 sessions" + + # Verify counts match + if [ "$SESSION_COUNT" -eq "$RETRIEVED_COUNT_2" ]; then + echo "[Session 2] βœ… Test PASSED: Session data is consistent" + else + echo "[Session 2] ❌ Test FAILED: Session count mismatch! Expected $SESSION_COUNT, got $RETRIEVED_COUNT_2" + echo "$SESSIONS_RESPONSE_2" > "${OUTPUT_DIR}/leak_sessions_2.json" + fi + + # Save full responses for detailed analysis + echo "$UPLOAD_RESPONSE" > "${OUTPUT_DIR}/upload_response_2.json" + echo "$SESSIONS_RESPONSE" > "${OUTPUT_DIR}/sessions_response_1_2.json" + echo "$SESSIONS_RESPONSE_2" > "${OUTPUT_DIR}/sessions_response_2_2.json" + + echo "[Session 2] Complete" +} + +# Function to verify that sessions don't contaminate each other +verify_session_isolation() { + echo "Verifying session isolation..." + + # Get the first session IDs from each session's responses + # Using simple grep/cut to extract IDs from the first few sessions + SESSION1_IDS=$(cat "${OUTPUT_DIR}/sessions_response_2_1.json" | grep -o '"id":"[^"]*"' | head -5 | sort) + SESSION2_IDS=$(cat "${OUTPUT_DIR}/sessions_response_2_2.json" | grep -o '"id":"[^"]*"' | head -5 | sort) + + # Check if there's any overlap in session IDs + OVERLAP=$(comm -12 <(echo "$SESSION1_IDS") <(echo "$SESSION2_IDS")) + + if [ -z "$OVERLAP" ]; then + echo "βœ… Session isolation test PASSED: No session data leakage detected" + else + echo "❌ Session isolation test FAILED: Found overlapping session IDs between sessions!" + echo "Overlapping IDs:" + echo "$OVERLAP" + fi + + echo "Test output saved to $OUTPUT_DIR" +} + +# Function for just uploading files in true parallel +upload_in_parallel() { + echo "[Parallel Upload] Starting simultaneous uploads..." + + # Prepare upload commands for both sessions + UPLOAD_CMD1="curl -s -X POST -F \"file=@$JSON_FILE1\" -F \"consent=false\" -c \"${OUTPUT_DIR}/cookies_1.txt\" -v \"${SERVER_URL}/api/upload\" 2> \"${OUTPUT_DIR}/curl_verbose_1.txt\" > \"${OUTPUT_DIR}/upload_response_1.txt\"" + UPLOAD_CMD2="curl -s -X POST -F \"file=@$JSON_FILE2\" -F \"consent=false\" -c \"${OUTPUT_DIR}/cookies_2.txt\" -v \"${SERVER_URL}/api/upload\" 2> \"${OUTPUT_DIR}/curl_verbose_2.txt\" > \"${OUTPUT_DIR}/upload_response_2.txt\"" + + # Run both uploads with minimal delay between them + eval "$UPLOAD_CMD1" & + UPLOAD_PID1=$! + sleep 0.1 # Very minimal delay to avoid exact same millisecond + eval "$UPLOAD_CMD2" & + UPLOAD_PID2=$! + + # Wait for uploads to complete + wait $UPLOAD_PID1 + wait $UPLOAD_PID2 + + echo "[Parallel Upload] Both uploads completed" + + # Display upload results + echo "[Parallel Upload] Session 1 uploaded:" + SESSION1_COUNT=$(cat "${OUTPUT_DIR}/upload_response_1.txt" | grep -o '"count":[0-9]*' | cut -d':' -f2) + echo "[Parallel Upload] Uploaded $SESSION1_COUNT sessions for Session 1" + + echo "[Parallel Upload] Session 2 uploaded:" + SESSION2_COUNT=$(cat "${OUTPUT_DIR}/upload_response_2.txt" | grep -o '"count":[0-9]*' | cut -d':' -f2) + echo "[Parallel Upload] Uploaded $SESSION2_COUNT sessions for Session 2" + + # Extract session cookies + SESSION_ID1=$(grep -A 10 "Set-Cookie:" "${OUTPUT_DIR}/curl_verbose_1.txt" | grep "session_id" | head -1 | sed -n 's/.*session_id=\([^;]*\).*/\1/p') + SESSION_ID2=$(grep -A 10 "Set-Cookie:" "${OUTPUT_DIR}/curl_verbose_2.txt" | grep "session_id" | head -1 | sed -n 's/.*session_id=\([^;]*\).*/\1/p') + + echo "[Parallel Upload] Session 1 ID: $SESSION_ID1" + echo "[Parallel Upload] Session 2 ID: $SESSION_ID2" + + # Check sessions immediately after parallel upload + check_sessions_after_parallel_upload +} + +# Function to check sessions immediately after parallel upload +check_sessions_after_parallel_upload() { + echo "[Parallel Test] Checking sessions immediately after parallel upload" + + # Get sessions for first upload + SESSION_ID1=$(grep -A 10 "Set-Cookie:" "${OUTPUT_DIR}/curl_verbose_1.txt" | grep "session_id" | head -1 | sed -n 's/.*session_id=\([^;]*\).*/\1/p') + SESSIONS_RESPONSE1=$(curl -s -b "${OUTPUT_DIR}/cookies_1.txt" -H "Cookie: session_id=$SESSION_ID1" "${SERVER_URL}/api/sessions") + echo "$SESSIONS_RESPONSE1" > "${OUTPUT_DIR}/sessions_immediate_1.json" + RETRIEVED_COUNT1=$(echo "$SESSIONS_RESPONSE1" | grep -o '"avg_power"' | wc -l | tr -d ' ') + + # Get sessions for second upload + SESSION_ID2=$(grep -A 10 "Set-Cookie:" "${OUTPUT_DIR}/curl_verbose_2.txt" | grep "session_id" | head -1 | sed -n 's/.*session_id=\([^;]*\).*/\1/p') + SESSIONS_RESPONSE2=$(curl -s -b "${OUTPUT_DIR}/cookies_2.txt" -H "Cookie: session_id=$SESSION_ID2" "${SERVER_URL}/api/sessions") + echo "$SESSIONS_RESPONSE2" > "${OUTPUT_DIR}/sessions_immediate_2.json" + RETRIEVED_COUNT2=$(echo "$SESSIONS_RESPONSE2" | grep -o '"avg_power"' | wc -l | tr -d ' ') + + # Compare to upload counts + SESSION1_COUNT=$(cat "${OUTPUT_DIR}/upload_response_1.txt" | grep -o '"count":[0-9]*' | cut -d':' -f2) + SESSION2_COUNT=$(cat "${OUTPUT_DIR}/upload_response_2.txt" | grep -o '"count":[0-9]*' | cut -d':' -f2) + + echo "[Parallel Test] Session 1: Expected $SESSION1_COUNT, Retrieved $RETRIEVED_COUNT1" + echo "[Parallel Test] Session 2: Expected $SESSION2_COUNT, Retrieved $RETRIEVED_COUNT2" + + # Check for any data leakage + if [ "$SESSION1_COUNT" -eq "$RETRIEVED_COUNT1" ]; then + echo "[Parallel Test] βœ… Session 1 data consistent after parallel upload" + else + echo "[Parallel Test] ❌ Session 1 data inconsistent after parallel upload! Expected $SESSION1_COUNT, got $RETRIEVED_COUNT1" + fi + + if [ "$SESSION2_COUNT" -eq "$RETRIEVED_COUNT2" ]; then + echo "[Parallel Test] βœ… Session 2 data consistent after parallel upload" + else + echo "[Parallel Test] ❌ Session 2 data inconsistent after parallel upload! Expected $SESSION2_COUNT, got $RETRIEVED_COUNT2" + fi +} + +# Run the full isolation test with true parallel uploads first +upload_in_parallel + +# Then run the full test with sessions as before +run_session_1 & +SESSION1_PID=$! + +run_session_2 & +SESSION2_PID=$! + +# Wait for both processes to finish +wait $SESSION1_PID +wait $SESSION2_PID + +# Verify isolation between sessions +verify_session_isolation + +echo "Test complete. Results saved in $OUTPUT_DIR" diff --git a/go-rewrite/traefik/README.md b/go-rewrite/traefik/README.md index 3a8a0b5..20f8a67 100644 --- a/go-rewrite/traefik/README.md +++ b/go-rewrite/traefik/README.md @@ -5,9 +5,9 @@ This project now uses Traefik as a reverse proxy instead of Nginx + Certbot. ## Setup Instructions 1. Make sure the `acme.json` file has the proper permissions: - ``` + ***REMOVED*** chmod 600 traefik/acme.json - ``` + ***REMOVED*** 2. Update the domain name in `docker-compose.yaml` if needed: - The default domain is set to `bmwtools.localhost` @@ -19,9 +19,9 @@ This project now uses Traefik as a reverse proxy instead of Nginx + Certbot. ## Starting the Application -``` +***REMOVED*** docker-compose up -d -``` +***REMOVED*** ## How It Works