Optimize SQLite writes and add PostgreSQL migration script

- Enabled WAL mode and set synchronous to NORMAL for SQLite to reduce CPU usage during writes.
- Added support for PostgreSQL via the DATABASE_URL environment variable.
- Created a migration script in cmd/migrate to facilitate moving data from SQLite to PostgreSQL.
- Updated documentation and environment examples for the new features.

Co-authored-by: birabittoh <26506860+birabittoh@users.noreply.github.com>

Author: google-labs-jules[bot]

.env.example

@@ -1,3 +1,6 @@
 OWM_API_KEY=
 OWM_LATITUDE=
 OWM_LONGITUDE=
+
+# Optional: PostgreSQL connection string
+# DATABASE_URL=postgres://user:password@localhost:5432/dbname

README.md

@@ -61,6 +61,17 @@ docker compose up -d
 -------------|----------------
 `OWM_CRON`   |`0 0/30 * * * *`
 `APP_ADDRESS`|`:3000`
+`DATABASE_URL`| (If set, PostgreSQL is used instead of SQLite)
+
+## Migration to PostgreSQL
+If you want to migrate your existing SQLite data to PostgreSQL:
+
+1. Set `DATABASE_URL` in your `.env` file.
+2. (Optional) Set `SQLITE_PATH` if your database is not in `data/data.sqlite`.
+3. Run the migration script:
+```sh
+go run ./cmd/migrate
+```
 
 ## License
 Rainbbit is licensed under MIT.

cmd/migrate/main.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"log"
+	"os"
+
+	"github.com/birabittoh/rainbbit/src"
+	"github.com/glebarez/sqlite"
+	"github.com/joho/godotenv"
+	"gorm.io/driver/postgres"
+	"gorm.io/gorm"
+)
+
+func main() {
+	err := godotenv.Load()
+	if err != nil {
+		log.Println("No .env file found")
+	}
+
+	sqlitePath := os.Getenv("SQLITE_PATH")
+	if sqlitePath == "" {
+		sqlitePath = "data/data.sqlite"
+	}
+
+	pgDSN := os.Getenv("DATABASE_URL")
+	if pgDSN == "" {
+		log.Fatal("DATABASE_URL environment variable is required")
+	}
+
+	log.Printf("Connecting to SQLite: %s", sqlitePath)
+	sqliteDB, err := gorm.Open(sqlite.Open(sqlitePath), &gorm.Config{})
+	if err != nil {
+		log.Fatalf("Failed to connect to SQLite: %v", err)
+	}
+
+	log.Println("Connecting to PostgreSQL...")
+	pgDB, err := gorm.Open(postgres.Open(pgDSN), &gorm.Config{})
+	if err != nil {
+		log.Fatalf("Failed to connect to PostgreSQL: %v", err)
+	}
+
+	log.Println("Migrating schema to PostgreSQL...")
+	err = pgDB.AutoMigrate(&src.Record{})
+	if err != nil {
+		log.Fatalf("Failed to migrate PostgreSQL schema: %v", err)
+	}
+
+	var records []src.Record
+	log.Println("Fetching records from SQLite...")
+	err = sqliteDB.Find(&records).Error
+	if err != nil {
+		log.Fatalf("Failed to fetch records from SQLite: %v", err)
+	}
+
+	log.Printf("Found %d records. Starting migration...", len(records))
+
+	if len(records) > 0 {
+		// Batch insert to PostgreSQL
+		err = pgDB.CreateInBatches(records, 100).Error
+		if err != nil {
+			log.Fatalf("Failed to insert records into PostgreSQL: %v", err)
+		}
+	}
+
+	log.Println("Migration completed successfully!")
+}

go.mod

@@ -25,16 +25,23 @@ require (
 	github.com/glebarez/go-sqlite v1.22.0 // indirect
 	github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
+	github.com/jackc/pgx/v5 v5.6.0 // indirect
+	github.com/jackc/puddle/v2 v2.2.2 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	golang.org/x/crypto v0.31.0 // indirect
 	golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 // indirect
 	golang.org/x/image v0.27.0 // indirect
+	golang.org/x/sync v0.14.0 // indirect
 	golang.org/x/sys v0.33.0 // indirect
 	golang.org/x/text v0.25.0 // indirect
+	gorm.io/driver/postgres v1.6.0 // indirect
 	modernc.org/libc v1.65.6 // indirect
 	modernc.org/mathutil v1.7.1 // indirect
 	modernc.org/memory v1.10.0 // indirect

go.sum

@@ -23,6 +23,7 @@ github.com/briandowns/openweathermap v0.21.1 h1:TPbuixuF+aGJP1mpgTNny6eUkdbvj7gq
 github.com/briandowns/openweathermap v0.21.1/go.mod h1:0GLnknqicWxXnGi1IqoOaZIw+kIe5hkt+YM5WY3j8+0=
 github.com/campoy/embedmd v1.0.0 h1:V4kI2qTJJLf4J29RzI/MAt2c3Bl4dQSYPuflzwFH2hY=
 github.com/campoy/embedmd v1.0.0/go.mod h1:oxyr9RCiSXg0M3VJ3ks0UGfp98BpSSGr0kpiX3MzVl8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/glebarez/go-sqlite v1.22.0 h1:uAcMJhaA6r3LHMTFgP0SifzgXg46yJkgxqyuyec+ruQ=
@@ -37,6 +38,14 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
+github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
+github.com/jackc/pgx/v5 v5.6.0 h1:SWJzexBzPL5jb0GEsrPMLIsi/3jOo7RHlzTjcAeDrPY=
+github.com/jackc/pgx/v5 v5.6.0/go.mod h1:DNZ/vlrUnhWCoFGxHAG8U2ljioxukquj7utPDgtQdTw=
+github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo=
+github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
 github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -54,10 +63,15 @@ github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
 github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
 github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6 h1:y5zboxd6LQAqYIhHnB48p0ByQ/GnQx2BE33L8BOHQkI=
 golang.org/x/exp v0.0.0-20250506013437-ce4c2cf36ca6/go.mod h1:U6Lno4MTRCDY+Ba7aCcauB9T60gsv5s4ralQzP72ZoQ=
 golang.org/x/image v0.27.0 h1:C8gA4oWU/tKkdCfYT6T2u4faJu3MeNS5O8UPWlPF61w=
@@ -97,6 +111,10 @@ gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
 gonum.org/v1/plot v0.16.0 h1:dK28Qx/Ky4VmPUN/2zeW0ELyM6ucDnBAj5yun7M9n1g=
 gonum.org/v1/plot v0.16.0/go.mod h1:Xz6U1yDMi6Ni6aaXILqmVIb6Vro8E+K7Q/GeeH+Pn0c=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gorm.io/driver/postgres v1.6.0 h1:2dxzU8xJ+ivvqTRph34QX+WrRaJlmfyPqXmoGVjMBa4=
+gorm.io/driver/postgres v1.6.0/go.mod h1:vUw0mrGgrTK+uPHEhAdV4sfFELrByKVGnaVRkXDhtWo=
 gorm.io/gorm v1.26.1 h1:ghB2gUI9FkS46luZtn6DLZ0f6ooBJ5IbVej2ENFDjRw=
 gorm.io/gorm v1.26.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
 honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las=

src/db.go

@@ -10,14 +10,15 @@ import (
 
 	"github.com/glebarez/sqlite"
 	"github.com/hashicorp/golang-lru/v2/expirable"
+	"gorm.io/driver/postgres"
 	"gorm.io/gorm"
 	"gorm.io/gorm/schema"
 )
 
 const (
 	dataDir   = "data"
 	dbPath    = dataDir + string(os.PathSeparator) + "data.sqlite"
-	dbOptions = "?_pragma=foreign_keys(1)"
+	dbOptions = "?_pragma=foreign_keys(1)&_pragma=journal_mode(WAL)&_pragma=synchronous(NORMAL)"
 	zonePath  = dataDir + string(os.PathSeparator) + "zone.txt"
 )
 
@@ -173,28 +174,36 @@ func getDataPoints(requestedMeasures []string, f, t *int64) (dp []DataPoint, err
 }
 
 func initDB() (err error) {
-	// Assicuriamoci che la directory "data" esista
-	if err := os.MkdirAll(dataDir, os.ModePerm); err != nil {
-		return errors.New("Errore nella creazione della directory 'data': " + err.Error())
-	}
+	dsn := os.Getenv("DATABASE_URL")
+	if dsn != "" {
+		db, err = gorm.Open(postgres.Open(dsn), &gorm.Config{})
+		if err != nil {
+			return errors.New("Errore nell'apertura del database PostgreSQL: " + err.Error())
+		}
+	} else {
+		// Assicuriamoci che la directory "data" esista
+		if err := os.MkdirAll(dataDir, os.ModePerm); err != nil {
+			return errors.New("Errore nella creazione della directory 'data': " + err.Error())
+		}
 
-	// Inizializzazione del database SQLite con GORM
-	db, err = gorm.Open(sqlite.Open(dbPath+dbOptions), &gorm.Config{})
-	if err != nil {
-		return errors.New("Errore nell'apertura del database: " + err.Error())
+		// Inizializzazione del database SQLite con GORM
+		db, err = gorm.Open(sqlite.Open(dbPath+dbOptions), &gorm.Config{})
+		if err != nil {
+			return errors.New("Errore nell'apertura del database SQLite: " + err.Error())
+		}
+
+		// Limitazione delle connessioni per SQLite
+		sqlDB, err := db.DB()
+		if err == nil {
+			sqlDB.SetMaxOpenConns(1)
+		}
 	}
 
 	// Migrazione dello schema per il modello Record
 	if err := db.AutoMigrate(&Record{}); err != nil {
 		return errors.New("Errore nella migrazione del database: " + err.Error())
 	}
 
-	// Limitazione delle connessioni per SQLite
-	sqlDB, err := db.DB()
-	if err == nil {
-		sqlDB.SetMaxOpenConns(1)
-	}
-
 	// Inizializzazione delle colonne
 	s, err := schema.Parse(&Record{}, &sync.Map{}, schema.NamingStrategy{})
 	if err != nil {