diff --git a/.gitignore b/.gitignore index 81647d0..fa45e11 100644 --- a/.gitignore +++ b/.gitignore @@ -9,4 +9,5 @@ counts main ised_data hamcall.db -hamcall \ No newline at end of file +hamcall +hamcall-test diff --git a/README.md b/README.md index c299910..c8b0272 100644 --- a/README.md +++ b/README.md @@ -44,3 +44,40 @@ Since the hangup is moving the files to the CDN the next step was to write them - LOTW https://lotw.arrl.org/ - RadioID https://www.radioid.net/ - Location https://haminfo.tetranz.com + +## Backup System + +The application includes a backup system for secondary data sources to ensure resilient processing even when external services are unavailable (such as during ARRL LOTW maintenance). + +### Configuration + +Set the `BACKUP_PATH` environment variable to enable backup functionality: + +```bash +export BACKUP_PATH="backups" # Path in B2 bucket where backup files are stored +export B2_KEYID="your_b2_key_id" +export B2_APPKEY="your_b2_application_key" +``` + +### How It Works + +1. **Primary Download**: The system first attempts to download from the original source +2. **Backup on Success**: If successful, the file is automatically uploaded to the backup path in B2 +3. **Fallback on Failure**: If the primary source fails, the system attempts to download from the backup +4. **Graceful Degradation**: If both primary and backup fail, processing continues without that data source + +### Behavior by Service + +- **ULS (Primary)**: Always required - fatal error if unavailable +- **LOTW, RadioID, GEO, ISED (Secondary)**: Resilient with backup system - warnings on failure but processing continues + +### Example Output + +``` +Backup system enabled with path: backups +Downloading lotw data +Primary download failed for lotw.csv, trying backup... +Successfully downloaded lotw.csv from backup +Warning: Error downloading GEO data: connection refused +Continuing without GEO data due to download failure +``` diff --git a/downloader/backup.go b/downloader/backup.go new file mode 100644 index 0000000..d27aa4f --- /dev/null +++ b/downloader/backup.go @@ -0,0 +1,107 @@ +package downloader + +import ( + "fmt" + "io" + "os" + "path/filepath" + + "gopkg.in/kothar/go-backblaze.v0" +) + +// BackupDownloader handles backup operations with B2 +type BackupDownloader struct { + bucket *backblaze.Bucket + path string +} + +// NewBackupDownloader creates a new backup downloader +func NewBackupDownloader(keyID, applicationKey, backupPath string) (*BackupDownloader, error) { + if keyID == "" || applicationKey == "" || backupPath == "" { + return nil, fmt.Errorf("backup credentials or path not configured") + } + + b, err := backblaze.NewB2(backblaze.Credentials{ + KeyID: keyID, + ApplicationKey: applicationKey, + }) + if err != nil { + return nil, err + } + + bucket, err := b.Bucket("hamcall") + if err != nil { + return nil, err + } + + return &BackupDownloader{ + bucket: bucket, + path: backupPath, + }, nil +} + +// UploadBackup uploads a file to the backup location +func (bd *BackupDownloader) UploadBackup(localFile, remoteFile string) error { + file, err := os.Open(localFile) + if err != nil { + return err + } + defer file.Close() + + backupPath := filepath.Join(bd.path, remoteFile) + _, err = bd.bucket.UploadFile(backupPath, nil, file) + return err +} + +// DownloadBackup downloads a file from the backup location +func (bd *BackupDownloader) DownloadBackup(remoteFile, localFile string) error { + backupPath := filepath.Join(bd.path, remoteFile) + + _, reader, err := bd.bucket.DownloadFileByName(backupPath) + if err != nil { + return err + } + defer reader.Close() + + out, err := os.Create(localFile) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, reader) + return err +} + +// FetchWithBackup tries to fetch from primary URL, falls back to backup, and updates backup on success +func FetchWithBackup(localFile, primaryURL, backupFile string, backup *BackupDownloader) error { + // Try primary download first + err := FetchHttp(localFile, primaryURL) + if err == nil { + // Primary succeeded, upload to backup if backup system is available + if backup != nil { + if uploadErr := backup.UploadBackup(localFile, backupFile); uploadErr != nil { + fmt.Printf("Warning: Failed to upload %s to backup: %v\n", backupFile, uploadErr) + } else { + fmt.Printf("Successfully backed up %s to B2\n", backupFile) + } + } + return nil + } + + // Primary failed, try backup if available + fmt.Printf("Primary download failed for %s: %v\n", localFile, err) + if backup != nil { + fmt.Printf("Attempting to download %s from backup...\n", localFile) + if backupErr := backup.DownloadBackup(backupFile, localFile); backupErr == nil { + fmt.Printf("Successfully restored %s from backup\n", localFile) + return nil + } else { + fmt.Printf("Backup download also failed for %s: %v\n", localFile, backupErr) + return backupErr + } + } + + // Primary failed and no backup system available + return err +} \ No newline at end of file diff --git a/main.go b/main.go index d8b7e8d..f5ffbb9 100644 --- a/main.go +++ b/main.go @@ -20,6 +20,7 @@ import ( "github.com/manifoldco/promptui" "github.com/pcunning/hamcall/b2" "github.com/pcunning/hamcall/data" + "github.com/pcunning/hamcall/downloader" "github.com/pcunning/hamcall/source/geo" "github.com/pcunning/hamcall/source/ised" "github.com/pcunning/hamcall/source/lotw" @@ -50,7 +51,7 @@ func main() { uploadWorkers := 200 if *downloadDataFiles { - downloadFiles() + downloadFiles(keyID, applicationKey) fmt.Printf("download finished at %s\n", time.Since(start).String()) } else { @@ -84,16 +85,52 @@ func main() { fmt.Printf("total runtime %s\n", time.Since(start).String()) } -func downloadFiles() { +func downloadFiles(keyID, applicationKey string) { var wg sync.WaitGroup + // Create backup downloader if B2 credentials and backup path are configured + backupPath := os.Getenv("BACKUP_PATH") + var backup *downloader.BackupDownloader + if backupPath != "" && keyID != "" && applicationKey != "" { + var err error + backup, err = downloader.NewBackupDownloader(keyID, applicationKey, backupPath) + if err != nil { + fmt.Printf("Warning: Failed to initialize backup downloader: %v\n", err) + fmt.Printf("Continuing without backup system - downloads will not be resilient\n") + backup = nil + } else { + fmt.Printf("Backup system enabled with path: %s\n", backupPath) + } + } else { + fmt.Printf("Backup system not configured (set BACKUP_PATH, B2_KEYID, B2_APPKEY to enable)\n") + } + wg.Add(5) + // ULS is critical - must succeed go uls.Download(&wg) - go ised.Download(&wg) - go radioid.Download(&wg) - go lotw.Download(&wg) - go geo.Download(&wg) + + // Secondary services - try primary, fallback to backup if available + go func() { + if _, err := ised.Download(&wg, backup); err != nil { + fmt.Printf("Warning: Continuing without ISED data\n") + } + }() + go func() { + if err := radioid.Download(&wg, backup); err != nil { + fmt.Printf("Warning: Continuing without RadioID data\n") + } + }() + go func() { + if err := lotw.Download(&wg, backup); err != nil { + fmt.Printf("Warning: Continuing without LOTW data\n") + } + }() + go func() { + if err := geo.Download(&wg, backup); err != nil { + fmt.Printf("Warning: Continuing without GEO data\n") + } + }() wg.Wait() } diff --git a/main_test.go b/main_test.go index 162fd76..b965c52 100644 --- a/main_test.go +++ b/main_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/pcunning/hamcall/data" + "github.com/pcunning/hamcall/downloader" ) func TestWebHandler(t *testing.T) { @@ -130,11 +131,36 @@ func TestProcessFunction(t *testing.T) { func TestDownloadFilesFunction(t *testing.T) { // Test that downloadFiles function exists and can be referenced - // We skip actual execution to avoid network dependencies and log.Fatalf calls + // We skip actual execution to avoid network dependencies // Just verify we can reference the function (if we couldn't, this wouldn't compile) - _ = downloadFiles + _ = func() { downloadFiles("", "") } - // Skip actual execution to avoid network dependencies - t.Skip("Skipping actual download test to avoid network dependencies and log.Fatalf calls") + // Skip actual execution to avoid network dependencies and B2 credentials + t.Skip("Skipping actual download test to avoid network dependencies and B2 credentials") +} + +func TestBackupSystemIntegration(t *testing.T) { + // Test that backup system is properly integrated without requiring actual network access + + // Since ULS download will cause log.Fatalf on network failure, we'll just test + // the backup downloader initialization logic in isolation + + // Test with empty credentials (should disable backup) + backup, err := downloader.NewBackupDownloader("", "", "test-path") + if err == nil { + t.Fatalf("NewBackupDownloader with empty credentials should return error") + } + if backup != nil { + t.Fatalf("NewBackupDownloader with empty credentials should return nil backup") + } + + // Test with empty path (should disable backup) + backup2, err2 := downloader.NewBackupDownloader("test-key", "test-app-key", "") + if err2 == nil { + t.Fatalf("NewBackupDownloader with empty path should return error") + } + if backup2 != nil { + t.Fatalf("NewBackupDownloader with empty path should return nil backup") + } } \ No newline at end of file diff --git a/source/geo/geo.go b/source/geo/geo.go index acdb3f5..2225b86 100644 --- a/source/geo/geo.go +++ b/source/geo/geo.go @@ -4,7 +4,6 @@ import ( "encoding/csv" "fmt" "io" - "log" "os" "strconv" "sync" @@ -14,12 +13,20 @@ import ( "github.com/pcunning/hamcall/downloader" ) -func Download(wg *sync.WaitGroup) error { +func Download(wg *sync.WaitGroup, backup *downloader.BackupDownloader) error { defer wg.Done() fmt.Println("Downloading GEO data") - err := downloader.FetchHttp("ham-stations.csv", os.Getenv("GEO_URL")) + + var err error + if backup != nil { + err = downloader.FetchWithBackup("ham-stations.csv", os.Getenv("GEO_URL"), "ham-stations.csv", backup) + } else { + err = downloader.FetchHttp("ham-stations.csv", os.Getenv("GEO_URL")) + } + if err != nil { - log.Fatalf("Error downloading GEO data: %v", err) + fmt.Printf("Warning: Error downloading GEO data: %v\n", err) + return err } return nil } diff --git a/source/ised/ised.go b/source/ised/ised.go index 87eabf5..0479b23 100644 --- a/source/ised/ised.go +++ b/source/ised/ised.go @@ -3,7 +3,6 @@ package ised import ( "bufio" "fmt" - "log" "os" "strings" "sync" @@ -13,17 +12,26 @@ import ( "github.com/pcunning/hamcall/downloader" ) -func Download(wg *sync.WaitGroup) (string, error) { +func Download(wg *sync.WaitGroup, backup *downloader.BackupDownloader) (string, error) { defer wg.Done() fmt.Println("Downloading ISED (Canada) data") - err := downloader.FetchHttp("ised.zip", "https://apc-cap.ic.gc.ca/datafiles/amateur_delim.zip") + + var err error + if backup != nil { + err = downloader.FetchWithBackup("ised.zip", "https://apc-cap.ic.gc.ca/datafiles/amateur_delim.zip", "ised.zip", backup) + } else { + err = downloader.FetchHttp("ised.zip", "https://apc-cap.ic.gc.ca/datafiles/amateur_delim.zip") + } + if err != nil { - log.Fatalf("Error downloading ISED (Canada) data: %v", err) + fmt.Printf("Warning: Error downloading ISED (Canada) data: %v\n", err) + return "", err } _, err = downloader.Unzip("ised.zip", "ised_data") if err != nil { - log.Fatalf("Error unzipping ISED (Canada) data: %v", err) + fmt.Printf("Warning: Error unzipping ISED (Canada) data: %v\n", err) + return "", err } return "ised_data/amateur_delim.txt", nil diff --git a/source/lotw/lotw.go b/source/lotw/lotw.go index 4522404..d465e47 100644 --- a/source/lotw/lotw.go +++ b/source/lotw/lotw.go @@ -4,7 +4,6 @@ import ( "encoding/csv" "fmt" "io" - "log" "os" "sync" "time" @@ -13,12 +12,20 @@ import ( "github.com/pcunning/hamcall/downloader" ) -func Download(wg *sync.WaitGroup) error { +func Download(wg *sync.WaitGroup, backup *downloader.BackupDownloader) error { defer wg.Done() fmt.Println("Downloading lotw data") - err := downloader.FetchHttp("lotw.csv", "https://lotw.arrl.org/lotw-user-activity.csv") + + var err error + if backup != nil { + err = downloader.FetchWithBackup("lotw.csv", "https://lotw.arrl.org/lotw-user-activity.csv", "lotw.csv", backup) + } else { + err = downloader.FetchHttp("lotw.csv", "https://lotw.arrl.org/lotw-user-activity.csv") + } + if err != nil { - log.Fatalf("Error downloading LOTW data: %v", err) + fmt.Printf("Warning: Error downloading LOTW data: %v\n", err) + return err } return nil } diff --git a/source/radioid/radioid.go b/source/radioid/radioid.go index a8e40bc..1004d96 100644 --- a/source/radioid/radioid.go +++ b/source/radioid/radioid.go @@ -4,7 +4,6 @@ import ( "encoding/csv" "fmt" "io" - "log" "os" "strconv" "sync" @@ -14,12 +13,20 @@ import ( "github.com/pcunning/hamcall/downloader" ) -func Download(wg *sync.WaitGroup) error { +func Download(wg *sync.WaitGroup, backup *downloader.BackupDownloader) error { defer wg.Done() fmt.Println("Downloading radioid data") - err := downloader.FetchHttp("dmrid.dat", "https://www.radioid.net/static/dmrid.dat") + + var err error + if backup != nil { + err = downloader.FetchWithBackup("dmrid.dat", "https://www.radioid.net/static/dmrid.dat", "dmrid.dat", backup) + } else { + err = downloader.FetchHttp("dmrid.dat", "https://www.radioid.net/static/dmrid.dat") + } + if err != nil { - log.Fatalf("Error downloading RadioID data: %v", err) + fmt.Printf("Warning: Error downloading RadioID data: %v\n", err) + return err } return nil }