Make processing resilient to missing secondary services with B2 backup system

.gitignore

@@ -9,4 +9,5 @@ counts
 main
 ised_data
 hamcall.db
-hamcall
+hamcall
+hamcall-test

README.md

@@ -44,3 +44,40 @@ Since the hangup is moving the files to the CDN the next step was to write them
 - LOTW https://lotw.arrl.org/
 - RadioID https://www.radioid.net/
 - Location https://haminfo.tetranz.com
+
+## Backup System
+
+The application includes a backup system for secondary data sources to ensure resilient processing even when external services are unavailable (such as during ARRL LOTW maintenance).
+
+### Configuration
+
+Set the `BACKUP_PATH` environment variable to enable backup functionality:
+
+```bash
+export BACKUP_PATH="backups"  # Path in B2 bucket where backup files are stored
+export B2_KEYID="your_b2_key_id"
+export B2_APPKEY="your_b2_application_key"
+```
+
+### How It Works
+
+1. **Primary Download**: The system first attempts to download from the original source
+2. **Backup on Success**: If successful, the file is automatically uploaded to the backup path in B2
+3. **Fallback on Failure**: If the primary source fails, the system attempts to download from the backup
+4. **Graceful Degradation**: If both primary and backup fail, processing continues without that data source
+
+### Behavior by Service
+
+- **ULS (Primary)**: Always required - fatal error if unavailable
+- **LOTW, RadioID, GEO, ISED (Secondary)**: Resilient with backup system - warnings on failure but processing continues
+
+### Example Output
+
+```
+Backup system enabled with path: backups
+Downloading lotw data
+Primary download failed for lotw.csv, trying backup...
+Successfully downloaded lotw.csv from backup
+Warning: Error downloading GEO data: connection refused
+Continuing without GEO data due to download failure
+```

downloader/backup.go

@@ -0,0 +1,107 @@
+package downloader
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/kothar/go-backblaze.v0"
+)
+
+// BackupDownloader handles backup operations with B2
+type BackupDownloader struct {
+	bucket *backblaze.Bucket
+	path   string
+}
+
+// NewBackupDownloader creates a new backup downloader
+func NewBackupDownloader(keyID, applicationKey, backupPath string) (*BackupDownloader, error) {
+	if keyID == "" || applicationKey == "" || backupPath == "" {
+		return nil, fmt.Errorf("backup credentials or path not configured")
+	}
+
+	b, err := backblaze.NewB2(backblaze.Credentials{
+		KeyID:          keyID,
+		ApplicationKey: applicationKey,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	bucket, err := b.Bucket("hamcall")
+	if err != nil {
+		return nil, err
+	}
+
+	return &BackupDownloader{
+		bucket: bucket,
+		path:   backupPath,
+	}, nil
+}
+
+// UploadBackup uploads a file to the backup location
+func (bd *BackupDownloader) UploadBackup(localFile, remoteFile string) error {
+	file, err := os.Open(localFile)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	backupPath := filepath.Join(bd.path, remoteFile)
+	_, err = bd.bucket.UploadFile(backupPath, nil, file)
+	return err
+}
+
+// DownloadBackup downloads a file from the backup location
+func (bd *BackupDownloader) DownloadBackup(remoteFile, localFile string) error {
+	backupPath := filepath.Join(bd.path, remoteFile)
+
+	_, reader, err := bd.bucket.DownloadFileByName(backupPath)
+	if err != nil {
+		return err
+	}
+	defer reader.Close()
+
+	out, err := os.Create(localFile)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	_, err = io.Copy(out, reader)
+	return err
+}
+
+// FetchWithBackup tries to fetch from primary URL, falls back to backup, and updates backup on success
+func FetchWithBackup(localFile, primaryURL, backupFile string, backup *BackupDownloader) error {
+	// Try primary download first
+	err := FetchHttp(localFile, primaryURL)
+	if err == nil {
+		// Primary succeeded, upload to backup if backup system is available
+		if backup != nil {
+			if uploadErr := backup.UploadBackup(localFile, backupFile); uploadErr != nil {
+				fmt.Printf("Warning: Failed to upload %s to backup: %v\n", backupFile, uploadErr)
+			} else {
+				fmt.Printf("Successfully backed up %s to B2\n", backupFile)
+			}
+		}
+		return nil
+	}
+
+	// Primary failed, try backup if available
+	fmt.Printf("Primary download failed for %s: %v\n", localFile, err)
+	if backup != nil {
+		fmt.Printf("Attempting to download %s from backup...\n", localFile)
+		if backupErr := backup.DownloadBackup(backupFile, localFile); backupErr == nil {
+			fmt.Printf("Successfully restored %s from backup\n", localFile)
+			return nil
+		} else {
+			fmt.Printf("Backup download also failed for %s: %v\n", localFile, backupErr)
+			return backupErr
+		}
+	}
+
+	// Primary failed and no backup system available
+	return err
+}

main.go

@@ -20,6 +20,7 @@ import (
 	"github.com/manifoldco/promptui"
 	"github.com/pcunning/hamcall/b2"
 	"github.com/pcunning/hamcall/data"
+	"github.com/pcunning/hamcall/downloader"
 	"github.com/pcunning/hamcall/source/geo"
 	"github.com/pcunning/hamcall/source/ised"
 	"github.com/pcunning/hamcall/source/lotw"
@@ -50,7 +51,7 @@ func main() {
 	uploadWorkers := 200
 
 	if *downloadDataFiles {
-		downloadFiles()
+		downloadFiles(keyID, applicationKey)
 		fmt.Printf("download finished at %s\n", time.Since(start).String())
 
 	} else {
@@ -84,16 +85,52 @@ func main() {
 	fmt.Printf("total runtime %s\n", time.Since(start).String())
 }
 
-func downloadFiles() {
+func downloadFiles(keyID, applicationKey string) {
 	var wg sync.WaitGroup
 
+	// Create backup downloader if B2 credentials and backup path are configured
+	backupPath := os.Getenv("BACKUP_PATH")
+	var backup *downloader.BackupDownloader
+	if backupPath != "" && keyID != "" && applicationKey != "" {
+		var err error
+		backup, err = downloader.NewBackupDownloader(keyID, applicationKey, backupPath)
+		if err != nil {
+			fmt.Printf("Warning: Failed to initialize backup downloader: %v\n", err)
+			fmt.Printf("Continuing without backup system - downloads will not be resilient\n")
+			backup = nil
+		} else {
+			fmt.Printf("Backup system enabled with path: %s\n", backupPath)
+		}
+	} else {
+		fmt.Printf("Backup system not configured (set BACKUP_PATH, B2_KEYID, B2_APPKEY to enable)\n")
+	}
+
 	wg.Add(5)
 
+	// ULS is critical - must succeed
 	go uls.Download(&wg)
-	go ised.Download(&wg)
-	go radioid.Download(&wg)
-	go lotw.Download(&wg)
-	go geo.Download(&wg)
+
+	// Secondary services - try primary, fallback to backup if available
+	go func() {
+		if _, err := ised.Download(&wg, backup); err != nil {
+			fmt.Printf("Warning: Continuing without ISED data\n")
+		}
+	}()
+	go func() {
+		if err := radioid.Download(&wg, backup); err != nil {
+			fmt.Printf("Warning: Continuing without RadioID data\n")
+		}
+	}()
+	go func() {
+		if err := lotw.Download(&wg, backup); err != nil {
+			fmt.Printf("Warning: Continuing without LOTW data\n")
+		}
+	}()
+	go func() {
+		if err := geo.Download(&wg, backup); err != nil {
+			fmt.Printf("Warning: Continuing without GEO data\n")
+		}
+	}()
 
 	wg.Wait()
 }

main_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/pcunning/hamcall/data"
+	"github.com/pcunning/hamcall/downloader"
 )
 
 func TestWebHandler(t *testing.T) {
@@ -130,11 +131,36 @@ func TestProcessFunction(t *testing.T) {
 
 func TestDownloadFilesFunction(t *testing.T) {
 	// Test that downloadFiles function exists and can be referenced
-	// We skip actual execution to avoid network dependencies and log.Fatalf calls
+	// We skip actual execution to avoid network dependencies
 
 	// Just verify we can reference the function (if we couldn't, this wouldn't compile)
-	_ = downloadFiles
+	_ = func() { downloadFiles("", "") }
 
-	// Skip actual execution to avoid network dependencies
-	t.Skip("Skipping actual download test to avoid network dependencies and log.Fatalf calls")
+	// Skip actual execution to avoid network dependencies and B2 credentials
+	t.Skip("Skipping actual download test to avoid network dependencies and B2 credentials")
+}
+
+func TestBackupSystemIntegration(t *testing.T) {
+	// Test that backup system is properly integrated without requiring actual network access
+
+	// Since ULS download will cause log.Fatalf on network failure, we'll just test 
+	// the backup downloader initialization logic in isolation
+
+	// Test with empty credentials (should disable backup)
+	backup, err := downloader.NewBackupDownloader("", "", "test-path")
+	if err == nil {
+		t.Fatalf("NewBackupDownloader with empty credentials should return error")
+	}
+	if backup != nil {
+		t.Fatalf("NewBackupDownloader with empty credentials should return nil backup")
+	}
+
+	// Test with empty path (should disable backup)
+	backup2, err2 := downloader.NewBackupDownloader("test-key", "test-app-key", "")
+	if err2 == nil {
+		t.Fatalf("NewBackupDownloader with empty path should return error")
+	}
+	if backup2 != nil {
+		t.Fatalf("NewBackupDownloader with empty path should return nil backup")
+	}
 }

source/geo/geo.go

@@ -4,7 +4,6 @@ import (
 	"encoding/csv"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"strconv"
 	"sync"
@@ -14,12 +13,20 @@ import (
 	"github.com/pcunning/hamcall/downloader"
 )
 
-func Download(wg *sync.WaitGroup) error {
+func Download(wg *sync.WaitGroup, backup *downloader.BackupDownloader) error {
 	defer wg.Done()
 	fmt.Println("Downloading GEO data")
-	err := downloader.FetchHttp("ham-stations.csv", os.Getenv("GEO_URL"))
+
+	var err error
+	if backup != nil {
+		err = downloader.FetchWithBackup("ham-stations.csv", os.Getenv("GEO_URL"), "ham-stations.csv", backup)
+	} else {
+		err = downloader.FetchHttp("ham-stations.csv", os.Getenv("GEO_URL"))
+	}
+
 	if err != nil {
-		log.Fatalf("Error downloading GEO data: %v", err)
+		fmt.Printf("Warning: Error downloading GEO data: %v\n", err)
+		return err
 	}
 	return nil
 }

source/ised/ised.go

@@ -3,7 +3,6 @@ package ised
 import (
 	"bufio"
 	"fmt"
-	"log"
 	"os"
 	"strings"
 	"sync"
@@ -13,17 +12,26 @@ import (
 	"github.com/pcunning/hamcall/downloader"
 )
 
-func Download(wg *sync.WaitGroup) (string, error) {
+func Download(wg *sync.WaitGroup, backup *downloader.BackupDownloader) (string, error) {
 	defer wg.Done()
 	fmt.Println("Downloading ISED (Canada) data")
-	err := downloader.FetchHttp("ised.zip", "https://apc-cap.ic.gc.ca/datafiles/amateur_delim.zip")
+
+	var err error
+	if backup != nil {
+		err = downloader.FetchWithBackup("ised.zip", "https://apc-cap.ic.gc.ca/datafiles/amateur_delim.zip", "ised.zip", backup)
+	} else {
+		err = downloader.FetchHttp("ised.zip", "https://apc-cap.ic.gc.ca/datafiles/amateur_delim.zip")
+	}
+
 	if err != nil {
-		log.Fatalf("Error downloading ISED (Canada) data: %v", err)
+		fmt.Printf("Warning: Error downloading ISED (Canada) data: %v\n", err)
+		return "", err
 	}
 
 	_, err = downloader.Unzip("ised.zip", "ised_data")
 	if err != nil {
-		log.Fatalf("Error unzipping ISED (Canada) data: %v", err)
+		fmt.Printf("Warning: Error unzipping ISED (Canada) data: %v\n", err)
+		return "", err
 	}
 
 	return "ised_data/amateur_delim.txt", nil

source/lotw/lotw.go

@@ -4,7 +4,6 @@ import (
 	"encoding/csv"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"sync"
 	"time"
@@ -13,12 +12,20 @@ import (
 	"github.com/pcunning/hamcall/downloader"
 )
 
-func Download(wg *sync.WaitGroup) error {
+func Download(wg *sync.WaitGroup, backup *downloader.BackupDownloader) error {
 	defer wg.Done()
 	fmt.Println("Downloading lotw data")
-	err := downloader.FetchHttp("lotw.csv", "https://lotw.arrl.org/lotw-user-activity.csv")
+
+	var err error
+	if backup != nil {
+		err = downloader.FetchWithBackup("lotw.csv", "https://lotw.arrl.org/lotw-user-activity.csv", "lotw.csv", backup)
+	} else {
+		err = downloader.FetchHttp("lotw.csv", "https://lotw.arrl.org/lotw-user-activity.csv")
+	}
+
 	if err != nil {
-		log.Fatalf("Error downloading LOTW data: %v", err)
+		fmt.Printf("Warning: Error downloading LOTW data: %v\n", err)
+		return err
 	}
 	return nil
 }