Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 41 additions & 16 deletions internal/cmd/cicd/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,39 @@ var BackupCmd = &cli.Command{
Action: cmdtools.Wrap(BackupWorkflow),
}

// filterBackupableResources returns resources that support automated backups
// based on the backup.enabled flag and provider compatibility.
func filterBackupableResources(resources []appdef.Resource) []appdef.Resource {
var result []appdef.Resource
for _, resource := range resources {
if !resource.IsBackupEnabled() {
continue
}

// Check provider compatibility.
switch resource.Type {
case appdef.ResourceTypeS3:
// S3 backup only compatible with DigitalOcean Spaces.
if resource.Provider != appdef.ResourceProviderDigitalOcean {
continue
}
case appdef.ResourceTypeSQLite:
// SQLite backup only compatible with Turso.
if resource.Provider != appdef.ResourceProviderTurso {
continue
}
case appdef.ResourceTypePostgres:
// Postgres supports all providers.
default:
// Skip unknown resource types.
continue
}

result = append(result, resource)
}
return result
}

// BackupWorkflow creates backup workflows for every resource if the
// backup config is enabled.
func BackupWorkflow(_ context.Context, input cmdtools.CommandInput) error {
Expand All @@ -29,10 +62,13 @@ func BackupWorkflow(_ context.Context, input cmdtools.CommandInput) error {
tpl := templates.MustLoadTemplate(filepath.Join(workflowsPath, "backup.yaml.tmpl"))
path := filepath.Join(workflowsPath, "backup.yaml")

// Filter resources that support backup.
backupableResources := filterBackupableResources(appDef.Resources)

// Build nested data map with all resource secrets grouped by resource name.
// This allows cleaner template access: {{ index $.Data .Name "DatabaseURL" }}
secretData := make(map[string]map[string]string)
for _, resource := range appDef.Resources {
for _, resource := range backupableResources {
resourceSecrets := make(map[string]string)

switch resource.Type {
Expand All @@ -41,41 +77,30 @@ func BackupWorkflow(_ context.Context, input cmdtools.CommandInput) error {
resourceSecrets["DatabaseID"] = resource.GitHubSecretName(enviro, "id")

case appdef.ResourceTypeS3:
// NOTE: S3 backup is currently only compatible with DigitalOcean Spaces.
// Backblaze B2 and other providers are not yet supported.
if resource.Provider != appdef.ResourceProviderDigitalOcean {
continue
}

resourceSecrets["AccessKey"] = resource.GitHubSecretName(enviro, "access_key")
resourceSecrets["SecretKey"] = resource.GitHubSecretName(enviro, "secret_key")
resourceSecrets["Region"] = resource.GitHubSecretName(enviro, "region")
resourceSecrets["BucketName"] = resource.GitHubSecretName(enviro, "bucket_name")

case appdef.ResourceTypeSQLite:
// NOTE: SQLite backup is currently only compatible with Turso.
// The database name is constructed as ${project_name}-${resource_name} (same as in terraform).
// Authentication is handled via TURSO_API_TOKEN environment variable.
if resource.Provider != appdef.ResourceProviderTurso {
continue
}
// SQLite backup uses TURSO_API_TOKEN environment variable.
}

secretData[resource.Name] = resourceSecrets
}

data := map[string]any{
"Resources": appDef.Resources,
"Resources": backupableResources,
"Data": secretData,
"MonitoringEnabled": appDef.Monitoring.IsEnabled(),
// TODO: This may change at some point, see workflow for more details.
"BucketName": appDef.Project.Name,
"Env": enviro,
}

// Track all resources as sources for this workflow.
// Track backupable resources as sources for this workflow.
var trackingOptions []scaffold.Option
for _, resource := range appDef.Resources {
for _, resource := range backupableResources {
trackingOptions = append(trackingOptions, scaffold.WithTracking(manifest.SourceResource(resource.Name)))
}

Expand Down
143 changes: 143 additions & 0 deletions internal/cmd/cicd/backup_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -413,6 +413,149 @@ func TestBackupWorkflow(t *testing.T) {
assert.Contains(t, content, "PROD_CODEBASE_BACKUP_PING_URL")
})

t.Run("Backup disabled for single resource", func(t *testing.T) {
t.Parallel()

appDef := &appdef.Definition{
Project: appdef.Project{
Name: "test-project",
},
Resources: []appdef.Resource{
{
Name: "store",
Type: appdef.ResourceTypeS3,
Provider: appdef.ResourceProviderDigitalOcean,
Backup: appdef.ResourceBackupConfig{
Enabled: ptr.BoolPtr(false),
},
},
},
}

input := setup(t, afero.NewMemMapFs(), appDef)

got := BackupWorkflow(t.Context(), input)
assert.NoError(t, got)

file, err := afero.ReadFile(input.FS, filepath.Join(workflowsPath, "backup.yaml"))
require.NoError(t, err)

err = validateGithubYaml(t, file, false)
assert.NoError(t, err)

// Verify that no backup job is created for the resource with backup disabled
content := string(file)
assert.NotContains(t, content, "backup-resource-store:")
assert.NotContains(t, content, "PROD_STORE_BACKUP_PING_URL")
// Codebase backup should still exist
assert.Contains(t, content, "backup-codebase:")
})

t.Run("Mixed backup enabled and disabled resources", func(t *testing.T) {
t.Parallel()

appDef := &appdef.Definition{
Project: appdef.Project{
Name: "test-project",
},
Resources: []appdef.Resource{
{
Name: "db",
Type: appdef.ResourceTypePostgres,
Provider: appdef.ResourceProviderDigitalOcean,
Monitoring: ptr.BoolPtr(true),
Backup: appdef.ResourceBackupConfig{
Enabled: ptr.BoolPtr(true),
},
},
{
Name: "store",
Type: appdef.ResourceTypeS3,
Provider: appdef.ResourceProviderDigitalOcean,
Backup: appdef.ResourceBackupConfig{
Enabled: ptr.BoolPtr(false),
},
},
{
Name: "cache",
Type: appdef.ResourceTypeSQLite,
Provider: appdef.ResourceProviderTurso,
Backup: appdef.ResourceBackupConfig{
Enabled: ptr.BoolPtr(true),
},
},
},
}

input := setup(t, afero.NewMemMapFs(), appDef)

got := BackupWorkflow(t.Context(), input)
assert.NoError(t, got)

file, err := afero.ReadFile(input.FS, filepath.Join(workflowsPath, "backup.yaml"))
require.NoError(t, err)

err = validateGithubYaml(t, file, false)
assert.NoError(t, err)

content := string(file)
// Verify that backup jobs are created only for resources with backup enabled
assert.Contains(t, content, "backup-resource-db:")
assert.Contains(t, content, "backup-resource-cache:")
assert.NotContains(t, content, "backup-resource-store:")

// Verify sync-to-gdrive only depends on enabled backup jobs
assert.Contains(t, content, "- backup-resource-db")
assert.Contains(t, content, "- backup-resource-cache")
assert.NotContains(t, content, "- backup-resource-store")
})

t.Run("All resources with backup disabled", func(t *testing.T) {
t.Parallel()

appDef := &appdef.Definition{
Project: appdef.Project{
Name: "test-project",
},
Resources: []appdef.Resource{
{
Name: "db",
Type: appdef.ResourceTypePostgres,
Provider: appdef.ResourceProviderDigitalOcean,
Backup: appdef.ResourceBackupConfig{
Enabled: ptr.BoolPtr(false),
},
},
{
Name: "store",
Type: appdef.ResourceTypeS3,
Provider: appdef.ResourceProviderDigitalOcean,
Backup: appdef.ResourceBackupConfig{
Enabled: ptr.BoolPtr(false),
},
},
},
}

input := setup(t, afero.NewMemMapFs(), appDef)

got := BackupWorkflow(t.Context(), input)
assert.NoError(t, got)

file, err := afero.ReadFile(input.FS, filepath.Join(workflowsPath, "backup.yaml"))
require.NoError(t, err)

err = validateGithubYaml(t, file, false)
assert.NoError(t, err)

content := string(file)
// Verify that no resource backup jobs are created
assert.NotContains(t, content, "backup-resource-db:")
assert.NotContains(t, content, "backup-resource-store:")
// Only codebase backup should exist
assert.Contains(t, content, "backup-codebase:")
})

t.Run("FS Failure", func(t *testing.T) {
t.Parallel()

Expand Down
2 changes: 0 additions & 2 deletions internal/templates/.github/workflows/backup.yaml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -325,9 +325,7 @@ jobs:
timeout-minutes: 120 # 2 Hours
needs:
{{- range .Resources }}
{{- if index $.Data .Name }}
- backup-resource-{{ .Name }}
{{- end }}
{{- end }}
- backup-codebase
env:
Expand Down
Loading