feat: Add support for pre-loading environment variables from comet.yaml and enhance documentation

Author: speier

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Added
+- **Config-based environment variables** - Pre-load environment variables from `comet.yaml` before any command runs. Perfect for setting `SOPS_AGE_KEY` and other secrets needed during stack parsing. Supports secret resolution via `op://` and `sops://` prefixes. Shell environment variables take precedence.
 - **`comet init` command** - Initialize backends and providers without running plan/apply operations. Useful for read-only operations like `comet output` or troubleshooting provider/backend initialization issues.
 - **DSL Improvements** - Two core enhancements to reduce boilerplate by ~30%:
   - Bulk environment variables: `envs({})` accepts objects to set multiple vars at once

Makefile

@@ -1,4 +1,4 @@
-VERSION=v0.4.2
+VERSION=v0.4.3
 
 tag:
 	@git tag -a ${VERSION} -m "version ${VERSION}" && git push origin ${VERSION}

README.md

@@ -345,7 +345,40 @@ For more examples, see the [docs](https://github.com/moonwalker/comet/tree/main/
 
 ## Configuration
 
-Comet can be configured using `comet.yaml` in your project directory. 
+Comet can be configured using `comet.yaml` in your project directory.
+
+### Basic Configuration
+
+```yaml
+# comet.yaml
+stacks_dir: stacks              # Directory containing stack files
+work_dir: stacks/_components    # Working directory for components
+generate_backend: false         # Auto-generate backend.tf.json
+log_level: INFO                 # Log verbosity
+tf_command: tofu                # Use 'tofu' or 'terraform'
+```
+
+### Environment Variables
+
+Pre-load environment variables before any command runs. Perfect for secrets needed during stack parsing (like SOPS_AGE_KEY):
+
+```yaml
+# comet.yaml
+env:
+  # Load SOPS AGE key from 1Password (or other secret manager)
+  SOPS_AGE_KEY: op://ci-cd/sops-age-key/private
+  
+  # Plain values work too
+  TF_LOG: DEBUG
+  AWS_REGION: us-west-2
+```
+
+**Features:**
+- Supports `op://` (1Password) and `sops://` secret resolution
+- Shell environment variables take precedence
+- Loaded before stack parsing begins
+
+See [Best Practices](docs/best-practices.md) for more configuration examples. 
 
 ## Development
 

cmd/root.go

@@ -3,6 +3,7 @@ package cmd
 import (
 	"fmt"
 	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
 
@@ -11,6 +12,7 @@ import (
 	"github.com/moonwalker/comet/internal/env"
 	"github.com/moonwalker/comet/internal/log"
 	"github.com/moonwalker/comet/internal/schema"
+	"github.com/moonwalker/comet/internal/secrets"
 )
 
 const (
@@ -32,6 +34,7 @@ var (
 func init() {
 	env.Load()
 	cfg.Read(cfgFile, config)
+	loadConfigEnv(config)
 	log.SetLevel(config.LogLevel)
 
 	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", cfgFile, "config file")
@@ -43,6 +46,28 @@ func init() {
 	})
 }
 
+func loadConfigEnv(config *schema.Config) {
+	for key, value := range config.Env {
+		// Skip if already set in shell environment (shell wins)
+		if os.Getenv(key) != "" {
+			continue
+		}
+
+		// Resolve secrets if value starts with op:// or sops://
+		if strings.HasPrefix(value, "op://") || strings.HasPrefix(value, "sops://") {
+			resolved, err := secrets.Get(value)
+			if err != nil {
+				log.Error(fmt.Sprintf("failed to resolve env var %s: %v", key, err))
+				continue
+			}
+			os.Setenv(key, resolved)
+		} else {
+			// Plain value
+			os.Setenv(key, value)
+		}
+	}
+}
+
 func Execute() {
 	if len(os.Args) == 1 {
 		cli.PrintStyledText(name)

comet.yaml

@@ -1,3 +1,8 @@
 stacks_dir: stacks
 work_dir: stacks/_components
 generate_backend: false
+
+# Pre-load environment variables before any command runs
+# Perfect for SOPS AGE key which is needed during stack parsing
+env:
+  SOPS_AGE_KEY: op://ci-cd/sops-age-key/private

docs/best-practices.md

@@ -153,6 +153,30 @@ const db = component('database', 'modules/db', {
 })
 ```
 
+### SOPS AGE Key Setup
+
+SOPS requires the `SOPS_AGE_KEY` to be set **before** stack parsing begins. Use `comet.yaml` to pre-load it:
+
+**DO:** Configure in comet.yaml (recommended)
+```yaml
+# comet.yaml
+env:
+  # Load SOPS AGE key from 1Password before stack parsing
+  SOPS_AGE_KEY: op://ci-cd/sops-age-key/private
+```
+
+**Alternative:** Export in shell
+```bash
+# In your shell profile or CI/CD
+export SOPS_AGE_KEY="AGE-SECRET-KEY-1..."
+```
+
+**Why use comet.yaml?**
+- ✅ Automatic - no manual shell setup needed
+- ✅ Team-consistent - everyone uses the same config
+- ✅ CI/CD friendly - works seamlessly in pipelines
+- ✅ Secure - secrets fetched from 1Password on-demand
+
 ### Use Path-Based Organization
 
 ```yaml
@@ -371,7 +395,12 @@ comet apply app webapp  # Now vpc outputs are available
 
 **Issue:** SOPS decryption fails
 ```bash
-# Solution: Ensure SOPS_AGE_KEY is set
+# Solution 1: Set SOPS_AGE_KEY in comet.yaml (recommended)
+# comet.yaml:
+# env:
+#   SOPS_AGE_KEY: op://ci-cd/sops-age-key/private
+
+# Solution 2: Export in shell
 export SOPS_AGE_KEY="your-age-key"
 comet apply dev
 ```

internal/schema/config.go

@@ -1,9 +1,10 @@
 package schema
 
 type Config struct {
-	LogLevel        string `mapstructure:"log_level"`
-	Command         string `mapstructure:"tf_command"`
-	StacksDir       string `mapstructure:"stacks_dir"`
-	WorkDir         string `mapstructure:"work_dir"`
-	GenerateBackend bool   `mapstructure:"generate_backend"`
+	LogLevel        string            `mapstructure:"log_level"`
+	Command         string            `mapstructure:"tf_command"`
+	StacksDir       string            `mapstructure:"stacks_dir"`
+	WorkDir         string            `mapstructure:"work_dir"`
+	GenerateBackend bool              `mapstructure:"generate_backend"`
+	Env             map[string]string `mapstructure:"env"`
 }

stacks/_examples/complete-secrets-workflow.stack.js

@@ -0,0 +1,57 @@
+// Example: Complete secrets workflow
+//
+// This demonstrates the difference between:
+// 1. Pre-loaded env vars (comet.yaml) - needed BEFORE parsing
+// 2. Stack-level secrets - loaded DURING parsing, used by Terraform
+
+// ============================================================
+// In comet.yaml:
+// ============================================================
+// env:
+//   # SOPS AGE key must be available before stack parsing
+//   SOPS_AGE_KEY: op://ci-cd/sops-age-key/private
+//
+//   # Any other early-stage environment variables
+//   TF_LOG: DEBUG
+// ============================================================
+
+stack({
+  name: 'complete-secrets-example',
+  backend: {
+    type: 'gcs',
+    bucket: 'my-terraform-state',
+    prefix: 'complete-example'
+  }
+})
+
+// Now that SOPS_AGE_KEY is set, we can use sops:// references
+component('database', {
+  source: './modules/database',
+  vars: {
+    // SOPS secret (requires SOPS_AGE_KEY from comet.yaml)
+    admin_password: secret('sops://secrets/db.yaml#admin_password'),
+
+    // 1Password secret (loaded on-demand during stack parsing)
+    backup_credentials: secret('op://production/database/backup-key'),
+
+    // Plain values work too
+    database_name: 'myapp_production'
+  }
+})
+
+component('app', {
+  source: './modules/app',
+  vars: {
+    // Mix and match secret sources
+    api_key: secret('sops://secrets/app.yaml#api_key'),
+    oauth_client_secret: secret('op://production/oauth/client-secret'),
+
+    // Reference outputs from other components
+    database_host: state('database', 'host'),
+    database_port: state('database', 'port')
+  },
+  envs: {
+    // Environment variables for Terraform execution
+    TF_VAR_region: 'us-west-2'
+  }
+})

stacks/_examples/sops-with-age.stack.js

@@ -0,0 +1,27 @@
+// Example: Using SOPS with AGE encryption
+//
+// The SOPS_AGE_KEY is pre-loaded from comet.yaml before this stack is parsed,
+// allowing sops:// references to work immediately.
+//
+// In comet.yaml:
+//   env:
+//     SOPS_AGE_KEY: op://ci-cd/sops-age-key/private
+
+stack({
+  name: 'sops-example',
+  backend: {
+    type: 'gcs',
+    bucket: 'my-terraform-state',
+    prefix: 'sops-example'
+  }
+})
+
+// SOPS_AGE_KEY is already available, so this works:
+component('app', {
+  source: './modules/app',
+  vars: {
+    // These secrets are decrypted using the AGE key set in comet.yaml
+    database_password: secret('sops://secrets/prod.yaml#database_password'),
+    api_key: secret('sops://secrets/prod.yaml#api_key')
+  }
+})