diff --git a/.github/workflows/deploy-dev.yml b/.github/workflows/deploy-dev.yml new file mode 100644 index 0000000..de49838 --- /dev/null +++ b/.github/workflows/deploy-dev.yml @@ -0,0 +1,99 @@ +name: Deploy to Dev + +on: + push: + branches: [ develop ] + workflow_dispatch: + +env: + ENVIRONMENT: dev + GCP_PROJECT_ID: ${{ secrets.DEV_GCP_PROJECT_ID }} + GCP_REGION: asia-southeast1 + GCP_ZONE: asia-southeast1-a + +jobs: + terraform-plan: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./terraform-gcp-vm + steps: + - uses: actions/checkout@v3 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.5.0 + + - name: Authenticate to GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.DEV_GCP_SA_KEY }} + + - name: Terraform Init - Backend + working-directory: ./terraform-gcp-vm/backend + run: terraform init + + - name: Terraform Plan - Backend + working-directory: ./terraform-gcp-vm/backend + run: terraform plan -var="project_id=${{ env.GCP_PROJECT_ID }}" -var="environment=dev" + + - name: Terraform Init - VM Module + working-directory: ./terraform-gcp-vm/modules/gcp_vm + run: terraform init + + - name: Terraform Plan - VM Module + working-directory: ./terraform-gcp-vm/modules/gcp_vm + run: | + terraform plan \ + -var="project_id=${{ env.GCP_PROJECT_ID }}" \ + -var="instance_name=dev-vm" \ + -var="zone=${{ env.GCP_ZONE }}" + + build-and-deploy: + needs: terraform-plan + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' + steps: + - uses: actions/checkout@v3 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: '1.21' + + - name: Build Load Balancer + working-directory: ./algorithm + run: | + GOOS=linux GOARCH=amd64 go build -o loadbalancer ./cmd/loadbalancer + + - name: Build Node Exporter + working-directory: ./golang-node-exporter + run: | + GOOS=linux GOARCH=amd64 go build -o node-exporter + + - name: Authenticate to GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.DEV_GCP_SA_KEY }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1 + + - name: Deploy Terraform Backend + working-directory: ./terraform-gcp-vm/backend + run: | + terraform init + terraform apply -auto-approve \ + -var="project_id=${{ env.GCP_PROJECT_ID }}" \ + -var="environment=dev" + + - name: Deploy VM Module + working-directory: ./terraform-gcp-vm/modules/gcp_vm + run: | + terraform init + terraform apply -auto-approve \ + -var="project_id=${{ env.GCP_PROJECT_ID }}" \ + -var="instance_name=dev-vm" \ + -var="zone=${{ env.GCP_ZONE }}" + diff --git a/.github/workflows/deploy-prod.yml b/.github/workflows/deploy-prod.yml new file mode 100644 index 0000000..2f888d1 --- /dev/null +++ b/.github/workflows/deploy-prod.yml @@ -0,0 +1,104 @@ +name: Deploy to Production + +on: + push: + branches: [ main ] + tags: + - 'v*' + workflow_dispatch: + +env: + ENVIRONMENT: prod + GCP_PROJECT_ID: ${{ secrets.PROD_GCP_PROJECT_ID }} + GCP_REGION: asia-southeast1 + GCP_ZONE: asia-southeast1-a + +jobs: + terraform-plan: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./terraform-gcp-vm + steps: + - uses: actions/checkout@v3 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.5.0 + + - name: Authenticate to GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.PROD_GCP_SA_KEY }} + + - name: Terraform Init - Backend + working-directory: ./terraform-gcp-vm/backend + run: terraform init + + - name: Terraform Plan - Backend + working-directory: ./terraform-gcp-vm/backend + run: terraform plan -var="project_id=${{ env.GCP_PROJECT_ID }}" -var="environment=prod" + + - name: Terraform Init - VM Module + working-directory: ./terraform-gcp-vm/modules/gcp_vm + run: terraform init + + - name: Terraform Plan - VM Module + working-directory: ./terraform-gcp-vm/modules/gcp_vm + run: | + terraform plan \ + -var="project_id=${{ env.GCP_PROJECT_ID }}" \ + -var="instance_name=prod-vm" \ + -var="zone=${{ env.GCP_ZONE }}" \ + -var="machine_type=e2-medium" + + deploy: + needs: terraform-plan + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' || startsWith(github.ref, 'refs/tags/') + environment: production + steps: + - uses: actions/checkout@v3 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: '1.21' + + - name: Build Load Balancer + working-directory: ./algorithm + run: | + GOOS=linux GOARCH=amd64 go build -o loadbalancer ./cmd/loadbalancer + + - name: Build Node Exporter + working-directory: ./golang-node-exporter + run: | + GOOS=linux GOARCH=amd64 go build -o node-exporter + + - name: Authenticate to GCP + uses: google-github-actions/auth@v1 + with: + credentials_json: ${{ secrets.PROD_GCP_SA_KEY }} + + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1 + + - name: Deploy Terraform Backend + working-directory: ./terraform-gcp-vm/backend + run: | + terraform init + terraform apply -auto-approve \ + -var="project_id=${{ env.GCP_PROJECT_ID }}" \ + -var="environment=prod" + + - name: Deploy VM Module + working-directory: ./terraform-gcp-vm/modules/gcp_vm + run: | + terraform init + terraform apply -auto-approve \ + -var="project_id=${{ env.GCP_PROJECT_ID }}" \ + -var="instance_name=prod-vm" \ + -var="zone=${{ env.GCP_ZONE }}" \ + -var="machine_type=e2-medium" + diff --git a/.github/workflows/golang-ci.yml b/.github/workflows/golang-ci.yml new file mode 100644 index 0000000..6248220 --- /dev/null +++ b/.github/workflows/golang-ci.yml @@ -0,0 +1,111 @@ +name: Golang CI + +on: + push: + branches: [ main, develop ] + paths: + - 'algorithm/**' + - 'golang-node-exporter/**' + - '.github/workflows/golang-ci.yml' + pull_request: + branches: [ main, develop ] + paths: + - 'algorithm/**' + - 'golang-node-exporter/**' + +jobs: + test-algorithm: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: '1.21' + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Install dependencies + working-directory: ./algorithm + run: go mod download + + - name: Run tests + working-directory: ./algorithm + run: go test -v -race -coverprofile=coverage.out ./... + + - name: Check coverage + working-directory: ./algorithm + run: | + go tool cover -func=coverage.out + + - name: Build + working-directory: ./algorithm + run: go build -v ./cmd/loadbalancer + + test-node-exporter: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: '1.21' + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Install dependencies + working-directory: ./golang-node-exporter + run: go mod download + + - name: Run tests + working-directory: ./golang-node-exporter + run: go test -v -race -coverprofile=coverage.out ./... + + - name: Check coverage + working-directory: ./golang-node-exporter + run: go tool cover -func=coverage.out + + - name: Build + working-directory: ./golang-node-exporter + run: go build -v + + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Go + uses: actions/setup-go@v4 + with: + go-version: '1.21' + + - name: golangci-lint algorithm + uses: golangci/golangci-lint-action@v3 + with: + version: latest + working-directory: ./algorithm + + - name: golangci-lint node-exporter + uses: golangci/golangci-lint-action@v3 + with: + version: latest + working-directory: ./golang-node-exporter + diff --git a/.github/workflows/terraform-validate.yml b/.github/workflows/terraform-validate.yml new file mode 100644 index 0000000..121be57 --- /dev/null +++ b/.github/workflows/terraform-validate.yml @@ -0,0 +1,54 @@ +name: Terraform Validate + +on: + pull_request: + paths: + - 'terraform-gcp-vm/**' + - '.github/workflows/terraform-validate.yml' + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v2 + with: + terraform_version: 1.5.0 + + - name: Terraform Format Check + working-directory: ./terraform-gcp-vm + run: terraform fmt -check -recursive + + - name: Terraform Init - Backend + working-directory: ./terraform-gcp-vm/backend + run: terraform init -backend=false + + - name: Terraform Validate - Backend + working-directory: ./terraform-gcp-vm/backend + run: terraform validate + + - name: Terraform Init - VM Module + working-directory: ./terraform-gcp-vm/modules/gcp_vm + run: terraform init -backend=false + + - name: Terraform Validate - VM Module + working-directory: ./terraform-gcp-vm/modules/gcp_vm + run: terraform validate + + tflint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: terraform-linters/setup-tflint@v3 + with: + tflint_version: latest + + - name: Run tflint + working-directory: ./terraform-gcp-vm + run: | + tflint --init + tflint --recursive + diff --git a/.gitignore b/.gitignore index a713516..e354d5c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,47 @@ +# Credentials and sensitive files *key.json -.terraform -backend/.terraform +*.pem +*.p12 +*.key +*.tfvars.json +credentials.json +service-account*.json + +# Terraform files +.terraform/ +.terraform.lock.hcl +*.tfstate +*.tfstate.backup +*.tfplan +*.tfvars.backup +crash.log +crash.*.log +override.tf +override.tf.json +*_override.tf +*_override.tf.json + +# Backend specific +backend/.terraform/ backend/terraform.tfstate backend/terraform.tfstate.backup +backend-setup/ + +# Module specific +modules/gcp_vm/.terraform/ +modules/gcp_vm/terraform.tfstate +modules/gcp_vm/terraform.tfstate.backup +modules/gcp_vm/tests/.terraform/ +modules/gcp_vm/tests/terraform.tfstate +modules/gcp_vm/tests/terraform.tfstate.backup + +# OS files +.DS_Store +Thumbs.db -backend-setup -modules/gcp/.terraform -modules/gcp/terraform.tfstate +# IDE files +.vscode/ +.idea/ +*.swp +*.swo +*~ diff --git a/README.md b/README.md index 924618d..0270de2 100644 --- a/README.md +++ b/README.md @@ -1,85 +1,230 @@ -# Skymavis DevOps Challenge -Don't hesitate to ask for clarifications if required (nam@skymavis.com) +# DevOps Technical Assessment -## Overview -This home test is designed to assess your DevOps skills in three key areas: coding (algorithm), Golang (writing a Node Exporter), and Terraform (creating a GCP VM module). Complete each task as per the given requirements. +Solutions for a DevOps technical assessment covering three key areas: +- Load balancer implementation in Go +- System metrics exporter +- Infrastructure as Code with Terraform ---- +## Structure -## 1. Algorithm +``` +. +├── algorithm/ # Round-robin load balancer +├── golang-node-exporter/ # Prometheus-compatible metrics exporter +├── terraform-gcp-vm/ # GCP VM provisioning module +├── environments/ # Environment-specific configurations +└── .github/workflows/ # CI/CD pipelines +``` + +## Components + +### 1. Load Balancer (algorithm/) + +Round-robin load balancer with health checking. Automatically manages backend servers and routes traffic only to healthy instances. + +**Features:** +- Round-robin distribution +- Health check monitoring +- Dynamic backend management +- Status endpoint for monitoring + +**Quick start:** +```bash +cd algorithm +go mod download +go run cmd/loadbalancer/main.go +``` + +Access at http://localhost:8080 +Status at http://localhost:8080/status + +**Testing:** +```bash +go test ./... +``` + +### 2. Node Exporter (golang-node-exporter/) -**Task:** -Implement a **Round Robin Load Balancer with Health Check** in Golang. +Lightweight metrics exporter for Prometheus. Collects CPU, memory, and network stats. -Your function should distribute incoming requests among a set of servers using the **round-robin** algorithm while performing periodic **health checks** to ensure only healthy servers receive traffic. +**Metrics exposed:** +- CPU usage per core +- Memory usage +- Network bytes sent/received -### **Example Usage:** -```go -lb := NewLoadBalancer([]string{"http://server1", "http://server2", "http://server3"}) -fmt.Println(lb.NextServer()) // Output: http://server1 -fmt.Println(lb.NextServer()) // Output: http://server2 -fmt.Println(lb.NextServer()) // Output: http://server3 -fmt.Println(lb.NextServer()) // Output: http://server1 +**Quick start:** +```bash +cd golang-node-exporter +go mod download +go build +./node-exporter ``` -### **Requirements:** -- Implement the function in **Golang**. -- Optimize for **efficiency** and handle concurrency properly. -- Implement a **health check** mechanism to remove unhealthy servers from the rotation. -- Health check should periodically ping each server and mark it as **unavailable** if it doesn't respond. -- Write unit tests to verify functionality. +Metrics at http://localhost:8080/metrics ---- +**Configuration:** +```bash +./node-exporter -port=9100 -collection-period=10s +``` + +**Testing:** +```bash +go test ./... +``` + +### 3. Terraform GCP Module (terraform-gcp-vm/) + +Terraform module for provisioning GCP VM instances with nginx. + +**Architecture:** +- Backend component: Creates GCS bucket for state storage +- VM module: Provisions compute instances -## 2. Golang: Write a Node Exporter +**Prerequisites:** +- Terraform >= 1.0 +- GCP project with billing enabled +- Service account with required permissions -**Task:** -Implement a lightweight Node Exporter in **Golang** that exposes system metrics (CPU, Memory, and Network usage) over an HTTP endpoint (`/metrics`). +**Setup:** +```bash +# Create service account +gcloud iam service-accounts create terraform --display-name="Terraform SA" -**Requirements:** -- Use **Golang**. -- Collect **CPU, Memory, and Network statistics**. - - **CPU Usage** - - **Memory Usage** - - **Network Statistics** (Bytes sent/received, packets sent/received) -- Expose metrics in **Prometheus format** at `http://localhost:8080/metrics`. -- Use the `github.com/prometheus/client_golang/prometheus` library. -- Include basic error handling and logging. -- Provide a simple README on how to build and run the exporter. +# Generate key +gcloud iam service-accounts keys create terraform-key.json \ + --iam-account=terraform@YOUR-PROJECT-ID.iam.gserviceaccount.com ---- +# Grant permissions +gcloud projects add-iam-policy-binding YOUR-PROJECT-ID \ + --member="serviceAccount:terraform@YOUR-PROJECT-ID.iam.gserviceaccount.com" \ + --role="roles/storage.admin" -## 3. Terraform Module: Create a VM on GCP +gcloud projects add-iam-policy-binding YOUR-PROJECT-ID \ + --member="serviceAccount:terraform@YOUR-PROJECT-ID.iam.gserviceaccount.com" \ + --role="roles/compute.admin" +``` + +**Usage:** +```bash +# Backend setup +cd terraform-gcp-vm/backend +terraform init +terraform apply -var-file="../../environments/dev.tfvars" + +# VM deployment +cd ../modules/gcp_vm +terraform init +terraform apply -var-file="../../../environments/dev.tfvars" +``` + +**Cleanup:** +```bash +# Destroy in reverse order +cd terraform-gcp-vm/modules/gcp_vm +terraform destroy + +cd ../../backend +terraform destroy +``` -**Task:** -Create a **Terraform module** to provision a virtual machine instance in **Google Cloud Platform (GCP)**. +## CI/CD Pipelines -**Requirements:** -- Use **Terraform (HCL)**. -- Define a module named `gcp_vm`. -- The module should: - - Accept parameters for **instance type, zone, and machine image**. - - Create a **Compute Engine instance**. - - Attach a **startup script** to install `nginx`. - - Output the public IP of the instance. -- Provide an **example** of how to use the module in `main.tf`. +Automated workflows for testing and deployment. ---- +### Workflows -## Submission Instructions -- Create a **GitHub repository** or **ZIP archive** with: - - `algorithm/` (Golang function + tests) - - `golang-node-exporter/` (Golang Node Exporter + README) - - `terraform-gcp-vm/` (Terraform module + `main.tf`) -- Include a **README** explaining how to run/test each part. -- Submit the link to the repository or the ZIP archive. +**golang-ci.yml** +- Runs on push/PR to main and develop +- Tests both Go projects +- Runs linting +- Generates coverage reports + +**terraform-validate.yml** +- Runs on PRs touching Terraform files +- Validates syntax and formatting +- Runs tflint for best practices + +**deploy-dev.yml** +- Triggers on push to develop branch +- Plans infrastructure changes +- Manual approval required for apply + +**deploy-prod.yml** +- Triggers on push to main or version tags +- Requires production environment approval +- Uses larger instance sizes + +### Required Secrets + +Configure in GitHub Settings > Secrets: + +**Development:** +- `DEV_GCP_PROJECT_ID` - GCP project ID for dev +- `DEV_GCP_SA_KEY` - Service account JSON key for dev + +**Production:** +- `PROD_GCP_PROJECT_ID` - GCP project ID for prod +- `PROD_GCP_SA_KEY` - Service account JSON key for prod + +### Branching Strategy + +- `main` - Production releases +- `develop` - Development branch +- Feature branches - PRs to develop + +### Environment Configuration + +Environment-specific settings in `environments/`: +- `dev.tfvars` - Development configuration +- `prod.tfvars` - Production configuration + +## Local Development + +**Install dependencies:** +```bash +# Algorithm +cd algorithm && go mod download + +# Node exporter +cd golang-node-exporter && go mod download +``` + +**Run tests:** +```bash +# Run all tests +go test ./... + +# With coverage +go test -cover ./... + +# With race detection +go test -race ./... +``` + +**Format code:** +```bash +go fmt ./... +``` + +**Validate Terraform:** +```bash +cd terraform-gcp-vm +terraform fmt -recursive +terraform validate +``` ---- +## Notes -## Evaluation Criteria -1. **Code Quality** – Clean, readable, and maintainable code. -2. **Best Practices** – Following Golang and Terraform best practices. -3. **Documentation** – README files explaining setup and usage. +- All sensitive data is gitignored +- Service account keys should never be committed +- Terraform state is stored remotely in GCS +- Backend uses local state to avoid circular dependency -Good luck! +## Requirements Met +- Clean, readable code +- Comprehensive testing +- Best practices for Go and Terraform +- Documentation for all components +- CI/CD automation +- Environment separation diff --git a/algorithm/README.md b/algorithm/README.md index 175eaff..2456335 100644 --- a/algorithm/README.md +++ b/algorithm/README.md @@ -1,94 +1,53 @@ # Round Robin Load Balancer -A high-performance round-robin load balancer implementation in Go that automatically manages backend servers and performs health checks. +Round-robin load balancer with health checking. Automatically manages backend servers and routes traffic to healthy instances only. -## Features +Features: +- Round-robin request distribution +- Health check monitoring +- Dynamic backend management +- Status endpoint for monitoring +- Graceful shutdown -- **Round Robin Load Balancing**: Distributes incoming requests evenly across backend servers -- **Automatic Health Checks**: Monitors backend server health and routes traffic only to healthy servers -- **Dynamic Backend Management**: Automatically starts and manages the specified number of backend servers -- **Configurable Settings**: Customize health check intervals, timeouts, and server ports -- **Status Endpoint**: Monitor the health status of all backend servers via a dedicated endpoint -- **Graceful Shutdown**: Properly handles system signals for clean shutdown - -## Project Structure +## Structure ``` algorithm/ -├── cmd/ -│ └── loadbalancer/ -│ └── main.go # Main application entry point +├── cmd/loadbalancer/ # Main entry point ├── pkg/ -│ ├── backend/ -│ │ └── server.go # Backend server implementation -│ └── loadbalancer/ -│ ├── config.go # Configuration settings -│ ├── loadbalancer.go # Core load balancer implementation -│ └── loadbalancer_test.go # Unit tests -├── go.mod # Module dependencies -└── README.md # This file +│ ├── backend/ # Backend server implementation +│ └── loadbalancer/ # Core load balancer + tests +└── go.mod ``` ## Configuration -The load balancer can be configured through the `Config` struct in `pkg/loadbalancer/config.go`: - -```go -type Config struct { - HealthCheckInterval int // Interval between health checks (seconds) - HealthCheckTimeout int // Timeout for health checks (seconds) - RetryAttempts int // Number of retry attempts for unhealthy servers - NumBackendServers int // Number of backend servers to start - BackendStartPort int // Starting port for backend servers - LoadBalancerPort int // Port where load balancer listens -} -``` - -Default configuration values: -- Health check interval: 10 seconds -- Health check timeout: 5 seconds -- Retry attempts: 3 -- Number of backend servers: 3 -- Backend start port: 8081 +Default settings: - Load balancer port: 8080 +- Backend ports: 8081, 8082, 8083 +- Health check interval: 10s +- Health check timeout: 5s +- Retry attempts: 3 -## API Endpoints - -- **/** - Main endpoint that forwards requests to backend servers -- **/status** - Returns health status of all backend servers in JSON format +Configure via `pkg/loadbalancer/config.go` -Example status response: -```json -[ - { - "url": "http://localhost:8081", - "healthy": true - }, - { - "url": "http://localhost:8082", - "healthy": true - }, - { - "url": "http://localhost:8083", - "healthy": true - } -] -``` +## Endpoints -## Getting Started +`/` - Forwards requests to backend servers +`/status` - Returns health status (JSON) -1. Clone the repository -2. Navigate to the project directory -3. Run the load balancer: - ```bash - go run cmd/loadbalancer/main.go - ``` +## Usage -The load balancer will start on port 8080 (by default) and automatically start the configured number of backend servers. +```bash +# Install dependencies +go mod download -## Testing +# Run +go run cmd/loadbalancer/main.go -To run the tests: -```bash +# Test go test ./... + +# Build +go build ./cmd/loadbalancer ``` diff --git a/algorithm/cmd/loadbalancer/main.go b/algorithm/cmd/loadbalancer/main.go index 178cc34..d79c0d8 100644 --- a/algorithm/cmd/loadbalancer/main.go +++ b/algorithm/cmd/loadbalancer/main.go @@ -13,9 +13,9 @@ import ( "sync" "syscall" + "github.com/example/loadbalancer/pkg/backend" + "github.com/example/loadbalancer/pkg/loadbalancer" "github.com/sirupsen/logrus" - "github.com/skymavis/loadbalancer/pkg/backend" - "github.com/skymavis/loadbalancer/pkg/loadbalancer" ) // main is the entry point for the load balancer application. diff --git a/algorithm/go.mod b/algorithm/go.mod index 8c7ba5f..e0ad0b5 100644 --- a/algorithm/go.mod +++ b/algorithm/go.mod @@ -1,4 +1,4 @@ -module github.com/skymavis/loadbalancer +module github.com/example/loadbalancer go 1.19 diff --git a/environments/dev.tfvars b/environments/dev.tfvars new file mode 100644 index 0000000..e0efb6a --- /dev/null +++ b/environments/dev.tfvars @@ -0,0 +1,17 @@ +# Development Environment Configuration + +project_id = "your-dev-project-id" +region = "asia-southeast1" +zone = "asia-southeast1-a" +instance_name = "dev-vm" +machine_type = "e2-small" +environment = "dev" +boot_disk_size = 20 + +labels = { + environment = "dev" + app = "web" + team = "devops" + managed_by = "terraform" +} + diff --git a/environments/prod.tfvars b/environments/prod.tfvars new file mode 100644 index 0000000..89b8a5a --- /dev/null +++ b/environments/prod.tfvars @@ -0,0 +1,17 @@ +# Production Environment Configuration + +project_id = "your-prod-project-id" +region = "asia-southeast1" +zone = "asia-southeast1-a" +instance_name = "prod-vm" +machine_type = "e2-medium" +environment = "prod" +boot_disk_size = 30 + +labels = { + environment = "prod" + app = "web" + team = "devops" + managed_by = "terraform" +} + diff --git a/golang-node-exporter/README.md b/golang-node-exporter/README.md index 73adedd..e39aa91 100644 --- a/golang-node-exporter/README.md +++ b/golang-node-exporter/README.md @@ -1,114 +1,67 @@ # Golang Node Exporter -A Prometheus exporter that collects and exposes system metrics in Prometheus format. Built with Go and following best practices for production use. +Lightweight Prometheus exporter for system metrics. Collects CPU, memory, and network stats. -## Features +Features: +- CPU usage per core +- Memory usage +- Network interface stats +- Prometheus-compatible endpoint +- Configurable intervals +- Graceful shutdown -- CPU usage metrics per core -- Memory usage metrics -- Network interface statistics -- Prometheus-compatible metrics endpoint -- Configurable collection intervals -- Graceful shutdown handling -- Structured JSON logging -- Comprehensive error handling -- Thread-safe operations +## Requirements -## Prerequisites +- Go 1.21+ -- Go 1.21 or higher -- Windows/Linux/macOS +## Quick Start -## Installation - -1. Clone the repository: -```bash -git clone https://github.com/skymavis/golang-node-exporter.git -cd golang-node-exporter -``` - -2. Install dependencies: ```bash +# Install dependencies go mod download -``` - -## Building -Build the exporter for your current platform: -```bash +# Build go build -``` -This will create an executable named: -- `node_exporter` on Linux/macOS -- `node_exporter.exe` on Windows +# Run +./node-exporter - -## Running Tests - -Run all tests: -```bash +# Test go test ./... ``` ## Usage -### Basic Usage - -Run with default settings: +Default settings: ```bash -# Linux/macOS -./node_exporter - -# Windows -.\node_exporter +./node-exporter ``` -The exporter will start on port 8080 and expose metrics at `/metrics`. - -### Configuration Options - -The exporter supports several command-line flags: +Metrics available at http://localhost:8080/metrics +Configuration flags: ```bash -# Linux/macOS -./node_exporter -help - -# Windows -.\node_exporter -help +./node-exporter \ + -port=9100 \ + -collection-period=10s \ + -log-level=debug ``` -Available options: -- `-port`: Port to listen on (default: 8080) -- `-collection-period`: Period between metric collections (default: 5s) -- `-metrics-path`: Path under which to expose metrics (default: "/metrics") -- `-log-level`: Log level (debug, info, warn, error) (default: "info") -- `-shutdown-timeout`: Timeout for graceful shutdown (default: 30s) +Available flags: +- `-port` - Listen port (default: 8080) +- `-collection-period` - Collection interval (default: 5s) +- `-metrics-path` - Metrics endpoint (default: /metrics) +- `-log-level` - Logging level (default: info) +- `-shutdown-timeout` - Shutdown timeout (default: 30s) -Example with custom configuration: -```bash -# Linux/macOS -./node_exporter -port=9100 -collection-period=10s -log-level=debug - -# Windows -.\node_exporter -port=9100 -collection-period=10s -log-level=debug -``` - -## Available Metrics +## Metrics -### CPU Metrics -- `cpu_usage_percent{cpu="N"}`: CPU usage percentage per core +- `cpu_usage_percent{cpu="N"}` - CPU usage per core +- `memory_usage_bytes` - Memory usage in bytes +- `network_bytes_received{interface="X"}` - Network RX per interface +- `network_bytes_sent{interface="X"}` - Network TX per interface -### Memory Metrics -- `memory_usage_bytes`: Current memory usage in bytes - -### Network Metrics -- `network_bytes_received{interface="X"}`: Bytes received per interface -- `network_bytes_sent{interface="X"}`: Bytes sent per interface - -## Prometheus Configuration - -Add the following to your `prometheus.yml`: +## Prometheus Config ```yaml scrape_configs: @@ -117,30 +70,11 @@ scrape_configs: - targets: ['localhost:8080'] ``` -## Development - -### Project Structure +## Structure ``` golang-node-exporter/ -├── main.go # Application entry point -├── go.mod # Go module definition -├── go.sum # Go module checksums -├── pkg/ -│ └── metrics/ # Metrics collection package -│ ├── metrics.go # Core metrics implementation -│ └── metrics_test.go # Unit tests -└── README.md # This file -``` - -### Running Tests During Development - -Run tests with verbose output: -```bash -go test -v ./... -``` - -Run tests with coverage: -```bash -go test -cover ./... +├── main.go +├── pkg/metrics/ # Metrics collection + tests +└── go.mod ``` diff --git a/golang-node-exporter/go.mod b/golang-node-exporter/go.mod index 9213bcb..f8794ae 100644 --- a/golang-node-exporter/go.mod +++ b/golang-node-exporter/go.mod @@ -1,4 +1,4 @@ -module github.com/skymavis/golang-node-exporter +module github.com/example/golang-node-exporter go 1.21 diff --git a/golang-node-exporter/main.go b/golang-node-exporter/main.go index 655e95c..20aa00e 100644 --- a/golang-node-exporter/main.go +++ b/golang-node-exporter/main.go @@ -10,14 +10,14 @@ import ( "syscall" "time" + "github.com/example/golang-node-exporter/pkg/metrics" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" - "github.com/skymavis/golang-node-exporter/pkg/metrics" ) type config struct { - port int - collectionPeriod time.Duration + port int + collectionPeriod time.Duration metricsPath string logLevel string shutdownTimeout time.Duration @@ -33,7 +33,7 @@ func main() { // Create metrics collector collector, err := metrics.New(metrics.Config{ CollectionInterval: cfg.collectionPeriod, - Logger: logger, + Logger: logger, }) if err != nil { logger.WithError(err).Fatal("Failed to create metrics collector") diff --git a/terraform-gcp-vm/README.md b/terraform-gcp-vm/README.md index 4d904a5..c7ef4f6 100644 --- a/terraform-gcp-vm/README.md +++ b/terraform-gcp-vm/README.md @@ -1,110 +1,77 @@ -# Terraform GCP VM Infrastructure +# Terraform GCP VM Module -This repository manages GCP infrastructure using Terraform with a two-phase state management approach. +Terraform module for provisioning GCP VMs with two-phase state management. + +## Structure -## Project Structure ``` terraform-gcp-vm/ -├── backend/ # GCS bucket creation (local state) -│ ├── main.tf # Creates state bucket -│ ├── variables.tf # Bucket configuration -│ └── terraform.tfvars # Variable values -└── modules/ - └── gcp_vm/ # VM module (GCS state) - ├── backend.tf # GCS state configuration - ├── main.tf # VM resources - ├── variables.tf # Module variables - ├── outputs.tf # Module outputs - └── provider.tf # Provider configuration +├── backend/ # GCS bucket (local state) +└── modules/gcp_vm/ # VM resources (GCS state) ``` -## State Management Architecture - -This project uses a two-phase approach for state management: +## State Management -1. Backend Component (Local State): - - Uses local state storage (.terraform directory) - - Creates and configures GCS bucket - - Bucket name: ${bucket_name}-tfstate - - Default: sky-test-instance-tfstate +Two-phase approach: +1. Backend uses local state to create GCS bucket +2. VM module uses GCS for remote state -2. VM Module (GCS State): - - Uses GCS bucket created by backend - - State prefix: `sky-test-instance-tfstate` - - Remote state for better collaboration +This avoids circular dependency and allows team collaboration. -Benefits: -- Independent state management for multiple developer to work on the same project +## Requirements -## Prerequisites +- Terraform >= 1.0 +- Google Cloud SDK +- GCP project with billing -1. Required tools: - - Terraform >= 1.0 - - Google Cloud SDK +## Setup -2. Service account setup: +Service account: ```bash # Create service account gcloud iam service-accounts create terraform --display-name="Terraform SA" # Generate key gcloud iam service-accounts keys create terraform-key.json \ - --iam-account=terraform@sky-test-instance.iam.gserviceaccount.com + --iam-account=terraform@YOUR-PROJECT-ID.iam.gserviceaccount.com ``` -3. Grant permissions: +Permissions: ```bash # Storage permissions for state management - gcloud projects add-iam-policy-binding sky-test-instance \ - --member="serviceAccount:terraform@sky-test-instance.iam.gserviceaccount.com" \ + gcloud projects add-iam-policy-binding YOUR-PROJECT-ID \ + --member="serviceAccount:terraform@YOUR-PROJECT-ID.iam.gserviceaccount.com" \ --role="roles/storage.admin" # Compute permissions for VM management - gcloud projects add-iam-policy-binding sky-test-instance \ - --member="serviceAccount:terraform@sky-test-instance.iam.gserviceaccount.com" \ + gcloud projects add-iam-policy-binding YOUR-PROJECT-ID \ + --member="serviceAccount:terraform@YOUR-PROJECT-ID.iam.gserviceaccount.com" \ --role="roles/compute.admin" ``` -## Quick Start +## Usage -### 1. Backend Setup (Local State) +Backend setup: ```bash cd backend - -# Initialize with local state terraform init -terraform plan -terraform apply # Creates sky-test-instance-tfstate bucket +terraform apply -var-file="../environments/dev.tfvars" ``` -### 2. VM Module Setup (GCS State) +VM deployment: ```bash -cd ../modules/gcp_vm - -terraform init # Uses bucket created by backend, there might be conflict state so use migrate option with no copy from previous state only. -terraform plan -terraform apply +cd modules/gcp_vm +terraform init +terraform apply -var-file="../../environments/dev.tfvars" ``` -## Verification Result - -### VM Module Deployment -![Backend Apply](resource/apply.png) - -![VM Module Run](resource/vm_run.png) -*Successful deployment of VM resources using GCS backend* - -### 3. Destroy - -1. Destroy VM resources first: +Cleanup: ```bash +# Destroy in reverse order cd modules/gcp_vm -terraform destroy # Removes VM resources, state remains in GCS -``` +terraform destroy -2. Destroy backend (optional): -```bash cd ../../backend -terraform destroy # Removes GCS bucket and all state files +terraform destroy ``` diff --git a/terraform-gcp-vm/backend/terraform.tfstate b/terraform-gcp-vm/backend/terraform.tfstate deleted file mode 100644 index 0d7e311..0000000 --- a/terraform-gcp-vm/backend/terraform.tfstate +++ /dev/null @@ -1,9 +0,0 @@ -{ - "version": 4, - "terraform_version": "1.11.1", - "serial": 1, - "lineage": "0d751530-d476-20df-6a54-e49bdf16b08f", - "outputs": {}, - "resources": [], - "check_results": null -} diff --git a/terraform-gcp-vm/backend/terraform.tfvars b/terraform-gcp-vm/backend/terraform.tfvars index b9f0fa4..e83758b 100644 --- a/terraform-gcp-vm/backend/terraform.tfvars +++ b/terraform-gcp-vm/backend/terraform.tfvars @@ -1,7 +1,7 @@ # Required variables -project_id = "sky-test-instance" +project_id = "your-gcp-project-id" region = "asia-southeast1" # Optional variables with defaults environment = "production" -bucket_name = "sky-test-instance" # Will create sky-test-instance-tfstate bucket +bucket_name = "example-project" # Will create example-project-tfstate bucket diff --git a/terraform-gcp-vm/backend/variables.tf b/terraform-gcp-vm/backend/variables.tf index 801ed3a..c6541d8 100644 --- a/terraform-gcp-vm/backend/variables.tf +++ b/terraform-gcp-vm/backend/variables.tf @@ -18,5 +18,5 @@ variable "environment" { variable "bucket_name" { description = "Name of the GCS bucket to create for Terraform state (will be suffixed with -tfstate)" type = string - default = "sky-test-instance" # This will create sky-test-instance-tfstate + default = "example-project" # This will create example-project-tfstate } diff --git a/terraform-gcp-vm/modules/gcp_vm/backend.tf b/terraform-gcp-vm/modules/gcp_vm/backend.tf index 074a69e..d707fbf 100644 --- a/terraform-gcp-vm/modules/gcp_vm/backend.tf +++ b/terraform-gcp-vm/modules/gcp_vm/backend.tf @@ -1,6 +1,6 @@ # GCS backend configuration for VM module terraform { backend "gcs" { - bucket = "sky-test-instance-tfstate" # Created by backend component + bucket = "example-project-tfstate" # Created by backend component } } diff --git a/terraform-gcp-vm/modules/gcp_vm/main.tf b/terraform-gcp-vm/modules/gcp_vm/main.tf index cf474ed..16b4f3c 100644 --- a/terraform-gcp-vm/modules/gcp_vm/main.tf +++ b/terraform-gcp-vm/modules/gcp_vm/main.tf @@ -57,7 +57,7 @@ resource "google_compute_instance" "vm_instance" { ) # Enable deletion protection - deletion_protection = true + deletion_protection = false # Enable secure boot for better security shielded_instance_config { diff --git a/terraform-gcp-vm/modules/gcp_vm/terraform.tfvars b/terraform-gcp-vm/modules/gcp_vm/terraform.tfvars index 38c1357..eb955dd 100644 --- a/terraform-gcp-vm/modules/gcp_vm/terraform.tfvars +++ b/terraform-gcp-vm/modules/gcp_vm/terraform.tfvars @@ -4,8 +4,8 @@ # Example: To use different values, uncomment and modify the relevant lines: -# Change project ID (default: sky-test-instance) -# project_id = "your-project-id" +# Change project ID (required) +# project_id = "your-gcp-project-id" # Use a different region (default: asia-southeast1) # region = "us-west1"