diff --git a/approved/environment/environment.yaml b/approved/environment/environment.yaml new file mode 100644 index 0000000..6e8d744 --- /dev/null +++ b/approved/environment/environment.yaml @@ -0,0 +1,82 @@ +# Basic Environment YAML Definition for V1 + +environment: + name: envin + id: envin ## optional + tags: {} ## optional + type: production | non-production + org: default #optional + project: TcSvcOverrideTest # optional + + +## Environment Group in V1 YAML Definition + +environmentGroup: + name: dev + identifier: dev + description: "" + tags: {} + org: default + project: PM_Signoff + environments: + - SAM + - asg + - ecs + - k8sdev + - k8sprod + - qa + - serverless + + + +## Example Usage Below + +# simple stage, with multi-service, multi-environment + +pipeline: + stages: + - steps: + - run: + script: go build + service: + items: + - petstore-frontend + - petstore-backend + environment: + parallel: true + items: + - name: prod + deploy-to: all + - name: stage + deploy-to: + - infra1 + - infra2 + - name: dev + deploy-to: infra3 + +--- + +# service and environment at the pipeline level, +# allows us to remove propagation configuration. + + +pipeline: + service: + items: + - petstore-frontend + - petstore-backend + environment: + parallel: true + items: + - name: prod + deploy-to: all + stages: + # override the service and environment + # at the stage level. + - service: petstore + environment: prod + steps: + - run: + script: go build + + \ No newline at end of file diff --git a/approved/http.yaml b/approved/http.yaml index 5215eef..9592ad5 100644 --- a/approved/http.yaml +++ b/approved/http.yaml @@ -10,7 +10,8 @@ headers: x-api-key: <+secret.getValue("api_key")> content-type: "application/json" - assertion: "expression" + assertion: + code: 200 body: | identifier: "test" name: "test" diff --git a/approved/infrastructure/infra.yaml b/approved/infrastructure/infra.yaml new file mode 100644 index 0000000..59e9fed --- /dev/null +++ b/approved/infrastructure/infra.yaml @@ -0,0 +1,55 @@ +## Kubernetes + +infrastructure: + name: prod-platformdemo-k8scluster + id: prodplatformdemok8scluster + description: "" + tags: {} + org: default + project: Platform_Demo + uses: gke | eks | aks | kubernetes + with: + connector: platformdemok8s + namespace: e2e-prod + release: release-<+INFRA_KEY> + parallel-deployment: false + +--- +## ECS + +infrastructure: + name: ECS + id: ECS + description: "ECS Deployment Infrastructure" + tags: + account:dev + org: default + project: Platform_Demo + uses: ecs + with: + connector: AWSSalesDanF + region: us-west-2 + cluster: lg-fargate + parallel-deployment: false + +--- +## SSH + +infrastructure: + name: dev-ssh-aws + id: devsshaws + org: default + project: CD_Demo + uses: ssh-aws | ssh-azure | ssh-pdc | winrm-azure | winrm-aws | winrm-pdc + with: + credentials: sshawsdemo + connector: account.AWS_Sales_Account + region: us-west-2 + # instance-filter: Remove Instance Filter + tags: + name: sshdemo-instance + vpcs: [] + host-connection-type: public-ip + instance-type: aws + parallel-deployment: false + diff --git a/approved/pipeline/failure-strategy.yaml b/approved/pipeline/failure-strategy.yaml new file mode 100644 index 0000000..10b5465 --- /dev/null +++ b/approved/pipeline/failure-strategy.yaml @@ -0,0 +1,168 @@ +# step with failure strategy + +pipeline: + stages: + - steps: + - run: + script: go test + on-failure: + errors: all + action: ignore +--- +# sample pipeline with a retry failure strategy +# that fails if all retry attempts fail. + +pipeline: + stages: + - steps: + - run: + script: go test + container: golang + on-failure: + errors: [ unknown ] + action: + retry: + attempts: 5 + interval: 10s + failure-action: fail + +--- + +# sample pipeline with a retry failure strategy +# that demonstrates multiple, staggered intervals. + +pipeline: + stages: + - steps: + - run: + script: go test + container: golang + on-failure: + errors: [ unknown ] + action: + retry: + attempts: 5 + interval: [ 10s, 30s, 1m, 5m, 10m ] + failure-action: fail + +--- + +# sample pipeline with retry failure strategy with a +# complex failure action. + +pipeline: + stages: + - steps: + - run: + script: go test + container: golang + on-failure: + errors: [ unknown ] + action: + retry: + attempts: 5 + interval: 10s + failure-action: + manual-intervention: + timeout: 60m + timeout-action: fail + +--- + +# sample pipeline with simplified retry strategy +# syntax that should apply sane defaults. + +pipeline: + stages: + - steps: + - run: + script: go test + container: golang + on-failure: + errors: all + action: retry + +--- + +# sample pipeline with manual-intervention +# failure strategy that fails on timeout. + +pipeline: + stages: + - steps: + - run: + script: go test + container: golang + on-failure: + errors: [ all ] + action: + manual-intervention: + timeout: 30m + timeout-action: fail + +--- + +# sample pipeline with manual-intervention +# failure strategy with a complex timeout +# action. + +pipeline: + stages: + - steps: + - run: + script: go test + container: golang + on-failure: + errors: [ all ] + action: + manual-intervention: + timeout: 30m + timeout-action: + retry: + attempts: 10 + interval: 30s + failure-action: success +--- + +# sample pipeline with a basic failure strategy at the stage +# level that aborts on all errors. + +pipeline: + stages: + - steps: + - run: + script: go test + container: golang + on-failure: + errors: all + action: abort + +--- + +# sample pipeline with a basic failure strategy at the step +# level that aborts on all errors. + +pipeline: + stages: + - steps: + - run: + script: go test + container: golang + on-failure: + errors: all + action: abort + +--- + +# sample pipeline with a retry failure strategy +# that aborts for enumerated error types. + +pipeline: + stages: + - steps: + - run: + script: go test + container: golang + on-failure: + errors: [ unknown, connectivity ] + action: abort diff --git a/approved/pipeline/pipeline.yaml b/approved/pipeline/pipeline.yaml new file mode 100644 index 0000000..a4c72bd --- /dev/null +++ b/approved/pipeline/pipeline.yaml @@ -0,0 +1,75 @@ +# sample pipeline with run step + +pipeline: + stages: + - steps: + - run: + script: go build + +--- + +# sample pipeline with run step, short syntax + +pipeline: + stages: + - steps: + - run: go build + +--- + +# sample pipeline with run step, shortest syntax + +pipeline: + stages: + - steps: + - go build + +--- + +# sample pipeline with conditional execution + +pipeline: + if: ${{ branch == "main" }} + stages: + - steps: + - run: + script: go build + +--- + +# sample pipeline with global envs + +pipeline: + env: + GOOS: linux + GOARCH: amd64 + stages: + - steps: + - go build + +--- + +# sample pipeline with optional repository override + +pipeline: + # repository should be optional. If undefined, + # the repository and conector are the same as + # where the yaml was stored. + repo: + name: drone/drone + connector: account.github + stages: + - steps: + - go build + +--- + +# sample pipeline, github compatible + +jobs: + test: + runs-on: ubuntu + steps: + - run: go build + +--- \ No newline at end of file diff --git a/approved/pipeline/stage.yaml b/approved/pipeline/stage.yaml new file mode 100644 index 0000000..1cca37a --- /dev/null +++ b/approved/pipeline/stage.yaml @@ -0,0 +1,155 @@ +# simple stage. + +pipeline: + stages: + - steps: + - run: + script: go build + +--- + +# simple stage, with id + +pipeline: + stages: + - id: build + steps: + - run: + script: go build + +--- + +# simple stage, with name + +pipeline: + stages: + - name: build + steps: + - run: + script: go build + +--- + +# simple stage, with conditions + +pipeline: + stages: + - if: ${{ branch == "main" }} + steps: + - run: + script: go build + + +--- + +# simple stage, pinned to delegate + +pipeline: + stages: + - delegate: some-delegate + steps: + - run: + script: go build + +--- + +# simple stage, with simple failure strategy + +pipeline: + stages: + - steps: + - run: + script: go build + on-failure: + errors: all + action: ignore + +--- + +# simple stage, with matrix strategy + +pipeline: + stages: + - steps: + - run: + script: go build + container: golang:${{ matrix.version }} + strategy: + matrix: + version: + - "1.19" + - "1.20" + +--- + +# simple stage, with cache intelligence settings + +pipeline: + stages: + - steps: + - run: + script: go build + cache: + path: /path/to/file + +--- + +# simple stage, with single service, single environment + +pipeline: + stages: + - steps: + - run: + script: go build + service: petstore + environment: prod + +--- + +# simple stage, with multi-service, multi-environment + +pipeline: + stages: + - steps: + - run: + script: go build + service: + items: + - petstore-frontend + - petstore-backend + environment: + parallel: true + items: + - name: prod + deploy-to: all + - name: stage + deploy-to: + - infra1 + - infra2 + - name: dev + deploy-to: infra3 + +--- + +# service and environment at the pipeline level, +# allows us to remove propagation configuration. + + +pipeline: + service: + items: + - petstore-frontend + - petstore-backend + environment: + parallel: true + items: + - name: prod + deploy-to: all + stages: + # override the service and environment + # at the stage level. + - service: petstore + environment: prod + steps: + - run: + script: go build diff --git a/approved/pipeline/step-group.yaml b/approved/pipeline/step-group.yaml new file mode 100644 index 0000000..1eb3f0e --- /dev/null +++ b/approved/pipeline/step-group.yaml @@ -0,0 +1,30 @@ +# simple group step. + +pipeline: + stages: + - steps: + - group: + steps: + - run: + container: golang + script: go build + - run: + container: golang + script: go test + +--- + +# simple group step with conditional + +pipeline: + stages: + - steps: + - if: ${{ branch == "main" }} + group: + steps: + - run: + container: golang + script: go build + - run: + container: golang + script: go test \ No newline at end of file diff --git a/approved/pipeline/step-parallel.yaml b/approved/pipeline/step-parallel.yaml new file mode 100644 index 0000000..850df38 --- /dev/null +++ b/approved/pipeline/step-parallel.yaml @@ -0,0 +1,30 @@ +# simple parallel step. + +pipeline: + stages: + - steps: + - parallel: + steps: + - run: + container: golang + script: go build + - run: + container: golang + script: go test + +--- + +# simple parallel step with conditional + +pipeline: + stages: + - steps: + - if: ${{ branch == "main" }} + parallel: + steps: + - run: + container: golang + script: go build + - run: + container: golang + script: go test diff --git a/approved/pipeline/step.yaml b/approved/pipeline/step.yaml new file mode 100644 index 0000000..8218190 --- /dev/null +++ b/approved/pipeline/step.yaml @@ -0,0 +1,52 @@ +# simple step. + +pipeline: + stages: + - steps: + - run: + script: go build + +--- + +# step with conditions + +pipeline: + stages: + - steps: + - if: ${{ branch == "main" }} + run: + script: go build + +--- + +# step with id + +pipeline: + stages: + - steps: + - id: build + run: + script: go build + +--- + +# step with name + +pipeline: + stages: + - steps: + - name: build + run: + script: go build + +--- + +# step with timeout (10 minutes) + +pipeline: + stages: + - steps: + - name: build + timeout: 10m + run: + script: go build diff --git a/approved/service/service.yaml b/approved/service/service.yaml new file mode 100644 index 0000000..b7ef52f --- /dev/null +++ b/approved/service/service.yaml @@ -0,0 +1,66 @@ +# Service V1 definition of a Kubernetes Service with a primary artifact from Google Artifact Registry + +service: + id: artifact_service_gar + name: artifact_service_gar + uses: kubernetes + with: + artifacts: + primary: nginx_stable + sources: + - id: nginx_stable + uses: google-artifact-registry + with: + connector: account.gcp + type: docker + region: us-east1 + repo: docker-test + package: nginx + project: qa-target + version: 1 + +--- + +# Service V1 with K8s Manifest + +service: + id: s2 + name: s2 + uses: kubernetes + with: + artifacts: + primary: nginx_stable + sources: + - id: nginx_stable + uses: google-artifact-registry + with: + connector: account.gcp + type: docker + region: us-east1 + repo: docker-test + package: nginx + project: qa-target + version: 1 + - id: nginx_stable_sidecar_1 + uses: google-artifact-registry + with: + connector: account.gcp + type: docker + region: us-east1 + repo: docker-test + package: nginx + project: qa-target + version: 2 + manifests: + sources: + - id: manifest_1 + uses: k8s + with: + store: + uses: github + with: + connector: account.yaml + type: branch + path: k8s/examples/simple/templates + branch: main + diff --git a/samples/actions/helm-rollback.yaml b/samples/actions/helm-rollback.yaml new file mode 100644 index 0000000..3f99bff --- /dev/null +++ b/samples/actions/helm-rollback.yaml @@ -0,0 +1,5 @@ +- action: + uses: helm-rollback + with: + steady-state: false + timeout: 10m \ No newline at end of file diff --git a/samples/actions/kubernetes-scale.yaml b/samples/actions/kubernetes-scale.yaml new file mode 100644 index 0000000..ced91c7 --- /dev/null +++ b/samples/actions/kubernetes-scale.yaml @@ -0,0 +1,9 @@ +- action: + uses: kubernetes-scale + with: + workload: default/Deployment/harness-example + replica: 2 + percentage: 100 # optional (either replica or percentage) + steady-state-check: false # optional - default is false + + diff --git a/samples/actions/terraform-plan.yaml b/samples/actions/terraform-plan.yaml new file mode 100644 index 0000000..2a9f9dc --- /dev/null +++ b/samples/actions/terraform-plan.yaml @@ -0,0 +1,101 @@ +## Cloning the Repository +- action: + uses: git-clone + with: + branch: main + repo-name: Product-Management + connector: cd-demo + + +## Terraform Plan +- action: + uses: terraform-plan + with: + command: apply | destroy + aws-provider: account.aws_connector + workspace: dev + backendConfig: |- + terraform { + backend "gcs" { + bucket = "tf-state-prod" + prefix = "terraform/state" + } + } + secrets-manager: Harness Secrets Manager + env: + - TF_LOG_PATH: ./terraform.log + args: + - refresh: --args + export-plan: true + human-readable-plan: true + state-storage: false + skip-refresh: false + + + + + + +### Curent NG Step Design +# - step: +# type: TerraformPlan +# name: TerraformPlan_1 +# identifier: TerraformPlan_1 +# spec: +# provisionerIdentifier: dev +# configuration: +# command: Apply +# configFiles: +# store: +# spec: +# connectorRef: account.CDNGAuto_GithubRepoPipelinesNgAutomationiDYaC0PbFx +# folderPath: dev +# gitFetchType: Branch +# branch: prod +# type: Github +# providerCredential: +# type: Aws +# spec: +# connectorRef: <+input> +# region: <+input> +# roleArn: <+input> +# backendConfig: +# type: Inline +# spec: +# content: |- +# terraform { +# backend "gcs" { +# bucket = "tf-state-prod" +# prefix = "terraform/state" +# } +# } +# environmentVariables: +# - name: TF_LOG_PATH +# value: ./terraform.log +# type: String +# targets: +# - module.s3_bucket +# commandFlags: +# - commandType: WORKSPACE +# flag: <+input> +# varFiles: +# - varFile: +# type: Remote +# identifier: dev +# spec: +# store: +# type: Bitbucket +# spec: +# gitFetchType: Branch +# repoName: prod +# branch: proad +# paths: +# - /prod/tf.vars +# connectorRef: <+input> +# secretManagerRef: org.pl_hashicorp_withsecret_jSlq +# workspace: <+input> +# exportTerraformPlanJson: <+input> +# skipStateStorage: <+input> +# exportTerraformHumanReadablePlan: <+input> +# skipRefreshCommand: <+input> +# timeout: 10m \ No newline at end of file diff --git a/samples/pipeline-background-services.yaml b/samples/pipeline-background-services.yaml new file mode 100644 index 0000000..caa2ad5 --- /dev/null +++ b/samples/pipeline-background-services.yaml @@ -0,0 +1,366 @@ +# Pipeline with Background Services for Integration Testing +# Demonstrates: background containers, service dependencies, health checks, network configuration +version: 1 + +pipeline: + name: integration-test-with-services + + inputs: + test_suite: + type: choice + options: [all, unit, integration, e2e] + default: all + debug_services: + type: boolean + description: "Keep services running for debugging" + default: false + + env: + DATABASE_URL: postgres://testuser:testpass@localhost:5432/testdb + REDIS_URL: redis://localhost:6379 + ELASTICSEARCH_URL: http://localhost:9200 + KAFKA_BROKERS: localhost:9092 + MINIO_ENDPOINT: localhost:9000 + + stages: + - name: integration-tests + runtime: + type: cloud + spec: + machine: large + + volumes: + - temp: + name: postgres-data + path: /var/lib/postgresql/data + - temp: + name: redis-data + path: /data + - temp: + name: es-data + path: /usr/share/elasticsearch/data + - temp: + name: kafka-data + path: /var/lib/kafka/data + - temp: + name: minio-data + path: /data + + steps: + # Start PostgreSQL + - name: postgres + background: + container: + image: postgres:15-alpine + env: + POSTGRES_USER: testuser + POSTGRES_PASSWORD: testpass + POSTGRES_DB: testdb + PGDATA: /var/lib/postgresql/data/pgdata + ports: + - 5432:5432 + volumes: + - postgres-data:/var/lib/postgresql/data + memory: 512m + cpu: 0.5 + + # Start Redis + - name: redis + background: + container: + image: redis:7-alpine + args: + - --appendonly + - "yes" + ports: + - 6379:6379 + volumes: + - redis-data:/data + memory: 256m + + # Start Elasticsearch + - name: elasticsearch + background: + container: + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 + env: + discovery.type: single-node + xpack.security.enabled: "false" + ES_JAVA_OPTS: "-Xms512m -Xmx512m" + ports: + - 9200:9200 + - 9300:9300 + volumes: + - es-data:/usr/share/elasticsearch/data + memory: 1024m + + # Start Kafka with Zookeeper + - name: zookeeper + background: + container: + image: confluentinc/cp-zookeeper:7.5.0 + env: + ZOOKEEPER_CLIENT_PORT: "2181" + ZOOKEEPER_TICK_TIME: "2000" + ports: + - 2181:2181 + memory: 256m + + - name: kafka + background: + container: + image: confluentinc/cp-kafka:7.5.0 + env: + KAFKA_BROKER_ID: "1" + KAFKA_ZOOKEEPER_CONNECT: localhost:2181 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + ports: + - 9092:9092 + volumes: + - kafka-data:/var/lib/kafka/data + memory: 512m + + # Start MinIO (S3-compatible storage) + - name: minio + background: + container: + image: minio/minio:latest + args: + - server + - /data + - --console-address + - ":9001" + env: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: minioadmin + ports: + - 9000:9000 + - 9001:9001 + volumes: + - minio-data:/data + memory: 256m + + # Start mock services + - name: mock-external-api + background: + container: + image: wiremock/wiremock:3.3.1 + args: + - --verbose + ports: + - 8081:8080 + volumes: + - bind: + source: ./test/wiremock + target: /home/wiremock + + # Wait for all services to be healthy + - name: wait-for-services + run: | + echo "Waiting for PostgreSQL..." + timeout 60 bash -c 'until pg_isready -h localhost -p 5432 -U testuser; do sleep 1; done' + + echo "Waiting for Redis..." + timeout 30 bash -c 'until redis-cli -h localhost ping | grep -q PONG; do sleep 1; done' + + echo "Waiting for Elasticsearch..." + timeout 120 bash -c 'until curl -s http://localhost:9200/_cluster/health | grep -q "status"; do sleep 2; done' + + echo "Waiting for Kafka..." + timeout 90 bash -c 'until nc -z localhost 9092; do sleep 2; done' + + echo "Waiting for MinIO..." + timeout 30 bash -c 'until curl -s http://localhost:9000/minio/health/live; do sleep 1; done' + + echo "All services are ready!" + timeout: 5m + + # Initialize databases and services + - name: init-postgres + run: | + psql ${{ env.DATABASE_URL }} -f ./scripts/init-db.sql + psql ${{ env.DATABASE_URL }} -f ./scripts/seed-test-data.sql + + - name: init-elasticsearch + run: | + curl -X PUT "localhost:9200/test-index" -H 'Content-Type: application/json' -d' + { + "settings": {"number_of_shards": 1, "number_of_replicas": 0}, + "mappings": {"properties": {"title": {"type": "text"}, "content": {"type": "text"}}} + }' + + - name: init-minio + run: | + mc alias set local http://localhost:9000 minioadmin minioadmin + mc mb local/test-bucket + mc cp ./test/fixtures/sample-files/* local/test-bucket/ + + - name: init-kafka-topics + run: | + kafka-topics --create --bootstrap-server localhost:9092 --topic test-events --partitions 3 + kafka-topics --create --bootstrap-server localhost:9092 --topic test-commands --partitions 1 + + # Run the actual tests + - name: run-unit-tests + if: ${{ inputs.test_suite == 'all' || inputs.test_suite == 'unit' }} + run: go test -v -race ./internal/... + report: + type: junit + path: test-results/unit.xml + + - name: run-integration-tests + if: ${{ inputs.test_suite == 'all' || inputs.test_suite == 'integration' }} + run: | + go test -v -tags=integration ./tests/integration/... + env: + TEST_DATABASE_URL: ${{ env.DATABASE_URL }} + TEST_REDIS_URL: ${{ env.REDIS_URL }} + TEST_ELASTICSEARCH_URL: ${{ env.ELASTICSEARCH_URL }} + TEST_KAFKA_BROKERS: ${{ env.KAFKA_BROKERS }} + TEST_S3_ENDPOINT: ${{ env.MINIO_ENDPOINT }} + TEST_S3_ACCESS_KEY: minioadmin + TEST_S3_SECRET_KEY: minioadmin + MOCK_API_URL: http://localhost:8081 + report: + type: junit + path: test-results/integration.xml + + - name: run-e2e-tests + if: ${{ inputs.test_suite == 'all' || inputs.test_suite == 'e2e' }} + run: | + # Start the application + ./bin/app & + APP_PID=$! + sleep 5 + + # Run E2E tests + npm run test:e2e + + # Cleanup + kill $APP_PID || true + container: + image: mcr.microsoft.com/playwright:v1.40.0 + network-mode: host + report: + type: junit + path: test-results/e2e.xml + + # Debug step - keeps services running + - name: debug-wait + if: ${{ inputs.debug_services && failure() }} + run: | + echo "Services are running for debugging. Press Ctrl+C to exit." + echo "PostgreSQL: localhost:5432" + echo "Redis: localhost:6379" + echo "Elasticsearch: localhost:9200" + echo "Kafka: localhost:9092" + echo "MinIO: localhost:9000 (console: localhost:9001)" + sleep 3600 + timeout: 1h + + # Contract testing stage with separate services + - name: contract-tests + if: ${{ inputs.test_suite == 'all' }} + + steps: + - name: pact-broker + background: + container: + image: pactfoundation/pact-broker:latest + env: + PACT_BROKER_DATABASE_URL: sqlite:///tmp/pact_broker.sqlite3 + ports: + - 9292:9292 + + - name: wait-for-pact-broker + run: | + timeout 30 bash -c 'until curl -s http://localhost:9292/; do sleep 1; done' + + - name: run-consumer-tests + run: | + go test -v -tags=consumer ./tests/contract/consumer/... + env: + PACT_BROKER_URL: http://localhost:9292 + + - name: publish-pacts + run: | + pact-broker publish ./pacts \ + --broker-base-url http://localhost:9292 \ + --consumer-app-version ${{ build.number }} + + - name: run-provider-tests + run: | + go test -v -tags=provider ./tests/contract/provider/... + env: + PACT_BROKER_URL: http://localhost:9292 + + # Performance testing with dedicated resources + - name: performance-tests + if: ${{ inputs.test_suite == 'all' && branch == 'main' }} + + runtime: + type: cloud + spec: + machine: xlarge + + steps: + - name: influxdb + background: + container: + image: influxdb:2.7 + env: + DOCKER_INFLUXDB_INIT_MODE: setup + DOCKER_INFLUXDB_INIT_USERNAME: admin + DOCKER_INFLUXDB_INIT_PASSWORD: adminpass + DOCKER_INFLUXDB_INIT_ORG: test + DOCKER_INFLUXDB_INIT_BUCKET: k6 + ports: + - 8086:8086 + + - name: grafana + background: + container: + image: grafana/grafana:10.2.0 + env: + GF_AUTH_ANONYMOUS_ENABLED: "true" + GF_AUTH_ANONYMOUS_ORG_ROLE: Admin + ports: + - 3000:3000 + + - name: start-app-under-test + background: + script: | + ./bin/app serve --port 8080 + container: + image: golang:1.21 + + - name: wait-for-app + run: | + timeout 30 bash -c 'until curl -s http://localhost:8080/health; do sleep 1; done' + + - name: run-load-tests + run: | + k6 run \ + --out influxdb=http://localhost:8086/k6 \ + --vus 50 \ + --duration 5m \ + ./tests/performance/load-test.js + report: + type: custom + path: k6-results.json + + - name: run-stress-tests + run: | + k6 run \ + --out influxdb=http://localhost:8086/k6 \ + --stages "1m:100,5m:100,1m:0" \ + ./tests/performance/stress-test.js + + - name: export-grafana-snapshot + run: | + curl -X POST http://localhost:3000/api/snapshots \ + -H "Content-Type: application/json" \ + -d '{"dashboard": {"title": "K6 Results"}}' \ + > grafana-snapshot.json diff --git a/samples/pipeline-caching-parallel.yaml b/samples/pipeline-caching-parallel.yaml new file mode 100644 index 0000000..7826c13 --- /dev/null +++ b/samples/pipeline-caching-parallel.yaml @@ -0,0 +1,271 @@ +# Advanced Caching and Parallel Execution Pipeline +# Demonstrates: multi-layer caching, parallel stages, parallel steps, cache policies, workspace sharing +version: 1 + +pipeline: + name: optimized-build-pipeline + + inputs: + skip_cache: + type: boolean + description: "Skip cache for fresh build" + default: false + + env: + PNPM_HOME: /root/.local/share/pnpm + GOPATH: /go + GOCACHE: /go-cache + + stages: + # Parallel dependency installation with caching + - parallel: + # Node.js dependencies + - name: install-node-deps + cache: + - path: node_modules + key: node-${{ hashFiles('pnpm-lock.yaml') }} + policy: pull-push + - path: ${{ env.PNPM_HOME }} + key: pnpm-store-${{ hashFiles('pnpm-lock.yaml') }} + policy: pull-push + - path: .next/cache + key: nextjs-${{ hashFiles('pnpm-lock.yaml') }}-${{ hashFiles('src/**/*.ts', 'src/**/*.tsx') }} + policy: pull-push + + steps: + - name: install-pnpm + run: | + npm install -g pnpm@8 + pnpm config set store-dir ${{ env.PNPM_HOME }}/store + + - name: install-deps + run: pnpm install --frozen-lockfile + + # Go dependencies + - name: install-go-deps + cache: + - path: ${{ env.GOPATH }}/pkg/mod + key: go-mod-${{ hashFiles('go.sum') }} + policy: pull-push + - path: ${{ env.GOCACHE }} + key: go-build-${{ hashFiles('**/*.go') }} + policy: pull-push + + steps: + - name: download-go-modules + run: go mod download -x + + # Python dependencies + - name: install-python-deps + cache: + - path: ~/.cache/pip + key: pip-${{ hashFiles('requirements.txt', 'requirements-dev.txt') }} + policy: pull-push + - path: .venv + key: venv-${{ hashFiles('requirements.txt', 'requirements-dev.txt') }} + policy: pull-push + + steps: + - name: setup-venv + run: | + python -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + pip install -r requirements.txt -r requirements-dev.txt + + # Parallel build stage + - name: build + steps: + # Build all components in parallel + - parallel: + - name: build-frontend + run: | + pnpm run build:frontend + env: + NODE_ENV: production + NEXT_TELEMETRY_DISABLED: "1" + + - name: build-backend-api + run: | + CGO_ENABLED=0 go build -ldflags="-w -s" -o bin/api ./cmd/api + + - name: build-backend-worker + run: | + CGO_ENABLED=0 go build -ldflags="-w -s" -o bin/worker ./cmd/worker + + - name: build-python-package + run: | + source .venv/bin/activate + python -m build + + # Parallel test execution with different test types + - name: test + steps: + - parallel: + # Frontend tests + - group: + name: frontend-tests + steps: + - parallel: + - name: unit-tests-frontend + run: pnpm test:unit --coverage + report: + type: junit + path: coverage/junit.xml + + - name: component-tests + run: pnpm test:components + container: + image: mcr.microsoft.com/playwright:v1.40.0 + + # Backend tests with parallel shards + - group: + name: backend-tests + strategy: + matrix: + shard: [1, 2, 3, 4] + max-parallel: 4 + steps: + - name: go-tests-shard-${{ matrix.shard }} + run: | + go test -v -race -coverprofile=coverage-${{ matrix.shard }}.out \ + $(go list ./... | awk "NR % 4 == ${{ matrix.shard }} - 1") + report: + type: junit + path: test-results/shard-${{ matrix.shard }}.xml + + # Python tests + - group: + name: python-tests + steps: + - name: pytest + run: | + source .venv/bin/activate + pytest -n auto --dist loadfile \ + --cov=src --cov-report=xml \ + --junitxml=test-results/pytest.xml + + # Parallel linting and static analysis + - name: quality-checks + steps: + - parallel: + - name: eslint + run: pnpm lint + on-failure: + errors: [all] + action: + ignore: {} + + - name: golangci-lint + run: golangci-lint run --timeout 5m + on-failure: + errors: [all] + action: + ignore: {} + + - name: ruff + run: | + source .venv/bin/activate + ruff check . + + - name: type-check-frontend + run: pnpm typecheck + + - name: type-check-python + run: | + source .venv/bin/activate + mypy src/ + + # Parallel Docker builds with layer caching + - name: docker-builds + runtime: + type: cloud + spec: + machine: large + + cache: + - path: /var/lib/docker + key: docker-layers-${{ hashFiles('**/Dockerfile') }} + policy: pull-push + + steps: + - parallel: + - name: build-api-image + run: | + docker build \ + --cache-from type=local,src=/var/lib/docker/buildcache/api \ + --cache-to type=local,dest=/var/lib/docker/buildcache/api,mode=max \ + -t myapp/api:${{ build.number }} \ + -f docker/api.Dockerfile . + + - name: build-worker-image + run: | + docker build \ + --cache-from type=local,src=/var/lib/docker/buildcache/worker \ + --cache-to type=local,dest=/var/lib/docker/buildcache/worker,mode=max \ + -t myapp/worker:${{ build.number }} \ + -f docker/worker.Dockerfile . + + - name: build-frontend-image + run: | + docker build \ + --cache-from type=local,src=/var/lib/docker/buildcache/frontend \ + --cache-to type=local,dest=/var/lib/docker/buildcache/frontend,mode=max \ + -t myapp/frontend:${{ build.number }} \ + -f docker/frontend.Dockerfile . + + # Parallel integration tests with shared workspace + - name: integration-tests + workspace: + path: /workspace/integration + + volumes: + - bind: + source: /var/run/docker.sock + target: /var/run/docker.sock + + steps: + - name: start-services + run: docker-compose -f docker-compose.test.yaml up -d + + - parallel: + - name: api-integration-tests + run: | + go test -v -tags=integration ./tests/integration/api/... + env: + API_URL: http://localhost:8080 + + - name: e2e-tests + run: pnpm test:e2e + container: + image: mcr.microsoft.com/playwright:v1.40.0 + network-mode: host + + - name: python-integration + run: | + source .venv/bin/activate + pytest tests/integration/ -v + + - name: stop-services + run: docker-compose -f docker-compose.test.yaml down -v + if: ${{ always() }} + + # Merge coverage reports + - name: coverage-report + cache: + - path: coverage-merged + key: coverage-${{ build.number }} + policy: push + + steps: + - name: merge-go-coverage + run: | + gocovmerge coverage-*.out > coverage-merged/go-coverage.out + go tool cover -html=coverage-merged/go-coverage.out -o coverage-merged/go-coverage.html + + - name: upload-coverage + uses: codecov/codecov-action@v4 + with: + files: coverage-merged/*.out,coverage/*.xml + flags: unittests + fail_ci_if_error: false diff --git a/samples/pipeline-ci-cd-matrix.yaml b/samples/pipeline-ci-cd-matrix.yaml new file mode 100644 index 0000000..b32ffa5 --- /dev/null +++ b/samples/pipeline-ci-cd-matrix.yaml @@ -0,0 +1,219 @@ +# Multi-Stage CI/CD Pipeline with Matrix Builds +# Demonstrates: matrix strategy, parallel stages, caching, artifacts, and conditional execution +version: 1 + +pipeline: + name: multi-platform-ci-cd + + inputs: + version: + type: string + description: "Version to build and deploy" + default: "1.0.0" + skip_tests: + type: boolean + description: "Skip test execution" + default: false + deploy_env: + type: choice + description: "Target deployment environment" + options: [staging, production] + default: staging + + env: + GO_VERSION: "1.21" + NODE_VERSION: "20" + + stages: + # Build stage with matrix strategy for multiple platforms + - name: build + strategy: + matrix: + os: [linux, darwin, windows] + arch: [amd64, arm64] + exclude: + - os: windows + arch: arm64 + fail-fast: false + max-parallel: 4 + + cache: + - path: /go/pkg/mod + key: go-mod-${{ hashFiles('go.sum') }} + - path: node_modules + key: node-modules-${{ hashFiles('package-lock.json') }} + + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: setup-go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + + - name: build-binary + run: + script: | + echo "Building for ${{ matrix.os }}/${{ matrix.arch }}" + GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -ldflags="-X main.Version=${{ inputs.version }}" -o bin/app-${{ matrix.os }}-${{ matrix.arch }} ./cmd/app + shell: bash + env: + CGO_ENABLED: "0" + + - name: upload-artifact + uses: actions/upload-artifact@v4 + with: + name: binary-${{ matrix.os }}-${{ matrix.arch }} + path: bin/ + retention-days: 7 + + # Parallel test stages + - parallel: + - name: unit-tests + if: ${{ inputs.skip_tests != true }} + steps: + - name: run-unit-tests + run: go test -v -race -coverprofile=coverage.out ./... + report: + type: junit + path: reports/junit.xml + + - name: lint + steps: + - name: golangci-lint + uses: golangci/golangci-lint-action@v4 + with: + version: latest + + - name: security-scan + steps: + - name: gosec + run: | + go install github.com/securego/gosec/v2/cmd/gosec@latest + gosec -fmt=json -out=security-report.json ./... + + # Integration tests with service dependencies + - name: integration-tests + if: ${{ inputs.skip_tests != true && branch == "main" }} + + runtime: + type: cloud + spec: + machine: large + + steps: + - name: postgres + background: + container: + image: postgres:15 + env: + POSTGRES_USER: test + POSTGRES_PASSWORD: test + POSTGRES_DB: testdb + ports: + - 5432:5432 + + - name: redis + background: + container: + image: redis:7-alpine + ports: + - 6379:6379 + + - name: wait-for-services + run: | + timeout 60 bash -c 'until pg_isready -h localhost -p 5432; do sleep 1; done' + timeout 60 bash -c 'until redis-cli -h localhost ping; do sleep 1; done' + + - name: run-integration-tests + run: go test -v -tags=integration ./... + env: + DATABASE_URL: postgres://test:test@localhost:5432/testdb + REDIS_URL: redis://localhost:6379 + + # Deploy to staging + - name: deploy-staging + if: ${{ inputs.deploy_env == "staging" || branch == "main" }} + environment: staging + + approval: + uses: harness + with: + timeout: 1h + min-approvers: 1 + groups: [developers] + + steps: + - name: download-artifacts + uses: actions/download-artifact@v4 + with: + pattern: binary-linux-* + path: dist/ + + - name: deploy + run: | + echo "Deploying version ${{ inputs.version }} to staging" + ./scripts/deploy.sh staging + env: + DEPLOY_TOKEN: ${{ secrets.STAGING_DEPLOY_TOKEN }} + + rollback: + - run: ./scripts/rollback.sh staging + + # Deploy to production (conditional) + - name: deploy-production + if: ${{ inputs.deploy_env == "production" && branch == "main" }} + environment: production + + approval: + uses: harness + with: + timeout: 24h + min-approvers: 2 + groups: [senior-engineers, devops] + + on-failure: + errors: [all] + action: + manual-intervention: + timeout: 30m + failure-action: + pipeline-rollback: {} + + steps: + - name: health-check-pre + run: ./scripts/health-check.sh production + + - name: deploy + run: | + echo "Deploying version ${{ inputs.version }} to production" + ./scripts/deploy.sh production + env: + DEPLOY_TOKEN: ${{ secrets.PROD_DEPLOY_TOKEN }} + + - name: health-check-post + run: | + sleep 30 + ./scripts/health-check.sh production + + - name: notify + uses: slackapi/slack-github-action@v1 + with: + channel-id: deployments + payload: | + { + "text": "Deployed ${{ inputs.version }} to production" + } + + rollback: + - run: ./scripts/rollback.sh production + - uses: slackapi/slack-github-action@v1 + with: + channel-id: deployments + payload: | + { + "text": "ROLLBACK: ${{ inputs.version }} in production" + } diff --git a/samples/pipeline-deployment-environments.yaml b/samples/pipeline-deployment-environments.yaml new file mode 100644 index 0000000..ea49161 --- /dev/null +++ b/samples/pipeline-deployment-environments.yaml @@ -0,0 +1,319 @@ +# Multi-Environment Deployment Pipeline with Approvals and Rollbacks +# Demonstrates: environments, services, approval gates, deployment strategies, rollback handling +version: 1 + +pipeline: + name: multi-environment-deployment + + inputs: + service_name: + type: string + description: "Service to deploy" + required: true + image_tag: + type: string + description: "Docker image tag" + required: true + deploy_strategy: + type: choice + options: [rolling, canary, blue-green] + default: rolling + canary_percentage: + type: number + description: "Canary traffic percentage (only for canary strategy)" + default: 10 + + env: + REGISTRY: gcr.io/my-project + + stages: + # Pre-deployment validation + - name: validation + steps: + - name: verify-image + run: | + echo "Verifying image exists: ${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }}" + docker manifest inspect ${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }} + + - name: security-scan + run: | + trivy image --severity HIGH,CRITICAL \ + ${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }} + + - name: validate-k8s-manifests + run: | + kubeval --strict manifests/*.yaml + conftest test manifests/*.yaml + + # Deploy to Development + - name: deploy-development + environment: + name: development + type: non-production + + steps: + - name: deploy + uses: template/k8s-deploy@1.0.0 + with: + namespace: development + image: ${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }} + replicas: 1 + + - name: smoke-test + run: ./scripts/smoke-test.sh development + timeout: 5m + + # Deploy to QA with parallel infrastructure targets + - name: deploy-qa + environment: + name: qa + type: non-production + deploy-to: all + + steps: + - name: deploy + uses: template/k8s-deploy@1.0.0 + with: + namespace: qa + image: ${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }} + replicas: 2 + + - name: integration-tests + run: ./scripts/integration-tests.sh qa + timeout: 15m + report: + type: junit + path: test-results/integration.xml + + # Deploy to Staging with approval + - name: deploy-staging + environment: + name: staging + type: non-production + + approval: + uses: harness + with: + timeout: 4h + min-approvers: 1 + groups: [qa-team] + message: "Approve deployment of ${{ inputs.service_name }}:${{ inputs.image_tag }} to staging" + + on-failure: + errors: [all] + action: + stage-rollback: {} + + steps: + - name: pre-deploy-snapshot + run: | + kubectl get deployment ${{ inputs.service_name }} -n staging -o yaml > rollback-snapshot.yaml + + - name: deploy + id: deploy-staging + uses: template/k8s-deploy@1.0.0 + with: + namespace: staging + image: ${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }} + replicas: 3 + strategy: ${{ inputs.deploy_strategy }} + + - name: e2e-tests + run: ./scripts/e2e-tests.sh staging + timeout: 30m + + - name: performance-test + run: | + k6 run --out influxdb=http://metrics:8086/k6 \ + -e TARGET_URL=https://staging.example.com \ + scripts/load-test.js + timeout: 20m + + rollback: + - name: restore-previous + run: kubectl apply -f rollback-snapshot.yaml + - name: verify-rollback + run: | + kubectl rollout status deployment/${{ inputs.service_name }} -n staging --timeout=5m + + # Production deployment with strict approvals + - name: deploy-production + environment: + name: production + type: production + + approval: + uses: jira + with: + project: DEPLOY + issue-type: Change Request + timeout: 72h + auto-approve: + if: ${{ inputs.deploy_strategy == "rolling" && build.source == "release/*" }} + + concurrency: + group: production-${{ inputs.service_name }} + limit: 1 + + on-failure: + errors: [all] + action: + manual-intervention: + timeout: 1h + failure-action: + pipeline-rollback: {} + + steps: + # Pre-deployment checks + - name: check-maintenance-window + run: ./scripts/check-maintenance-window.sh + + - name: create-change-record + id: change-record + run: | + CHANGE_ID=$(./scripts/create-change-record.sh "${{ inputs.service_name }}" "${{ inputs.image_tag }}") + echo "change_id=$CHANGE_ID" >> $GITHUB_OUTPUT + + # Strategy-based deployment + - group: + name: canary-deployment + if: ${{ inputs.deploy_strategy == "canary" }} + steps: + - name: deploy-canary + run: | + kubectl apply -f manifests/canary.yaml + kubectl set image deployment/${{ inputs.service_name }}-canary \ + app=${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }} \ + -n production + + - name: configure-traffic-split + run: | + kubectl patch virtualservice ${{ inputs.service_name }} \ + --type=merge \ + -p '{"spec":{"http":[{"route":[{"destination":{"host":"${{ inputs.service_name }}","subset":"stable"},"weight":${{ 100 - inputs.canary_percentage }}},{"destination":{"host":"${{ inputs.service_name }}","subset":"canary"},"weight":${{ inputs.canary_percentage }}}]}]}}' + + - name: monitor-canary + run: ./scripts/monitor-canary.sh ${{ inputs.service_name }} 15m + timeout: 20m + + - name: approval-promote-canary + uses: harness + with: + timeout: 2h + min-approvers: 2 + groups: [sre-team, product-team] + message: "Canary metrics look good. Promote to full production?" + + - name: promote-canary + run: | + kubectl set image deployment/${{ inputs.service_name }} \ + app=${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }} \ + -n production + kubectl rollout status deployment/${{ inputs.service_name }} -n production + + - name: cleanup-canary + run: kubectl delete -f manifests/canary.yaml + + - group: + name: blue-green-deployment + if: ${{ inputs.deploy_strategy == "blue-green" }} + steps: + - name: deploy-green + run: | + kubectl apply -f manifests/green.yaml + kubectl set image deployment/${{ inputs.service_name }}-green \ + app=${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }} \ + -n production + kubectl rollout status deployment/${{ inputs.service_name }}-green -n production + + - name: test-green + run: ./scripts/smoke-test.sh production-green + + - name: switch-traffic + run: | + kubectl patch service ${{ inputs.service_name }} \ + -p '{"spec":{"selector":{"version":"green"}}}' \ + -n production + + - name: verify-switch + run: ./scripts/verify-traffic.sh ${{ inputs.service_name }} green + + - name: cleanup-blue + run: kubectl delete deployment ${{ inputs.service_name }}-blue -n production --ignore-not-found + + - group: + name: rolling-deployment + if: ${{ inputs.deploy_strategy == "rolling" }} + steps: + - name: rolling-update + run: | + kubectl set image deployment/${{ inputs.service_name }} \ + app=${{ env.REGISTRY }}/${{ inputs.service_name }}:${{ inputs.image_tag }} \ + -n production + kubectl rollout status deployment/${{ inputs.service_name }} -n production --timeout=10m + + # Post-deployment verification + - name: health-check + run: ./scripts/health-check.sh production + timeout: 5m + on-failure: + errors: [all] + action: + retry: + attempts: 3 + interval: [30s, 1m, 2m] + failure-action: fail + + - name: close-change-record + run: ./scripts/close-change-record.sh ${{ steps.change-record.outputs.change_id }} success + + rollback: + - name: rollback-deployment + run: kubectl rollout undo deployment/${{ inputs.service_name }} -n production + + - name: verify-rollback + run: | + kubectl rollout status deployment/${{ inputs.service_name }} -n production --timeout=5m + + - name: close-change-record-failed + run: ./scripts/close-change-record.sh ${{ steps.change-record.outputs.change_id }} failed + + - name: notify-failure + uses: slackapi/slack-github-action@v1 + with: + channel-id: incidents + payload: | + { + "text": ":rotating_light: Production deployment failed and rolled back", + "attachments": [{ + "color": "danger", + "fields": [ + {"title": "Service", "value": "${{ inputs.service_name }}", "short": true}, + {"title": "Version", "value": "${{ inputs.image_tag }}", "short": true} + ] + }] + } + + # Post-deployment notifications + - name: notifications + steps: + - name: update-deployment-tracker + run: | + curl -X POST https://deployments.internal/api/record \ + -H "Authorization: Bearer ${{ secrets.DEPLOY_API_TOKEN }}" \ + -d '{ + "service": "${{ inputs.service_name }}", + "version": "${{ inputs.image_tag }}", + "environment": "production", + "strategy": "${{ inputs.deploy_strategy }}", + "status": "success" + }' + + - name: notify-slack + uses: slackapi/slack-github-action@v1 + with: + channel-id: deployments + payload: | + { + "text": ":rocket: Successfully deployed ${{ inputs.service_name }}:${{ inputs.image_tag }} to production" + } diff --git a/samples/pipeline-failure-strategies.yaml b/samples/pipeline-failure-strategies.yaml new file mode 100644 index 0000000..c09b937 --- /dev/null +++ b/samples/pipeline-failure-strategies.yaml @@ -0,0 +1,388 @@ +# Complex Failure Handling Strategies Pipeline +# Demonstrates: retry strategies, manual intervention, rollbacks, error types, failure actions +version: 1 + +pipeline: + name: resilient-deployment-pipeline + + inputs: + auto_rollback: + type: boolean + description: "Automatically rollback on failure" + default: true + max_retries: + type: number + description: "Maximum retry attempts for flaky operations" + default: 3 + + env: + ENVIRONMENT: production + HEALTH_CHECK_URL: https://api.example.com/health + + stages: + # Stage with comprehensive retry strategy + - name: external-api-integration + on-failure: + errors: [connectivity, timeout] + action: + retry: + attempts: 5 + interval: [5s, 15s, 30s, 1m, 2m] + failure-action: fail + + steps: + - name: fetch-configuration + run: | + curl -f --retry 3 --retry-delay 5 \ + -H "Authorization: Bearer ${{ secrets.CONFIG_API_TOKEN }}" \ + https://config.example.com/api/v1/config > config.json + timeout: 2m + on-failure: + errors: [timeout, connectivity] + action: + retry: + attempts: 3 + interval: [10s, 30s, 1m] + failure-action: + manual-intervention: + timeout: 30m + failure-action: fail + + - name: sync-with-partner + run: ./scripts/sync-partner-api.sh + timeout: 5m + on-failure: + errors: [all] + action: + retry: + attempts: ${{ inputs.max_retries }} + interval: 30s + failure-action: + ignore: {} + + # Stage with authentication error handling + - name: secure-operations + on-failure: + errors: [authentication, authorization] + action: + manual-intervention: + timeout: 1h + failure-action: + abort: {} + + steps: + - name: rotate-credentials + run: | + # Attempt to use existing credentials + if ! ./scripts/verify-credentials.sh; then + echo "Credentials invalid, requesting rotation" + exit 1 + fi + on-failure: + errors: [authentication] + action: + retry: + attempts: 1 + interval: 0s + failure-action: + manual-intervention: + timeout: 2h + failure-action: fail + + - name: access-secure-resource + run: ./scripts/access-vault.sh + on-failure: + errors: [authorization] + action: + manual-intervention: + timeout: 30m + failure-action: + abort: {} + + # Deployment stage with multiple failure strategies + - name: deploy-application + environment: production + + on-failure: + errors: [all] + action: + manual-intervention: + timeout: 1h + failure-action: + stage-rollback: {} + + steps: + - name: pre-deployment-backup + id: backup + run: | + BACKUP_ID=$(./scripts/create-backup.sh) + echo "backup_id=$BACKUP_ID" >> $GITHUB_OUTPUT + on-failure: + errors: [all] + action: + retry: + attempts: 2 + interval: 1m + failure-action: fail + + - name: deploy-database-migrations + run: ./scripts/run-migrations.sh + timeout: 10m + on-failure: + errors: [timeout] + action: + retry: + attempts: 2 + interval: 2m + failure-action: + manual-intervention: + timeout: 30m + failure-action: + abort: {} + errors: [unknown] + action: + manual-intervention: + timeout: 1h + failure-action: + stage-rollback: {} + + - name: deploy-services + run: | + kubectl apply -f k8s/ + kubectl rollout status deployment/app --timeout=5m + timeout: 10m + on-failure: + errors: [all] + action: + retry: + attempts: 2 + interval: 30s + failure-action: + manual-intervention: + timeout: 30m + failure-action: + stage-rollback: {} + + - name: verify-deployment + run: | + for i in {1..10}; do + if curl -sf ${{ env.HEALTH_CHECK_URL }}; then + echo "Health check passed" + exit 0 + fi + sleep 10 + done + echo "Health check failed after 10 attempts" + exit 1 + timeout: 5m + on-failure: + errors: [all] + action: + retry: + attempts: 3 + interval: 30s + failure-action: + stage-rollback: {} + + rollback: + - name: restore-database + run: ./scripts/restore-backup.sh ${{ steps.backup.outputs.backup_id }} + on-failure: + errors: [all] + action: + manual-intervention: + timeout: 2h + failure-action: + abort: {} + + - name: rollback-services + run: kubectl rollout undo deployment/app + + - name: verify-rollback + run: | + kubectl rollout status deployment/app --timeout=5m + curl -sf ${{ env.HEALTH_CHECK_URL }} + + # Stage demonstrating delegate provisioning failures + - name: infrastructure-operations + delegate: k8s-delegate + + on-failure: + errors: [delegate-provisioning, delegate-restart] + action: + retry: + attempts: 3 + interval: [1m, 2m, 5m] + failure-action: + manual-intervention: + timeout: 1h + failure-action: + abort: {} + + steps: + - name: provision-resources + run: terraform apply -auto-approve + timeout: 30m + on-failure: + errors: [timeout] + action: + manual-intervention: + timeout: 1h + failure-action: + abort: {} + + # Stage with approval rejection handling + - name: production-release + approval: + uses: harness + with: + timeout: 24h + min-approvers: 2 + groups: [release-managers] + + on-failure: + errors: [approval-rejection] + action: + manual-intervention: + timeout: 48h + failure-action: + abort: {} + errors: [input-timeout] + action: + retry: + attempts: 1 + interval: 0s + failure-action: + abort: {} + + steps: + - name: release + run: ./scripts/release.sh + on-failure: + errors: [all] + action: + pipeline-rollback: {} + + # Stage with policy evaluation failures + - name: compliance-check + on-failure: + errors: [policy-evaluation] + action: + manual-intervention: + timeout: 4h + failure-action: + abort: {} + + steps: + - name: run-compliance-scan + run: ./scripts/compliance-scan.sh + on-failure: + errors: [policy-evaluation] + action: + ignore: {} + + - name: generate-compliance-report + run: ./scripts/generate-compliance-report.sh + + # Parallel stage with independent failure handling + - parallel: + - name: notify-slack + on-failure: + errors: [all] + action: + ignore: {} + steps: + - run: | + curl -X POST ${{ secrets.SLACK_WEBHOOK }} \ + -d '{"text": "Deployment completed"}' + + - name: notify-pagerduty + on-failure: + errors: [connectivity, timeout] + action: + retry: + attempts: 3 + interval: 10s + failure-action: + ignore: {} + steps: + - run: ./scripts/notify-pagerduty.sh success + + - name: update-status-page + on-failure: + errors: [all] + action: + retry: + attempts: 2 + interval: 5s + failure-action: + ignore: {} + steps: + - run: ./scripts/update-status-page.sh operational + + # Final stage with verification failure handling + - name: post-deployment-verification + on-failure: + errors: [verification] + action: + manual-intervention: + timeout: 2h + failure-action: + pipeline-rollback: {} + errors: [user-mark-fail] + action: + pipeline-rollback: {} + + steps: + - name: smoke-tests + run: ./scripts/smoke-tests.sh + timeout: 15m + on-failure: + errors: [all] + action: + retry: + attempts: 2 + interval: 1m + failure-action: + manual-intervention: + timeout: 30m + failure-action: + pipeline-rollback: {} + + - name: synthetic-monitoring + run: ./scripts/synthetic-tests.sh + timeout: 10m + + - name: canary-analysis + run: ./scripts/canary-analysis.sh + on-failure: + errors: [all] + action: + manual-intervention: + timeout: 1h + failure-action: + pipeline-rollback: {} + + # Cleanup stage that always runs + - name: cleanup + if: ${{ always() }} + + on-failure: + errors: [all] + action: + ignore: {} + + steps: + - name: cleanup-temp-resources + run: ./scripts/cleanup.sh + on-failure: + errors: [all] + action: + ignore: {} + + - name: archive-logs + run: ./scripts/archive-logs.sh + on-failure: + errors: [all] + action: + ignore: {} diff --git a/samples/pipeline-inputs-validation.yaml b/samples/pipeline-inputs-validation.yaml new file mode 100644 index 0000000..f6bdec1 --- /dev/null +++ b/samples/pipeline-inputs-validation.yaml @@ -0,0 +1,337 @@ +# Input-Driven Pipeline with Validations +# Demonstrates: all input types, validation patterns, conditional logic based on inputs, defaults +version: 1 + +pipeline: + name: configurable-deployment + + inputs: + # String input with pattern validation + version: + type: string + description: "Semantic version to deploy (e.g., 1.2.3)" + required: true + pattern: "^\\d+\\.\\d+\\.\\d+(-[a-zA-Z0-9]+)?$" + + # String with enum constraint + environment: + type: string + description: "Target deployment environment" + required: true + enum: [development, staging, production] + default: development + + # Choice type (GitHub Actions compatible) + region: + type: choice + description: "Deployment region" + options: + - us-east-1 + - us-west-2 + - eu-west-1 + - ap-southeast-1 + default: us-east-1 + + # Number input with validation + replicas: + type: number + description: "Number of replicas to deploy" + default: 2 + + # Boolean inputs + enable_monitoring: + type: boolean + description: "Enable enhanced monitoring" + default: true + + run_tests: + type: boolean + description: "Run test suite before deployment" + default: true + + skip_approval: + type: boolean + description: "Skip approval gate (only for non-production)" + default: false + + # Duration input + health_check_timeout: + type: duration + description: "Health check timeout duration" + default: 5m + + deployment_timeout: + type: duration + description: "Overall deployment timeout" + default: 30m + + # Array inputs + services: + type: array + description: "List of services to deploy" + default: ["api", "web", "worker"] + + notify_channels: + type: array + description: "Slack channels for notifications" + default: ["deployments"] + + feature_flags: + type: array + description: "Feature flags to enable" + default: [] + + # Secret input + deploy_token: + type: secret + description: "Deployment authentication token" + required: true + + database_password: + type: secret + description: "Database password for migrations" + + env: + DEPLOY_VERSION: ${{ inputs.version }} + DEPLOY_ENV: ${{ inputs.environment }} + DEPLOY_REGION: ${{ inputs.region }} + + stages: + # Validation stage + - name: validate-inputs + steps: + - name: validate-version-format + run: | + VERSION="${{ inputs.version }}" + if [[ ! "$VERSION" =~ ^[0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z0-9]+)?$ ]]; then + echo "Error: Invalid version format. Expected semver (e.g., 1.2.3 or 1.2.3-beta)" + exit 1 + fi + echo "Version $VERSION is valid" + + - name: validate-environment-constraints + run: | + ENV="${{ inputs.environment }}" + SKIP_APPROVAL="${{ inputs.skip_approval }}" + + if [[ "$ENV" == "production" && "$SKIP_APPROVAL" == "true" ]]; then + echo "Error: Cannot skip approval for production deployments" + exit 1 + fi + + if [[ "$ENV" == "production" && ${{ inputs.replicas }} -lt 3 ]]; then + echo "Warning: Production should have at least 3 replicas" + fi + + - name: validate-services + run: | + SERVICES='${{ join(inputs.services, ',') }}' + VALID_SERVICES="api,web,worker,scheduler,gateway" + + IFS=',' read -ra SERVICE_ARRAY <<< "$SERVICES" + for service in "${SERVICE_ARRAY[@]}"; do + if [[ ! "$VALID_SERVICES" =~ "$service" ]]; then + echo "Error: Unknown service '$service'. Valid services: $VALID_SERVICES" + exit 1 + fi + done + echo "All services are valid: $SERVICES" + + # Conditional test stage + - name: run-tests + if: ${{ inputs.run_tests }} + + steps: + - name: unit-tests + run: npm test + report: + type: junit + path: test-results/junit.xml + + - name: integration-tests + if: ${{ inputs.environment != 'development' }} + run: npm run test:integration + + - name: e2e-tests + if: ${{ inputs.environment == 'production' }} + run: npm run test:e2e + + # Build stage with input-driven configuration + - name: build + steps: + - name: build-services + strategy: + matrix: + service: ${{ inputs.services }} + + run: | + echo "Building ${{ matrix.service }} version ${{ inputs.version }}" + docker build \ + --build-arg VERSION=${{ inputs.version }} \ + --build-arg ENV=${{ inputs.environment }} \ + -t myapp/${{ matrix.service }}:${{ inputs.version }} \ + ./services/${{ matrix.service }} + + - name: configure-feature-flags + if: ${{ length(inputs.feature_flags) > 0 }} + run: | + echo "Configuring feature flags: ${{ join(inputs.feature_flags, ', ') }}" + for flag in ${{ join(inputs.feature_flags, ' ') }}; do + echo "$flag=true" >> .env.flags + done + + # Environment-specific deployment stages + - name: deploy-development + if: ${{ inputs.environment == 'development' }} + + steps: + - name: deploy + run: | + ./scripts/deploy.sh \ + --env development \ + --region ${{ inputs.region }} \ + --version ${{ inputs.version }} \ + --replicas ${{ inputs.replicas }} \ + --services "${{ join(inputs.services, ',') }}" + env: + DEPLOY_TOKEN: ${{ inputs.deploy_token }} + timeout: ${{ inputs.deployment_timeout }} + + - name: deploy-staging + if: ${{ inputs.environment == 'staging' }} + + approval: + if: ${{ !inputs.skip_approval }} + uses: harness + with: + timeout: 4h + min-approvers: 1 + groups: [developers] + + steps: + - name: deploy + run: | + ./scripts/deploy.sh \ + --env staging \ + --region ${{ inputs.region }} \ + --version ${{ inputs.version }} \ + --replicas ${{ inputs.replicas }} \ + --services "${{ join(inputs.services, ',') }}" \ + --monitoring ${{ inputs.enable_monitoring }} + env: + DEPLOY_TOKEN: ${{ inputs.deploy_token }} + DB_PASSWORD: ${{ inputs.database_password }} + timeout: ${{ inputs.deployment_timeout }} + + - name: health-check + run: ./scripts/health-check.sh staging + timeout: ${{ inputs.health_check_timeout }} + + - name: deploy-production + if: ${{ inputs.environment == 'production' }} + + approval: + uses: harness + with: + timeout: 24h + min-approvers: 2 + groups: [sre-team, product-owners] + message: | + Deploying version ${{ inputs.version }} to production + Region: ${{ inputs.region }} + Services: ${{ join(inputs.services, ', ') }} + Replicas: ${{ inputs.replicas }} + + steps: + - name: pre-deployment-checks + run: | + echo "Running pre-deployment checks for production" + ./scripts/pre-deploy-checks.sh production + + - name: deploy + run: | + ./scripts/deploy.sh \ + --env production \ + --region ${{ inputs.region }} \ + --version ${{ inputs.version }} \ + --replicas ${{ inputs.replicas }} \ + --services "${{ join(inputs.services, ',') }}" \ + --monitoring true \ + --canary true + env: + DEPLOY_TOKEN: ${{ inputs.deploy_token }} + DB_PASSWORD: ${{ inputs.database_password }} + timeout: ${{ inputs.deployment_timeout }} + + - name: health-check + run: ./scripts/health-check.sh production + timeout: ${{ inputs.health_check_timeout }} + on-failure: + errors: [all] + action: + retry: + attempts: 3 + interval: 30s + failure-action: + stage-rollback: {} + + - name: smoke-tests + run: ./scripts/smoke-tests.sh production + + rollback: + - run: ./scripts/rollback.sh production ${{ inputs.version }} + + # Monitoring setup based on input + - name: setup-monitoring + if: ${{ inputs.enable_monitoring }} + + steps: + - name: configure-alerts + run: | + ./scripts/configure-alerts.sh \ + --env ${{ inputs.environment }} \ + --version ${{ inputs.version }} \ + --services "${{ join(inputs.services, ',') }}" + + - name: setup-dashboards + run: | + ./scripts/setup-dashboards.sh \ + --env ${{ inputs.environment }} \ + --region ${{ inputs.region }} + + # Notifications based on input channels + - name: notifications + steps: + - name: notify-slack + strategy: + matrix: + channel: ${{ inputs.notify_channels }} + + run: | + curl -X POST ${{ secrets.SLACK_WEBHOOK }} \ + -H "Content-Type: application/json" \ + -d '{ + "channel": "#${{ matrix.channel }}", + "text": "Deployed ${{ inputs.version }} to ${{ inputs.environment }} (${{ inputs.region }})", + "attachments": [{ + "color": "good", + "fields": [ + {"title": "Services", "value": "${{ join(inputs.services, ', ') }}", "short": true}, + {"title": "Replicas", "value": "${{ inputs.replicas }}", "short": true} + ] + }] + }' + + - name: update-deployment-tracker + run: | + curl -X POST https://deployments.internal/api/record \ + -H "Authorization: Bearer ${{ inputs.deploy_token }}" \ + -d '{ + "version": "${{ inputs.version }}", + "environment": "${{ inputs.environment }}", + "region": "${{ inputs.region }}", + "services": ${{ inputs.services }}, + "replicas": ${{ inputs.replicas }}, + "monitoring": ${{ inputs.enable_monitoring }}, + "feature_flags": ${{ inputs.feature_flags }} + }' diff --git a/samples/pipeline-monorepo.yaml b/samples/pipeline-monorepo.yaml new file mode 100644 index 0000000..1b01a01 --- /dev/null +++ b/samples/pipeline-monorepo.yaml @@ -0,0 +1,396 @@ +# Monorepo Multi-Service Pipeline +# Demonstrates: path-based triggers, selective builds, shared dependencies, workspace management +version: 1 + +pipeline: + name: monorepo-ci-cd + + # Trigger configuration for monorepo + on: + push: + branches: [main, develop, "release/*"] + pull_request: + branches: [main] + types: [opened, synchronize, reopened] + + inputs: + force_all: + type: boolean + description: "Force build all services regardless of changes" + default: false + target_services: + type: array + description: "Specific services to build (overrides change detection)" + default: [] + + env: + REGISTRY: gcr.io/myproject + MONOREPO_ROOT: /workspace + + stages: + # Detect which services have changes + - name: detect-changes + outputs: + - changed_services + - changed_libs + - needs_shared_build + + steps: + - name: checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: detect-service-changes + id: changes + run: | + # Get the base commit for comparison + if [[ "${{ build.event }}" == "pull_request" ]]; then + BASE_SHA=$(git merge-base HEAD origin/${{ build.target_branch }}) + else + BASE_SHA=${{ build.before }} + fi + + # Detect changes in each service directory + CHANGED_SERVICES="" + + for service in api web worker scheduler gateway admin; do + if git diff --name-only $BASE_SHA HEAD | grep -q "^services/$service/"; then + CHANGED_SERVICES="$CHANGED_SERVICES $service" + fi + done + + # Detect changes in shared libraries + CHANGED_LIBS="" + NEEDS_SHARED_BUILD="false" + + for lib in common auth database utils; do + if git diff --name-only $BASE_SHA HEAD | grep -q "^libs/$lib/"; then + CHANGED_LIBS="$CHANGED_LIBS $lib" + NEEDS_SHARED_BUILD="true" + fi + done + + # Check for root config changes that affect all + if git diff --name-only $BASE_SHA HEAD | grep -qE "^(package.json|pnpm-lock.yaml|tsconfig.json|.eslintrc)"; then + NEEDS_SHARED_BUILD="true" + fi + + # Output results + echo "changed_services=${CHANGED_SERVICES# }" >> $GITHUB_OUTPUT + echo "changed_libs=${CHANGED_LIBS# }" >> $GITHUB_OUTPUT + echo "needs_shared_build=$NEEDS_SHARED_BUILD" >> $GITHUB_OUTPUT + + echo "Changed services: $CHANGED_SERVICES" + echo "Changed libs: $CHANGED_LIBS" + echo "Needs shared build: $NEEDS_SHARED_BUILD" + + # Build shared libraries first if needed + - name: build-shared + if: ${{ stages.detect-changes.outputs.needs_shared_build == 'true' || inputs.force_all }} + + cache: + - path: node_modules + key: monorepo-deps-${{ hashFiles('pnpm-lock.yaml') }} + - path: libs/**/dist + key: libs-${{ hashFiles('libs/**/*.ts') }} + + steps: + - name: install-dependencies + run: pnpm install --frozen-lockfile + + - name: build-common + run: pnpm --filter @myorg/common build + + - name: build-auth + run: pnpm --filter @myorg/auth build + + - name: build-database + run: pnpm --filter @myorg/database build + + - name: build-utils + run: pnpm --filter @myorg/utils build + + # Parallel service builds + - name: build-services + parallel: true + + steps: + # API Service + - group: + name: api-service + if: ${{ contains(stages.detect-changes.outputs.changed_services, 'api') || stages.detect-changes.outputs.needs_shared_build == 'true' || inputs.force_all || contains(inputs.target_services, 'api') }} + + steps: + - name: build-api + run: pnpm --filter @myorg/api build + + - name: test-api + run: pnpm --filter @myorg/api test + report: + type: junit + path: services/api/test-results/junit.xml + + - name: lint-api + run: pnpm --filter @myorg/api lint + + - name: docker-build-api + run: | + docker build \ + -f services/api/Dockerfile \ + -t ${{ env.REGISTRY }}/api:${{ build.number }} \ + --build-arg VERSION=${{ build.number }} \ + . + + # Web Service + - group: + name: web-service + if: ${{ contains(stages.detect-changes.outputs.changed_services, 'web') || stages.detect-changes.outputs.needs_shared_build == 'true' || inputs.force_all || contains(inputs.target_services, 'web') }} + + steps: + - name: build-web + run: pnpm --filter @myorg/web build + env: + NEXT_PUBLIC_API_URL: ${{ build.event == 'pull_request' && 'https://preview.example.com' || 'https://api.example.com' }} + + - name: test-web + run: pnpm --filter @myorg/web test + report: + type: junit + path: services/web/test-results/junit.xml + + - name: docker-build-web + run: | + docker build \ + -f services/web/Dockerfile \ + -t ${{ env.REGISTRY }}/web:${{ build.number }} \ + . + + # Worker Service + - group: + name: worker-service + if: ${{ contains(stages.detect-changes.outputs.changed_services, 'worker') || stages.detect-changes.outputs.needs_shared_build == 'true' || inputs.force_all || contains(inputs.target_services, 'worker') }} + + steps: + - name: build-worker + run: pnpm --filter @myorg/worker build + + - name: test-worker + run: pnpm --filter @myorg/worker test + + - name: docker-build-worker + run: | + docker build \ + -f services/worker/Dockerfile \ + -t ${{ env.REGISTRY }}/worker:${{ build.number }} \ + . + + # Scheduler Service + - group: + name: scheduler-service + if: ${{ contains(stages.detect-changes.outputs.changed_services, 'scheduler') || stages.detect-changes.outputs.needs_shared_build == 'true' || inputs.force_all || contains(inputs.target_services, 'scheduler') }} + + steps: + - name: build-scheduler + run: pnpm --filter @myorg/scheduler build + + - name: test-scheduler + run: pnpm --filter @myorg/scheduler test + + - name: docker-build-scheduler + run: | + docker build \ + -f services/scheduler/Dockerfile \ + -t ${{ env.REGISTRY }}/scheduler:${{ build.number }} \ + . + + # Gateway Service + - group: + name: gateway-service + if: ${{ contains(stages.detect-changes.outputs.changed_services, 'gateway') || stages.detect-changes.outputs.needs_shared_build == 'true' || inputs.force_all || contains(inputs.target_services, 'gateway') }} + + steps: + - name: build-gateway + run: pnpm --filter @myorg/gateway build + + - name: test-gateway + run: pnpm --filter @myorg/gateway test + + - name: docker-build-gateway + run: | + docker build \ + -f services/gateway/Dockerfile \ + -t ${{ env.REGISTRY }}/gateway:${{ build.number }} \ + . + + # Cross-service integration tests + - name: integration-tests + if: ${{ branch == 'main' || branch == 'develop' || inputs.force_all }} + + runtime: + type: cloud + spec: + machine: large + + steps: + # Start all services for integration testing + - name: start-dependencies + background: + script: docker-compose -f docker-compose.test.yaml up + + - name: wait-for-services + run: ./scripts/wait-for-services.sh + + - name: run-integration-tests + run: pnpm test:integration + report: + type: junit + path: test-results/integration.xml + + - name: run-e2e-tests + run: pnpm test:e2e + container: + image: mcr.microsoft.com/playwright:v1.40.0 + network-mode: host + + # Push images to registry + - name: push-images + if: ${{ branch == 'main' || branch == 'develop' || startsWith(branch, 'release/') }} + + steps: + - name: login-to-registry + run: | + echo "${{ secrets.GCR_KEY }}" | docker login -u _json_key --password-stdin gcr.io + + - name: push-changed-images + run: | + CHANGED="${{ stages.detect-changes.outputs.changed_services }}" + NEEDS_SHARED="${{ stages.detect-changes.outputs.needs_shared_build }}" + FORCE="${{ inputs.force_all }}" + + push_if_built() { + local service=$1 + if [[ "$FORCE" == "true" ]] || [[ "$NEEDS_SHARED" == "true" ]] || [[ "$CHANGED" == *"$service"* ]]; then + echo "Pushing $service..." + docker push ${{ env.REGISTRY }}/$service:${{ build.number }} + + if [[ "${{ branch }}" == "main" ]]; then + docker tag ${{ env.REGISTRY }}/$service:${{ build.number }} ${{ env.REGISTRY }}/$service:latest + docker push ${{ env.REGISTRY }}/$service:latest + fi + fi + } + + push_if_built api + push_if_built web + push_if_built worker + push_if_built scheduler + push_if_built gateway + + # Deploy to preview environment for PRs + - name: deploy-preview + if: ${{ build.event == 'pull_request' }} + environment: + name: preview-pr-${{ build.pr_number }} + type: non-production + + steps: + - name: deploy-preview-environment + run: | + ./scripts/deploy-preview.sh \ + --pr ${{ build.pr_number }} \ + --services "${{ stages.detect-changes.outputs.changed_services }}" \ + --version ${{ build.number }} + + - name: comment-preview-url + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.createComment({ + issue_number: ${{ build.pr_number }}, + owner: context.repo.owner, + repo: context.repo.repo, + body: '🚀 Preview deployed: https://pr-${{ build.pr_number }}.preview.example.com' + }) + + # Deploy to staging + - name: deploy-staging + if: ${{ branch == 'develop' }} + environment: + name: staging + type: non-production + + steps: + - name: deploy-changed-services + run: | + ./scripts/deploy.sh staging \ + --services "${{ stages.detect-changes.outputs.changed_services }}" \ + --version ${{ build.number }} + + # Deploy to production + - name: deploy-production + if: ${{ branch == 'main' }} + environment: + name: production + type: production + + approval: + uses: harness + with: + timeout: 24h + min-approvers: 2 + groups: [platform-team, service-owners] + message: | + Production deployment for build ${{ build.number }} + Changed services: ${{ stages.detect-changes.outputs.changed_services }} + Changed libs: ${{ stages.detect-changes.outputs.changed_libs }} + + steps: + - name: deploy-production + run: | + ./scripts/deploy.sh production \ + --services "${{ stages.detect-changes.outputs.changed_services }}" \ + --version ${{ build.number }} \ + --canary true + + - name: verify-deployment + run: ./scripts/verify-deployment.sh production + + rollback: + - name: rollback-services + run: | + ./scripts/rollback.sh production \ + --services "${{ stages.detect-changes.outputs.changed_services }}" + + # Cleanup preview environments for merged PRs + - name: cleanup-preview + if: ${{ build.event == 'pull_request' && build.action == 'closed' }} + + steps: + - name: delete-preview-environment + run: ./scripts/cleanup-preview.sh --pr ${{ build.pr_number }} + + # Notify on completion + - name: notify + if: ${{ branch == 'main' || branch == 'develop' }} + + steps: + - name: slack-notification + run: | + STATUS="${{ status }}" + COLOR=$([[ "$STATUS" == "success" ]] && echo "good" || echo "danger") + + curl -X POST ${{ secrets.SLACK_WEBHOOK }} \ + -H "Content-Type: application/json" \ + -d '{ + "attachments": [{ + "color": "'"$COLOR"'", + "title": "Monorepo Build '"$STATUS"'", + "fields": [ + {"title": "Branch", "value": "${{ branch }}", "short": true}, + {"title": "Build", "value": "${{ build.number }}", "short": true}, + {"title": "Changed Services", "value": "${{ stages.detect-changes.outputs.changed_services || 'None' }}", "short": false}, + {"title": "Changed Libs", "value": "${{ stages.detect-changes.outputs.changed_libs || 'None' }}", "short": false} + ] + }] + }' diff --git a/samples/pipeline-with-templates.yaml b/samples/pipeline-with-templates.yaml new file mode 100644 index 0000000..c974329 --- /dev/null +++ b/samples/pipeline-with-templates.yaml @@ -0,0 +1,165 @@ +# Pipeline Using Reusable Templates +# Demonstrates: template references, versioned templates, step templates, stage templates +version: 1 + +pipeline: + name: microservices-build-pipeline + + inputs: + services: + type: array + description: "Services to build" + default: ["api", "worker", "frontend"] + environment: + type: choice + options: [dev, staging, prod] + default: dev + + env: + REGISTRY: gcr.io/my-project + + stages: + # Use a stage template for setup + - name: setup + uses: account.org/ci-setup-stage@2.0.0 + with: + node_version: "20" + go_version: "1.21" + install_tools: true + + # Build multiple services using step templates + - name: build-services + strategy: + matrix: + service: ${{ inputs.services }} + + steps: + # Use versioned step template for Docker builds + - name: build-${{ matrix.service }} + uses: template/docker-build-push@1.0.0 + with: + dockerfile: ./services/${{ matrix.service }}/Dockerfile + context: ./services/${{ matrix.service }} + image_name: my-project/${{ matrix.service }} + tags: + - ${{ build.number }} + - ${{ branch == 'main' && 'latest' || branch }} + registry: ${{ env.REGISTRY }} + platforms: + - linux/amd64 + - linux/arm64 + build_args: + - VERSION=${{ build.number }} + - BUILD_DATE=${{ timestamp }} + push: true + scan: true + + # Run tests using test template + - name: test-services + strategy: + matrix: + service: ${{ inputs.services }} + parallel: true + + steps: + - name: unit-tests-${{ matrix.service }} + uses: account.org/run-tests@1.5.0 + with: + working_directory: ./services/${{ matrix.service }} + test_command: npm test + coverage_threshold: 80 + report_format: junit + + - name: integration-tests-${{ matrix.service }} + uses: account.org/integration-test@1.2.0 + with: + service: ${{ matrix.service }} + config: ./test-configs/${{ matrix.service }}.yaml + timeout: 10m + + # Security scanning using template + - name: security-scanning + steps: + - name: sast-scan + uses: account.org/security-scan@3.0.0 + with: + scan_type: sast + target: . + fail_on: critical + report_path: security-reports/sast.sarif + + - name: dependency-scan + uses: account.org/security-scan@3.0.0 + with: + scan_type: sca + target: . + fail_on: high + report_path: security-reports/sca.sarif + + - name: container-scan + uses: account.org/security-scan@3.0.0 + with: + scan_type: container + images: ${{ inputs.services }} + registry: ${{ env.REGISTRY }} + tag: ${{ build.number }} + + # Deploy using stage template + - name: deploy-dev + if: ${{ inputs.environment == 'dev' }} + uses: account.org/k8s-deploy-stage@2.1.0 + with: + environment: development + namespace: dev + services: ${{ inputs.services }} + image_tag: ${{ build.number }} + registry: ${{ env.REGISTRY }} + replicas: 1 + health_check_timeout: 5m + + - name: deploy-staging + if: ${{ inputs.environment == 'staging' }} + uses: account.org/k8s-deploy-stage@2.1.0 + with: + environment: staging + namespace: staging + services: ${{ inputs.services }} + image_tag: ${{ build.number }} + registry: ${{ env.REGISTRY }} + replicas: 2 + health_check_timeout: 10m + approval_required: true + approval_groups: [qa-team] + + - name: deploy-prod + if: ${{ inputs.environment == 'prod' }} + uses: account.org/k8s-deploy-stage@2.1.0 + with: + environment: production + namespace: production + services: ${{ inputs.services }} + image_tag: ${{ build.number }} + registry: ${{ env.REGISTRY }} + replicas: 3 + strategy: canary + canary_percentage: 10 + health_check_timeout: 15m + approval_required: true + approval_groups: [sre-team, product-owners] + enable_rollback: true + + # Notification using template + - name: notify + uses: account.org/notification-stage@1.0.0 + with: + channels: + - type: slack + webhook: ${{ secrets.SLACK_WEBHOOK }} + channel: deployments + - type: email + recipients: ${{ inputs.environment == 'prod' && 'team@example.com,ops@example.com' || 'team@example.com' }} + message_template: deployment_complete + variables: + services: ${{ join(inputs.services, ', ') }} + environment: ${{ inputs.environment }} + version: ${{ build.number }} diff --git a/samples/pipeline.yaml b/samples/pipeline.yaml index 208a7e8..c1bc759 100644 --- a/samples/pipeline.yaml +++ b/samples/pipeline.yaml @@ -74,18 +74,3 @@ jobs: --- -# sample github pipeline, with the extended -# harness sytnax. This pipeline includes a -# Harness template step, even though GitHub -# has zero concept of templates. - -jobs: - test: - runs-on: ubuntu - steps: - - run: go build - - template: - uses: account.docker - with: - repo: harness/hello-world - tags: latest diff --git a/samples/template-docker-build.yaml b/samples/template-docker-build.yaml new file mode 100644 index 0000000..d25573b --- /dev/null +++ b/samples/template-docker-build.yaml @@ -0,0 +1,102 @@ +# Reusable Template: Docker Build and Push +# This template can be used across multiple pipelines for consistent Docker workflows +version: 1 + +template: + name: docker-build-push + description: "Build and push Docker images with multi-arch support" + + inputs: + dockerfile: + type: string + description: "Path to Dockerfile" + default: "./Dockerfile" + context: + type: string + description: "Build context path" + default: "." + image_name: + type: string + description: "Image name without tag" + required: true + tags: + type: array + description: "List of tags to apply" + default: ["latest"] + registry: + type: string + description: "Container registry URL" + default: "docker.io" + platforms: + type: array + description: "Target platforms for multi-arch builds" + default: ["linux/amd64"] + build_args: + type: array + description: "Build arguments in KEY=value format" + default: [] + push: + type: boolean + description: "Push image after build" + default: true + cache_from: + type: string + description: "Cache source for builds" + default: "" + scan: + type: boolean + description: "Run security scan on built image" + default: true + + steps: + - name: setup-buildx + uses: docker/setup-buildx-action@v3 + + - name: setup-qemu + if: ${{ length(inputs.platforms) > 1 }} + uses: docker/setup-qemu-action@v3 + + - name: login-registry + uses: docker/login-action@v3 + with: + registry: ${{ inputs.registry }} + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: extract-metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ inputs.registry }}/${{ inputs.image_name }} + tags: ${{ inputs.tags }} + + - name: build-and-push + id: build + uses: docker/build-push-action@v5 + with: + context: ${{ inputs.context }} + file: ${{ inputs.dockerfile }} + platforms: ${{ join(inputs.platforms, ',') }} + push: ${{ inputs.push }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: ${{ join(inputs.build_args, '\n') }} + cache-from: ${{ inputs.cache_from != '' && inputs.cache_from || format('type=registry,ref={0}/{1}:buildcache', inputs.registry, inputs.image_name) }} + cache-to: type=registry,ref=${{ inputs.registry }}/${{ inputs.image_name }}:buildcache,mode=max + + - name: security-scan + if: ${{ inputs.scan }} + run: | + trivy image --severity HIGH,CRITICAL \ + --exit-code 1 \ + ${{ inputs.registry }}/${{ inputs.image_name }}:${{ inputs.tags[0] }} + on-failure: + errors: [all] + action: + ignore: {} + + - name: output-digest + run: | + echo "Image built: ${{ inputs.registry }}/${{ inputs.image_name }}" + echo "Digest: ${{ steps.build.outputs.digest }}" + echo "image=${{ inputs.registry }}/${{ inputs.image_name }}@${{ steps.build.outputs.digest }}" >> $GITHUB_OUTPUT diff --git a/schema/steps.ts b/schema/steps.ts index 3f3802b..9402361 100644 --- a/schema/steps.ts +++ b/schema/steps.ts @@ -42,6 +42,12 @@ export interface StepLong { */ action?: StepAction; + /** + * Codebase Clone step + */ + + clone?: StepClone; + /** * Approval defines an approval step. */