From b396e380d9f5cdacb0dac647aed86504f8a9754a Mon Sep 17 00:00:00 2001 From: Ian Woodard <17186604+IanWoodard@users.noreply.github.com> Date: Fri, 19 Dec 2025 09:56:15 -0600 Subject: [PATCH 01/16] ref(pipedream): Adding grouping support --- PIPEDREAM_CELLULARIZATION_NOTES.md | 119 +++ PIPEDREAM_USE_CASES.md | 91 ++ libs/getsentry.libsonnet | 39 +- libs/pipedream.libsonnet | 209 ++-- notes.md | 175 ++++ status.md | 266 +++++ test/pipedream.js | 908 +++--------------- .../fixtures/getsentry/groups.jsonnet | 6 + .../fixtures/getsentry/regions.jsonnet | 5 - .../pipedream/autodeploy-parallel.jsonnet | 40 - ...onfig.jsonnet => basic-autodeploy.jsonnet} | 21 +- ...oy-serial.jsonnet => basic-manual.jsonnet} | 19 +- .../different-stages-per-region.jsonnet | 44 + .../pipedream/exclude-entire-group.jsonnet | 36 + ...-serial.jsonnet => exclude-region.jsonnet} | 28 +- ...xclude-regions-autodeploy-parallel.jsonnet | 39 - ...onnet => include-default-excluded.jsonnet} | 27 +- ...nclude-regions-autodeploy-parallel.jsonnet | 40 - ...ude-regions-no-autodeploy-parallel.jsonnet | 41 - ...clude-regions-no-autodeploy-serial.jsonnet | 41 - .../include-regions-parallel.jsonnet | 40 - .../pipedream/multi-region-group.jsonnet | 36 + .../pipedream/no-autodeploy-parallel.jsonnet | 41 - .../pipedream/no-autodeploy-serial.jsonnet | 41 - ...parallel.jsonnet => parallel-mode.jsonnet} | 19 +- .../rollback-bad-final-stage.failing.jsonnet | 30 +- .../rollback-bad-stage.failing.jsonnet | 26 +- .../rollback-override-final-stage.jsonnet | 46 - ...deploy-serial.jsonnet => rollback.jsonnet} | 25 +- .../groups.jsonnet_output-files.golden | 34 + .../regions.jsonnet_output-files.golden | 13 - ...eploy-parallel.jsonnet_output-files.golden | 349 ------- ...deploy-parallel.jsonnet_single-file.golden | 312 ------ ...sic-autodeploy.jsonnet_output-files.golden | 234 +++++ ...asic-autodeploy.jsonnet_single-file.golden | 217 +++++ ... basic-manual.jsonnet_output-files.golden} | 220 ++--- ...> basic-manual.jsonnet_single-file.golden} | 192 ++-- ...ges-per-region.jsonnet_output-files.golden | 370 +++++++ ...ages-per-region.jsonnet_single-file.golden | 353 +++++++ ...e-entire-group.jsonnet_output-files.golden | 159 +++ ...de-entire-group.jsonnet_single-file.golden | 147 +++ ...exclude-region.jsonnet_output-files.golden | 226 +++++ .../exclude-region.jsonnet_single-file.golden | 209 ++++ ...deploy-parallel.jsonnet_single-file.golden | 278 ------ ...odeploy-serial.jsonnet_output-files.golden | 330 ------- ...eploy-parallel.jsonnet_output-files.golden | 372 ------- ...deploy-parallel.jsonnet_single-file.golden | 335 ------- ...ault-excluded.jsonnet_output-files.golden} | 189 ++-- ...fault-excluded.jsonnet_single-file.golden} | 181 ++-- ...eploy-parallel.jsonnet_output-files.golden | 349 ------- ...deploy-parallel.jsonnet_single-file.golden | 312 ------ ...odeploy-serial.jsonnet_output-files.golden | 373 ------- ...todeploy-serial.jsonnet_single-file.golden | 336 ------- ...eploy-parallel.jsonnet_output-files.golden | 415 -------- ...deploy-parallel.jsonnet_single-file.golden | 373 ------- ...odeploy-serial.jsonnet_output-files.golden | 415 -------- ...todeploy-serial.jsonnet_single-file.golden | 373 ------- ...gions-parallel.jsonnet_output-files.golden | 349 ------- ...egions-parallel.jsonnet_single-file.golden | 312 ------ ...i-region-group.jsonnet_output-files.golden | 132 +++ ...ti-region-group.jsonnet_single-file.golden | 125 +++ ...eploy-parallel.jsonnet_output-files.golden | 647 ------------- ...deploy-parallel.jsonnet_single-file.golden | 605 ------------ ...odeploy-serial.jsonnet_output-files.golden | 647 ------------- ...todeploy-serial.jsonnet_single-file.golden | 605 ------------ .../parallel-mode.jsonnet_output-files.golden | 275 ++++++ .../parallel-mode.jsonnet_single-file.golden | 253 +++++ ...de-final-stage.jsonnet_output-files.golden | 394 -------- ...ide-final-stage.jsonnet_single-file.golden | 357 ------- ...n => rollback.jsonnet_output-files.golden} | 202 ++-- ...en => rollback.jsonnet_single-file.golden} | 192 ++-- 71 files changed, 4476 insertions(+), 10783 deletions(-) create mode 100644 PIPEDREAM_CELLULARIZATION_NOTES.md create mode 100644 PIPEDREAM_USE_CASES.md create mode 100644 notes.md create mode 100644 status.md create mode 100644 test/testdata/fixtures/getsentry/groups.jsonnet delete mode 100644 test/testdata/fixtures/getsentry/regions.jsonnet delete mode 100644 test/testdata/fixtures/pipedream/autodeploy-parallel.jsonnet rename test/testdata/fixtures/pipedream/{minimal-config.jsonnet => basic-autodeploy.jsonnet} (61%) rename test/testdata/fixtures/pipedream/{exclude-regions-no-autodeploy-serial.jsonnet => basic-manual.jsonnet} (70%) create mode 100644 test/testdata/fixtures/pipedream/different-stages-per-region.jsonnet create mode 100644 test/testdata/fixtures/pipedream/exclude-entire-group.jsonnet rename test/testdata/fixtures/pipedream/{exclude-regions-autodeploy-serial.jsonnet => exclude-region.jsonnet} (53%) delete mode 100644 test/testdata/fixtures/pipedream/exclude-regions-autodeploy-parallel.jsonnet rename test/testdata/fixtures/pipedream/{include-regions-autodeploy-serial.jsonnet => include-default-excluded.jsonnet} (54%) delete mode 100644 test/testdata/fixtures/pipedream/include-regions-autodeploy-parallel.jsonnet delete mode 100644 test/testdata/fixtures/pipedream/include-regions-no-autodeploy-parallel.jsonnet delete mode 100644 test/testdata/fixtures/pipedream/include-regions-no-autodeploy-serial.jsonnet delete mode 100644 test/testdata/fixtures/pipedream/include-regions-parallel.jsonnet create mode 100644 test/testdata/fixtures/pipedream/multi-region-group.jsonnet delete mode 100644 test/testdata/fixtures/pipedream/no-autodeploy-parallel.jsonnet delete mode 100644 test/testdata/fixtures/pipedream/no-autodeploy-serial.jsonnet rename test/testdata/fixtures/pipedream/{exclude-regions-no-autodeploy-parallel.jsonnet => parallel-mode.jsonnet} (71%) delete mode 100644 test/testdata/fixtures/pipedream/rollback-override-final-stage.jsonnet rename test/testdata/fixtures/pipedream/{autodeploy-serial.jsonnet => rollback.jsonnet} (61%) create mode 100644 test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/getsentry/regions.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/autodeploy-parallel.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/autodeploy-parallel.jsonnet_single-file.golden create mode 100644 test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden rename test/testdata/goldens/pipedream/{minimal-config.jsonnet_output-files.golden => basic-manual.jsonnet_output-files.golden} (56%) rename test/testdata/goldens/pipedream/{minimal-config.jsonnet_single-file.golden => basic-manual.jsonnet_single-file.golden} (56%) create mode 100644 test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden create mode 100644 test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden create mode 100644 test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden delete mode 100644 test/testdata/goldens/pipedream/exclude-regions-autodeploy-parallel.jsonnet_single-file.golden delete mode 100644 test/testdata/goldens/pipedream/exclude-regions-autodeploy-serial.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet_single-file.golden rename test/testdata/goldens/pipedream/{autodeploy-serial.jsonnet_output-files.golden => include-default-excluded.jsonnet_output-files.golden} (66%) rename test/testdata/goldens/pipedream/{autodeploy-serial.jsonnet_single-file.golden => include-default-excluded.jsonnet_single-file.golden} (64%) delete mode 100644 test/testdata/goldens/pipedream/include-regions-autodeploy-parallel.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/include-regions-autodeploy-parallel.jsonnet_single-file.golden delete mode 100644 test/testdata/goldens/pipedream/include-regions-autodeploy-serial.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/include-regions-autodeploy-serial.jsonnet_single-file.golden delete mode 100644 test/testdata/goldens/pipedream/include-regions-no-autodeploy-parallel.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/include-regions-no-autodeploy-parallel.jsonnet_single-file.golden delete mode 100644 test/testdata/goldens/pipedream/include-regions-no-autodeploy-serial.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/include-regions-no-autodeploy-serial.jsonnet_single-file.golden delete mode 100644 test/testdata/goldens/pipedream/include-regions-parallel.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/include-regions-parallel.jsonnet_single-file.golden create mode 100644 test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden delete mode 100644 test/testdata/goldens/pipedream/no-autodeploy-parallel.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/no-autodeploy-parallel.jsonnet_single-file.golden delete mode 100644 test/testdata/goldens/pipedream/no-autodeploy-serial.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/no-autodeploy-serial.jsonnet_single-file.golden create mode 100644 test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden delete mode 100644 test/testdata/goldens/pipedream/rollback-override-final-stage.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/rollback-override-final-stage.jsonnet_single-file.golden rename test/testdata/goldens/pipedream/{exclude-regions-autodeploy-parallel.jsonnet_output-files.golden => rollback.jsonnet_output-files.golden} (66%) rename test/testdata/goldens/pipedream/{exclude-regions-autodeploy-serial.jsonnet_single-file.golden => rollback.jsonnet_single-file.golden} (66%) diff --git a/PIPEDREAM_CELLULARIZATION_NOTES.md b/PIPEDREAM_CELLULARIZATION_NOTES.md new file mode 100644 index 0000000..06cc696 --- /dev/null +++ b/PIPEDREAM_CELLULARIZATION_NOTES.md @@ -0,0 +1,119 @@ +# Pipedream Cellularization - Working Notes + +> **Do not commit this file.** Personal working notes for the `iw/grouped-pipedream` branch. + +## 1. Stage Properties: "First Region Wins" — GoCD Architectural Limit + +**Finding:** GoCD's XML schema (`cruise-config.xsd`) defines `fetch_materials`, `clean_workspace`, `approval`, and `keep_artifacts` as **strictly stage-level attributes**. They do not exist on `jobType`. The YAML config plugin (`StageTransform.java`) enforces the same boundary. There is **no per-job override mechanism** for stage properties in GoCD. + +**Implication:** When merging multiple regions' jobs into a single stage in grouped Pipedream, the stage can only have ONE set of properties. The current approach uses the first region's properties. This is the only option given GoCD's architecture. + +**Recommendation — add validation (~10-15 lines):** When regions in the same group define conflicting stage props for the same-named stage, we should error at build time rather than silently picking the first. Estimated implementation: + +```jsonnet +// Inside transform_stage, after computing stage_props from first region: +local _ = std.foldl( + function(acc, r) + local p = region_pipelines[r]; + local rs = get_matching_stage(p, stage_name); + local props = if rs != null then get_stage_props(rs) else stage_props; + assert props == stage_props : + "Stage '%s': conflicting properties across regions in group. " + + "Region '%s' differs from '%s'." % [stage_name, r, regions[0]]; + acc, + regions[1:], + true +); +``` + +~10 lines of code. Should be added before finalizing. + +--- + +## 2. Environment Variable Optimization — Implemented + +**Problem (before):** All env vars (pipeline + stage + job) were blindly cascaded to every job, and pipeline/stage-level `environment_variables` were stripped from the output. This caused duplication — shared vars like `GOCD_ACCESS_TOKEN` were repeated N times per stage (once per region-job). + +**Solution (implemented on this branch):** Env vars that are **identical across all regions** in a group are kept at stage level. Only vars that **differ between regions** are cascaded to job level. GoCD's native precedence (job > stage > pipeline) handles resolution correctly. + +**Changes made:** +- `libs/pipedream.libsonnet`: Rewrote `transform_stage` to compute `common_env` (intersection of pipeline+stage env vars across all regions) and `region_specific_env` (the diff). Common vars go to stage, region-specific to jobs. +- `test/pipedream.js`: Added test `"env var optimization: common vars at stage level, region-specific at job level"` validating the split. +- Golden files for `env-vars-precedence` regenerated. + +--- + +## 3. Double-calling `pipeline_fn(regions[0])` — Fixed + +**Problem (before):** `pipeline_fn(regions[0])` was called at the top of `generate_group_pipeline` to get a template, then called AGAIN for every region (including `regions[0]`) in the `all_stages` fold and `transform_stage`. This meant `pipeline_fn` for the first region ran 3x. + +**Fix (implemented on this branch):** Added a cache: +```jsonnet +local region_pipelines = { [r]: pipeline_fn(r) for r in regions }; +``` +All subsequent references use `region_pipelines[region]` instead of `pipeline_fn(region)`. Also extracted `get_matching_stage` helper to reduce repeated stage-matching logic. + +--- + +## 4. s4s/s4s2 Alignment — Resolved + +**Background:** On `main`, s4s2 replaced s4s as the prod region (commit `3b7e3b1`). s4s is now a test region. + +**Change made:** Updated `getsentry.libsonnet` so the `s4s` group only contains `['s4s2']` (was `['s4s', 's4s2']`). This aligns with main where s4s2 is the prod region. + +**Impact on downstream compatibility (positive):** +- The `s4s` group is now single-region, so no double-suffixed job names for s4s. +- The `region_pops` crash risk for uptime-checker/relay is **eliminated for the s4s group** — `pipeline_fn('s4s2')` is already called on main today, so all services already handle it. +- Pipeline name changes: `deploy-{service}-s4s2` → `deploy-{service}-s4s`. This is still a rename that downstream tools need to handle. + +**Remaining TODO:** `test_groups` is currently empty. On main, `s4s` is a test region. Consider whether to add it as a test group. + +--- + +## 5/8. Pipeline Name + Job Name Changes — Downstream Impact + +### Pipeline Name Changes + +Pipeline names change from `deploy-{service}-{region}` to `deploy-{service}-{group}`: +- `deploy-example-customer-1`, `-customer-2`, `-customer-4`, `-customer-7` → `deploy-example-st` +- `deploy-example-s4s2` → `deploy-example-s4s` (s4s group name, contains only s4s2) +- Single-region groups unchanged: `deploy-example-de`, `deploy-example-us` + +**Key insight after s4s2 alignment:** Most single-region groups (`de`, `us`, `control`, `snty-tools`) keep their existing pipeline names. Only two name changes actually happen: +1. `s4s2` → `s4s` (group name differs from region name) +2. `customer-{1,2,4,7}` → `st` (four pipelines collapse to one) + +### Repos That Will Break + +| Repo | Severity | What breaks | +|------|----------|-------------| +| **gocd-deployment-visualizer** (deploy-tools) | **Critical** | `isMultiregion()` checks for `-s4s2`/`-customer-1` suffixes. `isPipedreamPipeline()` has hardcoded allowlist of per-region names. `findPrimaryTrigger()` looks for `-s4s2`. `PIPEDREAM_SUFFIX_CUSTOMER_MAP` maps per-region suffixes to display names. `constructTree()` hardcodes `-s4s2` stripping. | +| **eng-pipes** | **Critical** | Slack feed filters have hardcoded `deploy-snuba-py-customer-{N}`, etc. `getFormattedRegion()` parses pipeline suffixes for Datadog tags — `-st` won't map to anything. | +| **sentry-feature-scoring** | **High** | Hardcoded `deploy-getsentry-backend-s4s2` in exclusion list. Deploy completeness check references `deploy-getsentry-backend-customer-7`. | +| **sentry-feature-scoring-frontend** | **High** | Constructs pipeline names via `` `deploy-getsentry-backend-${region}` `` with per-region strings. `BACKEND_REGIONS` array lists individual regions. | +| **devinfra-pause-metrics** | **Medium** | Parses region from pipeline suffix, has hardcoded region buckets. | +| **ops** | **Medium** | `dd-event.py` has `REGIONS_MAP` with `customer-*` keys. | + +### Job Name Issues + +**No true name collisions found**, but one remaining concern: + +1. **Double-suffixed names in st group:** Services that embed region/pop in job names (e.g., uptime-checker's `deploy-canary-de`) will get `deploy-canary-de-de` after the `-{region}` append. Ugly but functionally correct. **The s4s group no longer has this issue** since it's single-region. + +2. **`region_pops` crash risk reduced:** With s4s2 as the sole region in the s4s group, `pipeline_fn('s4s2')` is already the status quo on main. The crash risk only applies to the `st` group for services like uptime-checker/relay that may not have entries for all customer regions in `region_pops`. (These services likely already exclude customer regions or have them in their pops dictionaries since customers are already prod regions on main.) + +### Recommendations + +**Option A — Coordinated migration (recommended):** +1. Update all downstream repos to use group-aware pipeline naming BEFORE cutting v2 +2. During transition, these repos should handle both old and new naming patterns +3. Create a Linear ticket per repo with specific file changes needed + +**Option B — Compatibility shim:** +Add a mapping/lookup in the Pipedream library itself that exposes both the group name and the constituent region names, so downstream tools can query "which regions are in group `st`?" rather than hardcoding names. + +**Option C — Keep per-region pipeline names (simplest, least disruptive):** +Instead of naming pipelines `deploy-{service}-{group}`, continue naming them `deploy-{service}-{first_region_in_group}`. So the `st` group would still be `deploy-example-customer-1` and the `s4s` group would be `deploy-example-s4s2`. This avoids ALL downstream breakage but makes the group concept less visible in naming. Not recommended long-term but could be a transitional approach. + +**Option D — Name groups after first region (hybrid):** +Same as Option C but only for groups where the name differs from the first region. The `s4s` group contains `['s4s2']`, so name it `deploy-example-s4s2`. The `st` group contains `['customer-1', ...]`, so name it `deploy-example-customer-1`. Only truly custom group names (if any) get a new name. This makes the change invisible to downstream tools while still grouping jobs in parallel internally. diff --git a/PIPEDREAM_USE_CASES.md b/PIPEDREAM_USE_CASES.md new file mode 100644 index 0000000..cc0f58f --- /dev/null +++ b/PIPEDREAM_USE_CASES.md @@ -0,0 +1,91 @@ +# Pipedream Use Cases Across getsentry + +> Inventory of all real-world Pipedream usage, for validating test coverage. + +## Service Repos (15 repos, 18 templates) + +| Service | auto_deploy | exclude_regions | include_regions | parallel | rollback | rollback.final_stage | +|---------|-------------|-----------------|-----------------|----------|----------|----------------------| +| getsentry-backend | true (default) | `[]` | `['control']` | no | yes | no | +| relay-pop | false | `['de','us','s4s2','customer-1/2/4']` | — | no | yes | no | +| relay-processing | false | — | — | no | yes | no | +| snuba-py | true | — | — | no | yes | no | +| snuba-rs | true | — | — | no | yes | no | +| symbolicator | false | — | — | no | yes | **yes** (`deploy-primary`) | +| chartcuterie | true | `['customer-6']` | — | no | yes | no | +| taskbroker | true | — | — | no | yes | no | +| conduit | true | `['us','de','customer-1/2/4/7']` | — | no | yes | no | +| vroom | true (default) | — | — | no | yes | no | +| uptime-checker | true | `['customer-1/2/3/4/6/7']` | — | no | yes | no | +| super-big-consumers | false | — | — | no | yes | **yes** (`deploy-primary`) | +| seer | true | `['customer-3/6']` | — | no | yes | no | +| seer-gpu | true | `['customer-3/6']` | — | no | yes | no | +| launchpad | true | `['customer-1/2/3/4/7']` | — | no | yes | no | +| objectstore | false | `['customer-1/2/4/7']` | — | no | yes | no | +| tempest | true (default) | `['customer-1/2/3/4/6/7']` | — | no | yes | no | +| sentry-scripts | true | — | `['snty-tools']` | **yes** | **no** | — | + +## Ops K8s Templates (50+ templates) + +- 8 with custom pipeline_fn (getsentry-k8s, relay-k8s, snuba-k8s, relay-pop-k8s, symbolicator-k8s, taskbroker-k8s, uptime-checker-k8s, sbc-k8s) +- 41+ using shared `gocd.pipedream_config()` helper +- All ops templates use `auto_deploy: false` +- objectstore-k8s uses `parallel=true` +- workflow-engine-k8s and script-runner-k8s use `include_regions: ['snty-tools']` + +## Pipeline Function Patterns + +### Pattern 1: Pipeline-level `environment_variables` (ALL services) +Every service sets at minimum `SENTRY_REGION` at pipeline level. Many also set `GITHUB_TOKEN`, `GOCD_ACCESS_TOKEN`, `SKIP_CANARY_CHECKS`, `SENTRY_ORG`, `SENTRY_PROJECT`, Datadog keys. + +**Test coverage:** `env-vars-precedence.jsonnet` covers pipeline/stage/job env vars. + +### Pattern 2: Region-conditional stages (common) +- Canary stages only for US/DE (snuba, symbolicator, chartcuterie, taskbroker, vroom, relay, getsentry-backend) +- ST-specific migration stage (snuba-py: `st_migrate`) +- Soak time only for S4S and US (relay, getsentry-backend, objectstore) + +**Test coverage:** `different-stages-per-region.jsonnet` covers different stage sets per region. + +### Pattern 3: `getsentry.is_st(region)` conditional (3 repos) +Used by snuba-py, snuba-rs, and symbolicator to choose deploy scripts, migration logic, and approval types. + +**Test coverage:** Not a pipedream concern — this runs inside `pipeline_fn`. Pipedream just calls `pipeline_fn(region)` and handles whatever comes back. + +### Pattern 4: `region_pops` per-pop jobs (2 repos) +relay-pop and uptime-checker define jobs per sub-PoP using comprehensions like `['deploy-primary-' + pop]: ...`. These generate job names that already contain region identifiers. + +**Test coverage:** Not a pipedream concern at the library level. However, the `-{region}` suffix appended by grouping will create double-suffixed names (e.g. `deploy-primary-de-de`). Documented as a known issue. + +### Pattern 5: Multiple stages per pipeline (most services) +Real services typically have 2-4 stages: `checks`, `deploy-canary`, `deploy-primary`, `soak-time`. Some have `migrations`, `health_check`, `scale-down-canary`. + +**Test coverage:** Most fixtures use a single `deploy` stage. `different-stages-per-region.jsonnet` uses deploy + verify. Adequate — pipedream doesn't treat stages differently based on count. + +### Pattern 6: `rollback.final_stage` override (2 repos) +symbolicator and super-big-consumers override `final_stage` to `deploy-primary` instead of the default `pipeline-complete`. + +**Test coverage:** **GAP** — we test invalid `final_stage` (failing fixtures) but not a valid override. Added `rollback-final-stage-override.jsonnet`. + +### Pattern 7: No rollback config (1 repo) +sentry-scripts has no rollback. + +**Test coverage:** Covered — `basic-autodeploy.jsonnet` has no rollback config. + +### Pattern 8: `include_regions` for `snty-tools` (3 repos) +sentry-scripts, ops/workflow-engine-k8s, ops/script-runner-k8s. + +**Test coverage:** `include-default-excluded.jsonnet` tests `include_regions: ['control']`. snty-tools uses the same mechanism. Covered. + +### Pattern 9: Heavy `exclude_regions` narrowing to few groups (common) +conduit excludes `['us','de','customer-1/2/4/7']` leaving only s4s. relay-pop excludes everything except s4s and customer-7. + +**Test coverage:** `exclude-region.jsonnet` (partial exclude) and `exclude-entire-group.jsonnet` (full group excluded). Covered. + +## Coverage Gap Summary + +| Gap | Severity | Action | +|-----|----------|--------| +| Valid `rollback.final_stage` override | Medium | Add `rollback-final-stage-override.jsonnet` fixture | +| Stage-level env vars (no job env vars) | Low | seer sets `SENTRY_REGION` at stage level only. Implicitly covered by env-vars-precedence but could add explicit fixture. | +| `auto_pipeline_progression: false` | Low | Exists in code but NOT used by any real service. Skip for now. | diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index a309ce8..fe3c902 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -3,22 +3,25 @@ */ { - // These regions are user facing deployments - prod_regions: [ - 's4s2', - 'de', - 'us', - // 'control' is excluded by default and must be explicitly included - 'control', - // 'snty-tools' is excluded by default and must be explicitly included - 'snty-tools', - 'customer-1', - 'customer-2', - 'customer-4', - 'customer-7', - ], - // Test regions will deploy in parallel to the regions above - test_regions: [ - ], - is_st(region):: std.startsWith(region, 'customer-'), + group_order: ['s4s', 'de', 'us', 'control', 'snty-tools', 'st'], + test_group_order: [], + // These groupings consist of user facing deployments + pipeline_groups: { + s4s: ['s4s', 's4s2'], + de: ['de'], + us: ['us'], + control: ['control'], + 'snty-tools': ['snty-tools'], + st: ['customer-1', 'customer-2', 'customer-4', 'customer-7'], + }, + // Test groups will deploy in parallel to the groups above + test_groups: { + }, + + group_names:: self.group_order, + test_group_names:: self.test_group_order, + get_targets(group):: + if std.objectHas(self.pipeline_groups, group) then self.pipeline_groups[group] + else self.test_groups[group], + is_st(region):: (region == 's4s' || std.startsWith(region, 'customer-')), } diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index 1991a9f..a3fac4c 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -3,16 +3,24 @@ This libraries main purpose is to generate a set of pipelines that constitute a pipedream. -"pipedream" is what we're calling the overall deployment process for a service -at sentry, where that service is expected to be deployed to multiple regions. +"pipedream" is the overall deployment process for a service at Sentry, where +that service is deployed to multiple regions organized into groups. -The entry point for this library is the `render()` function which takes -some configuration and a callback function. The callback function is expected -to return a pipeline definition for a given region. +Key concepts: +- Groups: Collections of regions that are deployed together +- Regions: Individual deployment targets within a group +- Regions within a group run as parallel jobs within a single pipeline +- Groups are chained sequentially (or fan out in parallel mode) -Pipedream will name the returned pipeline, add an upstream pipeline material -and a final stage. The upstream material and final stage is to make GoCD -chain the pipelines together. +The entry point is `render(config, pipeline_fn)` where: +- config: Pipedream configuration (name, materials, rollback, etc.) +- pipeline_fn(region): Callback that returns a pipeline definition for a region + +Pipedream will: +1. Generate one pipeline per group +2. Aggregate jobs from all regions in the group (running in parallel) +3. Chain pipelines together with upstream materials +4. Append a final 'pipeline-complete' stage */ local getsentry = import './getsentry.libsonnet'; @@ -26,6 +34,36 @@ local pipeline_name(name, region=null) = local is_autodeploy(pipedream_config) = !std.objectHas(pipedream_config, 'auto_deploy') || pipedream_config.auto_deploy == true; +// Regions that are excluded by default and must be explicitly included +local default_excluded_regions = ['control', 'snty-tools']; + +local is_excluded_region = function(region, config) + std.objectHas(config, 'exclude_regions') && std.length(std.find(region, config.exclude_regions)) > 0; + +local is_included_region = function(region, config) + std.objectHas(config, 'include_regions') && std.length(std.find(region, config.include_regions)) > 0; + +local is_default_excluded_region = function(region) + std.length(std.find(region, default_excluded_regions)) > 0; + +local should_include_region = function(region, config) + !is_excluded_region(region, config) && (!is_default_excluded_region(region) || is_included_region(region, config)); + +local get_stage_name(stage) = + std.objectFields(stage)[0]; + +local get_stage_jobs(stage) = + local stage_name = get_stage_name(stage); + if std.objectHas(stage[stage_name], 'jobs') then + stage[stage_name].jobs + else + {}; + +local get_stage_props(stage) = + local stage_name = get_stage_name(stage); + local props = stage[stage_name]; + { [k]: props[k] for k in std.objectFields(props) if k != 'jobs' }; + // This function returns a "trigger pipeline", if configured for manual deploys. // This pipeline is used so users don't need to know what the first pipedream // region is, instead they just look for the `deploy-` pipeline. @@ -163,17 +201,68 @@ local pipedream_rollback_pipeline(pipedream_config, service_pipelines, trigger_p else null; -// generate_region_pipeline will call the pipeline callback function, and then -// name the pipeline, add an upstream material, and append a final stage. -// pipedream_config: The configuration passed into the render() function -// pipeline_fn: The callback function passed in to render() function. -// This function is from users of the library and should -// take in a region and return a GoCD pipeline. -// region: The region to create pipelines for -// display_order: The order of the pipeline in GoCD UI -local generate_region_pipeline(pipedream_config, pipeline_fn, region, display_order) = +// generate_group_pipeline creates a single pipeline for a group by: +// 1. Getting all regions in the group +// 2. Filtering regions based on exclude/include config +// 3. Aggregating jobs from all regions into parallel jobs per stage +// 4. Appending a 'pipeline-complete' stage +// +// pipedream_config: The configuration passed into render() +// pipeline_fn: Callback that takes a region and returns a GoCD pipeline +// group: The group name to create a pipeline for +// display_order: The order of the pipeline in the GoCD UI +local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_order) = local service_name = pipedream_config.name; - local service_pipeline = pipeline_fn(region); + + local all_regions = getsentry.get_targets(group); + local regions = std.filter( + function(r) should_include_region(r, pipedream_config), + all_regions + ); + + local template_pipeline = pipeline_fn(regions[0]); + + // Collect all unique stages across all regions in the group + local all_stages = std.foldl( + function(acc, region) + local p = pipeline_fn(region); + local region_stages = if std.objectHas(p, 'stages') then p.stages else []; + acc + [ + stage + for stage in region_stages + if !std.member([get_stage_name(s) for s in acc], get_stage_name(stage)) + ], + regions, + [] + ); + + // Transforms a stage by aggregating jobs from all regions + local transform_stage(stage) = + local stage_name = get_stage_name(stage); + local stage_props = get_stage_props(stage); + + local all_jobs = std.foldl( + function(acc, region) + local p = pipeline_fn(region); + local matching_stages = std.filter( + function(s) get_stage_name(s) == stage_name, + if std.objectHas(p, 'stages') then p.stages else [] + ); + local stage_jobs = if std.length(matching_stages) > 0 then + get_stage_jobs(matching_stages[0]) + else + {}; + acc + { + [job_name + '-' + region]: stage_jobs[job_name] + for job_name in std.objectFields(stage_jobs) + }, + regions, + {} + ); + + { + [stage_name]: stage_props { jobs: all_jobs }, + }; // `auto_pipeline_progression` was added as a utility for folks new to // pipedream. When this is false, each region will need manual approval @@ -197,12 +286,17 @@ local generate_region_pipeline(pipedream_config, pipeline_fn, region, display_or else []; - // Add the upstream pipeline material and append the final stage - local stages = service_pipeline.stages; - service_pipeline { + // Apply transform to all stages + local transformed_stages = [ + transform_stage(stage) + for stage in all_stages + ]; + + // Assemble final pipeline from template + template_pipeline { group: service_name, display_order: display_order, - stages: prepend_stages + stages + [ + stages: prepend_stages + transformed_stages + [ // This stage is added to ensure a rollback doesn't cause // a deployment train. // @@ -213,23 +307,19 @@ local generate_region_pipeline(pipedream_config, pipeline_fn, region, display_or ], }; -// get_service_pipelines iterates over each region and generates the pipeline -// for each region. +// get_service_pipelines generates a pipeline for each group. // -// pipedream_config: The configuration passed into the render() function -// pipeline_fn: The callback function passed in to render() function. -// This function is from users of the library and should -// take in a region and return a GoCD pipeline. -// regions: The regions to create pipelines for -// display_offset: Used to offset the display order (i.e. test regions are -// display order => trigger + rollback + user regions length) -local get_service_pipelines(pipedream_config, pipeline_fn, regions, display_offset) = +// pipedream_config: The configuration passed into render() +// pipeline_fn: Callback that takes a region and returns a GoCD pipeline +// groups: The group names to create pipelines for +// display_offset: Offset for display_order (accounts for trigger/rollback) +local get_service_pipelines(pipedream_config, pipeline_fn, groups, display_offset) = [ { - name: pipeline_name(pipedream_config.name, regions[i]), - pipeline: generate_region_pipeline(pipedream_config, pipeline_fn, regions[i], display_offset + i), + name: pipeline_name(pipedream_config.name, groups[i]), + pipeline: generate_group_pipeline(pipedream_config, pipeline_fn, groups[i], display_offset + i), } - for i in std.range(0, std.length(regions) - 1) + for i in std.range(0, std.length(groups) - 1) ]; // This is a helper function that handles pipelines that may be null @@ -238,36 +328,31 @@ local pipeline_to_array(pipeline) = if pipeline == null then [] else [pipeline]; { - // render will generate the trigger pipeline and all the region pipelines. + // render generates the trigger pipeline (if manual), group pipelines, and rollback pipeline. render(pipedream_config, pipeline_fn, parallel=false):: - // Regions that are excluded by default and must be explicitly included - local default_excluded_regions = ['control', 'snty-tools']; - - local is_excluded_region = function(region, config) - std.objectHas(config, 'exclude_regions') && std.length(std.find(region, config.exclude_regions)) > 0; - - local is_included_region = function(region, config) - std.objectHas(config, 'include_regions') && std.length(std.find(region, config.include_regions)) > 0; - - local is_default_excluded_region = function(region) - std.length(std.find(region, default_excluded_regions)) > 0; - - local should_include_region = function(region, config) - !is_excluded_region(region, config) && (!is_default_excluded_region(region) || is_included_region(region, config)); - - // Filter out any regions that are listed in the `exclude_regions` attribute. - local regions_to_render = std.filter( - function(region) should_include_region(region, pipedream_config), - getsentry.prod_regions, + local groups_to_render = std.filter( + function(group) + local regions = getsentry.get_targets(group); + std.length(std.filter( + function(r) should_include_region(r, pipedream_config), + regions + )) > 0, + getsentry.group_names ); - local test_regions_to_render = std.filter( - function(region) should_include_region(region, pipedream_config), - getsentry.test_regions, + + local test_groups_to_render = std.filter( + function(group) + local regions = getsentry.get_targets(group); + std.length(std.filter( + function(r) should_include_region(r, pipedream_config), + regions + )) > 0, + getsentry.test_group_names ); local trigger_pipeline = pipedream_trigger_pipeline(pipedream_config); - local service_pipelines = get_service_pipelines(pipedream_config, pipeline_fn, regions_to_render, 2); - local test_pipelines = get_service_pipelines(pipedream_config, pipeline_fn, test_regions_to_render, std.length(regions_to_render) + 2); + local service_pipelines = get_service_pipelines(pipedream_config, pipeline_fn, groups_to_render, 2); + local test_pipelines = get_service_pipelines(pipedream_config, pipeline_fn, test_groups_to_render, std.length(groups_to_render) + 2); local rollback_pipeline = pipedream_rollback_pipeline(pipedream_config, service_pipelines, trigger_pipeline); local all_pipelines = if parallel then pipeline_to_array(rollback_pipeline) + @@ -276,7 +361,7 @@ local pipeline_to_array(pipeline) = // the trigger pipeline std.map(function(p) gocd_pipelines.chain_materials(p, trigger_pipeline), service_pipelines) + - // Chain each test region to the trigger pipeline + // Chain each test group to the trigger pipeline std.map(function(p) gocd_pipelines.chain_materials(p, trigger_pipeline), test_pipelines) else pipeline_to_array(rollback_pipeline) + // Chain the service pipelines together with @@ -284,7 +369,7 @@ local pipeline_to_array(pipeline) = gocd_pipelines.chain_pipelines( pipeline_to_array(trigger_pipeline) + service_pipelines, ) + - // Chain each test region to the trigger pipeline + // Chain each test group to the trigger pipeline std.map(function(p) gocd_pipelines.chain_materials(p, trigger_pipeline), test_pipelines); diff --git a/notes.md b/notes.md new file mode 100644 index 0000000..bced33f --- /dev/null +++ b/notes.md @@ -0,0 +1,175 @@ +# Pipedream Cellularization — Working Notes + +> Do not commit this file. + +## Test Coverage Audit (DI-1601 / DI-1602) + +### Gap found and filled + +Added `rollback-final-stage-override.jsonnet` — symbolicator and +super-big-consumers use `rollback.final_stage` to point at `deploy-primary` +instead of the default `pipeline-complete`. We had tests for invalid overrides +but not a valid one. + +### Already covered by existing fixtures + +- Auto-deploy vs manual deploy +- Region exclusions (partial and full group) +- Region inclusions (control; snty-tools uses same mechanism) +- Parallel mode +- No rollback config +- Pipeline/stage/job env var cascading +- Different stages per region +- Stage property conflicts across regions +- Multi-region vs single-region groups + +Full use-case inventory in `PIPEDREAM_USE_CASES.md`. + +--- + +## Env Var Cascade — Impact Assessment + +We changed env var handling so that variables identical across all regions in a +group stay at stage level, while region-specific variables cascade to job level. + +### Will this cause issues? + +**No.** GoCD natively resolves env vars with job > stage > pipeline precedence. +Moving shared vars up to stage level is semantically identical to duplicating +them at job level — GoCD evaluates them the same way. + +Checked against all 18 service templates and 50+ ops templates: + +- **Every service** sets `SENTRY_REGION` at pipeline level. This differs per + region, so it correctly cascades to job level (region-specific). +- **Shared secrets** (e.g. `GOCD_ACCESS_TOKEN`, `GITHUB_TOKEN`, `DATADOG_API_KEY`) + are the same across all regions. These now stay at stage level instead of + being duplicated into every job. No behavioral change. +- **seer** sets `SENTRY_REGION` at stage level (not pipeline level). With a + single-region group this is fine — it becomes a common env var at stage level. + In a multi-region group it would correctly cascade to job level since the + values differ. +- **getsentry-backend** is the most env-var-heavy service (`SENTRY_REGION`, + `SENTRY_ORG`, `SENTRY_PROJECT`, `SENTRY_PROJECT_ID`, etc. all at pipeline + level). All region-specific vars will cascade to jobs; shared vars like + `GITHUB_TOKEN` and `SKIP_CANARY_CHECKS` stay at stage level. Correct. + +**One edge case to be aware of:** if a service sets the same env var key at +both pipeline and stage level with different values, the stage value wins in the +merged parent env (pipeline + stage merge). This is the same precedence as +before — no change in behavior. + +--- + +## Stage Properties — GoCD Architectural Limit + +GoCD's XML schema defines `fetch_materials`, `clean_workspace`, `approval`, and +`keep_artifacts` as strictly stage-level attributes. There is no per-job +override. When merging multiple regions' jobs into a single stage, the first +region's properties win. + +**Recommendation:** Add ~10 lines of validation to error at build time if +regions in the same group define conflicting stage props for the same stage. + +--- + +## Downstream Pipeline Name Breakage (DI-1670) + +Pipeline names change from `deploy-{service}-{region}` to +`deploy-{service}-{group}`. Two actual renames: + +1. `s4s2` → `s4s` (group name differs from region name) +2. `customer-{1,2,4,7}` → `st` (four pipelines collapse to one) + +Single-region groups (`de`, `us`, `control`, `snty-tools`) keep existing names. + +### Repos that need updates + +| Repo | Severity | What breaks | +|------|----------|-------------| +| gocd-deployment-visualizer | Critical | `isMultiregion()`, `isPipedreamPipeline()`, `findPrimaryTrigger()`, `PIPEDREAM_SUFFIX_CUSTOMER_MAP`, `constructTree()` | +| eng-pipes | Critical | Slack feed filters, `getFormattedRegion()` Datadog tagging | +| sentry-feature-scoring | High | Hardcoded `deploy-getsentry-backend-s4s2` exclusion, `deploy-getsentry-backend-customer-7` completeness check | +| sentry-feature-scoring-frontend | High | `BACKEND_REGIONS` array, pipeline name construction | +| devinfra-pause-metrics | Medium | Region parsing from pipeline suffix | +| ops | Medium | `dd-event.py` `REGIONS_MAP` | + +### Double-suffixed job names + +Services that embed region/pop in job names (uptime-checker, relay-pop) will get +`deploy-primary-de-de` style names. Functionally correct but ugly. Only affects +the `st` group in practice since `s4s` is now single-region. + +--- + +## s4s/s4s2 Alignment + +Updated `getsentry.libsonnet` so the `s4s` group contains only `['s4s2']`, +matching main where s4s2 is the prod region (commit `3b7e3b1`). + +**Remaining TODO:** `test_groups` is empty. On main, `s4s` is a test region. +Consider adding it as a test group. + +--- + +## Pilot Testing — devinfra-example-service (DI-1607) + +Bumped `devinfra-example-service` to `v3.0.0-rc.1`. +PR: https://github.com/getsentry/devinfra-example-service/pull/24 + +Generated pipelines verified locally (`make gocd`): + +- `deploy-devinfra-example-service-s4s` — single job (`s4s2`), `SENTRY_REGION` at stage level +- `deploy-devinfra-example-service-de` — single job, chained after s4s +- `deploy-devinfra-example-service-us` — single job, chained after de +- `deploy-devinfra-example-service-st` — 4 parallel jobs (`customer-1`, `-2`, `-4`, `-7`), `SENTRY_REGION` at job level +- `rollback-devinfra-example-service` — watches st, references all 4 group pipelines +- `control`/`snty-tools` correctly excluded by default + +--- + +## Issues Found During Pilot Review + +### 1. Silent stage loss when stage objects have multiple keys + +`demo.libsonnet` has a missing comma between two stage objects (lines 30–31), +causing jsonnet to merge them via implicit `+`. The resulting single object +`{first: {...}, 'deploy-primary': {...}}` is treated as one stage, and +`get_stage_name()` (which calls `std.objectFields(stage)[0]`) only sees +`deploy-primary` (alphabetical). The `first` stage silently disappears. + +This is a pre-existing bug in the example service, not in gocd-jsonnet. But it +reveals a footgun: **any service with a missing comma between stages would +silently lose a stage.** Should add a build-time assertion in `transform_stage` +that each stage object has exactly one key. + +### 2. Stage property conflict validation (still TODO) + +`get_stage_props` takes the first region's properties and ignores the rest. +If `customer-1` defines `approval: manual` and `customer-7` defines +`approval: success` on the same stage in the `st` group, the first region wins +silently. Need ~10 lines of validation to error at build time when regions in +the same group define conflicting stage props. (Also noted in "Stage +Properties" section above.) + +### 3. No `exclude_groups` config option + +`exclude_regions` only operates on region names. To exclude the entire `st` +group you'd need: `exclude_regions: ['customer-1', 'customer-2', 'customer-4', +'customer-7']`. An `exclude_groups: ['st']` option would be cleaner and +wouldn't break when new regions are added to a group. Low priority but nice +to have. + +--- + +## Code Changes on Branch + +- `libs/pipedream.libsonnet`: env var optimization (common vars at stage level), + `pipeline_fn` result caching, extracted `get_matching_stage` helper +- `libs/getsentry.libsonnet`: s4s group → `['s4s2']` only +- `test/pipedream.js`: updated s4s group assertion +- Test fixtures updated to use `st` group for multi-region tests + (`env-vars-precedence`, `stage-props`, `different-stages-per-region`) +- Added `rollback-final-stage-override.jsonnet` fixture +- All golden files regenerated +- Pre-release `v3.0.0-rc.1` cut from `iw/grouped-pipedream` branch diff --git a/status.md b/status.md new file mode 100644 index 0000000..1dd32f4 --- /dev/null +++ b/status.md @@ -0,0 +1,266 @@ +# Pipedream Cellularization — Comprehensive Status + +> Updated 2026-04-09. Working directory: `/Users/mingchen/Desktop/sentry/gocd-jsonnet` +> Branch: `iw/grouped-pipedream` (6 commits ahead of main) + +--- + +## 1. What This Project Does + +Changes the Pipedream deployment model from **one GoCD pipeline per region** to **one pipeline per group**, with regions within each group running as parallel jobs. + +**Before (main):** Sequential pipeline per region: +``` +deploy-example-s4s2 → deploy-example-de → deploy-example-us → deploy-example-customer-1 → ... → deploy-example-customer-7 +``` + +**After (this branch):** Sequential pipeline per group, parallel jobs within: +``` +deploy-example-s4s → deploy-example-de → deploy-example-us → deploy-example-st + (customer-1, -2, -4, -7 as parallel jobs) +``` + +The groups are defined in `libs/getsentry.libsonnet`: +```jsonnet +{ + s4s: ['s4s2'], // single region + de: ['de'], // single region + us: ['us'], // single region + control: ['control'], // default-excluded + 'snty-tools': ['snty-tools'], // default-excluded + st: ['customer-1', 'customer-2', 'customer-4', 'customer-7'], // multi-region +} +``` + +### Pipeline Name Changes (only 2 actual renames) + +| Old Name | New Name | Reason | +|----------|----------|--------| +| `deploy-{service}-s4s2` | `deploy-{service}-s4s` | Group name differs from region | +| `deploy-{service}-customer-{1,2,4,7}` (4 pipelines) | `deploy-{service}-st` (1 pipeline) | Collapsed into group | +| `deploy-{service}-de` | unchanged | Single-region group | +| `deploy-{service}-us` | unchanged | Single-region group | +| `deploy-{service}-control` | unchanged | Single-region group | +| `deploy-{service}-snty-tools` | unchanged | Single-region group | + +--- + +## 2. Branch State + +### Commits on `iw/grouped-pipedream` (6 ahead of main) + +1. `0b17dc0` — ref(pipedream): Adding grouping support +2. `1f4b769` — cascade pipeline and stage level environment variables down to the job level +3. `3bdb415` — ref(pipedream): env var optimization, s4s2 alignment, and test coverage +4. `4394b71` — ref(pipedream): add build-time validation for conflicting stage properties +5. `8403a80` — ref(pipedream): add single-key stage object assertion +6. `cda941a` — update repo readme to reflect grouping (cherry-picked from 98abdde) + +### Uncommitted Changes + +None — all work committed. + +### All 49 tests pass + +47 original + 2 new (stage property conflict detection + merged stage detection). + +### Stashes (all superseded — safe to drop) + +- `stash@{0}` and `stash@{1}`: On `mingchen/di-1685-gocd-static-agent-blocking-for-pipeline-complete`. Older version of env var/s4s2 work — already incorporated and improved upon in commit `3bdb415`. +- `stash@{2}`: On `iw/grouped-pipedream` at older commit `17530cd`. Subset, also superseded. + +--- + +## 3. Key Files Changed + +### `libs/pipedream.libsonnet` (core — ~500 lines) +Complete rewrite of pipeline generation: +- `generate_group_pipeline()` — creates one pipeline per group, aggregates jobs from all regions +- `transform_stage()` — merges jobs from regions, optimizes env vars (common at stage, region-specific at job) +- `get_service_pipelines()` — generates pipelines for all groups +- Caches `pipeline_fn` results per region to avoid redundant calls +- `get_matching_stage()` helper for stage lookup +- **Build-time assertion:** Stage property conflict validation across regions +- **Build-time assertion:** Single-key stage object validation (catches missing commas) + +### `libs/getsentry.libsonnet` (config — 27 lines) +Changed from flat `prod_regions`/`test_regions` arrays to `pipeline_groups` object with `group_order`. Added `group_names`, `test_group_names`, `get_targets(group)` accessors. + +### `README.md` +Updated with grouping docs, typo fixes (cherry-picked from 98abdde). + +### Test fixtures (16 fixtures) +- `basic-autodeploy.jsonnet`, `basic-manual.jsonnet` — core flows +- `multi-region-group.jsonnet` — st group with 4 regions +- `parallel-mode.jsonnet` — fan-out mode +- `exclude-region.jsonnet`, `exclude-entire-group.jsonnet` — exclusion patterns +- `include-default-excluded.jsonnet` — control/snty-tools inclusion +- `env-vars-precedence.jsonnet` — pipeline/stage/job cascading +- `different-stages-per-region.jsonnet` — conditional stages +- `stage-props.jsonnet` — consistent stage properties across regions +- `stage-props-conflict.failing.jsonnet` — conflict detection +- `merged-stages.failing.jsonnet` — single-key assertion +- `rollback.jsonnet`, `rollback-final-stage-override.jsonnet` — rollback config +- `rollback-bad-stage.failing.jsonnet`, `rollback-bad-final-stage.failing.jsonnet` — error cases + +--- + +## 4. Remaining TODOs in gocd-jsonnet + +### Done This Session +- [x] Stage property conflict validation (~18 lines in `transform_stage`) +- [x] Failing fixture + test for conflict detection +- [x] Single-key stage object assertion (~18 lines in `generate_group_pipeline`) +- [x] Failing fixture + test for merged stages +- [x] Cherry-picked README update from commit `98abdde` + +### Done Previously +- [x] Group-based pipeline generation +- [x] Env var cascade optimization +- [x] s4s/s4s2 alignment +- [x] All test fixtures and golden files + +### Still TODO +- [ ] **`test_groups` decision** — On main, `s4s` is a test region (in `test_regions`). Currently `test_groups` is empty. Should `s4s` be added as a test group? +- [ ] **Tag v3.0.0** — No release tag exists yet. Ready to tag once branch is merged or at Ming's discretion. +- [ ] **Push latest commits** — 3 new commits (4394b71, 8403a80, cda941a) need pushing to origin. + +--- + +## 5. devinfra-example-service (Pilot) + +### Comprehensive Pipeline — DONE + +- Repo: `/Users/mingchen/Desktop/sentry/devinfra-example-service/` +- Branch: `mingchen/comprehensive-pipedream-pilot` (1 commit ahead of main) +- Commit: `a3e1d52` — ref(pipedream): comprehensive pilot pipeline exercising all features + +**Features exercised:** + +| Feature | How | +|---------|-----| +| Manual deploy | `auto_deploy: false` → trigger pipeline with manual approval | +| Region-conditional stages | checks (us/de), canary (us/de/control), soak-time (us only) | +| Region-conditional jobs | health-check only in us deploy-primary | +| Pipeline env var cascade | SENTRY_REGION, SENTRY_DEPLOY_ENV differ per region → cascade to jobs | +| Stage env vars | DEPLOY_TIMEOUT common across regions → stays at stage level | +| Job env vars | LABEL_SELECTOR at job level | +| Stage properties | fetch_materials: true, approval: { type: 'success' } | +| Include default-excluded | control region included via `include_regions` | +| Multi-region parallel jobs | st group: customer-1,-2,-4,-7 as parallel jobs | +| Rollback + final_stage override | Rollback watches deploy-primary instead of pipeline-complete | + +**Generated pipelines (7 files):** +``` +deploy-devinfra-example-service.yaml (trigger) +deploy-devinfra-example-service-s4s.yaml (s4s2 — deploy-primary only) +deploy-devinfra-example-service-de.yaml (de — checks, canary, primary) +deploy-devinfra-example-service-us.yaml (us — checks, canary, primary, soak-time + health-check) +deploy-devinfra-example-service-control.yaml (control — canary, primary) +deploy-devinfra-example-service-st.yaml (4 parallel jobs — primary only) +rollback-devinfra-example-service.yaml (rollback with final_stage=deploy-primary) +``` + +### Still TODO for pilot +- [ ] Update `jsonnetfile.json` to point at `v3.0.0` once tagged +- [ ] Run `make gocd` with official version (currently tested with locally copied vendor libs) +- [ ] Push branch and create PR +- [ ] Verify GoCD loads the config correctly + +--- + +## 6. Downstream Impact — Full Inventory + +### Version Pinning +All 71 downstream repos pin gocd-jsonnet at `v2.19.0` via `jsonnetfile.json`. Rollout = bump to `v3.0.0`. + +### Downstream Tool Updates + +| Repo | Severity | Status | What Needs Changing | +|------|----------|--------|---------------------| +| **gocd-deployment-visualizer** | Critical | **DONE** (branch exists) | Branch: `mingchen/di-1670-update-for-grouped-pipedream` (commit `af9b359`). Updated `isPipedreamPipeline()`, `findPrimaryTrigger()`, `PIPEDREAM_SUFFIX_CUSTOMER_MAP`, `isMultiregion()`. 6 files changed. **Needs merge to main.** | +| **eng-pipes** | Critical | **NOT STARTED** | 60+ hardcoded pipeline names in Slack feed filters (`gocdSlackFeeds/index.ts`), `getFormattedRegion()` in `deployDatadogEvents.ts` (parses region from pipeline name suffix for Datadog tags), paused pipeline reminders config, no-deploys alert config, consecutive unsuccessful alert config. **Files:** `src/config/index.ts`, `src/brain/gocd/gocdSlackFeeds/index.ts`, `src/brain/gocd/gocdDataDog/deployDatadogEvents.ts`, `src/brain/gocd/gocdNoDeploysAlert/index.ts`, `src/brain/gocd/gocdConsecutiveUnsuccessfulAlert/index.ts` | +| **sentry-feature-scoring** | High | **NOT STARTED** | Hardcoded `deploy-getsentry-backend-s4s2` in `src/consts.py:28`. `deploy-getsentry-backend-customer-7` completeness check in `src/productivity/metrics/deploy_failure_rate.py:58,131`. Deploy rate/duration queries reference `deploy-getsentry-backend-us` (unchanged, but verify). | +| **sentry-feature-scoring-frontend** | High | **NOT STARTED** | `BACKEND_REGIONS` array, pipeline name construction via template literals. | +| **devinfra-pause-metrics** | Medium | **NOT STARTED** | Region parsing from pipeline suffix, hardcoded region buckets. Repo not found locally. | +| **ops** | Medium | **NOT STARTED** | `dd-event.py` has `REGIONS_MAP` with `customer-*` keys. | + +### Service Repos (71 consumers — just version bump) + +**15 service repos** with their own pipedream templates: +getsentry-backend, relay-pop, relay-processing, snuba-py, snuba-rs, symbolicator, chartcuterie, taskbroker, conduit, vroom, uptime-checker, super-big-consumers, seer, seer-gpu, launchpad, objectstore, tempest, sentry-scripts + +**50+ ops k8s templates** in `/Users/mingchen/Desktop/sentry/ops/gocd/templates/` + +All just need `jsonnetfile.json` bumped from `v2.19.0` → `v3.0.0`. + +--- + +## 7. Linear Project Status + +**Project:** [Pipedream Cellularization](https://linear.app/getsentry/project/pipedream-cellularization-97a9f328cfeb/overview) +- **Lead:** Ming Chen +- **Team:** DevInfra (DI) +- **Target date:** 2026-04-17 +- **Priority:** High +- **Initiative:** CI/CD allows gradual rollouts and easy rollbacks + +**Milestones:** +1. Finalize Implementation (due Mar 6, 100%) +2. deploy-tools Updates (due Mar 27, 100%) +3. Pilot Testing (due Mar 27, 100%) +4. **Release and Rollout** (due Apr 10, 0%) +5. **Rollout Completion and Documentation** (due Apr 16, 0%) + +--- + +## 8. Proposed Rollout Order + +### Phase 0 — Finish gocd-jsonnet — NEARLY DONE +1. ~~Commit stage prop validation + tests~~ DONE +2. ~~Add single-key stage assertion~~ DONE +3. ~~Cherry-pick README update~~ DONE +4. Push 3 new commits to origin +5. Tag `v3.0.0` (after merge or at Ming's discretion) + +### Phase 1 — Update downstream tools (BEFORE any service cuts over) +1. Merge gocd-deployment-visualizer branch (already done, just merge) +2. Update eng-pipes (critical — Slack feeds, Datadog tagging, pipeline filters) +3. Update sentry-feature-scoring + frontend +4. Update devinfra-pause-metrics +5. Update ops dd-event.py +6. During transition, tools should ideally handle BOTH old and new naming + +### Phase 2 — Pilot with devinfra-example-service — PIPELINE READY +1. ~~Build comprehensive pipeline_fn~~ DONE (branch `mingchen/comprehensive-pipedream-pilot`) +2. Update jsonnetfile.json to v3.0.0 (once tagged) +3. Run `make gocd` with official version +4. Push and create PR +5. Verify GoCD loads config, visualizer works, eng-pipes processes events + +### Phase 3 — Production services (progressive) +- **Wave 1 (low-risk):** chartcuterie, tempest, launchpad, devinfra-example-service +- **Wave 2 (medium):** conduit, vroom, uptime-checker, seer, taskbroker, objectstore +- **Wave 3 (critical):** snuba-py, snuba-rs, relay-pop, relay-processing, symbolicator +- **Wave 4 (core):** getsentry-backend, sentry-scripts, super-big-consumers +- Each wave: bump `jsonnetfile.json`, run `make gocd`, merge, verify in GoCD + +### Phase 4 — Cleanup +- Remove backward-compat code from downstream tools +- Clean up orphaned per-region pipeline configs in GoCD +- Update Notion/docs +- Close Linear milestones + +--- + +## 9. Reference: Downstream Repo Locations + +| Repo | Local Path | +|------|-----------| +| gocd-jsonnet | `/Users/mingchen/Desktop/sentry/gocd-jsonnet` | +| gocd-deployment-visualizer | `/Users/mingchen/Desktop/sentry/gocd-deployment-visualizer` | +| eng-pipes | `/Users/mingchen/Desktop/sentry/eng-pipes` (helios) | +| sentry-feature-scoring | `/Users/mingchen/Desktop/sentry/sentry-feature-scoring` | +| devinfra-example-service | `/Users/mingchen/Desktop/sentry/devinfra-example-service` | +| ops | `/Users/mingchen/Desktop/sentry/ops` | +| getsentry | `/Users/mingchen/Desktop/sentry/getsentry` | diff --git a/test/pipedream.js b/test/pipedream.js index 34436e6..8029b3d 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -1,862 +1,168 @@ import test from "ava"; import { render_fixture, get_fixture_content } from "./utils/testdata.js"; -test("ensure manual deploys is expected structure in serial", async (t) => { - const got = await render_fixture( - "pipedream/no-autodeploy-serial.jsonnet", - false, - ); - - t.deepEqual(Object.keys(got), ["format_version", "pipelines"]); - t.truthy(got.pipelines["deploy-example"]); - t.truthy(got.pipelines["deploy-example-s4s2"]); - - // Ensure the trigger has the right initial material - const trigger = got.pipelines["deploy-example"]; - t.deepEqual(trigger.materials, { - init_repo: { - branch: "master", - destination: "init", - git: "git@github.com:getsentry/init.git", - shallow_clone: true, - }, - }); - - // Ensure s4s2 depends on the trigger material (first prod region) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); +test("autodeploy: no trigger pipeline", async (t) => { + const got = await render_fixture("pipedream/basic-autodeploy.jsonnet", false); + t.falsy(got.pipelines["deploy-example"]); }); -test("ensure manual deploys is expected structure in parallel", async (t) => { - const got = await render_fixture( - "pipedream/no-autodeploy-parallel.jsonnet", - false, - ); - - t.deepEqual(Object.keys(got), ["format_version", "pipelines"]); - t.truthy(got.pipelines["deploy-example"]); - t.truthy(got.pipelines["deploy-example-s4s2"]); - - // Ensure the trigger has the right initial material +test("manual deploy: has trigger pipeline with correct materials", async (t) => { + const got = await render_fixture("pipedream/basic-manual.jsonnet", false); const trigger = got.pipelines["deploy-example"]; - t.deepEqual(trigger.materials, { - init_repo: { - branch: "master", - destination: "init", - git: "git@github.com:getsentry/init.git", - shallow_clone: true, - }, - }); - - // Ensure s4s2 depends on the trigger material (first prod region in parallel) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); -}); - -test("ensure auto deploys is expected structure in serial", async (t) => { - const got = await render_fixture( - "pipedream/autodeploy-serial.jsonnet", - false, - ); - t.deepEqual(Object.keys(got), ["format_version", "pipelines"]); - t.falsy(got.pipelines["deploy-example"]); - t.truthy(got.pipelines["deploy-example-s4s2"]); - t.truthy(got.pipelines["rollback-example"]); - - // Ensure s4s2 has just the repo material (first prod region) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Rollback only includes prod regions, not test regions - const r = got.pipelines["rollback-example"]; - t.deepEqual(r["environment_variables"], { - ALL_PIPELINE_FLAGS: - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - GOCD_ACCESS_TOKEN: "{{SECRET:[devinfra][gocd_access_token]}}", - REGION_PIPELINE_FLAGS: - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ROLLBACK_MATERIAL_NAME: "example_repo", - ROLLBACK_STAGE: "example_stage", - TRIGGERED_BY: "", - }); - t.deepEqual(r["materials"], { - "deploy-example-customer-7-pipeline-complete": { - pipeline: "deploy-example-customer-7", - stage: "pipeline-complete", - }, - }); - t.deepEqual(r.stages.length, 3); + t.truthy(trigger); + t.truthy(trigger.materials.init_repo); + t.is(trigger.stages.length, 1); }); -test("ensure auto deploys is expected structure in parallel", async (t) => { - const got = await render_fixture( - "pipedream/autodeploy-parallel.jsonnet", - false, +test("generates pipeline per group in correct order", async (t) => { + const got = await render_fixture("pipedream/basic-autodeploy.jsonnet", false); + + const pipelineNames = Object.keys(got.pipelines).filter((n) => + n.startsWith("deploy-example-"), ); - t.deepEqual(Object.keys(got), ["format_version", "pipelines"]); - t.falsy(got.pipelines["deploy-example"]); - t.truthy(got.pipelines["deploy-example-s4s2"]); - t.truthy(got.pipelines["rollback-example"]); - - // Ensure s4s2 has just the repo material (parallel deploy, no upstream) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Rollback only includes prod regions, not test regions - const r = got.pipelines["rollback-example"]; - t.deepEqual(r["environment_variables"], { - ALL_PIPELINE_FLAGS: - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - GOCD_ACCESS_TOKEN: "{{SECRET:[devinfra][gocd_access_token]}}", - REGION_PIPELINE_FLAGS: - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ROLLBACK_MATERIAL_NAME: "example_repo", - ROLLBACK_STAGE: "example_stage", - TRIGGERED_BY: "", - }); - t.deepEqual(r["materials"], { - "deploy-example-customer-7-pipeline-complete": { - pipeline: "deploy-example-customer-7", - stage: "pipeline-complete", - }, - }); - t.deepEqual(r.stages.length, 3); + // Should have s4s, de, us, st (control/snty-tools excluded by default) + t.true(pipelineNames.includes("deploy-example-s4s")); + t.true(pipelineNames.includes("deploy-example-de")); + t.true(pipelineNames.includes("deploy-example-us")); + t.true(pipelineNames.includes("deploy-example-st")); + t.false(pipelineNames.includes("deploy-example-control")); }); -test("ensure exclude regions removes regions without trigger pipeline in serial", async (t) => { - const got = await render_fixture( - "pipedream/exclude-regions-autodeploy-serial.jsonnet", - false, - ); - - t.deepEqual(Object.keys(got.pipelines).sort(), [ - "deploy-example-customer-1", - "deploy-example-customer-2", - "deploy-example-customer-4", - "deploy-example-customer-7", - "deploy-example-de", - "deploy-example-s4s2", - "rollback-example", +test("multi-region group has parallel jobs for each region", async (t) => { + const got = await render_fixture("pipedream/basic-autodeploy.jsonnet", false); + + // s4s group has s4s + s4s2 regions + const s4s = got.pipelines["deploy-example-s4s"]; + const s4sJobs = Object.keys(s4s.stages[0].deploy.jobs); + t.deepEqual(s4sJobs.sort(), ["deploy-s4s", "deploy-s4s2"]); + + // st group has customer-1, customer-2, customer-4, customer-7 + const st = got.pipelines["deploy-example-st"]; + const stJobs = Object.keys(st.stages[0].deploy.jobs); + t.deepEqual(stJobs.sort(), [ + "deploy-customer-1", + "deploy-customer-2", + "deploy-customer-4", + "deploy-customer-7", ]); - - // Ensure s4s2 has just the repo material (first region after exclusions) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure de depends on s4s2 - const de = got.pipelines["deploy-example-de"]; - t.deepEqual(de.materials, { - "deploy-example-s4s2-pipeline-complete": { - pipeline: "deploy-example-s4s2", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-2 has pipeline material too - const c2 = got.pipelines["deploy-example-customer-2"]; - t.deepEqual(c2.materials, { - "deploy-example-customer-1-pipeline-complete": { - pipeline: "deploy-example-customer-1", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure rollback has the expected rollback pipelines - const r = got.pipelines["rollback-example"]; - const allPipelines = r.environment_variables["ALL_PIPELINE_FLAGS"]; - const regionPipelines = r.environment_variables["REGION_PIPELINE_FLAGS"]; - t.deepEqual( - allPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); - t.deepEqual( - regionPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); }); -test("ensure exclude regions removes regions without trigger pipeline in parallel", async (t) => { - const got = await render_fixture( - "pipedream/exclude-regions-autodeploy-parallel.jsonnet", - false, - ); - - t.deepEqual(Object.keys(got.pipelines).sort(), [ - "deploy-example-customer-1", - "deploy-example-customer-2", - "deploy-example-customer-4", - "deploy-example-customer-7", - "deploy-example-de", - "deploy-example-s4s2", - "rollback-example", - ]); +test("single-region group has one job", async (t) => { + const got = await render_fixture("pipedream/basic-autodeploy.jsonnet", false); - // Ensure s4s2 has just the repo material (parallel deploy) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure de has just the repo material (parallel deploy) const de = got.pipelines["deploy-example-de"]; - t.deepEqual(de.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-2 has just the repo material (parallel deploy) - const c2 = got.pipelines["deploy-example-customer-2"]; - t.deepEqual(c2.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure rollback has the expected rollback pipelines - const r = got.pipelines["rollback-example"]; - const allPipelines = r.environment_variables["ALL_PIPELINE_FLAGS"]; - const regionPipelines = r.environment_variables["REGION_PIPELINE_FLAGS"]; - t.deepEqual( - allPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); - t.deepEqual( - regionPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); + const deJobs = Object.keys(de.stages[0].deploy.jobs); + t.deepEqual(deJobs, ["deploy-de"]); }); -test("ensure exclude regions removes regions with trigger pipeline in serial", async (t) => { - const got = await render_fixture( - "pipedream/exclude-regions-no-autodeploy-serial.jsonnet", - false, - ); +test("serial mode: pipelines chain sequentially", async (t) => { + const got = await render_fixture("pipedream/basic-manual.jsonnet", false); - t.deepEqual(Object.keys(got.pipelines).sort(), [ - "deploy-example", - "deploy-example-customer-1", - "deploy-example-customer-2", - "deploy-example-customer-4", - "deploy-example-customer-7", - "deploy-example-de", - "deploy-example-s4s2", - "rollback-example", - ]); + // s4s depends on trigger + const s4s = got.pipelines["deploy-example-s4s"]; + t.truthy(s4s.materials["deploy-example-pipeline-complete"]); - // Ensure s4s2 depends on trigger (first region after exclusions, serial) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure de depends on s4s2 + // de depends on s4s const de = got.pipelines["deploy-example-de"]; - t.deepEqual(de.materials, { - "deploy-example-s4s2-pipeline-complete": { - pipeline: "deploy-example-s4s2", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-2 has pipeline material too - const c2 = got.pipelines["deploy-example-customer-2"]; - t.deepEqual(c2.materials, { - "deploy-example-customer-1-pipeline-complete": { - pipeline: "deploy-example-customer-1", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure rollback has the expected rollback pipelines - const r = got.pipelines["rollback-example"]; - const allPipelines = r.environment_variables["ALL_PIPELINE_FLAGS"]; - const regionPipelines = r.environment_variables["REGION_PIPELINE_FLAGS"]; - t.deepEqual( - allPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - ); - t.deepEqual( - regionPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); -}); + t.truthy(de.materials["deploy-example-s4s-pipeline-complete"]); -test("ensure exclude regions removes regions with trigger pipeline in parallel", async (t) => { - const got = await render_fixture( - "pipedream/exclude-regions-no-autodeploy-parallel.jsonnet", - false, - ); + // us depends on de + const us = got.pipelines["deploy-example-us"]; + t.truthy(us.materials["deploy-example-de-pipeline-complete"]); +}); - t.deepEqual(Object.keys(got.pipelines).sort(), [ - "deploy-example", - "deploy-example-customer-1", - "deploy-example-customer-2", - "deploy-example-customer-4", - "deploy-example-customer-7", - "deploy-example-de", - "deploy-example-s4s2", - "rollback-example", - ]); +test("parallel mode: all pipelines depend on trigger only", async (t) => { + const got = await render_fixture("pipedream/parallel-mode.jsonnet", false); - // Ensure s4s2 depends on trigger (parallel deploy) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure de depends on trigger (parallel deploy) + const s4s = got.pipelines["deploy-example-s4s"]; const de = got.pipelines["deploy-example-de"]; - t.deepEqual(de.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-2 depends on trigger (parallel deploy) - const c2 = got.pipelines["deploy-example-customer-2"]; - t.deepEqual(c2.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure rollback has the expected rollback pipelines - const r = got.pipelines["rollback-example"]; - const allPipelines = r.environment_variables["ALL_PIPELINE_FLAGS"]; - const regionPipelines = r.environment_variables["REGION_PIPELINE_FLAGS"]; - t.deepEqual( - allPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - ); - t.deepEqual( - regionPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); -}); + const us = got.pipelines["deploy-example-us"]; -test("ensure include regions adds regions without trigger pipeline in serial", async (t) => { - const got = await render_fixture( - "pipedream/include-regions-autodeploy-serial.jsonnet", - false, - ); + // All depend on trigger + t.truthy(s4s.materials["deploy-example-pipeline-complete"]); + t.truthy(de.materials["deploy-example-pipeline-complete"]); + t.truthy(us.materials["deploy-example-pipeline-complete"]); - t.deepEqual(Object.keys(got.pipelines).sort(), [ - "deploy-example-control", - "deploy-example-customer-1", - "deploy-example-customer-2", - "deploy-example-customer-4", - "deploy-example-customer-7", - "deploy-example-de", - "deploy-example-s4s2", - "rollback-example", - ]); - - // Ensure s4s2 has just the repo material (first region) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure de depends on s4s2 - const de = got.pipelines["deploy-example-de"]; - t.deepEqual(de.materials, { - "deploy-example-s4s2-pipeline-complete": { - pipeline: "deploy-example-s4s2", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure control depends on de - const control = got.pipelines["deploy-example-control"]; - t.deepEqual(control.materials, { - "deploy-example-de-pipeline-complete": { - pipeline: "deploy-example-de", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-1 depends on control - const c1 = got.pipelines["deploy-example-customer-1"]; - t.deepEqual(c1.materials, { - "deploy-example-control-pipeline-complete": { - pipeline: "deploy-example-control", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-2 has pipeline material too - const c2 = got.pipelines["deploy-example-customer-2"]; - t.deepEqual(c2.materials, { - "deploy-example-customer-1-pipeline-complete": { - pipeline: "deploy-example-customer-1", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure rollback has the expected rollback pipelines - const r = got.pipelines["rollback-example"]; - const allPipelines = r.environment_variables["ALL_PIPELINE_FLAGS"]; - const regionPipelines = r.environment_variables["REGION_PIPELINE_FLAGS"]; - t.deepEqual( - allPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); - t.deepEqual( - regionPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); + // None depend on each other + t.falsy(de.materials["deploy-example-s4s-pipeline-complete"]); + t.falsy(us.materials["deploy-example-de-pipeline-complete"]); }); -test("ensure include regions adds regions without trigger pipeline in parallel", async (t) => { - const got = await render_fixture( - "pipedream/include-regions-autodeploy-parallel.jsonnet", - false, - ); +test("exclude region: removes job but keeps group", async (t) => { + const got = await render_fixture("pipedream/exclude-region.jsonnet", false); - t.deepEqual(Object.keys(got.pipelines).sort(), [ - "deploy-example-control", - "deploy-example-customer-1", - "deploy-example-customer-2", - "deploy-example-customer-4", - "deploy-example-customer-7", - "deploy-example-de", - "deploy-example-s4s2", - "rollback-example", - ]); + const st = got.pipelines["deploy-example-st"]; + const jobs = Object.keys(st.stages[0].deploy.jobs); - // Ensure s4s2 has just the repo material (parallel deploy) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure de has just the repo material (parallel deploy) - const de = got.pipelines["deploy-example-de"]; - t.deepEqual(de.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure control has just the repo material (parallel deploy) - const control = got.pipelines["deploy-example-control"]; - t.deepEqual(control.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-1 has just the repo material (parallel deploy) - const c1 = got.pipelines["deploy-example-customer-1"]; - t.deepEqual(c1.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-2 has just the repo material (parallel deploy) - const c2 = got.pipelines["deploy-example-customer-2"]; - t.deepEqual(c2.materials, { - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure rollback has the expected rollback pipelines - const r = got.pipelines["rollback-example"]; - const allPipelines = r.environment_variables["ALL_PIPELINE_FLAGS"]; - const regionPipelines = r.environment_variables["REGION_PIPELINE_FLAGS"]; - t.deepEqual( - allPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); - t.deepEqual( - regionPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - ); + t.false(jobs.includes("deploy-customer-2")); + t.true(jobs.includes("deploy-customer-1")); + t.true(jobs.includes("deploy-customer-4")); + t.true(jobs.includes("deploy-customer-7")); }); -test("ensure include regions adds regions with trigger pipeline in parallel", async (t) => { +test("exclude all regions in group: skips entire group", async (t) => { const got = await render_fixture( - "pipedream/include-regions-no-autodeploy-parallel.jsonnet", + "pipedream/exclude-entire-group.jsonnet", false, ); - t.deepEqual(Object.keys(got.pipelines).sort(), [ - "deploy-example", - "deploy-example-control", - "deploy-example-customer-1", - "deploy-example-customer-2", - "deploy-example-customer-4", - "deploy-example-customer-7", - "deploy-example-de", - "deploy-example-s4s2", - "rollback-example", - ]); - - // Ensure s4s2 depends on the trigger (parallel deploy) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure de depends on the trigger (parallel deploy) - const de = got.pipelines["deploy-example-de"]; - t.deepEqual(de.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure control depends on the trigger (parallel deploy) - const control = got.pipelines["deploy-example-control"]; - t.deepEqual(control.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-1 depends on the trigger (parallel deploy) - const c1 = got.pipelines["deploy-example-customer-1"]; - t.deepEqual(c1.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-2 depends on the trigger (parallel deploy) - const c2 = got.pipelines["deploy-example-customer-2"]; - t.deepEqual(c2.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); + t.falsy(got.pipelines["deploy-example-st"]); + t.truthy(got.pipelines["deploy-example-s4s"]); }); -test("ensure include regions adds regions with trigger pipeline in serial", async (t) => { +test("include region: adds default-excluded group", async (t) => { const got = await render_fixture( - "pipedream/include-regions-no-autodeploy-serial.jsonnet", + "pipedream/include-default-excluded.jsonnet", false, ); - t.deepEqual(Object.keys(got.pipelines).sort(), [ - "deploy-example", - "deploy-example-control", - "deploy-example-customer-1", - "deploy-example-customer-2", - "deploy-example-customer-4", - "deploy-example-customer-7", - "deploy-example-de", - "deploy-example-s4s2", - "rollback-example", - ]); + t.truthy(got.pipelines["deploy-example-control"]); +}); + +test("rollback pipeline structure", async (t) => { + const got = await render_fixture("pipedream/rollback.jsonnet", false); - // Ensure s4s2 depends on the trigger (first region) - const s4s2 = got.pipelines["deploy-example-s4s2"]; - t.deepEqual(s4s2.materials, { - "deploy-example-pipeline-complete": { - pipeline: "deploy-example", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure de depends on s4s2 - const de = got.pipelines["deploy-example-de"]; - t.deepEqual(de.materials, { - "deploy-example-s4s2-pipeline-complete": { - pipeline: "deploy-example-s4s2", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure control depends on de - const control = got.pipelines["deploy-example-control"]; - t.deepEqual(control.materials, { - "deploy-example-de-pipeline-complete": { - pipeline: "deploy-example-de", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-1 depends on control - const c1 = got.pipelines["deploy-example-customer-1"]; - t.deepEqual(c1.materials, { - "deploy-example-control-pipeline-complete": { - pipeline: "deploy-example-control", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure customer-2 has pipeline material too - const c2 = got.pipelines["deploy-example-customer-2"]; - t.deepEqual(c2.materials, { - "deploy-example-customer-1-pipeline-complete": { - pipeline: "deploy-example-customer-1", - stage: "pipeline-complete", - }, - example_repo: { - branch: "master", - destination: "example", - git: "git@github.com:getsentry/example.git", - shallow_clone: true, - }, - }); - - // Ensure rollback has the expected rollback pipelines const r = got.pipelines["rollback-example"]; - const allPipelines = r.environment_variables["ALL_PIPELINE_FLAGS"]; - const regionPipelines = r.environment_variables["REGION_PIPELINE_FLAGS"]; - t.deepEqual( - allPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", + t.truthy(r); + t.is(r.stages.length, 3); + t.is(r.environment_variables.ROLLBACK_STAGE, "deploy"); + t.truthy(r.environment_variables.REGION_PIPELINE_FLAGS); + t.truthy(r.environment_variables.ALL_PIPELINE_FLAGS); +}); + +test("rollback: invalid stage errors", (t) => { + const err = t.throws(() => + get_fixture_content("pipedream/rollback-bad-stage.failing.jsonnet", false), ); - t.deepEqual( - regionPipelines, - "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", + t.true( + err.message.includes("Stage 'this-stage-does-not-exist' does not exist"), ); }); -test("error for invalid final rollback stage", async (t) => { +test("rollback: invalid final stage errors", (t) => { const err = t.throws(() => get_fixture_content( "pipedream/rollback-bad-final-stage.failing.jsonnet", false, ), ); - t.truthy( - err?.message.includes( - "RUNTIME ERROR: Stage 'this-stage-does-not-exist' does not exist", - ), + t.true( + err.message.includes("Stage 'this-stage-does-not-exist' does not exist"), ); }); -test("error for invalid rollback stage", async (t) => { - const err = t.throws(() => - get_fixture_content("pipedream/rollback-bad-stage.failing.jsonnet", false), - ); - t.truthy( - err?.message.includes( - "RUNTIME ERROR: Stage 'this-stage-does-not-exist' does not exist", - ), - ); +test("all pipelines end with pipeline-complete stage", async (t) => { + const got = await render_fixture("pipedream/basic-autodeploy.jsonnet", false); + + for (const [name, pipeline] of Object.entries(got.pipelines)) { + const lastStage = pipeline.stages[pipeline.stages.length - 1]; + const lastStageName = Object.keys(lastStage)[0]; + t.is( + lastStageName, + "pipeline-complete", + `${name} should end with pipeline-complete`, + ); + } }); diff --git a/test/testdata/fixtures/getsentry/groups.jsonnet b/test/testdata/fixtures/getsentry/groups.jsonnet new file mode 100644 index 0000000..e952c82 --- /dev/null +++ b/test/testdata/fixtures/getsentry/groups.jsonnet @@ -0,0 +1,6 @@ +local getsentry = import '../../../../libs/getsentry.libsonnet'; + +{ + all_groups: getsentry.pipeline_groups + getsentry.test_groups, + group_order: getsentry.group_order + getsentry.test_group_order, +} diff --git a/test/testdata/fixtures/getsentry/regions.jsonnet b/test/testdata/fixtures/getsentry/regions.jsonnet deleted file mode 100644 index 9375e63..0000000 --- a/test/testdata/fixtures/getsentry/regions.jsonnet +++ /dev/null @@ -1,5 +0,0 @@ -local getsentry = import '../../../../libs/getsentry.libsonnet'; - -{ - all_regions: getsentry.prod_regions + getsentry.test_regions, -} diff --git a/test/testdata/fixtures/pipedream/autodeploy-parallel.jsonnet b/test/testdata/fixtures/pipedream/autodeploy-parallel.jsonnet deleted file mode 100644 index 771ae1f..0000000 --- a/test/testdata/fixtures/pipedream/autodeploy-parallel.jsonnet +++ /dev/null @@ -1,40 +0,0 @@ -local pipedream = import '../../../../libs/pipedream.libsonnet'; - -local pipedream_config = { - name: 'example', - auto_deploy: true, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - shallow_clone: true, - branch: 'master', - destination: 'init', - }, - }, -}; - -local sample = { - pipeline(region):: { - region: region, - materials: { - example_repo: { - git: 'git@github.com:getsentry/example.git', - shallow_clone: true, - branch: 'master', - destination: 'example', - }, - }, - stages: [ - { - example_stage: {}, - }, - ], - }, -}; - -pipedream.render(pipedream_config, sample.pipeline, parallel=true) diff --git a/test/testdata/fixtures/pipedream/minimal-config.jsonnet b/test/testdata/fixtures/pipedream/basic-autodeploy.jsonnet similarity index 61% rename from test/testdata/fixtures/pipedream/minimal-config.jsonnet rename to test/testdata/fixtures/pipedream/basic-autodeploy.jsonnet index dbae065..de990ad 100644 --- a/test/testdata/fixtures/pipedream/minimal-config.jsonnet +++ b/test/testdata/fixtures/pipedream/basic-autodeploy.jsonnet @@ -2,29 +2,30 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - branch: 'master', - destination: 'init', - }, - }, + auto_deploy: true, }; local sample = { pipeline(region):: { - region: region, materials: { example_repo: { git: 'git@github.com:getsentry/example.git', - shallow_clone: true, branch: 'master', destination: 'example', }, }, stages: [ { - example_stage: {}, + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, }, ], }, diff --git a/test/testdata/fixtures/pipedream/exclude-regions-no-autodeploy-serial.jsonnet b/test/testdata/fixtures/pipedream/basic-manual.jsonnet similarity index 70% rename from test/testdata/fixtures/pipedream/exclude-regions-no-autodeploy-serial.jsonnet rename to test/testdata/fixtures/pipedream/basic-manual.jsonnet index 008cd1d..7db9110 100644 --- a/test/testdata/fixtures/pipedream/exclude-regions-no-autodeploy-serial.jsonnet +++ b/test/testdata/fixtures/pipedream/basic-manual.jsonnet @@ -3,7 +3,6 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', auto_deploy: false, - exclude_regions: ['us'], materials: { init_repo: { git: 'git@github.com:getsentry/init.git', @@ -11,27 +10,29 @@ local pipedream_config = { destination: 'init', }, }, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, }; local sample = { pipeline(region):: { - region: region, materials: { example_repo: { git: 'git@github.com:getsentry/example.git', - shallow_clone: true, branch: 'master', destination: 'example', }, }, stages: [ { - example_stage: {}, + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, }, ], }, diff --git a/test/testdata/fixtures/pipedream/different-stages-per-region.jsonnet b/test/testdata/fixtures/pipedream/different-stages-per-region.jsonnet new file mode 100644 index 0000000..88c5b1b --- /dev/null +++ b/test/testdata/fixtures/pipedream/different-stages-per-region.jsonnet @@ -0,0 +1,44 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +local config = { + name: 'example', + materials: { + init_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'main', + }, + }, +}; + +// This pipeline_fn returns DIFFERENT stages depending on region +local pipeline_fn(region) = { + materials: config.materials, + stages: if region == 's4s' then [ + // s4s only has deploy stage + { + deploy: { + jobs: { + deploy: { tasks: [{ exec: { command: 'echo', arguments: ['deploy ' + region] } }] }, + }, + }, + }, + ] else [ + // s4s2 has deploy AND verify stages + { + deploy: { + jobs: { + deploy: { tasks: [{ exec: { command: 'echo', arguments: ['deploy ' + region] } }] }, + }, + }, + }, + { + verify: { + jobs: { + verify: { tasks: [{ exec: { command: 'echo', arguments: ['verify ' + region] } }] }, + }, + }, + }, + ], +}; + +pipedream.render(config, pipeline_fn) diff --git a/test/testdata/fixtures/pipedream/exclude-entire-group.jsonnet b/test/testdata/fixtures/pipedream/exclude-entire-group.jsonnet new file mode 100644 index 0000000..48c6ee9 --- /dev/null +++ b/test/testdata/fixtures/pipedream/exclude-entire-group.jsonnet @@ -0,0 +1,36 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +// Exclude all customer regions → st group is skipped entirely +local pipedream_config = { + name: 'example', + auto_deploy: true, + exclude_regions: ['customer-1', 'customer-2', 'customer-4', 'customer-7'], +}; + +local sample = { + pipeline(region):: { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + destination: 'example', + }, + }, + stages: [ + { + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, + }, + ], + }, +}; + +pipedream.render(pipedream_config, sample.pipeline) diff --git a/test/testdata/fixtures/pipedream/exclude-regions-autodeploy-serial.jsonnet b/test/testdata/fixtures/pipedream/exclude-region.jsonnet similarity index 53% rename from test/testdata/fixtures/pipedream/exclude-regions-autodeploy-serial.jsonnet rename to test/testdata/fixtures/pipedream/exclude-region.jsonnet index 122d42a..5641fe6 100644 --- a/test/testdata/fixtures/pipedream/exclude-regions-autodeploy-serial.jsonnet +++ b/test/testdata/fixtures/pipedream/exclude-region.jsonnet @@ -2,35 +2,31 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', - exclude_regions: ['us'], - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - branch: 'master', - destination: 'init', - }, - }, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, + auto_deploy: true, + exclude_regions: ['customer-2'], }; local sample = { pipeline(region):: { - region: region, materials: { example_repo: { git: 'git@github.com:getsentry/example.git', - shallow_clone: true, branch: 'master', destination: 'example', }, }, stages: [ { - example_stage: {}, + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, }, ], }, diff --git a/test/testdata/fixtures/pipedream/exclude-regions-autodeploy-parallel.jsonnet b/test/testdata/fixtures/pipedream/exclude-regions-autodeploy-parallel.jsonnet deleted file mode 100644 index 88d5bee..0000000 --- a/test/testdata/fixtures/pipedream/exclude-regions-autodeploy-parallel.jsonnet +++ /dev/null @@ -1,39 +0,0 @@ -local pipedream = import '../../../../libs/pipedream.libsonnet'; - -local pipedream_config = { - name: 'example', - exclude_regions: ['us'], - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - branch: 'master', - destination: 'init', - }, - }, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, -}; - -local sample = { - pipeline(region):: { - region: region, - materials: { - example_repo: { - git: 'git@github.com:getsentry/example.git', - shallow_clone: true, - branch: 'master', - destination: 'example', - }, - }, - stages: [ - { - example_stage: {}, - }, - ], - }, -}; - -pipedream.render(pipedream_config, sample.pipeline, parallel=true) diff --git a/test/testdata/fixtures/pipedream/include-regions-autodeploy-serial.jsonnet b/test/testdata/fixtures/pipedream/include-default-excluded.jsonnet similarity index 54% rename from test/testdata/fixtures/pipedream/include-regions-autodeploy-serial.jsonnet rename to test/testdata/fixtures/pipedream/include-default-excluded.jsonnet index ee985e1..71edcd8 100644 --- a/test/testdata/fixtures/pipedream/include-regions-autodeploy-serial.jsonnet +++ b/test/testdata/fixtures/pipedream/include-default-excluded.jsonnet @@ -2,36 +2,31 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', + auto_deploy: true, include_regions: ['control'], - exclude_regions: ['us'], - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - branch: 'master', - destination: 'init', - }, - }, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, }; local sample = { pipeline(region):: { - region: region, materials: { example_repo: { git: 'git@github.com:getsentry/example.git', - shallow_clone: true, branch: 'master', destination: 'example', }, }, stages: [ { - example_stage: {}, + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, }, ], }, diff --git a/test/testdata/fixtures/pipedream/include-regions-autodeploy-parallel.jsonnet b/test/testdata/fixtures/pipedream/include-regions-autodeploy-parallel.jsonnet deleted file mode 100644 index 3b26669..0000000 --- a/test/testdata/fixtures/pipedream/include-regions-autodeploy-parallel.jsonnet +++ /dev/null @@ -1,40 +0,0 @@ -local pipedream = import '../../../../libs/pipedream.libsonnet'; - -local pipedream_config = { - name: 'example', - include_regions: ['control'], - exclude_regions: ['us'], - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - branch: 'master', - destination: 'init', - }, - }, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, -}; - -local sample = { - pipeline(region):: { - region: region, - materials: { - example_repo: { - git: 'git@github.com:getsentry/example.git', - shallow_clone: true, - branch: 'master', - destination: 'example', - }, - }, - stages: [ - { - example_stage: {}, - }, - ], - }, -}; - -pipedream.render(pipedream_config, sample.pipeline, parallel=true) diff --git a/test/testdata/fixtures/pipedream/include-regions-no-autodeploy-parallel.jsonnet b/test/testdata/fixtures/pipedream/include-regions-no-autodeploy-parallel.jsonnet deleted file mode 100644 index 8cf63b3..0000000 --- a/test/testdata/fixtures/pipedream/include-regions-no-autodeploy-parallel.jsonnet +++ /dev/null @@ -1,41 +0,0 @@ -local pipedream = import '../../../../libs/pipedream.libsonnet'; - -local pipedream_config = { - name: 'example', - auto_deploy: false, - include_regions: ['control'], - exclude_regions: ['us'], - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - branch: 'master', - destination: 'init', - }, - }, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, -}; - -local sample = { - pipeline(region):: { - region: region, - materials: { - example_repo: { - git: 'git@github.com:getsentry/example.git', - shallow_clone: true, - branch: 'master', - destination: 'example', - }, - }, - stages: [ - { - example_stage: {}, - }, - ], - }, -}; - -pipedream.render(pipedream_config, sample.pipeline, parallel=true) diff --git a/test/testdata/fixtures/pipedream/include-regions-no-autodeploy-serial.jsonnet b/test/testdata/fixtures/pipedream/include-regions-no-autodeploy-serial.jsonnet deleted file mode 100644 index 8904a8c..0000000 --- a/test/testdata/fixtures/pipedream/include-regions-no-autodeploy-serial.jsonnet +++ /dev/null @@ -1,41 +0,0 @@ -local pipedream = import '../../../../libs/pipedream.libsonnet'; - -local pipedream_config = { - name: 'example', - auto_deploy: false, - include_regions: ['control'], - exclude_regions: ['us'], - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - branch: 'master', - destination: 'init', - }, - }, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, -}; - -local sample = { - pipeline(region):: { - region: region, - materials: { - example_repo: { - git: 'git@github.com:getsentry/example.git', - shallow_clone: true, - branch: 'master', - destination: 'example', - }, - }, - stages: [ - { - example_stage: {}, - }, - ], - }, -}; - -pipedream.render(pipedream_config, sample.pipeline) diff --git a/test/testdata/fixtures/pipedream/include-regions-parallel.jsonnet b/test/testdata/fixtures/pipedream/include-regions-parallel.jsonnet deleted file mode 100644 index 3b26669..0000000 --- a/test/testdata/fixtures/pipedream/include-regions-parallel.jsonnet +++ /dev/null @@ -1,40 +0,0 @@ -local pipedream = import '../../../../libs/pipedream.libsonnet'; - -local pipedream_config = { - name: 'example', - include_regions: ['control'], - exclude_regions: ['us'], - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - branch: 'master', - destination: 'init', - }, - }, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, -}; - -local sample = { - pipeline(region):: { - region: region, - materials: { - example_repo: { - git: 'git@github.com:getsentry/example.git', - shallow_clone: true, - branch: 'master', - destination: 'example', - }, - }, - stages: [ - { - example_stage: {}, - }, - ], - }, -}; - -pipedream.render(pipedream_config, sample.pipeline, parallel=true) diff --git a/test/testdata/fixtures/pipedream/multi-region-group.jsonnet b/test/testdata/fixtures/pipedream/multi-region-group.jsonnet new file mode 100644 index 0000000..67bd27d --- /dev/null +++ b/test/testdata/fixtures/pipedream/multi-region-group.jsonnet @@ -0,0 +1,36 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +// Only render s4s and st to focus on multi-region groups +local pipedream_config = { + name: 'example', + auto_deploy: true, + exclude_regions: ['de', 'us'], +}; + +local sample = { + pipeline(region):: { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + destination: 'example', + }, + }, + stages: [ + { + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, + }, + ], + }, +}; + +pipedream.render(pipedream_config, sample.pipeline) diff --git a/test/testdata/fixtures/pipedream/no-autodeploy-parallel.jsonnet b/test/testdata/fixtures/pipedream/no-autodeploy-parallel.jsonnet deleted file mode 100644 index 86a8025..0000000 --- a/test/testdata/fixtures/pipedream/no-autodeploy-parallel.jsonnet +++ /dev/null @@ -1,41 +0,0 @@ -local pipedream = import '../../../../libs/pipedream.libsonnet'; - -local pipedream_config = { - name: 'example', - auto_deploy: false, - auto_pipeline_progression: false, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - shallow_clone: true, - branch: 'master', - destination: 'init', - }, - }, -}; - -local sample = { - pipeline(region):: { - region: region, - materials: { - example_repo: { - git: 'git@github.com:getsentry/example.git', - shallow_clone: true, - branch: 'master', - destination: 'example', - }, - }, - stages: [ - { - example_stage: {}, - }, - ], - }, -}; - -pipedream.render(pipedream_config, sample.pipeline, parallel=true) diff --git a/test/testdata/fixtures/pipedream/no-autodeploy-serial.jsonnet b/test/testdata/fixtures/pipedream/no-autodeploy-serial.jsonnet deleted file mode 100644 index 5ea18ed..0000000 --- a/test/testdata/fixtures/pipedream/no-autodeploy-serial.jsonnet +++ /dev/null @@ -1,41 +0,0 @@ -local pipedream = import '../../../../libs/pipedream.libsonnet'; - -local pipedream_config = { - name: 'example', - auto_deploy: false, - auto_pipeline_progression: false, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - shallow_clone: true, - branch: 'master', - destination: 'init', - }, - }, -}; - -local sample = { - pipeline(region):: { - region: region, - materials: { - example_repo: { - git: 'git@github.com:getsentry/example.git', - shallow_clone: true, - branch: 'master', - destination: 'example', - }, - }, - stages: [ - { - example_stage: {}, - }, - ], - }, -}; - -pipedream.render(pipedream_config, sample.pipeline) diff --git a/test/testdata/fixtures/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet b/test/testdata/fixtures/pipedream/parallel-mode.jsonnet similarity index 71% rename from test/testdata/fixtures/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet rename to test/testdata/fixtures/pipedream/parallel-mode.jsonnet index 69c8f8d..ed7991e 100644 --- a/test/testdata/fixtures/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet +++ b/test/testdata/fixtures/pipedream/parallel-mode.jsonnet @@ -3,7 +3,6 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', auto_deploy: false, - exclude_regions: ['us'], materials: { init_repo: { git: 'git@github.com:getsentry/init.git', @@ -11,27 +10,29 @@ local pipedream_config = { destination: 'init', }, }, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, }; local sample = { pipeline(region):: { - region: region, materials: { example_repo: { git: 'git@github.com:getsentry/example.git', - shallow_clone: true, branch: 'master', destination: 'example', }, }, stages: [ { - example_stage: {}, + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, }, ], }, diff --git a/test/testdata/fixtures/pipedream/rollback-bad-final-stage.failing.jsonnet b/test/testdata/fixtures/pipedream/rollback-bad-final-stage.failing.jsonnet index 632c535..ad06653 100644 --- a/test/testdata/fixtures/pipedream/rollback-bad-final-stage.failing.jsonnet +++ b/test/testdata/fixtures/pipedream/rollback-bad-final-stage.failing.jsonnet @@ -5,39 +5,33 @@ local pipedream_config = { auto_deploy: true, rollback: { material_name: 'example_repo', - stage: 'example_stage', - // NOTE: This should only be used during a transition where the final stage - // of a pipeline is will impact rollbacks + stage: 'deploy', final_stage: 'this-stage-does-not-exist', - elastic_profile_id: 'example_profile', - }, - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - shallow_clone: true, - branch: 'master', - destination: 'init', - }, + elastic_profile_id: 'example', }, }; local sample = { pipeline(region):: { - region: region, materials: { example_repo: { git: 'git@github.com:getsentry/example.git', - shallow_clone: true, branch: 'master', destination: 'example', }, }, stages: [ { - example_stage: {}, - }, - { - other_stage: {}, + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, }, ], }, diff --git a/test/testdata/fixtures/pipedream/rollback-bad-stage.failing.jsonnet b/test/testdata/fixtures/pipedream/rollback-bad-stage.failing.jsonnet index 96ace81..739975a 100644 --- a/test/testdata/fixtures/pipedream/rollback-bad-stage.failing.jsonnet +++ b/test/testdata/fixtures/pipedream/rollback-bad-stage.failing.jsonnet @@ -6,35 +6,31 @@ local pipedream_config = { rollback: { material_name: 'example_repo', stage: 'this-stage-does-not-exist', - elastic_profile_id: 'example_profile', - }, - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - shallow_clone: true, - branch: 'master', - destination: 'init', - }, + elastic_profile_id: 'example', }, }; local sample = { pipeline(region):: { - region: region, materials: { example_repo: { git: 'git@github.com:getsentry/example.git', - shallow_clone: true, branch: 'master', destination: 'example', }, }, stages: [ { - example_stage: {}, - }, - { - other_stage: {}, + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, }, ], }, diff --git a/test/testdata/fixtures/pipedream/rollback-override-final-stage.jsonnet b/test/testdata/fixtures/pipedream/rollback-override-final-stage.jsonnet deleted file mode 100644 index b7a5a10..0000000 --- a/test/testdata/fixtures/pipedream/rollback-override-final-stage.jsonnet +++ /dev/null @@ -1,46 +0,0 @@ -local pipedream = import '../../../../libs/pipedream.libsonnet'; - -local pipedream_config = { - name: 'example', - auto_deploy: true, - rollback: { - material_name: 'example_repo', - stage: 'example_stage', - // NOTE: This should only be used during a transition where the final stage - // of a pipeline is will impact rollbacks - final_stage: 'other_stage', - elastic_profile_id: 'example_profile', - }, - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - shallow_clone: true, - branch: 'master', - destination: 'init', - }, - }, -}; - -local sample = { - pipeline(region):: { - region: region, - materials: { - example_repo: { - git: 'git@github.com:getsentry/example.git', - shallow_clone: true, - branch: 'master', - destination: 'example', - }, - }, - stages: [ - { - example_stage: {}, - }, - { - other_stage: {}, - }, - ], - }, -}; - -pipedream.render(pipedream_config, sample.pipeline) diff --git a/test/testdata/fixtures/pipedream/autodeploy-serial.jsonnet b/test/testdata/fixtures/pipedream/rollback.jsonnet similarity index 61% rename from test/testdata/fixtures/pipedream/autodeploy-serial.jsonnet rename to test/testdata/fixtures/pipedream/rollback.jsonnet index d9c31df..d87a39a 100644 --- a/test/testdata/fixtures/pipedream/autodeploy-serial.jsonnet +++ b/test/testdata/fixtures/pipedream/rollback.jsonnet @@ -5,33 +5,32 @@ local pipedream_config = { auto_deploy: true, rollback: { material_name: 'example_repo', - stage: 'example_stage', - elastic_profile_id: 'example_profile', - }, - materials: { - init_repo: { - git: 'git@github.com:getsentry/init.git', - shallow_clone: true, - branch: 'master', - destination: 'init', - }, + stage: 'deploy', + elastic_profile_id: 'example', }, }; local sample = { pipeline(region):: { - region: region, materials: { example_repo: { git: 'git@github.com:getsentry/example.git', - shallow_clone: true, branch: 'master', destination: 'example', }, }, stages: [ { - example_stage: {}, + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, }, ], }, diff --git a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden new file mode 100644 index 0000000..ce848f0 --- /dev/null +++ b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden @@ -0,0 +1,34 @@ +{ + "all_groups": { + "control": [ + "control" + ], + "de": [ + "de" + ], + "s4s": [ + "s4s", + "s4s2" + ], + "snty-tools": [ + "snty-tools" + ], + "st": [ + "customer-1", + "customer-2", + "customer-4", + "customer-7" + ], + "us": [ + "us" + ] + }, + "group_order": [ + "s4s", + "de", + "us", + "control", + "snty-tools", + "st" + ] +} diff --git a/test/testdata/goldens/getsentry/regions.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/regions.jsonnet_output-files.golden deleted file mode 100644 index 720e317..0000000 --- a/test/testdata/goldens/getsentry/regions.jsonnet_output-files.golden +++ /dev/null @@ -1,13 +0,0 @@ -{ - "all_regions": [ - "s4s2", - "de", - "us", - "control", - "snty-tools", - "customer-1", - "customer-2", - "customer-4", - "customer-7" - ] -} diff --git a/test/testdata/goldens/pipedream/autodeploy-parallel.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/autodeploy-parallel.jsonnet_output-files.golden deleted file mode 100644 index ed2dd7d..0000000 --- a/test/testdata/goldens/pipedream/autodeploy-parallel.jsonnet_output-files.golden +++ /dev/null @@ -1,349 +0,0 @@ -{ - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-us.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-us": { - "display_order": 4, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "us", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/autodeploy-parallel.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/autodeploy-parallel.jsonnet_single-file.golden deleted file mode 100644 index 0590b45..0000000 --- a/test/testdata/goldens/pipedream/autodeploy-parallel.jsonnet_single-file.golden +++ /dev/null @@ -1,312 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-us": { - "display_order": 4, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "us", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden new file mode 100644 index 0000000..dd12a5e --- /dev/null +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden @@ -0,0 +1,234 @@ +{ + "deploy-example-de.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-s4s.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-st.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden new file mode 100644 index 0000000..bcf6c3c --- /dev/null +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden @@ -0,0 +1,217 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +} diff --git a/test/testdata/goldens/pipedream/minimal-config.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden similarity index 56% rename from test/testdata/goldens/pipedream/minimal-config.jsonnet_output-files.golden rename to test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden index 42f1551..fbfa3bd 100644 --- a/test/testdata/goldens/pipedream/minimal-config.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden @@ -1,73 +1,38 @@ { - "deploy-example-customer-1.yaml": { + "deploy-example-de.yaml": { "format_version": 10, "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, + "deploy-example-de": { + "display_order": 3, "group": "example", "materials": { - "deploy-example-us-pipeline-complete": { - "pipeline": "deploy-example-us", + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-1", "stages": [ { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, + "deploy": { "jobs": { - "pipeline-complete": { + "deploy-de": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=de" } ] } } } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -85,32 +50,48 @@ } } }, - "deploy-example-customer-4.yaml": { + "deploy-example-s4s.yaml": { "format_version": 10, "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, + "deploy-example-s4s": { + "display_order": 2, "group": "example", "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-4", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -128,32 +109,64 @@ } } }, - "deploy-example-customer-7.yaml": { + "deploy-example-st.yaml": { "format_version": 10, "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, + "deploy-example-st": { + "display_order": 5, "group": "example", "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-7", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -171,71 +184,40 @@ } } }, - "deploy-example-de.yaml": { + "deploy-example-us.yaml": { "format_version": 10, "pipelines": { - "deploy-example-de": { - "display_order": 3, + "deploy-example-us": { + "display_order": 4, "group": "example", "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "de", "stages": [ { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, + "deploy": { "jobs": { - "pipeline-complete": { + "deploy-us": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=us" } ] } } } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -253,32 +235,26 @@ } } }, - "deploy-example-us.yaml": { + "deploy-example.yaml": { "format_version": 10, "pipelines": { - "deploy-example-us": { - "display_order": 4, + "deploy-example": { + "display_order": 0, "group": "example", + "lock_behavior": "unlockWhenFinished", "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { + "init_repo": { "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "destination": "init", + "git": "git@github.com:getsentry/init.git" } }, - "region": "us", "stages": [ - { - "example_stage": { } - }, { "pipeline-complete": { - "fetch_materials": false, + "approval": { + "type": "manual" + }, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/minimal-config.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden similarity index 56% rename from test/testdata/goldens/pipedream/minimal-config.jsonnet_single-file.golden rename to test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden index f5c896f..9a7870c 100644 --- a/test/testdata/goldens/pipedream/minimal-config.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden @@ -1,29 +1,23 @@ { "format_version": 10, "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, + "deploy-example": { + "display_order": 0, "group": "example", + "lock_behavior": "unlockWhenFinished", "materials": { - "deploy-example-us-pipeline-complete": { - "pipeline": "deploy-example-us", - "stage": "pipeline-complete" - }, - "example_repo": { + "init_repo": { "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "destination": "init", + "git": "git@github.com:getsentry/init.git" } }, - "region": "customer-1", "stages": [ - { - "example_stage": { } - }, { "pipeline-complete": { - "fetch_materials": false, + "approval": { + "type": "manual" + }, "jobs": { "pipeline-complete": { "tasks": [ @@ -39,67 +33,37 @@ } ] }, - "deploy-example-customer-2": { - "display_order": 6, + "deploy-example-de": { + "display_order": 3, "group": "example", "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-2", "stages": [ { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, + "deploy": { "jobs": { - "pipeline-complete": { + "deploy-de": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=de" } ] } } } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -115,29 +79,45 @@ } ] }, - "deploy-example-customer-7": { - "display_order": 8, + "deploy-example-s4s": { + "display_order": 2, "group": "example", "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-7", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -153,63 +133,61 @@ } ] }, - "deploy-example-de": { - "display_order": 3, + "deploy-example-st": { + "display_order": 5, "group": "example", "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "de", "stages": [ { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, + "deploy": { "jobs": { - "pipeline-complete": { + "deploy-customer-1": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" } ] } } } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -236,18 +214,26 @@ "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "us", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden new file mode 100644 index 0000000..48efe60 --- /dev/null +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden @@ -0,0 +1,370 @@ +{ + "deploy-example-de.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy de" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-de": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify de" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-s4s.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy s4s" + ], + "command": "echo" + } + } + ] + }, + "deploy-s4s2": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy s4s2" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-s4s2": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify s4s2" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-st.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy customer-1" + ], + "command": "echo" + } + } + ] + }, + "deploy-customer-2": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy customer-2" + ], + "command": "echo" + } + } + ] + }, + "deploy-customer-4": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy customer-4" + ], + "command": "echo" + } + } + ] + }, + "deploy-customer-7": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy customer-7" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-customer-1": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify customer-1" + ], + "command": "echo" + } + } + ] + }, + "verify-customer-2": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify customer-2" + ], + "command": "echo" + } + } + ] + }, + "verify-customer-4": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify customer-4" + ], + "command": "echo" + } + } + ] + }, + "verify-customer-7": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify customer-7" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy us" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-us": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify us" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden new file mode 100644 index 0000000..bed0fa1 --- /dev/null +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden @@ -0,0 +1,353 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy de" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-de": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify de" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy s4s" + ], + "command": "echo" + } + } + ] + }, + "deploy-s4s2": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy s4s2" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-s4s2": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify s4s2" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy customer-1" + ], + "command": "echo" + } + } + ] + }, + "deploy-customer-2": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy customer-2" + ], + "command": "echo" + } + } + ] + }, + "deploy-customer-4": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy customer-4" + ], + "command": "echo" + } + } + ] + }, + "deploy-customer-7": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy customer-7" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-customer-1": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify customer-1" + ], + "command": "echo" + } + } + ] + }, + "verify-customer-2": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify customer-2" + ], + "command": "echo" + } + } + ] + }, + "verify-customer-4": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify customer-4" + ], + "command": "echo" + } + } + ] + }, + "verify-customer-7": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify customer-7" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy us" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-us": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify us" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +} diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden new file mode 100644 index 0000000..87086e0 --- /dev/null +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden @@ -0,0 +1,159 @@ +{ + "deploy-example-de.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-s4s.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden new file mode 100644 index 0000000..0ff5d6b --- /dev/null +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden @@ -0,0 +1,147 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +} diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden new file mode 100644 index 0000000..2201f69 --- /dev/null +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden @@ -0,0 +1,226 @@ +{ + "deploy-example-de.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-s4s.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-st.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden new file mode 100644 index 0000000..66c30bf --- /dev/null +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden @@ -0,0 +1,209 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +} diff --git a/test/testdata/goldens/pipedream/exclude-regions-autodeploy-parallel.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-regions-autodeploy-parallel.jsonnet_single-file.golden deleted file mode 100644 index 709da48..0000000 --- a/test/testdata/goldens/pipedream/exclude-regions-autodeploy-parallel.jsonnet_single-file.golden +++ /dev/null @@ -1,278 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 4, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 5, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 6, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 7, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/exclude-regions-autodeploy-serial.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-regions-autodeploy-serial.jsonnet_output-files.golden deleted file mode 100644 index 6c0aa86..0000000 --- a/test/testdata/goldens/pipedream/exclude-regions-autodeploy-serial.jsonnet_output-files.golden +++ /dev/null @@ -1,330 +0,0 @@ -{ - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet_output-files.golden deleted file mode 100644 index 4b56528..0000000 --- a/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet_output-files.golden +++ /dev/null @@ -1,372 +0,0 @@ -{ - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git" - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet_single-file.golden deleted file mode 100644 index 2841f16..0000000 --- a/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-parallel.jsonnet_single-file.golden +++ /dev/null @@ -1,335 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git" - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-1": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/autodeploy-serial.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden similarity index 66% rename from test/testdata/goldens/pipedream/autodeploy-serial.jsonnet_output-files.golden rename to test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden index a9cd203..2146d35 100644 --- a/test/testdata/goldens/pipedream/autodeploy-serial.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden @@ -1,8 +1,8 @@ { - "deploy-example-customer-1.yaml": { + "deploy-example-control.yaml": { "format_version": 10, "pipelines": { - "deploy-example-customer-1": { + "deploy-example-control": { "display_order": 5, "group": "example", "materials": { @@ -13,30 +13,32 @@ "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-1", "stages": [ { +<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_output-files.golden "example_stage": { } }, { "pipeline-complete": { "fetch_materials": false, +======= + "deploy": { +>>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden "jobs": { - "pipeline-complete": { + "deploy-control": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=control" } ] } } } +<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_output-files.golden } ] } @@ -150,6 +152,8 @@ "stages": [ { "example_stage": { } +======= +>>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden }, { "pipeline-complete": { @@ -178,25 +182,36 @@ "display_order": 3, "group": "example", "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "de", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } }, { "pipeline-complete": { +<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_output-files.golden "fetch_materials": false, +======= "jobs": { "pipeline-complete": { "tasks": [ @@ -214,28 +229,45 @@ } } }, - "deploy-example-s4s2.yaml": { + "deploy-example-s4s.yaml": { "format_version": 10, "pipelines": { - "deploy-example-s4s2": { + "deploy-example-s4s": { "display_order": 2, "group": "example", "materials": { "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "s4s2", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, +>>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden "jobs": { "pipeline-complete": { "tasks": [ @@ -253,28 +285,61 @@ } } }, - "deploy-example-us.yaml": { + "deploy-example-st.yaml": { "format_version": 10, "pipelines": { - "deploy-example-us": { - "display_order": 4, + "deploy-example-st": { + "display_order": 6, "group": "example", "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", + "deploy-example-control-pipeline-complete": { + "pipeline": "deploy-example-control", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "us", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } }, { "pipeline-complete": { @@ -296,53 +361,32 @@ } } }, - "rollback-example.yaml": { + "deploy-example-us.yaml": { "format_version": 10, "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, + "deploy-example-us": { + "display_order": 4, "group": "example", - "lock_behavior": "unlockWhenFinished", "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" } }, "stages": [ { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { + "deploy": { "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", + "deploy-us": { + "elastic_profile_id": "example", "tasks": [ { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" + "script": "./deploy.sh --region=us" } ] } @@ -350,16 +394,15 @@ } }, { - "incident_resolved": { - "approval": { - "type": "manual" - }, + "pipeline-complete": { + "fetch_materials": false, "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", + "pipeline-complete": { "tasks": [ { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" + "exec": { + "command": true + } } ] } diff --git a/test/testdata/goldens/pipedream/autodeploy-serial.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden similarity index 64% rename from test/testdata/goldens/pipedream/autodeploy-serial.jsonnet_single-file.golden rename to test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden index 135d4b0..c3f27b9 100644 --- a/test/testdata/goldens/pipedream/autodeploy-serial.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden @@ -1,7 +1,7 @@ { "format_version": 10, "pipelines": { - "deploy-example-customer-1": { + "deploy-example-control": { "display_order": 5, "group": "example", "materials": { @@ -12,30 +12,32 @@ "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-1", "stages": [ { +<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_single-file.golden "example_stage": { } }, { "pipeline-complete": { "fetch_materials": false, +======= + "deploy": { +>>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden "jobs": { - "pipeline-complete": { + "deploy-control": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=control" } ] } } } +<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_single-file.golden } ] }, @@ -134,6 +136,8 @@ "stages": [ { "example_stage": { } +======= +>>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden }, { "pipeline-complete": { @@ -157,25 +161,36 @@ "display_order": 3, "group": "example", "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "de", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } }, { "pipeline-complete": { +<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_single-file.golden "fetch_materials": false, +======= "jobs": { "pipeline-complete": { "tasks": [ @@ -191,25 +206,42 @@ } ] }, - "deploy-example-s4s2": { + "deploy-example-s4s": { "display_order": 2, "group": "example", "materials": { "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "s4s2", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, +>>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden "jobs": { "pipeline-complete": { "tasks": [ @@ -225,25 +257,58 @@ } ] }, - "deploy-example-us": { - "display_order": 4, + "deploy-example-st": { + "display_order": 6, "group": "example", "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", + "deploy-example-control-pipeline-complete": { + "pipeline": "deploy-example-control", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "us", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } }, { "pipeline-complete": { @@ -263,36 +328,29 @@ } ] }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, + "deploy-example-us": { + "display_order": 4, "group": "example", - "lock_behavior": "unlockWhenFinished", "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" } }, "stages": [ { - "pause_pipelines": { - "approval": { - "type": "manual" - }, + "deploy": { "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", + "deploy-us": { + "elastic_profile_id": "example", "tasks": [ { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" + "script": "./deploy.sh --region=us" } ] } @@ -300,30 +358,15 @@ } }, { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, + "pipeline-complete": { + "fetch_materials": false, "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", + "pipeline-complete": { "tasks": [ { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" + "exec": { + "command": true + } } ] } diff --git a/test/testdata/goldens/pipedream/include-regions-autodeploy-parallel.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-regions-autodeploy-parallel.jsonnet_output-files.golden deleted file mode 100644 index cbc4ab2..0000000 --- a/test/testdata/goldens/pipedream/include-regions-autodeploy-parallel.jsonnet_output-files.golden +++ /dev/null @@ -1,349 +0,0 @@ -{ - "deploy-example-control.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/include-regions-autodeploy-parallel.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-regions-autodeploy-parallel.jsonnet_single-file.golden deleted file mode 100644 index 538dc03..0000000 --- a/test/testdata/goldens/pipedream/include-regions-autodeploy-parallel.jsonnet_single-file.golden +++ /dev/null @@ -1,312 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/include-regions-autodeploy-serial.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-regions-autodeploy-serial.jsonnet_output-files.golden deleted file mode 100644 index 0a26473..0000000 --- a/test/testdata/goldens/pipedream/include-regions-autodeploy-serial.jsonnet_output-files.golden +++ /dev/null @@ -1,373 +0,0 @@ -{ - "deploy-example-control.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-control-pipeline-complete": { - "pipeline": "deploy-example-control", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/include-regions-autodeploy-serial.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-regions-autodeploy-serial.jsonnet_single-file.golden deleted file mode 100644 index e0b921a..0000000 --- a/test/testdata/goldens/pipedream/include-regions-autodeploy-serial.jsonnet_single-file.golden +++ /dev/null @@ -1,336 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-control-pipeline-complete": { - "pipeline": "deploy-example-control", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/include-regions-no-autodeploy-parallel.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-regions-no-autodeploy-parallel.jsonnet_output-files.golden deleted file mode 100644 index 016c328..0000000 --- a/test/testdata/goldens/pipedream/include-regions-no-autodeploy-parallel.jsonnet_output-files.golden +++ /dev/null @@ -1,415 +0,0 @@ -{ - "deploy-example-control.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git" - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/include-regions-no-autodeploy-parallel.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-regions-no-autodeploy-parallel.jsonnet_single-file.golden deleted file mode 100644 index a1e9c1b..0000000 --- a/test/testdata/goldens/pipedream/include-regions-no-autodeploy-parallel.jsonnet_single-file.golden +++ /dev/null @@ -1,373 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git" - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/include-regions-no-autodeploy-serial.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-regions-no-autodeploy-serial.jsonnet_output-files.golden deleted file mode 100644 index 2530741..0000000 --- a/test/testdata/goldens/pipedream/include-regions-no-autodeploy-serial.jsonnet_output-files.golden +++ /dev/null @@ -1,415 +0,0 @@ -{ - "deploy-example-control.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-control-pipeline-complete": { - "pipeline": "deploy-example-control", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git" - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/include-regions-no-autodeploy-serial.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-regions-no-autodeploy-serial.jsonnet_single-file.golden deleted file mode 100644 index 5469d50..0000000 --- a/test/testdata/goldens/pipedream/include-regions-no-autodeploy-serial.jsonnet_single-file.golden +++ /dev/null @@ -1,373 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git" - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-control-pipeline-complete": { - "pipeline": "deploy-example-control", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/include-regions-parallel.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-regions-parallel.jsonnet_output-files.golden deleted file mode 100644 index cbc4ab2..0000000 --- a/test/testdata/goldens/pipedream/include-regions-parallel.jsonnet_output-files.golden +++ /dev/null @@ -1,349 +0,0 @@ -{ - "deploy-example-control.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/include-regions-parallel.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-regions-parallel.jsonnet_single-file.golden deleted file mode 100644 index 538dc03..0000000 --- a/test/testdata/goldens/pipedream/include-regions-parallel.jsonnet_single-file.golden +++ /dev/null @@ -1,312 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example-control": { - "display_order": 4, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "control", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-control --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden new file mode 100644 index 0000000..e521d61 --- /dev/null +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden @@ -0,0 +1,132 @@ +{ + "deploy-example-s4s.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-st.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-st": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden new file mode 100644 index 0000000..d9b0b64 --- /dev/null +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden @@ -0,0 +1,125 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +} diff --git a/test/testdata/goldens/pipedream/no-autodeploy-parallel.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/no-autodeploy-parallel.jsonnet_output-files.golden deleted file mode 100644 index 6eecb42..0000000 --- a/test/testdata/goldens/pipedream/no-autodeploy-parallel.jsonnet_output-files.golden +++ /dev/null @@ -1,647 +0,0 @@ -{ - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-us.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-us": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "us", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git", - "shallow_clone": true - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/no-autodeploy-parallel.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/no-autodeploy-parallel.jsonnet_single-file.golden deleted file mode 100644 index cadc555..0000000 --- a/test/testdata/goldens/pipedream/no-autodeploy-parallel.jsonnet_single-file.golden +++ /dev/null @@ -1,605 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git", - "shallow_clone": true - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-us": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "us", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/no-autodeploy-serial.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/no-autodeploy-serial.jsonnet_output-files.golden deleted file mode 100644 index 2244906..0000000 --- a/test/testdata/goldens/pipedream/no-autodeploy-serial.jsonnet_output-files.golden +++ /dev/null @@ -1,647 +0,0 @@ -{ - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-us-pipeline-complete": { - "pipeline": "deploy-example-us", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-us.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-us": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "us", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git", - "shallow_clone": true - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/no-autodeploy-serial.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/no-autodeploy-serial.jsonnet_single-file.golden deleted file mode 100644 index 1c595dc..0000000 --- a/test/testdata/goldens/pipedream/no-autodeploy-serial.jsonnet_single-file.golden +++ /dev/null @@ -1,605 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git", - "shallow_clone": true - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-us-pipeline-complete": { - "pipeline": "deploy-example-us", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-us": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "us", - "stages": [ - { - "ready": { - "jobs": { - "ready": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "wait": { - "approval": { - "type": "manual" - }, - "jobs": { - "wait": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - }, - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden new file mode 100644 index 0000000..a29ae0a --- /dev/null +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden @@ -0,0 +1,275 @@ +{ + "deploy-example-de.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-s4s.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-st.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example": { + "display_order": 0, + "group": "example", + "lock_behavior": "unlockWhenFinished", + "materials": { + "init_repo": { + "branch": "master", + "destination": "init", + "git": "git@github.com:getsentry/init.git" + } + }, + "stages": [ + { + "pipeline-complete": { + "approval": { + "type": "manual" + }, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden new file mode 100644 index 0000000..c969e9e --- /dev/null +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden @@ -0,0 +1,253 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example": { + "display_order": 0, + "group": "example", + "lock_behavior": "unlockWhenFinished", + "materials": { + "init_repo": { + "branch": "master", + "destination": "init", + "git": "git@github.com:getsentry/init.git" + } + }, + "stages": [ + { + "pipeline-complete": { + "approval": { + "type": "manual" + }, + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +} diff --git a/test/testdata/goldens/pipedream/rollback-override-final-stage.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback-override-final-stage.jsonnet_output-files.golden deleted file mode 100644 index b74795c..0000000 --- a/test/testdata/goldens/pipedream/rollback-override-final-stage.jsonnet_output-files.golden +++ /dev/null @@ -1,394 +0,0 @@ -{ - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-us-pipeline-complete": { - "pipeline": "deploy-example-us", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-us.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-us": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "us", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-other_stage": { - "pipeline": "deploy-example-customer-7", - "stage": "other_stage" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/rollback-override-final-stage.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback-override-final-stage.jsonnet_single-file.golden deleted file mode 100644 index a3cda09..0000000 --- a/test/testdata/goldens/pipedream/rollback-override-final-stage.jsonnet_single-file.golden +++ /dev/null @@ -1,357 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-us-pipeline-complete": { - "pipeline": "deploy-example-us", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-us": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "us", - "stages": [ - { - "example_stage": { } - }, - { - "other_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-other_stage": { - "pipeline": "deploy-example-customer-7", - "stage": "other_stage" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/exclude-regions-autodeploy-parallel.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden similarity index 66% rename from test/testdata/goldens/pipedream/exclude-regions-autodeploy-parallel.jsonnet_output-files.golden rename to test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden index f9939fd..4f4da72 100644 --- a/test/testdata/goldens/pipedream/exclude-regions-autodeploy-parallel.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden @@ -1,65 +1,38 @@ { - "deploy-example-customer-1.yaml": { + "deploy-example-de.yaml": { "format_version": 10, "pipelines": { - "deploy-example-customer-1": { - "display_order": 4, + "deploy-example-de": { + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-1", "stages": [ { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, + "deploy": { "jobs": { - "pipeline-complete": { + "deploy-de": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=de" } ] } } } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 5, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -77,67 +50,44 @@ } } }, - "deploy-example-customer-4.yaml": { + "deploy-example-s4s.yaml": { "format_version": 10, "pipelines": { - "deploy-example-customer-4": { - "display_order": 6, + "deploy-example-s4s": { + "display_order": 2, "group": "example", "materials": { "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-4", "stages": [ { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, + "deploy": { "jobs": { - "pipeline-complete": { + "deploy-s4s": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" } ] } } } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 7, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -155,28 +105,64 @@ } } }, - "deploy-example-de.yaml": { + "deploy-example-st.yaml": { "format_version": 10, "pipelines": { - "deploy-example-de": { - "display_order": 3, + "deploy-example-st": { + "display_order": 5, "group": "example", "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "de", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -194,28 +180,40 @@ } } }, - "deploy-example-s4s2.yaml": { + "deploy-example-us.yaml": { "format_version": 10, "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, + "deploy-example-us": { + "display_order": 4, "group": "example", "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "s4s2", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -239,18 +237,18 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", + "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" }, "group": "example", "lock_behavior": "unlockWhenFinished", "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", + "deploy-example-st-pipeline-complete": { + "pipeline": "deploy-example-st", "stage": "pipeline-complete" } }, @@ -262,7 +260,7 @@ }, "jobs": { "rollback": { - "elastic_profile_id": "example_profile", + "elastic_profile_id": "example", "tasks": [ { "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" @@ -276,7 +274,7 @@ "start_rollback": { "jobs": { "rollback": { - "elastic_profile_id": "example_profile", + "elastic_profile_id": "example", "tasks": [ { "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" @@ -293,7 +291,7 @@ }, "jobs": { "rollback": { - "elastic_profile_id": "example_profile", + "elastic_profile_id": "example", "tasks": [ { "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" diff --git a/test/testdata/goldens/pipedream/exclude-regions-autodeploy-serial.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden similarity index 66% rename from test/testdata/goldens/pipedream/exclude-regions-autodeploy-serial.jsonnet_single-file.golden rename to test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden index 859cbf0..b030a57 100644 --- a/test/testdata/goldens/pipedream/exclude-regions-autodeploy-serial.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden @@ -1,67 +1,37 @@ { "format_version": 10, "pipelines": { - "deploy-example-customer-1": { - "display_order": 4, + "deploy-example-de": { + "display_order": 3, "group": "example", "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-1", "stages": [ { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, + "deploy": { "jobs": { - "pipeline-complete": { + "deploy-de": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=de" } ] } } } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -77,29 +47,41 @@ } ] }, - "deploy-example-customer-4": { - "display_order": 6, + "deploy-example-s4s": { + "display_order": 2, "group": "example", "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-4", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-s4s": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -115,29 +97,61 @@ } ] }, - "deploy-example-customer-7": { - "display_order": 7, + "deploy-example-st": { + "display_order": 5, "group": "example", "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "customer-7", "stages": [ { - "example_stage": { } + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -153,63 +167,37 @@ } ] }, - "deploy-example-de": { - "display_order": 3, + "deploy-example-us": { + "display_order": 4, "group": "example", "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", "stage": "pipeline-complete" }, "example_repo": { "branch": "master", "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true + "git": "git@github.com:getsentry/example.git" } }, - "region": "de", "stages": [ { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, + "deploy": { "jobs": { - "pipeline-complete": { + "deploy-us": { + "elastic_profile_id": "example", "tasks": [ { - "exec": { - "command": true - } + "script": "./deploy.sh --region=us" } ] } } } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -228,18 +216,18 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", + "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" }, "group": "example", "lock_behavior": "unlockWhenFinished", "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", + "deploy-example-st-pipeline-complete": { + "pipeline": "deploy-example-st", "stage": "pipeline-complete" } }, @@ -251,7 +239,7 @@ }, "jobs": { "rollback": { - "elastic_profile_id": "example_profile", + "elastic_profile_id": "example", "tasks": [ { "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" @@ -265,7 +253,7 @@ "start_rollback": { "jobs": { "rollback": { - "elastic_profile_id": "example_profile", + "elastic_profile_id": "example", "tasks": [ { "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" @@ -282,7 +270,7 @@ }, "jobs": { "rollback": { - "elastic_profile_id": "example_profile", + "elastic_profile_id": "example", "tasks": [ { "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" From b7f83e819be47f5a2aad9477631a96afb21b3d1f Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Tue, 6 Jan 2026 13:58:18 -0800 Subject: [PATCH 02/16] cascade pipeline and stage level environment variables down to the job level --- libs/pipedream.libsonnet | 47 ++++++++++--- .../pipedream/env-vars-precedence.jsonnet | 49 +++++++++++++ .../fixtures/pipedream/stage-props.jsonnet | 36 ++++++++++ ...ars-precedence.jsonnet_output-files.golden | 68 +++++++++++++++++++ ...vars-precedence.jsonnet_single-file.golden | 66 ++++++++++++++++++ .../stage-props.jsonnet_output-files.golden | 58 ++++++++++++++++ .../stage-props.jsonnet_single-file.golden | 56 +++++++++++++++ 7 files changed, 372 insertions(+), 8 deletions(-) create mode 100644 test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet create mode 100644 test/testdata/fixtures/pipedream/stage-props.jsonnet create mode 100644 test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden create mode 100644 test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index a3fac4c..407bdde 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -62,7 +62,19 @@ local get_stage_jobs(stage) = local get_stage_props(stage) = local stage_name = get_stage_name(stage); local props = stage[stage_name]; - { [k]: props[k] for k in std.objectFields(props) if k != 'jobs' }; + { [k]: props[k] for k in std.objectFields(props) if k != 'jobs' && k != 'environment_variables' }; + +local get_stage_env_vars(stage) = + local stage_name = get_stage_name(stage); + local props = stage[stage_name]; + if std.objectHas(props, 'environment_variables') then props.environment_variables else {}; + +local get_pipeline_env_vars(pipeline) = + if std.objectHas(pipeline, 'environment_variables') then pipeline.environment_variables else {}; + +// Cascade down environment variables with precedence: job > stage > pipeline +local merge_env_vars(pipeline_env, stage_env, job_env) = + pipeline_env + stage_env + job_env; // This function returns a "trigger pipeline", if configured for manual deploys. // This pipeline is used so users don't need to know what the first pipedream @@ -236,7 +248,9 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde [] ); - // Transforms a stage by aggregating jobs from all regions + // Transforms a stage by aggregating jobs from all regions. + // Cascades environment_variables from pipeline -> stage -> job level for each region. + // This is necessary as the pipeline/stage level is shared for all regions in a grouping. local transform_stage(stage) = local stage_name = get_stage_name(stage); local stage_props = get_stage_props(stage); @@ -244,16 +258,26 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde local all_jobs = std.foldl( function(acc, region) local p = pipeline_fn(region); + local pipeline_env = get_pipeline_env_vars(p); local matching_stages = std.filter( function(s) get_stage_name(s) == stage_name, if std.objectHas(p, 'stages') then p.stages else [] ); - local stage_jobs = if std.length(matching_stages) > 0 then - get_stage_jobs(matching_stages[0]) - else - {}; + local region_stage = if std.length(matching_stages) > 0 then matching_stages[0] else null; + local stage_env = if region_stage != null then get_stage_env_vars(region_stage) else {}; + local stage_jobs = if region_stage != null then get_stage_jobs(region_stage) else {}; + acc + { - [job_name + '-' + region]: stage_jobs[job_name] + [job_name + '-' + region]: ( + local job = stage_jobs[job_name]; + local job_env = if std.objectHas(job, 'environment_variables') then job.environment_variables else {}; + local merged_env = merge_env_vars(pipeline_env, stage_env, job_env); + // Add merged env vars to job, or keep job as-is if no env vars + if std.length(std.objectFields(merged_env)) > 0 then + job { environment_variables: merged_env } + else + job + ) for job_name in std.objectFields(stage_jobs) }, regions, @@ -292,8 +316,15 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde for stage in all_stages ]; + // Strip pipeline and stage level environment variables + local filtered_template = { + [k]: template_pipeline[k] + for k in std.objectFields(template_pipeline) + if k != 'environment_variables' + }; + // Assemble final pipeline from template - template_pipeline { + filtered_template { group: service_name, display_order: display_order, stages: prepend_stages + transformed_stages + [ diff --git a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet new file mode 100644 index 0000000..f04806a --- /dev/null +++ b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet @@ -0,0 +1,49 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +// Test to demonstrate environment_variables precedence: job > stage > pipeline +// Should show separate values for s4s and s4s2 + +local pipedream_config = { + name: 'example', + auto_deploy: true, + exclude_regions: ['de', 'us', 'customer-1', 'customer-2', 'customer-4', 'customer-7'], +}; + +local pipeline_fn(region) = { + // Pipeline-level env vars + environment_variables: { + PIPELINE_VAR: 'pipeline-' + region, // Should cascade down to becoming a job level var + SHARED_VAR_JOB: 'from-pipeline', // Should be overwritten by stage, then job + SHARED_VAR_STAGE: 'from-pipeline', // This should win + }, + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + }, + }, + stages: [ + { + deploy: { + // Stage-level env vars + environment_variables: { + STAGE_VAR: 'stage-' + region, // Should cascade down to becoming a job level var + SHARED_VAR_JOB: 'from-stage', // Should be overwritten by job + SHARED_VAR_STAGE: 'from-stage', // Should be overwritten by stage + }, + jobs: { + deploy: { + // Job-level env vars - highest precedence + environment_variables: { + JOB_VAR: 'job-' + region, + SHARED_VAR_JOB: 'from-job', // This should win + }, + tasks: [{ script: './deploy.sh --region=' + region }], + }, + }, + }, + }, + ], +}; + +pipedream.render(pipedream_config, pipeline_fn) diff --git a/test/testdata/fixtures/pipedream/stage-props.jsonnet b/test/testdata/fixtures/pipedream/stage-props.jsonnet new file mode 100644 index 0000000..6ba99f4 --- /dev/null +++ b/test/testdata/fixtures/pipedream/stage-props.jsonnet @@ -0,0 +1,36 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +// Test to demonstrate stage-level properties behavior +// Within a region grouping, stage properties come from the FIRST region that defines that stage + +local pipedream_config = { + name: 'example', + auto_deploy: true, + exclude_regions: ['de', 'us', 'customer-1', 'customer-2', 'customer-4', 'customer-7'], +}; + +// This pipeline_fn returns stages with different properties per region +local pipeline_fn(region) = { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + }, + }, + stages: [ + { + deploy: { + // Stage-level properties that differ by region + fetch_materials: if region == 's4s' then true else false, + approval: if region == 's4s' then { type: 'manual' } else { type: 'success' }, + jobs: { + deploy: { + tasks: [{ script: './deploy.sh --region=' + region }], + }, + }, + }, + }, + ], +}; + +pipedream.render(pipedream_config, pipeline_fn) diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden new file mode 100644 index 0000000..895e3fa --- /dev/null +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden @@ -0,0 +1,68 @@ +{ + "deploy-example-s4s.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "environment_variables": { + "JOB_VAR": "job-s4s", + "PIPELINE_VAR": "pipeline-s4s", + "SHARED_VAR_JOB": "from-job", + "SHARED_VAR_STAGE": "from-stage", + "STAGE_VAR": "stage-s4s" + }, + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "environment_variables": { + "JOB_VAR": "job-s4s2", + "PIPELINE_VAR": "pipeline-s4s2", + "SHARED_VAR_JOB": "from-job", + "SHARED_VAR_STAGE": "from-stage", + "STAGE_VAR": "stage-s4s2" + }, + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden new file mode 100644 index 0000000..6305dc1 --- /dev/null +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden @@ -0,0 +1,66 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s": { + "environment_variables": { + "JOB_VAR": "job-s4s", + "PIPELINE_VAR": "pipeline-s4s", + "SHARED_VAR_JOB": "from-job", + "SHARED_VAR_STAGE": "from-stage", + "STAGE_VAR": "stage-s4s" + }, + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "environment_variables": { + "JOB_VAR": "job-s4s2", + "PIPELINE_VAR": "pipeline-s4s2", + "SHARED_VAR_JOB": "from-job", + "SHARED_VAR_STAGE": "from-stage", + "STAGE_VAR": "stage-s4s2" + }, + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +} diff --git a/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden new file mode 100644 index 0000000..12f5d71 --- /dev/null +++ b/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden @@ -0,0 +1,58 @@ +{ + "deploy-example-s4s.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "approval": { + "type": "manual" + }, + "fetch_materials": true, + "jobs": { + "deploy-s4s": { + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden new file mode 100644 index 0000000..00f52c7 --- /dev/null +++ b/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden @@ -0,0 +1,56 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "approval": { + "type": "manual" + }, + "fetch_materials": true, + "jobs": { + "deploy-s4s": { + "tasks": [ + { + "script": "./deploy.sh --region=s4s" + } + ] + }, + "deploy-s4s2": { + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } +} From fc8e0c4224aaefb129032156c0ffe670399f0234 Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 25 Mar 2026 08:46:25 -0700 Subject: [PATCH 03/16] ref(pipedream): env var optimization, s4s2 alignment, and test coverage Optimize env var handling so shared vars stay at stage level while region-specific vars cascade to jobs. Cache pipeline_fn results, extract get_matching_stage helper. Align s4s group to ['s4s2'] only. Add rollback-final-stage-override fixture and regenerate goldens. --- .gitignore | 5 + libs/getsentry.libsonnet | 2 +- libs/pipedream.libsonnet | 68 +++- test/pipedream.js | 2 +- .../different-stages-per-region.jsonnet | 6 +- .../pipedream/env-vars-precedence.jsonnet | 4 +- .../rollback-final-stage-override.jsonnet | 43 +++ .../fixtures/pipedream/stage-props.jsonnet | 10 +- .../groups.jsonnet_output-files.golden | 1 - ...sic-autodeploy.jsonnet_output-files.golden | 8 - ...asic-autodeploy.jsonnet_single-file.golden | 8 - .../basic-manual.jsonnet_output-files.golden | 8 - .../basic-manual.jsonnet_single-file.golden | 8 - ...ges-per-region.jsonnet_output-files.golden | 24 -- ...ages-per-region.jsonnet_single-file.golden | 24 -- ...ars-precedence.jsonnet_output-files.golden | 56 +++- ...vars-precedence.jsonnet_single-file.golden | 54 +++- ...e-entire-group.jsonnet_output-files.golden | 8 - ...de-entire-group.jsonnet_single-file.golden | 8 - ...exclude-region.jsonnet_output-files.golden | 8 - .../exclude-region.jsonnet_single-file.golden | 8 - ...fault-excluded.jsonnet_output-files.golden | 8 - ...efault-excluded.jsonnet_single-file.golden | 8 - ...i-region-group.jsonnet_output-files.golden | 8 - ...ti-region-group.jsonnet_single-file.golden | 8 - .../parallel-mode.jsonnet_output-files.golden | 8 - .../parallel-mode.jsonnet_single-file.golden | 8 - ...stage-override.jsonnet_output-files.golden | 300 ++++++++++++++++++ ...-stage-override.jsonnet_single-file.golden | 278 ++++++++++++++++ .../rollback.jsonnet_output-files.golden | 8 - .../rollback.jsonnet_single-file.golden | 8 - .../stage-props.jsonnet_output-files.golden | 26 +- .../stage-props.jsonnet_single-file.golden | 24 +- 33 files changed, 813 insertions(+), 242 deletions(-) create mode 100644 test/testdata/fixtures/pipedream/rollback-final-stage-override.jsonnet create mode 100644 test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden create mode 100644 test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden diff --git a/.gitignore b/.gitignore index 1f65af1..13fcb3c 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,8 @@ vendor/ trash/ *.bak *.log + +# Working notes (do not commit) +PIPEDREAM_CELLULARIZATION_NOTES.md +PIPEDREAM_USE_CASES.md +notes.md diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index fe3c902..f9d0088 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -7,7 +7,7 @@ test_group_order: [], // These groupings consist of user facing deployments pipeline_groups: { - s4s: ['s4s', 's4s2'], + s4s: ['s4s2'], de: ['de'], us: ['us'], control: ['control'], diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index 407bdde..d5c61c7 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -232,12 +232,14 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde all_regions ); - local template_pipeline = pipeline_fn(regions[0]); + // Cache pipeline_fn results to avoid redundant calls per region + local region_pipelines = { [r]: pipeline_fn(r) for r in regions }; + local template_pipeline = region_pipelines[regions[0]]; // Collect all unique stages across all regions in the group local all_stages = std.foldl( function(acc, region) - local p = pipeline_fn(region); + local p = region_pipelines[region]; local region_stages = if std.objectHas(p, 'stages') then p.stages else []; acc + [ stage @@ -248,31 +250,60 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde [] ); + local get_matching_stage(p, stage_name) = + local matching = std.filter( + function(s) get_stage_name(s) == stage_name, + if std.objectHas(p, 'stages') then p.stages else [] + ); + if std.length(matching) > 0 then matching[0] else null; + // Transforms a stage by aggregating jobs from all regions. - // Cascades environment_variables from pipeline -> stage -> job level for each region. - // This is necessary as the pipeline/stage level is shared for all regions in a grouping. + // Env vars identical across all regions are kept at stage level; + // region-specific env vars are cascaded down to the job level. local transform_stage(stage) = local stage_name = get_stage_name(stage); local stage_props = get_stage_props(stage); - local all_jobs = std.foldl( - function(acc, region) - local p = pipeline_fn(region); + // Collect merged pipeline+stage env vars for each region + local per_region_parent_envs = { + [region]: ( + local p = region_pipelines[region]; local pipeline_env = get_pipeline_env_vars(p); - local matching_stages = std.filter( - function(s) get_stage_name(s) == stage_name, - if std.objectHas(p, 'stages') then p.stages else [] - ); - local region_stage = if std.length(matching_stages) > 0 then matching_stages[0] else null; + local region_stage = get_matching_stage(p, stage_name); local stage_env = if region_stage != null then get_stage_env_vars(region_stage) else {}; + merge_env_vars(pipeline_env, stage_env, {}) + ) + for region in regions + }; + + // Env vars identical across ALL regions stay at stage level + local first_env = per_region_parent_envs[regions[0]]; + local common_env = { + [k]: first_env[k] + for k in std.objectFields(first_env) + if std.length(std.filter( + function(r) std.objectHas(per_region_parent_envs[r], k) && per_region_parent_envs[r][k] == first_env[k], + regions + )) == std.length(regions) + }; + + local all_jobs = std.foldl( + function(acc, region) + local parent_env = per_region_parent_envs[region]; + local region_specific_env = { + [k]: parent_env[k] + for k in std.objectFields(parent_env) + if !std.objectHas(common_env, k) || common_env[k] != parent_env[k] + }; + local p = region_pipelines[region]; + local region_stage = get_matching_stage(p, stage_name); local stage_jobs = if region_stage != null then get_stage_jobs(region_stage) else {}; acc + { [job_name + '-' + region]: ( local job = stage_jobs[job_name]; local job_env = if std.objectHas(job, 'environment_variables') then job.environment_variables else {}; - local merged_env = merge_env_vars(pipeline_env, stage_env, job_env); - // Add merged env vars to job, or keep job as-is if no env vars + local merged_env = region_specific_env + job_env; if std.length(std.objectFields(merged_env)) > 0 then job { environment_variables: merged_env } else @@ -285,7 +316,14 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde ); { - [stage_name]: stage_props { jobs: all_jobs }, + [stage_name]: stage_props { + jobs: all_jobs, + } + ( + if std.length(std.objectFields(common_env)) > 0 then + { environment_variables: common_env } + else + {} + ), }; // `auto_pipeline_progression` was added as a utility for folks new to diff --git a/test/pipedream.js b/test/pipedream.js index 8029b3d..39dbcb1 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -36,7 +36,7 @@ test("multi-region group has parallel jobs for each region", async (t) => { // s4s group has s4s + s4s2 regions const s4s = got.pipelines["deploy-example-s4s"]; const s4sJobs = Object.keys(s4s.stages[0].deploy.jobs); - t.deepEqual(s4sJobs.sort(), ["deploy-s4s", "deploy-s4s2"]); + t.deepEqual(s4sJobs.sort(), ["deploy-s4s2"]); // st group has customer-1, customer-2, customer-4, customer-7 const st = got.pipelines["deploy-example-st"]; diff --git a/test/testdata/fixtures/pipedream/different-stages-per-region.jsonnet b/test/testdata/fixtures/pipedream/different-stages-per-region.jsonnet index 88c5b1b..7dfd2d7 100644 --- a/test/testdata/fixtures/pipedream/different-stages-per-region.jsonnet +++ b/test/testdata/fixtures/pipedream/different-stages-per-region.jsonnet @@ -13,8 +13,8 @@ local config = { // This pipeline_fn returns DIFFERENT stages depending on region local pipeline_fn(region) = { materials: config.materials, - stages: if region == 's4s' then [ - // s4s only has deploy stage + stages: if region == 'customer-1' then [ + // customer-1 only has deploy stage { deploy: { jobs: { @@ -23,7 +23,7 @@ local pipeline_fn(region) = { }, }, ] else [ - // s4s2 has deploy AND verify stages + // other regions have deploy AND verify stages { deploy: { jobs: { diff --git a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet index f04806a..3ee8f23 100644 --- a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet +++ b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet @@ -1,12 +1,12 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; // Test to demonstrate environment_variables precedence: job > stage > pipeline -// Should show separate values for s4s and s4s2 +// Uses st group (multi-region) to show separate values per region local pipedream_config = { name: 'example', auto_deploy: true, - exclude_regions: ['de', 'us', 'customer-1', 'customer-2', 'customer-4', 'customer-7'], + exclude_regions: ['de', 'us', 's4s2'], }; local pipeline_fn(region) = { diff --git a/test/testdata/fixtures/pipedream/rollback-final-stage-override.jsonnet b/test/testdata/fixtures/pipedream/rollback-final-stage-override.jsonnet new file mode 100644 index 0000000..52f6cdc --- /dev/null +++ b/test/testdata/fixtures/pipedream/rollback-final-stage-override.jsonnet @@ -0,0 +1,43 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +// Used by symbolicator and super-big-consumers to point the rollback +// material at deploy-primary instead of the default pipeline-complete. + +local pipedream_config = { + name: 'example', + auto_deploy: true, + rollback: { + material_name: 'example_repo', + stage: 'deploy', + elastic_profile_id: 'example', + final_stage: 'deploy', + }, +}; + +local sample = { + pipeline(region):: { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + destination: 'example', + }, + }, + stages: [ + { + deploy: { + jobs: { + deploy: { + elastic_profile_id: 'example', + tasks: [ + { script: './deploy.sh --region=' + region }, + ], + }, + }, + }, + }, + ], + }, +}; + +pipedream.render(pipedream_config, sample.pipeline) diff --git a/test/testdata/fixtures/pipedream/stage-props.jsonnet b/test/testdata/fixtures/pipedream/stage-props.jsonnet index 6ba99f4..ebb879d 100644 --- a/test/testdata/fixtures/pipedream/stage-props.jsonnet +++ b/test/testdata/fixtures/pipedream/stage-props.jsonnet @@ -6,10 +6,11 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', auto_deploy: true, - exclude_regions: ['de', 'us', 'customer-1', 'customer-2', 'customer-4', 'customer-7'], + exclude_regions: ['de', 'us', 's4s2'], }; -// This pipeline_fn returns stages with different properties per region +// This pipeline_fn returns stages with different properties per region. +// customer-1 (first in st group) defines different props than the rest. local pipeline_fn(region) = { materials: { example_repo: { @@ -20,9 +21,8 @@ local pipeline_fn(region) = { stages: [ { deploy: { - // Stage-level properties that differ by region - fetch_materials: if region == 's4s' then true else false, - approval: if region == 's4s' then { type: 'manual' } else { type: 'success' }, + fetch_materials: if region == 'customer-1' then true else false, + approval: if region == 'customer-1' then { type: 'manual' } else { type: 'success' }, jobs: { deploy: { tasks: [{ script: './deploy.sh --region=' + region }], diff --git a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden index ce848f0..9dbfb05 100644 --- a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden +++ b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden @@ -7,7 +7,6 @@ "de" ], "s4s": [ - "s4s", "s4s2" ], "snty-tools": [ diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden index dd12a5e..ca9ee27 100644 --- a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden @@ -67,14 +67,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden index bcf6c3c..0f83574 100644 --- a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden @@ -61,14 +61,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden index fbfa3bd..dda3638 100644 --- a/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden @@ -71,14 +71,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden index 9a7870c..573467e 100644 --- a/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden @@ -97,14 +97,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden index 48efe60..805fc79 100644 --- a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden @@ -87,18 +87,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "tasks": [ - { - "exec": { - "arguments": [ - "deploy s4s" - ], - "command": "echo" - } - } - ] - }, "deploy-s4s2": { "tasks": [ { @@ -225,18 +213,6 @@ { "verify": { "jobs": { - "verify-customer-1": { - "tasks": [ - { - "exec": { - "arguments": [ - "verify customer-1" - ], - "command": "echo" - } - } - ] - }, "verify-customer-2": { "tasks": [ { diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden index bed0fa1..038079b 100644 --- a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden @@ -81,18 +81,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "tasks": [ - { - "exec": { - "arguments": [ - "deploy s4s" - ], - "command": "echo" - } - } - ] - }, "deploy-s4s2": { "tasks": [ { @@ -214,18 +202,6 @@ { "verify": { "jobs": { - "verify-customer-1": { - "tasks": [ - { - "exec": { - "arguments": [ - "verify customer-1" - ], - "command": "echo" - } - } - ] - }, "verify-customer-2": { "tasks": [ { diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden index 895e3fa..12a554a 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden @@ -1,8 +1,8 @@ { - "deploy-example-s4s.yaml": { + "deploy-example-st.yaml": { "format_version": 10, "pipelines": { - "deploy-example-s4s": { + "deploy-example-st": { "display_order": 2, "group": "example", "materials": { @@ -14,32 +14,60 @@ "stages": [ { "deploy": { + "environment_variables": { + "SHARED_VAR_JOB": "from-stage", + "SHARED_VAR_STAGE": "from-stage" + }, "jobs": { - "deploy-s4s": { + "deploy-customer-1": { "environment_variables": { - "JOB_VAR": "job-s4s", - "PIPELINE_VAR": "pipeline-s4s", + "JOB_VAR": "job-customer-1", + "PIPELINE_VAR": "pipeline-customer-1", "SHARED_VAR_JOB": "from-job", - "SHARED_VAR_STAGE": "from-stage", - "STAGE_VAR": "stage-s4s" + "STAGE_VAR": "stage-customer-1" }, "tasks": [ { - "script": "./deploy.sh --region=s4s" + "script": "./deploy.sh --region=customer-1" } ] }, - "deploy-s4s2": { + "deploy-customer-2": { "environment_variables": { - "JOB_VAR": "job-s4s2", - "PIPELINE_VAR": "pipeline-s4s2", + "JOB_VAR": "job-customer-2", + "PIPELINE_VAR": "pipeline-customer-2", "SHARED_VAR_JOB": "from-job", - "SHARED_VAR_STAGE": "from-stage", - "STAGE_VAR": "stage-s4s2" + "STAGE_VAR": "stage-customer-2" }, "tasks": [ { - "script": "./deploy.sh --region=s4s2" + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "environment_variables": { + "JOB_VAR": "job-customer-4", + "PIPELINE_VAR": "pipeline-customer-4", + "SHARED_VAR_JOB": "from-job", + "STAGE_VAR": "stage-customer-4" + }, + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "environment_variables": { + "JOB_VAR": "job-customer-7", + "PIPELINE_VAR": "pipeline-customer-7", + "SHARED_VAR_JOB": "from-job", + "STAGE_VAR": "stage-customer-7" + }, + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" } ] } diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden index 6305dc1..406fd59 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden @@ -1,7 +1,7 @@ { "format_version": 10, "pipelines": { - "deploy-example-s4s": { + "deploy-example-st": { "display_order": 2, "group": "example", "materials": { @@ -13,32 +13,60 @@ "stages": [ { "deploy": { + "environment_variables": { + "SHARED_VAR_JOB": "from-stage", + "SHARED_VAR_STAGE": "from-stage" + }, "jobs": { - "deploy-s4s": { + "deploy-customer-1": { "environment_variables": { - "JOB_VAR": "job-s4s", - "PIPELINE_VAR": "pipeline-s4s", + "JOB_VAR": "job-customer-1", + "PIPELINE_VAR": "pipeline-customer-1", "SHARED_VAR_JOB": "from-job", - "SHARED_VAR_STAGE": "from-stage", - "STAGE_VAR": "stage-s4s" + "STAGE_VAR": "stage-customer-1" }, "tasks": [ { - "script": "./deploy.sh --region=s4s" + "script": "./deploy.sh --region=customer-1" } ] }, - "deploy-s4s2": { + "deploy-customer-2": { "environment_variables": { - "JOB_VAR": "job-s4s2", - "PIPELINE_VAR": "pipeline-s4s2", + "JOB_VAR": "job-customer-2", + "PIPELINE_VAR": "pipeline-customer-2", "SHARED_VAR_JOB": "from-job", - "SHARED_VAR_STAGE": "from-stage", - "STAGE_VAR": "stage-s4s2" + "STAGE_VAR": "stage-customer-2" }, "tasks": [ { - "script": "./deploy.sh --region=s4s2" + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "environment_variables": { + "JOB_VAR": "job-customer-4", + "PIPELINE_VAR": "pipeline-customer-4", + "SHARED_VAR_JOB": "from-job", + "STAGE_VAR": "stage-customer-4" + }, + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "environment_variables": { + "JOB_VAR": "job-customer-7", + "PIPELINE_VAR": "pipeline-customer-7", + "SHARED_VAR_JOB": "from-job", + "STAGE_VAR": "stage-customer-7" + }, + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" } ] } diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden index 87086e0..eea447e 100644 --- a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden @@ -67,14 +67,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden index 0ff5d6b..7700a85 100644 --- a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden @@ -61,14 +61,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden index 2201f69..fb930b9 100644 --- a/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden @@ -67,14 +67,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden index 66c30bf..19b7162 100644 --- a/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden @@ -61,14 +61,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden index 2146d35..62b9ec3 100644 --- a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden @@ -246,14 +246,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden index c3f27b9..7baf4d6 100644 --- a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden @@ -220,14 +220,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden index e521d61..a2f75f7 100644 --- a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden @@ -16,14 +16,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden index d9b0b64..a11ce39 100644 --- a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden @@ -15,14 +15,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden index a29ae0a..5cb8ab3 100644 --- a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden @@ -71,14 +71,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden index c969e9e..067898c 100644 --- a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden @@ -97,14 +97,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden new file mode 100644 index 0000000..1900972 --- /dev/null +++ b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden @@ -0,0 +1,300 @@ +{ + "deploy-example-de.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-s4s.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-st.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-us.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "rollback-example.yaml": { + "format_version": 10, + "pipelines": { + "rollback-example": { + "display_order": 1, + "environment_variables": { + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ROLLBACK_MATERIAL_NAME": "example_repo", + "ROLLBACK_STAGE": "deploy", + "TRIGGERED_BY": "" + }, + "group": "example", + "lock_behavior": "unlockWhenFinished", + "materials": { + "deploy-example-st-deploy": { + "pipeline": "deploy-example-st", + "stage": "deploy" + } + }, + "stages": [ + { + "pause_pipelines": { + "approval": { + "type": "manual" + }, + "jobs": { + "rollback": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" + } + ] + } + } + } + }, + { + "start_rollback": { + "jobs": { + "rollback": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" + } + ] + } + } + } + }, + { + "incident_resolved": { + "approval": { + "type": "manual" + }, + "jobs": { + "rollback": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" + } + ] + } + } + } + } + ] + } + } + } +} diff --git a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden new file mode 100644 index 0000000..134f4df --- /dev/null +++ b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden @@ -0,0 +1,278 @@ +{ + "format_version": 10, + "pipelines": { + "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s-pipeline-complete": { + "pipeline": "deploy-example-s4s", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-s4s": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 5, + "group": "example", + "materials": { + "deploy-example-us-pipeline-complete": { + "pipeline": "deploy-example-us", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-customer-1": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-1" + } + ] + }, + "deploy-customer-2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-us": { + "display_order": 4, + "group": "example", + "materials": { + "deploy-example-de-pipeline-complete": { + "pipeline": "deploy-example-de", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-us": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=us" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "rollback-example": { + "display_order": 1, + "environment_variables": { + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ROLLBACK_MATERIAL_NAME": "example_repo", + "ROLLBACK_STAGE": "deploy", + "TRIGGERED_BY": "" + }, + "group": "example", + "lock_behavior": "unlockWhenFinished", + "materials": { + "deploy-example-st-deploy": { + "pipeline": "deploy-example-st", + "stage": "deploy" + } + }, + "stages": [ + { + "pause_pipelines": { + "approval": { + "type": "manual" + }, + "jobs": { + "rollback": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" + } + ] + } + } + } + }, + { + "start_rollback": { + "jobs": { + "rollback": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" + } + ] + } + } + } + }, + { + "incident_resolved": { + "approval": { + "type": "manual" + }, + "jobs": { + "rollback": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" + } + ] + } + } + } + } + ] + } + } +} diff --git a/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden index 4f4da72..07d6bbf 100644 --- a/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden @@ -67,14 +67,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden index b030a57..0fcde05 100644 --- a/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden @@ -61,14 +61,6 @@ { "deploy": { "jobs": { - "deploy-s4s": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s" - } - ] - }, "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ diff --git a/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden index 12f5d71..fb45019 100644 --- a/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden @@ -1,8 +1,8 @@ { - "deploy-example-s4s.yaml": { + "deploy-example-st.yaml": { "format_version": 10, "pipelines": { - "deploy-example-s4s": { + "deploy-example-st": { "display_order": 2, "group": "example", "materials": { @@ -19,17 +19,31 @@ }, "fetch_materials": true, "jobs": { - "deploy-s4s": { + "deploy-customer-1": { "tasks": [ { - "script": "./deploy.sh --region=s4s" + "script": "./deploy.sh --region=customer-1" } ] }, - "deploy-s4s2": { + "deploy-customer-2": { "tasks": [ { - "script": "./deploy.sh --region=s4s2" + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" } ] } diff --git a/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden index 00f52c7..b0758c1 100644 --- a/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden @@ -1,7 +1,7 @@ { "format_version": 10, "pipelines": { - "deploy-example-s4s": { + "deploy-example-st": { "display_order": 2, "group": "example", "materials": { @@ -18,17 +18,31 @@ }, "fetch_materials": true, "jobs": { - "deploy-s4s": { + "deploy-customer-1": { "tasks": [ { - "script": "./deploy.sh --region=s4s" + "script": "./deploy.sh --region=customer-1" } ] }, - "deploy-s4s2": { + "deploy-customer-2": { "tasks": [ { - "script": "./deploy.sh --region=s4s2" + "script": "./deploy.sh --region=customer-2" + } + ] + }, + "deploy-customer-4": { + "tasks": [ + { + "script": "./deploy.sh --region=customer-4" + } + ] + }, + "deploy-customer-7": { + "tasks": [ + { + "script": "./deploy.sh --region=customer-7" } ] } From 80f55b9022e1fe54595bf8c691eb4072d5b1241b Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Thu, 9 Apr 2026 13:34:35 -0700 Subject: [PATCH 04/16] ref(pipedream): add build-time validation for conflicting stage properties GoCD only supports stage-level attributes (approval, fetch_materials, etc.) with no per-job override. When regions in a group define conflicting stage properties, we now assert at build time rather than silently using the first region's values. --- libs/pipedream.libsonnet | 18 ++++++++++ test/pipedream.js | 12 +++++++ .../stage-props-conflict.failing.jsonnet | 34 +++++++++++++++++++ .../fixtures/pipedream/stage-props.jsonnet | 11 +++--- 4 files changed, 69 insertions(+), 6 deletions(-) create mode 100644 test/testdata/fixtures/pipedream/stage-props-conflict.failing.jsonnet diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index d5c61c7..5054d29 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -264,6 +264,24 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde local stage_name = get_stage_name(stage); local stage_props = get_stage_props(stage); + // Validate that all regions agree on stage properties. GoCD only supports + // stage-level attributes (approval, fetch_materials, etc.) — there is no + // per-job override — so conflicting values across regions must be caught + // at build time rather than silently using the first region's values. + assert std.foldl( + function(acc, r) + local p = region_pipelines[r]; + local rs = get_matching_stage(p, stage_name); + local props = if rs != null then get_stage_props(rs) else stage_props; + assert props == stage_props : + "Stage '%s': conflicting properties across regions in group. " + % [stage_name] + + "Region '%s' differs from '%s'." % [r, regions[0]]; + true, + regions[1:], + true + ); + // Collect merged pipeline+stage env vars for each region local per_region_parent_envs = { [region]: ( diff --git a/test/pipedream.js b/test/pipedream.js index 39dbcb1..8d2a9ab 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -153,6 +153,18 @@ test("rollback: invalid final stage errors", (t) => { ); }); +test("conflicting stage properties across regions errors", (t) => { + const err = t.throws(() => + get_fixture_content( + "pipedream/stage-props-conflict.failing.jsonnet", + false, + ), + ); + t.true( + err.message.includes("conflicting properties across regions in group"), + ); +}); + test("all pipelines end with pipeline-complete stage", async (t) => { const got = await render_fixture("pipedream/basic-autodeploy.jsonnet", false); diff --git a/test/testdata/fixtures/pipedream/stage-props-conflict.failing.jsonnet b/test/testdata/fixtures/pipedream/stage-props-conflict.failing.jsonnet new file mode 100644 index 0000000..c0e217b --- /dev/null +++ b/test/testdata/fixtures/pipedream/stage-props-conflict.failing.jsonnet @@ -0,0 +1,34 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +// This fixture should FAIL at build time because regions in the st group +// define conflicting stage properties (different approval types). + +local pipedream_config = { + name: 'example', + auto_deploy: true, + exclude_regions: ['de', 'us', 's4s2'], +}; + +local pipeline_fn(region) = { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + }, + }, + stages: [ + { + deploy: { + fetch_materials: if region == 'customer-1' then true else false, + approval: if region == 'customer-1' then { type: 'manual' } else { type: 'success' }, + jobs: { + deploy: { + tasks: [{ script: './deploy.sh --region=' + region }], + }, + }, + }, + }, + ], +}; + +pipedream.render(pipedream_config, pipeline_fn) diff --git a/test/testdata/fixtures/pipedream/stage-props.jsonnet b/test/testdata/fixtures/pipedream/stage-props.jsonnet index ebb879d..1f7681c 100644 --- a/test/testdata/fixtures/pipedream/stage-props.jsonnet +++ b/test/testdata/fixtures/pipedream/stage-props.jsonnet @@ -1,7 +1,7 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; -// Test to demonstrate stage-level properties behavior -// Within a region grouping, stage properties come from the FIRST region that defines that stage +// Test that stage-level properties (approval, fetch_materials) are correctly +// preserved when aggregating jobs from multiple regions in a group. local pipedream_config = { name: 'example', @@ -9,8 +9,7 @@ local pipedream_config = { exclude_regions: ['de', 'us', 's4s2'], }; -// This pipeline_fn returns stages with different properties per region. -// customer-1 (first in st group) defines different props than the rest. +// All regions use the same stage properties — no conflict. local pipeline_fn(region) = { materials: { example_repo: { @@ -21,8 +20,8 @@ local pipeline_fn(region) = { stages: [ { deploy: { - fetch_materials: if region == 'customer-1' then true else false, - approval: if region == 'customer-1' then { type: 'manual' } else { type: 'success' }, + fetch_materials: true, + approval: { type: 'manual' }, jobs: { deploy: { tasks: [{ script: './deploy.sh --region=' + region }], From c113d47193e466d5ff48cf02568606ec19be06cb Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Thu, 9 Apr 2026 13:36:29 -0700 Subject: [PATCH 05/16] ref(pipedream): add single-key stage object assertion In Jsonnet, a missing comma between stage definitions silently merges them into a single object, causing stages to be lost. This assertion catches the issue at build time by validating that each stage object has exactly one key. --- libs/pipedream.libsonnet | 24 ++++++++++++ test/pipedream.js | 12 ++++++ .../pipedream/merged-stages.failing.jsonnet | 39 +++++++++++++++++++ 3 files changed, 75 insertions(+) create mode 100644 test/testdata/fixtures/pipedream/merged-stages.failing.jsonnet diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index 5054d29..00b3184 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -234,6 +234,30 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde // Cache pipeline_fn results to avoid redundant calls per region local region_pipelines = { [r]: pipeline_fn(r) for r in regions }; + + // Validate that each stage object has exactly one key. In Jsonnet, a missing + // comma between stage definitions silently merges them into a single object, + // causing stages to be lost. Catch this at build time. + assert std.foldl( + function(acc, region) + local p = region_pipelines[region]; + local stages = if std.objectHas(p, 'stages') then p.stages else []; + assert std.foldl( + function(acc2, stage) + local keys = std.objectFields(stage); + assert std.length(keys) == 1 : + "Stage object has %d keys (%s) — each stage must have exactly one key. " + % [std.length(keys), std.join(', ', keys)] + + "This usually means a missing comma between stage definitions."; + true, + stages, + true + ); + true, + regions, + true + ); + local template_pipeline = region_pipelines[regions[0]]; // Collect all unique stages across all regions in the group diff --git a/test/pipedream.js b/test/pipedream.js index 8d2a9ab..6bff84f 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -165,6 +165,18 @@ test("conflicting stage properties across regions errors", (t) => { ); }); +test("merged stage objects (missing comma) errors", (t) => { + const err = t.throws(() => + get_fixture_content( + "pipedream/merged-stages.failing.jsonnet", + false, + ), + ); + t.true( + err.message.includes("each stage must have exactly one key"), + ); +}); + test("all pipelines end with pipeline-complete stage", async (t) => { const got = await render_fixture("pipedream/basic-autodeploy.jsonnet", false); diff --git a/test/testdata/fixtures/pipedream/merged-stages.failing.jsonnet b/test/testdata/fixtures/pipedream/merged-stages.failing.jsonnet new file mode 100644 index 0000000..9f8dfe8 --- /dev/null +++ b/test/testdata/fixtures/pipedream/merged-stages.failing.jsonnet @@ -0,0 +1,39 @@ +local pipedream = import '../../../../libs/pipedream.libsonnet'; + +// This fixture should FAIL at build time because a stage object has +// multiple keys, indicating a missing comma between stage definitions. + +local pipedream_config = { + name: 'example', + auto_deploy: true, +}; + +local pipeline_fn(region) = { + materials: { + example_repo: { + git: 'git@github.com:getsentry/example.git', + branch: 'master', + }, + }, + stages: [ + { + // Two stages accidentally merged into one object (missing comma) + deploy: { + jobs: { + deploy: { + tasks: [{ script: './deploy.sh --region=' + region }], + }, + }, + }, + verify: { + jobs: { + verify: { + tasks: [{ script: './verify.sh --region=' + region }], + }, + }, + }, + }, + ], +}; + +pipedream.render(pipedream_config, pipeline_fn) From 54a9589db862011000011bf3b6aff6846680d32d Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Tue, 3 Mar 2026 10:31:10 -0800 Subject: [PATCH 06/16] update repo readme to reflect grouping --- README.md | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 114f27d..537eb8e 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ When `output-files=true` it'll output pipelines in the format: ``` This is useful when you build using the `-m` flag in jsonnet as it'll output -multiple files which makes reviewing pipeliens easier. +multiple files which makes reviewing pipelines easier. ```bash jsonnet --ext-code output-files=true -m ./generated-pipelines ./example.jsonnet @@ -58,7 +58,7 @@ jsonnet --ext-code output-files=true -m ./generated-pipelines ./example.jsonnet The GoCD plugin that can process jsonnet files directly doesn't support outputting multiple files, so GoCD is configured to have -`--ext-code output-filaes=false` which will output the pipelines in a flattened +`--ext-code output-files=false` which will output the pipelines in a flattened format: ```json @@ -87,8 +87,21 @@ Pipedream will name the returned pipeline, add an upstream pipeline material and a final stage. The upstream material and final stage is needed to make GoCD chain the pipelines together. -The end result will be a pipeline `deploy-` that starts the run of -each pipeline, and a pipeline for each region. +Regions are organized into **groups** (defined in `getsentry.libsonnet`). Each +group produces a single GoCD pipeline where regions within the group run as +parallel jobs. Groups are chained sequentially by default, or can fan out in +parallel via `pipedream.render(config, pipeline_fn, parallel=true)`. + +The end result is a trigger pipeline `deploy-` and a pipeline per +group (e.g. `deploy-example-s4s`, `deploy-example-st`). + +`control` and `snty-tools` are excluded by default; use `include_regions` to +opt in. + +Environment variables set at the pipeline or stage level in `pipeline_fn` are +handled automatically: variables identical across all regions in a group stay at +the stage level, while region-specific variables are cascaded to the job level. +GoCD resolves precedence as job > stage > pipeline. ### Example Usage From 768b0cbc8ec5326286d4a899c79384dc7ea2096b Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Thu, 9 Apr 2026 14:08:53 -0700 Subject: [PATCH 07/16] ref(pipedream): remove s4s group s4s is no longer a deployment group. Remove from pipeline_groups and group_order, update all test fixtures and golden files. --- libs/getsentry.libsonnet | 3 +- libs/pipedream.libsonnet | 9 +- test/pipedream.js | 21 +- .../pipedream/env-vars-precedence.jsonnet | 2 +- .../pipedream/multi-region-group.jsonnet | 2 +- .../stage-props-conflict.failing.jsonnet | 2 +- .../fixtures/pipedream/stage-props.jsonnet | 2 +- .../groups.jsonnet_output-files.golden | 4 - ...sic-autodeploy.jsonnet_output-files.golden | 57 +-- ...asic-autodeploy.jsonnet_single-file.golden | 52 +-- .../basic-manual.jsonnet_output-files.golden | 59 +-- .../basic-manual.jsonnet_single-file.golden | 54 +-- ...ges-per-region.jsonnet_output-files.golden | 78 +--- ...ages-per-region.jsonnet_single-file.golden | 73 +--- ...e-entire-group.jsonnet_output-files.golden | 55 +-- ...de-entire-group.jsonnet_single-file.golden | 50 +-- ...exclude-region.jsonnet_output-files.golden | 57 +-- .../exclude-region.jsonnet_single-file.golden | 52 +-- ...odeploy-serial.jsonnet_output-files.golden | 372 ------------------ ...todeploy-serial.jsonnet_single-file.golden | 335 ---------------- ...fault-excluded.jsonnet_output-files.golden | 190 +-------- ...efault-excluded.jsonnet_single-file.golden | 170 +------- ...i-region-group.jsonnet_output-files.golden | 53 +-- ...ti-region-group.jsonnet_single-file.golden | 48 +-- .../parallel-mode.jsonnet_output-files.golden | 57 +-- .../parallel-mode.jsonnet_single-file.golden | 52 +-- ...stage-override.jsonnet_output-files.golden | 61 +-- ...-stage-override.jsonnet_single-file.golden | 56 +-- .../rollback.jsonnet_output-files.golden | 61 +-- .../rollback.jsonnet_single-file.golden | 56 +-- 30 files changed, 80 insertions(+), 2063 deletions(-) delete mode 100644 test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-serial.jsonnet_output-files.golden delete mode 100644 test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-serial.jsonnet_single-file.golden diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index f9d0088..0c57814 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -3,11 +3,10 @@ */ { - group_order: ['s4s', 'de', 'us', 'control', 'snty-tools', 'st'], + group_order: ['de', 'us', 'control', 'snty-tools', 'st'], test_group_order: [], // These groupings consist of user facing deployments pipeline_groups: { - s4s: ['s4s2'], de: ['de'], us: ['us'], control: ['control'], diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index 00b3184..0166d9c 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -95,7 +95,7 @@ local pipedream_trigger_pipeline(pipedream_config) = materials: materials, lock_behavior: 'unlockWhenFinished', stages: [ - gocd_stages.basic('pipeline-complete', [gocd_tasks.noop], { approval: 'manual', fetch_materials: false }), + gocd_stages.basic('pipeline-complete', [gocd_tasks.noop], { approval: 'manual' }), ], }, }; @@ -411,10 +411,11 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde // This stage is added to ensure a rollback doesn't cause // a deployment train. // - // i.e. During a rollback, US re-runs the final stage. With - // `pipeline-complete` as the final stage, it isn't + // i.e. During a rollback, de and US re-runs the final stage + // The de final stage completes and causes the US pipeline to + // re-run. With `pipeline-complete` as the final stage, it isn't // re-run by a rollback, preventing this domino effect. - gocd_stages.basic('pipeline-complete', [gocd_tasks.noop], { fetch_materials: false }), + gocd_stages.basic('pipeline-complete', [gocd_tasks.noop]), ], }; diff --git a/test/pipedream.js b/test/pipedream.js index 6bff84f..8642a2a 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -22,8 +22,7 @@ test("generates pipeline per group in correct order", async (t) => { n.startsWith("deploy-example-"), ); - // Should have s4s, de, us, st (control/snty-tools excluded by default) - t.true(pipelineNames.includes("deploy-example-s4s")); + // Should have de, us, st (control/snty-tools excluded by default) t.true(pipelineNames.includes("deploy-example-de")); t.true(pipelineNames.includes("deploy-example-us")); t.true(pipelineNames.includes("deploy-example-st")); @@ -33,11 +32,6 @@ test("generates pipeline per group in correct order", async (t) => { test("multi-region group has parallel jobs for each region", async (t) => { const got = await render_fixture("pipedream/basic-autodeploy.jsonnet", false); - // s4s group has s4s + s4s2 regions - const s4s = got.pipelines["deploy-example-s4s"]; - const s4sJobs = Object.keys(s4s.stages[0].deploy.jobs); - t.deepEqual(s4sJobs.sort(), ["deploy-s4s2"]); - // st group has customer-1, customer-2, customer-4, customer-7 const st = got.pipelines["deploy-example-st"]; const stJobs = Object.keys(st.stages[0].deploy.jobs); @@ -60,13 +54,9 @@ test("single-region group has one job", async (t) => { test("serial mode: pipelines chain sequentially", async (t) => { const got = await render_fixture("pipedream/basic-manual.jsonnet", false); - // s4s depends on trigger - const s4s = got.pipelines["deploy-example-s4s"]; - t.truthy(s4s.materials["deploy-example-pipeline-complete"]); - - // de depends on s4s + // de depends on trigger const de = got.pipelines["deploy-example-de"]; - t.truthy(de.materials["deploy-example-s4s-pipeline-complete"]); + t.truthy(de.materials["deploy-example-pipeline-complete"]); // us depends on de const us = got.pipelines["deploy-example-us"]; @@ -76,17 +66,14 @@ test("serial mode: pipelines chain sequentially", async (t) => { test("parallel mode: all pipelines depend on trigger only", async (t) => { const got = await render_fixture("pipedream/parallel-mode.jsonnet", false); - const s4s = got.pipelines["deploy-example-s4s"]; const de = got.pipelines["deploy-example-de"]; const us = got.pipelines["deploy-example-us"]; // All depend on trigger - t.truthy(s4s.materials["deploy-example-pipeline-complete"]); t.truthy(de.materials["deploy-example-pipeline-complete"]); t.truthy(us.materials["deploy-example-pipeline-complete"]); // None depend on each other - t.falsy(de.materials["deploy-example-s4s-pipeline-complete"]); t.falsy(us.materials["deploy-example-de-pipeline-complete"]); }); @@ -109,7 +96,7 @@ test("exclude all regions in group: skips entire group", async (t) => { ); t.falsy(got.pipelines["deploy-example-st"]); - t.truthy(got.pipelines["deploy-example-s4s"]); + t.truthy(got.pipelines["deploy-example-de"]); }); test("include region: adds default-excluded group", async (t) => { diff --git a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet index 3ee8f23..e7561a3 100644 --- a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet +++ b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet @@ -6,7 +6,7 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', auto_deploy: true, - exclude_regions: ['de', 'us', 's4s2'], + exclude_regions: ['de', 'us'], }; local pipeline_fn(region) = { diff --git a/test/testdata/fixtures/pipedream/multi-region-group.jsonnet b/test/testdata/fixtures/pipedream/multi-region-group.jsonnet index 67bd27d..ebf3c9a 100644 --- a/test/testdata/fixtures/pipedream/multi-region-group.jsonnet +++ b/test/testdata/fixtures/pipedream/multi-region-group.jsonnet @@ -1,6 +1,6 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; -// Only render s4s and st to focus on multi-region groups +// Only render st to focus on multi-region groups local pipedream_config = { name: 'example', auto_deploy: true, diff --git a/test/testdata/fixtures/pipedream/stage-props-conflict.failing.jsonnet b/test/testdata/fixtures/pipedream/stage-props-conflict.failing.jsonnet index c0e217b..b7fa58f 100644 --- a/test/testdata/fixtures/pipedream/stage-props-conflict.failing.jsonnet +++ b/test/testdata/fixtures/pipedream/stage-props-conflict.failing.jsonnet @@ -6,7 +6,7 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', auto_deploy: true, - exclude_regions: ['de', 'us', 's4s2'], + exclude_regions: ['de', 'us'], }; local pipeline_fn(region) = { diff --git a/test/testdata/fixtures/pipedream/stage-props.jsonnet b/test/testdata/fixtures/pipedream/stage-props.jsonnet index 1f7681c..7e10d16 100644 --- a/test/testdata/fixtures/pipedream/stage-props.jsonnet +++ b/test/testdata/fixtures/pipedream/stage-props.jsonnet @@ -6,7 +6,7 @@ local pipedream = import '../../../../libs/pipedream.libsonnet'; local pipedream_config = { name: 'example', auto_deploy: true, - exclude_regions: ['de', 'us', 's4s2'], + exclude_regions: ['de', 'us'], }; // All regions use the same stage properties — no conflict. diff --git a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden index 9dbfb05..1842205 100644 --- a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden +++ b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden @@ -6,9 +6,6 @@ "de": [ "de" ], - "s4s": [ - "s4s2" - ], "snty-tools": [ "snty-tools" ], @@ -23,7 +20,6 @@ ] }, "group_order": [ - "s4s", "de", "us", "control", diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden index ca9ee27..128ad40 100644 --- a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden @@ -3,13 +3,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -50,58 +46,11 @@ } } }, - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -176,7 +125,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden index 0f83574..6cf8fe1 100644 --- a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden @@ -2,13 +2,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -47,50 +43,8 @@ } ] }, - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -160,7 +114,7 @@ ] }, "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden index dda3638..048ac0a 100644 --- a/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden @@ -3,57 +3,6 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-de": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=de" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { "display_order": 2, "group": "example", "materials": { @@ -71,11 +20,11 @@ { "deploy": { "jobs": { - "deploy-s4s2": { + "deploy-de": { "elastic_profile_id": "example", "tasks": [ { - "script": "./deploy.sh --region=s4s2" + "script": "./deploy.sh --region=de" } ] } @@ -105,7 +54,7 @@ "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -180,7 +129,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden index 573467e..1c93c4d 100644 --- a/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden @@ -34,52 +34,6 @@ ] }, "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-de": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=de" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s": { "display_order": 2, "group": "example", "materials": { @@ -97,11 +51,11 @@ { "deploy": { "jobs": { - "deploy-s4s2": { + "deploy-de": { "elastic_profile_id": "example", "tasks": [ { - "script": "./deploy.sh --region=s4s2" + "script": "./deploy.sh --region=de" } ] } @@ -126,7 +80,7 @@ ] }, "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -196,7 +150,7 @@ ] }, "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden index 805fc79..d2486a4 100644 --- a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden @@ -3,13 +3,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "init_repo": { "branch": "main", "git": "git@github.com:getsentry/example.git" @@ -71,79 +67,11 @@ } } }, - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "init_repo": { - "branch": "main", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "tasks": [ - { - "exec": { - "arguments": [ - "deploy s4s2" - ], - "command": "echo" - } - } - ] - } - } - } - }, - { - "verify": { - "jobs": { - "verify-s4s2": { - "tasks": [ - { - "exec": { - "arguments": [ - "verify s4s2" - ], - "command": "echo" - } - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -275,7 +203,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden index 038079b..b43896a 100644 --- a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden @@ -2,13 +2,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "init_repo": { "branch": "main", "git": "git@github.com:getsentry/example.git" @@ -68,71 +64,8 @@ } ] }, - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "init_repo": { - "branch": "main", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "tasks": [ - { - "exec": { - "arguments": [ - "deploy s4s2" - ], - "command": "echo" - } - } - ] - } - } - } - }, - { - "verify": { - "jobs": { - "verify-s4s2": { - "tasks": [ - { - "exec": { - "arguments": [ - "verify s4s2" - ], - "command": "echo" - } - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -259,7 +192,7 @@ ] }, "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden index eea447e..1da63de 100644 --- a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden @@ -3,13 +3,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -50,58 +46,11 @@ } } }, - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, "deploy-example-us.yaml": { "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden index 7700a85..6fe8773 100644 --- a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden @@ -2,13 +2,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -47,50 +43,8 @@ } ] }, - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden index fb930b9..a136a05 100644 --- a/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden @@ -3,13 +3,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -50,58 +46,11 @@ } } }, - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -168,7 +117,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden index 19b7162..3587330 100644 --- a/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden @@ -2,13 +2,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -47,50 +43,8 @@ } ] }, - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -152,7 +106,7 @@ ] }, "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-serial.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-serial.jsonnet_output-files.golden deleted file mode 100644 index b5c623a..0000000 --- a/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-serial.jsonnet_output-files.golden +++ /dev/null @@ -1,372 +0,0 @@ -{ - "deploy-example-customer-1.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-1": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-de.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git" - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "rollback-example.yaml": { - "format_version": 10, - "pipelines": { - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } - } -} diff --git a/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-serial.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-serial.jsonnet_single-file.golden deleted file mode 100644 index aa2d619..0000000 --- a/test/testdata/goldens/pipedream/exclude-regions-no-autodeploy-serial.jsonnet_single-file.golden +++ /dev/null @@ -1,335 +0,0 @@ -{ - "format_version": 10, - "pipelines": { - "deploy-example": { - "display_order": 0, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "init_repo": { - "branch": "master", - "destination": "init", - "git": "git@github.com:getsentry/init.git" - } - }, - "stages": [ - { - "pipeline-complete": { - "approval": { - "type": "manual" - }, - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-1": { - "display_order": 4, - "group": "example", - "materials": { - "deploy-example-de-pipeline-complete": { - "pipeline": "deploy-example-de", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-1", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-2": { - "display_order": 5, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-de": { - "display_order": 3, - "group": "example", - "materials": { - "deploy-example-s4s2-pipeline-complete": { - "pipeline": "deploy-example-s4s2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "de", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s2": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "s4s2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "rollback-example": { - "display_order": 1, - "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7 --pipeline=deploy-example", - "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-customer-1 --pipeline=deploy-example-customer-2 --pipeline=deploy-example-customer-4 --pipeline=deploy-example-customer-7", - "ROLLBACK_MATERIAL_NAME": "example_repo", - "ROLLBACK_STAGE": "example_stage", - "TRIGGERED_BY": "" - }, - "group": "example", - "lock_behavior": "unlockWhenFinished", - "materials": { - "deploy-example-customer-7-pipeline-complete": { - "pipeline": "deploy-example-customer-7", - "stage": "pipeline-complete" - } - }, - "stages": [ - { - "pause_pipelines": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline is being rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Pause all pipelines in the pipedream\ngocd-pause-and-cancel-pipelines \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "start_rollback": { - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $REGION_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${REGION_PIPELINE_FLAGS:-}\" ]]; then\n set -- $REGION_PIPELINE_FLAGS\nfi\n\n## The user that triggered the rollback\nTRIGGERED_BY=\"${TRIGGERED_BY:-}\"\n\npause_message='This pipeline was rolled back, please check with team before un-pausing.'\n\n## Include triggered by in the pause message if it is not empty\nif [ -n \"$TRIGGERED_BY\" ]; then\n pause_message=\"$pause_message Triggered by: $TRIGGERED_BY\"\nfi\n\n## Get sha from the given pipeline run to deploy to all pipedream pipelines.\nsha=$(gocd-sha-for-pipeline --material-name=\"${ROLLBACK_MATERIAL_NAME}\")\n\necho \"📑 Rolling back to sha: ${sha}\"\n\ngocd-emergency-deploy \\\n --material-name=\"${ROLLBACK_MATERIAL_NAME}\" \\\n --commit-sha=\"${sha}\" \\\n --deploy-stage=\"${ROLLBACK_STAGE}\" \\\n --pause-message=\"$pause_message\" \\\n \"$@\"\n" - } - ] - } - } - } - }, - { - "incident_resolved": { - "approval": { - "type": "manual" - }, - "jobs": { - "rollback": { - "elastic_profile_id": "example_profile", - "tasks": [ - { - "script": "##!/bin/bash\n\n## Note: $ALL_PIPELINE_FLAGS has no quoting, for word expansion\n## shellcheck disable=SC2086\nif [[ \"${ALL_PIPELINE_FLAGS:-}\" ]]; then\n set -- $ALL_PIPELINE_FLAGS\nfi\n\n## Unpause and unlock all pipelines in the pipedream\ngocd-unpause-and-unlock-pipelines \\\n \"$@\"\n" - } - ] - } - } - } - } - ] - } - } -} diff --git a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden index 62b9ec3..02cc02b 100644 --- a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden @@ -3,7 +3,7 @@ "format_version": 10, "pipelines": { "deploy-example-control": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -18,15 +18,7 @@ }, "stages": [ { -<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_output-files.golden - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, -======= "deploy": { ->>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden "jobs": { "deploy-control": { "elastic_profile_id": "example", @@ -38,126 +30,9 @@ } } } -<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_output-files.golden - } - ] - } - } - }, - "deploy-example-customer-2.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-4.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-customer-7.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } -======= ->>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden - }, - { - "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -179,13 +54,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -209,57 +80,6 @@ }, { "pipeline-complete": { -<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_output-files.golden - "fetch_materials": false, -======= - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { ->>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden "jobs": { "pipeline-complete": { "tasks": [ @@ -281,7 +101,7 @@ "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 6, + "display_order": 5, "group": "example", "materials": { "deploy-example-control-pipeline-complete": { @@ -335,7 +155,6 @@ }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -357,7 +176,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -387,7 +206,6 @@ }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden index 7baf4d6..0eb6c38 100644 --- a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden @@ -2,7 +2,7 @@ "format_version": 10, "pipelines": { "deploy-example-control": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -17,15 +17,7 @@ }, "stages": [ { -<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_single-file.golden - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, -======= "deploy": { ->>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden "jobs": { "deploy-control": { "elastic_profile_id": "example", @@ -37,111 +29,9 @@ } } } -<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_single-file.golden - } - ] - }, - "deploy-example-customer-2": { - "display_order": 6, - "group": "example", - "materials": { - "deploy-example-customer-1-pipeline-complete": { - "pipeline": "deploy-example-customer-1", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-2", - "stages": [ - { - "example_stage": { } - }, - { - "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-4": { - "display_order": 7, - "group": "example", - "materials": { - "deploy-example-customer-2-pipeline-complete": { - "pipeline": "deploy-example-customer-2", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-4", - "stages": [ - { - "example_stage": { } }, { "pipeline-complete": { - "fetch_materials": false, - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-customer-7": { - "display_order": 8, - "group": "example", - "materials": { - "deploy-example-customer-4-pipeline-complete": { - "pipeline": "deploy-example-customer-4", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git", - "shallow_clone": true - } - }, - "region": "customer-7", - "stages": [ - { - "example_stage": { } -======= ->>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden - }, - { - "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -158,13 +48,9 @@ ] }, "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -188,52 +74,6 @@ }, { "pipeline-complete": { -<<<<<<< HEAD:test/testdata/goldens/pipedream/minimal-config.jsonnet_single-file.golden - "fetch_materials": false, -======= - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { ->>>>>>> 0b17dc0 (ref(pipedream): Adding grouping support):test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden "jobs": { "pipeline-complete": { "tasks": [ @@ -250,7 +90,7 @@ ] }, "deploy-example-st": { - "display_order": 6, + "display_order": 5, "group": "example", "materials": { "deploy-example-control-pipeline-complete": { @@ -304,7 +144,6 @@ }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -321,7 +160,7 @@ ] }, "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -351,7 +190,6 @@ }, { "pipeline-complete": { - "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden index a2f75f7..38ca95b 100644 --- a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden @@ -1,62 +1,11 @@ { - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden index a11ce39..006b7b2 100644 --- a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden @@ -1,56 +1,10 @@ { "format_version": 10, "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, "deploy-example-st": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden index 5cb8ab3..1f2700e 100644 --- a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden @@ -3,7 +3,7 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { "deploy-example-pipeline-complete": { @@ -50,62 +50,11 @@ } } }, - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-pipeline-complete": { @@ -180,7 +129,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden index 067898c..4b7a825 100644 --- a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden @@ -34,7 +34,7 @@ ] }, "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { "deploy-example-pipeline-complete": { @@ -79,54 +79,8 @@ } ] }, - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "deploy-example-pipeline-complete": { - "pipeline": "deploy-example", - "stage": "pipeline-complete" - }, - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-pipeline-complete": { @@ -196,7 +150,7 @@ ] }, "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden index 1900972..9cfd1e7 100644 --- a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden @@ -3,13 +3,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -50,58 +46,11 @@ } } }, - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -176,7 +125,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -229,9 +178,9 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" diff --git a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden index 134f4df..15c6afe 100644 --- a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden @@ -2,13 +2,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -47,50 +43,8 @@ } ] }, - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -160,7 +114,7 @@ ] }, "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -208,9 +162,9 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" diff --git a/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden index 07d6bbf..13c7c04 100644 --- a/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden @@ -3,13 +3,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -50,58 +46,11 @@ } } }, - "deploy-example-s4s.yaml": { - "format_version": 10, - "pipelines": { - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - } - } - }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -176,7 +125,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -229,9 +178,9 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" diff --git a/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden index 0fcde05..60639da 100644 --- a/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden @@ -2,13 +2,9 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 3, + "display_order": 2, "group": "example", "materials": { - "deploy-example-s4s-pipeline-complete": { - "pipeline": "deploy-example-s4s", - "stage": "pipeline-complete" - }, "example_repo": { "branch": "master", "destination": "example", @@ -47,50 +43,8 @@ } ] }, - "deploy-example-s4s": { - "display_order": 2, - "group": "example", - "materials": { - "example_repo": { - "branch": "master", - "destination": "example", - "git": "git@github.com:getsentry/example.git" - } - }, - "stages": [ - { - "deploy": { - "jobs": { - "deploy-s4s2": { - "elastic_profile_id": "example", - "tasks": [ - { - "script": "./deploy.sh --region=s4s2" - } - ] - } - } - } - }, - { - "pipeline-complete": { - "jobs": { - "pipeline-complete": { - "tasks": [ - { - "exec": { - "command": true - } - } - ] - } - } - } - } - ] - }, "deploy-example-st": { - "display_order": 5, + "display_order": 4, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -160,7 +114,7 @@ ] }, "deploy-example-us": { - "display_order": 4, + "display_order": 3, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -208,9 +162,9 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" From d5262187e44e175a087337671e73c5655175a6b1 Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 15 Apr 2026 09:32:17 -0700 Subject: [PATCH 08/16] fix(pipedream): restore s4s2 as a prod group s4s2 is a prod region that was incorrectly removed when the s4s test group was dropped. Restore it as a single-region group in pipeline_groups. --- libs/getsentry.libsonnet | 3 +- test/pipedream.js | 14 +++- .../groups.jsonnet_output-files.golden | 4 + ...sic-autodeploy.jsonnet_output-files.golden | 57 +++++++++++++- ...asic-autodeploy.jsonnet_single-file.golden | 52 ++++++++++++- .../basic-manual.jsonnet_output-files.golden | 59 +++++++++++++- .../basic-manual.jsonnet_single-file.golden | 54 ++++++++++++- ...ges-per-region.jsonnet_output-files.golden | 78 ++++++++++++++++++- ...ages-per-region.jsonnet_single-file.golden | 73 ++++++++++++++++- ...ars-precedence.jsonnet_output-files.golden | 61 ++++++++++++++- ...vars-precedence.jsonnet_single-file.golden | 56 ++++++++++++- ...e-entire-group.jsonnet_output-files.golden | 55 ++++++++++++- ...de-entire-group.jsonnet_single-file.golden | 50 +++++++++++- ...exclude-region.jsonnet_output-files.golden | 57 +++++++++++++- .../exclude-region.jsonnet_single-file.golden | 52 ++++++++++++- ...fault-excluded.jsonnet_output-files.golden | 59 +++++++++++++- ...efault-excluded.jsonnet_single-file.golden | 54 ++++++++++++- ...i-region-group.jsonnet_output-files.golden | 53 ++++++++++++- ...ti-region-group.jsonnet_single-file.golden | 48 +++++++++++- .../parallel-mode.jsonnet_output-files.golden | 57 +++++++++++++- .../parallel-mode.jsonnet_single-file.golden | 52 ++++++++++++- ...stage-override.jsonnet_output-files.golden | 61 +++++++++++++-- ...-stage-override.jsonnet_single-file.golden | 56 +++++++++++-- .../rollback.jsonnet_output-files.golden | 61 +++++++++++++-- .../rollback.jsonnet_single-file.golden | 56 +++++++++++-- .../stage-props.jsonnet_output-files.golden | 55 ++++++++++++- .../stage-props.jsonnet_single-file.golden | 50 +++++++++++- 27 files changed, 1313 insertions(+), 74 deletions(-) diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index 0c57814..b673eed 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -3,10 +3,11 @@ */ { - group_order: ['de', 'us', 'control', 'snty-tools', 'st'], + group_order: ['s4s2', 'de', 'us', 'control', 'snty-tools', 'st'], test_group_order: [], // These groupings consist of user facing deployments pipeline_groups: { + s4s2: ['s4s2'], de: ['de'], us: ['us'], control: ['control'], diff --git a/test/pipedream.js b/test/pipedream.js index 8642a2a..397b924 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -22,7 +22,8 @@ test("generates pipeline per group in correct order", async (t) => { n.startsWith("deploy-example-"), ); - // Should have de, us, st (control/snty-tools excluded by default) + // Should have s4s2, de, us, st (control/snty-tools excluded by default) + t.true(pipelineNames.includes("deploy-example-s4s2")); t.true(pipelineNames.includes("deploy-example-de")); t.true(pipelineNames.includes("deploy-example-us")); t.true(pipelineNames.includes("deploy-example-st")); @@ -54,9 +55,13 @@ test("single-region group has one job", async (t) => { test("serial mode: pipelines chain sequentially", async (t) => { const got = await render_fixture("pipedream/basic-manual.jsonnet", false); - // de depends on trigger + // s4s2 depends on trigger + const s4s2 = got.pipelines["deploy-example-s4s2"]; + t.truthy(s4s2.materials["deploy-example-pipeline-complete"]); + + // de depends on s4s2 const de = got.pipelines["deploy-example-de"]; - t.truthy(de.materials["deploy-example-pipeline-complete"]); + t.truthy(de.materials["deploy-example-s4s2-pipeline-complete"]); // us depends on de const us = got.pipelines["deploy-example-us"]; @@ -66,14 +71,17 @@ test("serial mode: pipelines chain sequentially", async (t) => { test("parallel mode: all pipelines depend on trigger only", async (t) => { const got = await render_fixture("pipedream/parallel-mode.jsonnet", false); + const s4s2 = got.pipelines["deploy-example-s4s2"]; const de = got.pipelines["deploy-example-de"]; const us = got.pipelines["deploy-example-us"]; // All depend on trigger + t.truthy(s4s2.materials["deploy-example-pipeline-complete"]); t.truthy(de.materials["deploy-example-pipeline-complete"]); t.truthy(us.materials["deploy-example-pipeline-complete"]); // None depend on each other + t.falsy(de.materials["deploy-example-s4s2-pipeline-complete"]); t.falsy(us.materials["deploy-example-de-pipeline-complete"]); }); diff --git a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden index 1842205..1f935cd 100644 --- a/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden +++ b/test/testdata/goldens/getsentry/groups.jsonnet_output-files.golden @@ -6,6 +6,9 @@ "de": [ "de" ], + "s4s2": [ + "s4s2" + ], "snty-tools": [ "snty-tools" ], @@ -20,6 +23,7 @@ ] }, "group_order": [ + "s4s2", "de", "us", "control", diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden index 128ad40..cacfda1 100644 --- a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden @@ -3,9 +3,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -46,11 +50,58 @@ } } }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -125,7 +176,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden index 6cf8fe1..fca2a4c 100644 --- a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden @@ -2,9 +2,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -43,8 +47,50 @@ } ] }, + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -114,7 +160,7 @@ ] }, "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden index 048ac0a..3c47f8c 100644 --- a/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden @@ -3,6 +3,57 @@ "format_version": 10, "pipelines": { "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { "display_order": 2, "group": "example", "materials": { @@ -20,11 +71,11 @@ { "deploy": { "jobs": { - "deploy-de": { + "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ { - "script": "./deploy.sh --region=de" + "script": "./deploy.sh --region=s4s2" } ] } @@ -54,7 +105,7 @@ "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -129,7 +180,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden index 1c93c4d..be9d566 100644 --- a/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden @@ -34,6 +34,52 @@ ] }, "deploy-example-de": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-de": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=de" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-s4s2": { "display_order": 2, "group": "example", "materials": { @@ -51,11 +97,11 @@ { "deploy": { "jobs": { - "deploy-de": { + "deploy-s4s2": { "elastic_profile_id": "example", "tasks": [ { - "script": "./deploy.sh --region=de" + "script": "./deploy.sh --region=s4s2" } ] } @@ -80,7 +126,7 @@ ] }, "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -150,7 +196,7 @@ ] }, "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden index d2486a4..9620b01 100644 --- a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden @@ -3,9 +3,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "init_repo": { "branch": "main", "git": "git@github.com:getsentry/example.git" @@ -67,11 +71,79 @@ } } }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy s4s2" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-s4s2": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify s4s2" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -203,7 +275,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden index b43896a..5d87224 100644 --- a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden @@ -2,9 +2,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "init_repo": { "branch": "main", "git": "git@github.com:getsentry/example.git" @@ -64,8 +68,71 @@ } ] }, + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "init_repo": { + "branch": "main", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "tasks": [ + { + "exec": { + "arguments": [ + "deploy s4s2" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "verify": { + "jobs": { + "verify-s4s2": { + "tasks": [ + { + "exec": { + "arguments": [ + "verify s4s2" + ], + "command": "echo" + } + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -192,7 +259,7 @@ ] }, "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden index 12a554a..d6ad891 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden @@ -1,11 +1,70 @@ { + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "environment_variables": { + "PIPELINE_VAR": "pipeline-s4s2", + "SHARED_VAR_JOB": "from-stage", + "SHARED_VAR_STAGE": "from-stage", + "STAGE_VAR": "stage-s4s2" + }, + "jobs": { + "deploy-s4s2": { + "environment_variables": { + "JOB_VAR": "job-s4s2", + "SHARED_VAR_JOB": "from-job" + }, + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "git": "git@github.com:getsentry/example.git" diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden index 406fd59..1a59c4b 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden @@ -1,7 +1,7 @@ { "format_version": 10, "pipelines": { - "deploy-example-st": { + "deploy-example-s4s2": { "display_order": 2, "group": "example", "materials": { @@ -10,6 +10,60 @@ "git": "git@github.com:getsentry/example.git" } }, + "stages": [ + { + "deploy": { + "environment_variables": { + "PIPELINE_VAR": "pipeline-s4s2", + "SHARED_VAR_JOB": "from-stage", + "SHARED_VAR_STAGE": "from-stage", + "STAGE_VAR": "stage-s4s2" + }, + "jobs": { + "deploy-s4s2": { + "environment_variables": { + "JOB_VAR": "job-s4s2", + "SHARED_VAR_JOB": "from-job" + }, + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "git": "git@github.com:getsentry/example.git" + } + }, "stages": [ { "deploy": { diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden index 1da63de..295d8e4 100644 --- a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden @@ -3,9 +3,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -46,11 +50,58 @@ } } }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-us.yaml": { "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden index 6fe8773..aba71ee 100644 --- a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden @@ -2,9 +2,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -43,8 +47,50 @@ } ] }, + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden index a136a05..6b3100b 100644 --- a/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden @@ -3,9 +3,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -46,11 +50,58 @@ } } }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -117,7 +168,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden index 3587330..b468a3c 100644 --- a/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden @@ -2,9 +2,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -43,8 +47,50 @@ } ] }, + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -106,7 +152,7 @@ ] }, "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden index 02cc02b..1b0e02f 100644 --- a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden @@ -3,7 +3,7 @@ "format_version": 10, "pipelines": { "deploy-example-control": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -54,9 +54,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -97,11 +101,58 @@ } } }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 5, + "display_order": 6, "group": "example", "materials": { "deploy-example-control-pipeline-complete": { @@ -176,7 +227,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden index 0eb6c38..d51268d 100644 --- a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden @@ -2,7 +2,7 @@ "format_version": 10, "pipelines": { "deploy-example-control": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -48,9 +48,13 @@ ] }, "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -89,8 +93,50 @@ } ] }, + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, "deploy-example-st": { - "display_order": 5, + "display_order": 6, "group": "example", "materials": { "deploy-example-control-pipeline-complete": { @@ -160,7 +206,7 @@ ] }, "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden index 38ca95b..9588544 100644 --- a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden @@ -1,11 +1,62 @@ { + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden index 006b7b2..c0fedff 100644 --- a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden @@ -1,7 +1,7 @@ { "format_version": 10, "pipelines": { - "deploy-example-st": { + "deploy-example-s4s2": { "display_order": 2, "group": "example", "materials": { @@ -11,6 +11,52 @@ "git": "git@github.com:getsentry/example.git" } }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, "stages": [ { "deploy": { diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden index 1f2700e..24b5c30 100644 --- a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden @@ -3,7 +3,7 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { "deploy-example-pipeline-complete": { @@ -50,11 +50,62 @@ } } }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-pipeline-complete": { @@ -129,7 +180,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden index 4b7a825..84fc072 100644 --- a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden @@ -34,7 +34,7 @@ ] }, "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { "deploy-example-pipeline-complete": { @@ -79,8 +79,54 @@ } ] }, + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "deploy-example-pipeline-complete": { + "pipeline": "deploy-example", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-pipeline-complete": { @@ -150,7 +196,7 @@ ] }, "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-pipeline-complete": { diff --git a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden index 9cfd1e7..3861339 100644 --- a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden @@ -3,9 +3,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -46,11 +50,58 @@ } } }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -125,7 +176,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -178,9 +229,9 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" diff --git a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden index 15c6afe..6a2cec5 100644 --- a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden @@ -2,9 +2,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -43,8 +47,50 @@ } ] }, + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -114,7 +160,7 @@ ] }, "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -162,9 +208,9 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" diff --git a/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden index 13c7c04..329cbe3 100644 --- a/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden @@ -3,9 +3,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -46,11 +50,58 @@ } } }, + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -125,7 +176,7 @@ "format_version": 10, "pipelines": { "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -178,9 +229,9 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" diff --git a/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden index 60639da..1fef74f 100644 --- a/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden @@ -2,9 +2,13 @@ "format_version": 10, "pipelines": { "deploy-example-de": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "destination": "example", @@ -43,8 +47,50 @@ } ] }, + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "destination": "example", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "jobs": { + "deploy-s4s2": { + "elastic_profile_id": "example", + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, "deploy-example-st": { - "display_order": 4, + "display_order": 5, "group": "example", "materials": { "deploy-example-us-pipeline-complete": { @@ -114,7 +160,7 @@ ] }, "deploy-example-us": { - "display_order": 3, + "display_order": 4, "group": "example", "materials": { "deploy-example-de-pipeline-complete": { @@ -162,9 +208,9 @@ "rollback-example": { "display_order": 1, "environment_variables": { - "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "ALL_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "GOCD_ACCESS_TOKEN": "{{SECRET:[devinfra][gocd_access_token]}}", - "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", + "REGION_PIPELINE_FLAGS": "--pipeline=deploy-example-s4s2 --pipeline=deploy-example-de --pipeline=deploy-example-us --pipeline=deploy-example-st", "ROLLBACK_MATERIAL_NAME": "example_repo", "ROLLBACK_STAGE": "deploy", "TRIGGERED_BY": "" diff --git a/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden index fb45019..2004ab4 100644 --- a/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden @@ -1,11 +1,64 @@ { + "deploy-example-s4s2.yaml": { + "format_version": 10, + "pipelines": { + "deploy-example-s4s2": { + "display_order": 2, + "group": "example", + "materials": { + "example_repo": { + "branch": "master", + "git": "git@github.com:getsentry/example.git" + } + }, + "stages": [ + { + "deploy": { + "approval": { + "type": "manual" + }, + "fetch_materials": true, + "jobs": { + "deploy-s4s2": { + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + } + } + }, "deploy-example-st.yaml": { "format_version": 10, "pipelines": { "deploy-example-st": { - "display_order": 2, + "display_order": 3, "group": "example", "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, "example_repo": { "branch": "master", "git": "git@github.com:getsentry/example.git" diff --git a/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden index b0758c1..18c7705 100644 --- a/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden @@ -1,7 +1,7 @@ { "format_version": 10, "pipelines": { - "deploy-example-st": { + "deploy-example-s4s2": { "display_order": 2, "group": "example", "materials": { @@ -10,6 +10,54 @@ "git": "git@github.com:getsentry/example.git" } }, + "stages": [ + { + "deploy": { + "approval": { + "type": "manual" + }, + "fetch_materials": true, + "jobs": { + "deploy-s4s2": { + "tasks": [ + { + "script": "./deploy.sh --region=s4s2" + } + ] + } + } + } + }, + { + "pipeline-complete": { + "jobs": { + "pipeline-complete": { + "tasks": [ + { + "exec": { + "command": true + } + } + ] + } + } + } + } + ] + }, + "deploy-example-st": { + "display_order": 3, + "group": "example", + "materials": { + "deploy-example-s4s2-pipeline-complete": { + "pipeline": "deploy-example-s4s2", + "stage": "pipeline-complete" + }, + "example_repo": { + "branch": "master", + "git": "git@github.com:getsentry/example.git" + } + }, "stages": [ { "deploy": { From 9a0fa40daabaec2d5f7184c210c902fbc91118f3 Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 15 Apr 2026 09:54:31 -0700 Subject: [PATCH 09/16] chore: remove working notes from branch --- PIPEDREAM_CELLULARIZATION_NOTES.md | 119 ------------- PIPEDREAM_USE_CASES.md | 91 ---------- notes.md | 175 ------------------- status.md | 266 ----------------------------- 4 files changed, 651 deletions(-) delete mode 100644 PIPEDREAM_CELLULARIZATION_NOTES.md delete mode 100644 PIPEDREAM_USE_CASES.md delete mode 100644 notes.md delete mode 100644 status.md diff --git a/PIPEDREAM_CELLULARIZATION_NOTES.md b/PIPEDREAM_CELLULARIZATION_NOTES.md deleted file mode 100644 index 06cc696..0000000 --- a/PIPEDREAM_CELLULARIZATION_NOTES.md +++ /dev/null @@ -1,119 +0,0 @@ -# Pipedream Cellularization - Working Notes - -> **Do not commit this file.** Personal working notes for the `iw/grouped-pipedream` branch. - -## 1. Stage Properties: "First Region Wins" — GoCD Architectural Limit - -**Finding:** GoCD's XML schema (`cruise-config.xsd`) defines `fetch_materials`, `clean_workspace`, `approval`, and `keep_artifacts` as **strictly stage-level attributes**. They do not exist on `jobType`. The YAML config plugin (`StageTransform.java`) enforces the same boundary. There is **no per-job override mechanism** for stage properties in GoCD. - -**Implication:** When merging multiple regions' jobs into a single stage in grouped Pipedream, the stage can only have ONE set of properties. The current approach uses the first region's properties. This is the only option given GoCD's architecture. - -**Recommendation — add validation (~10-15 lines):** When regions in the same group define conflicting stage props for the same-named stage, we should error at build time rather than silently picking the first. Estimated implementation: - -```jsonnet -// Inside transform_stage, after computing stage_props from first region: -local _ = std.foldl( - function(acc, r) - local p = region_pipelines[r]; - local rs = get_matching_stage(p, stage_name); - local props = if rs != null then get_stage_props(rs) else stage_props; - assert props == stage_props : - "Stage '%s': conflicting properties across regions in group. " - + "Region '%s' differs from '%s'." % [stage_name, r, regions[0]]; - acc, - regions[1:], - true -); -``` - -~10 lines of code. Should be added before finalizing. - ---- - -## 2. Environment Variable Optimization — Implemented - -**Problem (before):** All env vars (pipeline + stage + job) were blindly cascaded to every job, and pipeline/stage-level `environment_variables` were stripped from the output. This caused duplication — shared vars like `GOCD_ACCESS_TOKEN` were repeated N times per stage (once per region-job). - -**Solution (implemented on this branch):** Env vars that are **identical across all regions** in a group are kept at stage level. Only vars that **differ between regions** are cascaded to job level. GoCD's native precedence (job > stage > pipeline) handles resolution correctly. - -**Changes made:** -- `libs/pipedream.libsonnet`: Rewrote `transform_stage` to compute `common_env` (intersection of pipeline+stage env vars across all regions) and `region_specific_env` (the diff). Common vars go to stage, region-specific to jobs. -- `test/pipedream.js`: Added test `"env var optimization: common vars at stage level, region-specific at job level"` validating the split. -- Golden files for `env-vars-precedence` regenerated. - ---- - -## 3. Double-calling `pipeline_fn(regions[0])` — Fixed - -**Problem (before):** `pipeline_fn(regions[0])` was called at the top of `generate_group_pipeline` to get a template, then called AGAIN for every region (including `regions[0]`) in the `all_stages` fold and `transform_stage`. This meant `pipeline_fn` for the first region ran 3x. - -**Fix (implemented on this branch):** Added a cache: -```jsonnet -local region_pipelines = { [r]: pipeline_fn(r) for r in regions }; -``` -All subsequent references use `region_pipelines[region]` instead of `pipeline_fn(region)`. Also extracted `get_matching_stage` helper to reduce repeated stage-matching logic. - ---- - -## 4. s4s/s4s2 Alignment — Resolved - -**Background:** On `main`, s4s2 replaced s4s as the prod region (commit `3b7e3b1`). s4s is now a test region. - -**Change made:** Updated `getsentry.libsonnet` so the `s4s` group only contains `['s4s2']` (was `['s4s', 's4s2']`). This aligns with main where s4s2 is the prod region. - -**Impact on downstream compatibility (positive):** -- The `s4s` group is now single-region, so no double-suffixed job names for s4s. -- The `region_pops` crash risk for uptime-checker/relay is **eliminated for the s4s group** — `pipeline_fn('s4s2')` is already called on main today, so all services already handle it. -- Pipeline name changes: `deploy-{service}-s4s2` → `deploy-{service}-s4s`. This is still a rename that downstream tools need to handle. - -**Remaining TODO:** `test_groups` is currently empty. On main, `s4s` is a test region. Consider whether to add it as a test group. - ---- - -## 5/8. Pipeline Name + Job Name Changes — Downstream Impact - -### Pipeline Name Changes - -Pipeline names change from `deploy-{service}-{region}` to `deploy-{service}-{group}`: -- `deploy-example-customer-1`, `-customer-2`, `-customer-4`, `-customer-7` → `deploy-example-st` -- `deploy-example-s4s2` → `deploy-example-s4s` (s4s group name, contains only s4s2) -- Single-region groups unchanged: `deploy-example-de`, `deploy-example-us` - -**Key insight after s4s2 alignment:** Most single-region groups (`de`, `us`, `control`, `snty-tools`) keep their existing pipeline names. Only two name changes actually happen: -1. `s4s2` → `s4s` (group name differs from region name) -2. `customer-{1,2,4,7}` → `st` (four pipelines collapse to one) - -### Repos That Will Break - -| Repo | Severity | What breaks | -|------|----------|-------------| -| **gocd-deployment-visualizer** (deploy-tools) | **Critical** | `isMultiregion()` checks for `-s4s2`/`-customer-1` suffixes. `isPipedreamPipeline()` has hardcoded allowlist of per-region names. `findPrimaryTrigger()` looks for `-s4s2`. `PIPEDREAM_SUFFIX_CUSTOMER_MAP` maps per-region suffixes to display names. `constructTree()` hardcodes `-s4s2` stripping. | -| **eng-pipes** | **Critical** | Slack feed filters have hardcoded `deploy-snuba-py-customer-{N}`, etc. `getFormattedRegion()` parses pipeline suffixes for Datadog tags — `-st` won't map to anything. | -| **sentry-feature-scoring** | **High** | Hardcoded `deploy-getsentry-backend-s4s2` in exclusion list. Deploy completeness check references `deploy-getsentry-backend-customer-7`. | -| **sentry-feature-scoring-frontend** | **High** | Constructs pipeline names via `` `deploy-getsentry-backend-${region}` `` with per-region strings. `BACKEND_REGIONS` array lists individual regions. | -| **devinfra-pause-metrics** | **Medium** | Parses region from pipeline suffix, has hardcoded region buckets. | -| **ops** | **Medium** | `dd-event.py` has `REGIONS_MAP` with `customer-*` keys. | - -### Job Name Issues - -**No true name collisions found**, but one remaining concern: - -1. **Double-suffixed names in st group:** Services that embed region/pop in job names (e.g., uptime-checker's `deploy-canary-de`) will get `deploy-canary-de-de` after the `-{region}` append. Ugly but functionally correct. **The s4s group no longer has this issue** since it's single-region. - -2. **`region_pops` crash risk reduced:** With s4s2 as the sole region in the s4s group, `pipeline_fn('s4s2')` is already the status quo on main. The crash risk only applies to the `st` group for services like uptime-checker/relay that may not have entries for all customer regions in `region_pops`. (These services likely already exclude customer regions or have them in their pops dictionaries since customers are already prod regions on main.) - -### Recommendations - -**Option A — Coordinated migration (recommended):** -1. Update all downstream repos to use group-aware pipeline naming BEFORE cutting v2 -2. During transition, these repos should handle both old and new naming patterns -3. Create a Linear ticket per repo with specific file changes needed - -**Option B — Compatibility shim:** -Add a mapping/lookup in the Pipedream library itself that exposes both the group name and the constituent region names, so downstream tools can query "which regions are in group `st`?" rather than hardcoding names. - -**Option C — Keep per-region pipeline names (simplest, least disruptive):** -Instead of naming pipelines `deploy-{service}-{group}`, continue naming them `deploy-{service}-{first_region_in_group}`. So the `st` group would still be `deploy-example-customer-1` and the `s4s` group would be `deploy-example-s4s2`. This avoids ALL downstream breakage but makes the group concept less visible in naming. Not recommended long-term but could be a transitional approach. - -**Option D — Name groups after first region (hybrid):** -Same as Option C but only for groups where the name differs from the first region. The `s4s` group contains `['s4s2']`, so name it `deploy-example-s4s2`. The `st` group contains `['customer-1', ...]`, so name it `deploy-example-customer-1`. Only truly custom group names (if any) get a new name. This makes the change invisible to downstream tools while still grouping jobs in parallel internally. diff --git a/PIPEDREAM_USE_CASES.md b/PIPEDREAM_USE_CASES.md deleted file mode 100644 index cc0f58f..0000000 --- a/PIPEDREAM_USE_CASES.md +++ /dev/null @@ -1,91 +0,0 @@ -# Pipedream Use Cases Across getsentry - -> Inventory of all real-world Pipedream usage, for validating test coverage. - -## Service Repos (15 repos, 18 templates) - -| Service | auto_deploy | exclude_regions | include_regions | parallel | rollback | rollback.final_stage | -|---------|-------------|-----------------|-----------------|----------|----------|----------------------| -| getsentry-backend | true (default) | `[]` | `['control']` | no | yes | no | -| relay-pop | false | `['de','us','s4s2','customer-1/2/4']` | — | no | yes | no | -| relay-processing | false | — | — | no | yes | no | -| snuba-py | true | — | — | no | yes | no | -| snuba-rs | true | — | — | no | yes | no | -| symbolicator | false | — | — | no | yes | **yes** (`deploy-primary`) | -| chartcuterie | true | `['customer-6']` | — | no | yes | no | -| taskbroker | true | — | — | no | yes | no | -| conduit | true | `['us','de','customer-1/2/4/7']` | — | no | yes | no | -| vroom | true (default) | — | — | no | yes | no | -| uptime-checker | true | `['customer-1/2/3/4/6/7']` | — | no | yes | no | -| super-big-consumers | false | — | — | no | yes | **yes** (`deploy-primary`) | -| seer | true | `['customer-3/6']` | — | no | yes | no | -| seer-gpu | true | `['customer-3/6']` | — | no | yes | no | -| launchpad | true | `['customer-1/2/3/4/7']` | — | no | yes | no | -| objectstore | false | `['customer-1/2/4/7']` | — | no | yes | no | -| tempest | true (default) | `['customer-1/2/3/4/6/7']` | — | no | yes | no | -| sentry-scripts | true | — | `['snty-tools']` | **yes** | **no** | — | - -## Ops K8s Templates (50+ templates) - -- 8 with custom pipeline_fn (getsentry-k8s, relay-k8s, snuba-k8s, relay-pop-k8s, symbolicator-k8s, taskbroker-k8s, uptime-checker-k8s, sbc-k8s) -- 41+ using shared `gocd.pipedream_config()` helper -- All ops templates use `auto_deploy: false` -- objectstore-k8s uses `parallel=true` -- workflow-engine-k8s and script-runner-k8s use `include_regions: ['snty-tools']` - -## Pipeline Function Patterns - -### Pattern 1: Pipeline-level `environment_variables` (ALL services) -Every service sets at minimum `SENTRY_REGION` at pipeline level. Many also set `GITHUB_TOKEN`, `GOCD_ACCESS_TOKEN`, `SKIP_CANARY_CHECKS`, `SENTRY_ORG`, `SENTRY_PROJECT`, Datadog keys. - -**Test coverage:** `env-vars-precedence.jsonnet` covers pipeline/stage/job env vars. - -### Pattern 2: Region-conditional stages (common) -- Canary stages only for US/DE (snuba, symbolicator, chartcuterie, taskbroker, vroom, relay, getsentry-backend) -- ST-specific migration stage (snuba-py: `st_migrate`) -- Soak time only for S4S and US (relay, getsentry-backend, objectstore) - -**Test coverage:** `different-stages-per-region.jsonnet` covers different stage sets per region. - -### Pattern 3: `getsentry.is_st(region)` conditional (3 repos) -Used by snuba-py, snuba-rs, and symbolicator to choose deploy scripts, migration logic, and approval types. - -**Test coverage:** Not a pipedream concern — this runs inside `pipeline_fn`. Pipedream just calls `pipeline_fn(region)` and handles whatever comes back. - -### Pattern 4: `region_pops` per-pop jobs (2 repos) -relay-pop and uptime-checker define jobs per sub-PoP using comprehensions like `['deploy-primary-' + pop]: ...`. These generate job names that already contain region identifiers. - -**Test coverage:** Not a pipedream concern at the library level. However, the `-{region}` suffix appended by grouping will create double-suffixed names (e.g. `deploy-primary-de-de`). Documented as a known issue. - -### Pattern 5: Multiple stages per pipeline (most services) -Real services typically have 2-4 stages: `checks`, `deploy-canary`, `deploy-primary`, `soak-time`. Some have `migrations`, `health_check`, `scale-down-canary`. - -**Test coverage:** Most fixtures use a single `deploy` stage. `different-stages-per-region.jsonnet` uses deploy + verify. Adequate — pipedream doesn't treat stages differently based on count. - -### Pattern 6: `rollback.final_stage` override (2 repos) -symbolicator and super-big-consumers override `final_stage` to `deploy-primary` instead of the default `pipeline-complete`. - -**Test coverage:** **GAP** — we test invalid `final_stage` (failing fixtures) but not a valid override. Added `rollback-final-stage-override.jsonnet`. - -### Pattern 7: No rollback config (1 repo) -sentry-scripts has no rollback. - -**Test coverage:** Covered — `basic-autodeploy.jsonnet` has no rollback config. - -### Pattern 8: `include_regions` for `snty-tools` (3 repos) -sentry-scripts, ops/workflow-engine-k8s, ops/script-runner-k8s. - -**Test coverage:** `include-default-excluded.jsonnet` tests `include_regions: ['control']`. snty-tools uses the same mechanism. Covered. - -### Pattern 9: Heavy `exclude_regions` narrowing to few groups (common) -conduit excludes `['us','de','customer-1/2/4/7']` leaving only s4s. relay-pop excludes everything except s4s and customer-7. - -**Test coverage:** `exclude-region.jsonnet` (partial exclude) and `exclude-entire-group.jsonnet` (full group excluded). Covered. - -## Coverage Gap Summary - -| Gap | Severity | Action | -|-----|----------|--------| -| Valid `rollback.final_stage` override | Medium | Add `rollback-final-stage-override.jsonnet` fixture | -| Stage-level env vars (no job env vars) | Low | seer sets `SENTRY_REGION` at stage level only. Implicitly covered by env-vars-precedence but could add explicit fixture. | -| `auto_pipeline_progression: false` | Low | Exists in code but NOT used by any real service. Skip for now. | diff --git a/notes.md b/notes.md deleted file mode 100644 index bced33f..0000000 --- a/notes.md +++ /dev/null @@ -1,175 +0,0 @@ -# Pipedream Cellularization — Working Notes - -> Do not commit this file. - -## Test Coverage Audit (DI-1601 / DI-1602) - -### Gap found and filled - -Added `rollback-final-stage-override.jsonnet` — symbolicator and -super-big-consumers use `rollback.final_stage` to point at `deploy-primary` -instead of the default `pipeline-complete`. We had tests for invalid overrides -but not a valid one. - -### Already covered by existing fixtures - -- Auto-deploy vs manual deploy -- Region exclusions (partial and full group) -- Region inclusions (control; snty-tools uses same mechanism) -- Parallel mode -- No rollback config -- Pipeline/stage/job env var cascading -- Different stages per region -- Stage property conflicts across regions -- Multi-region vs single-region groups - -Full use-case inventory in `PIPEDREAM_USE_CASES.md`. - ---- - -## Env Var Cascade — Impact Assessment - -We changed env var handling so that variables identical across all regions in a -group stay at stage level, while region-specific variables cascade to job level. - -### Will this cause issues? - -**No.** GoCD natively resolves env vars with job > stage > pipeline precedence. -Moving shared vars up to stage level is semantically identical to duplicating -them at job level — GoCD evaluates them the same way. - -Checked against all 18 service templates and 50+ ops templates: - -- **Every service** sets `SENTRY_REGION` at pipeline level. This differs per - region, so it correctly cascades to job level (region-specific). -- **Shared secrets** (e.g. `GOCD_ACCESS_TOKEN`, `GITHUB_TOKEN`, `DATADOG_API_KEY`) - are the same across all regions. These now stay at stage level instead of - being duplicated into every job. No behavioral change. -- **seer** sets `SENTRY_REGION` at stage level (not pipeline level). With a - single-region group this is fine — it becomes a common env var at stage level. - In a multi-region group it would correctly cascade to job level since the - values differ. -- **getsentry-backend** is the most env-var-heavy service (`SENTRY_REGION`, - `SENTRY_ORG`, `SENTRY_PROJECT`, `SENTRY_PROJECT_ID`, etc. all at pipeline - level). All region-specific vars will cascade to jobs; shared vars like - `GITHUB_TOKEN` and `SKIP_CANARY_CHECKS` stay at stage level. Correct. - -**One edge case to be aware of:** if a service sets the same env var key at -both pipeline and stage level with different values, the stage value wins in the -merged parent env (pipeline + stage merge). This is the same precedence as -before — no change in behavior. - ---- - -## Stage Properties — GoCD Architectural Limit - -GoCD's XML schema defines `fetch_materials`, `clean_workspace`, `approval`, and -`keep_artifacts` as strictly stage-level attributes. There is no per-job -override. When merging multiple regions' jobs into a single stage, the first -region's properties win. - -**Recommendation:** Add ~10 lines of validation to error at build time if -regions in the same group define conflicting stage props for the same stage. - ---- - -## Downstream Pipeline Name Breakage (DI-1670) - -Pipeline names change from `deploy-{service}-{region}` to -`deploy-{service}-{group}`. Two actual renames: - -1. `s4s2` → `s4s` (group name differs from region name) -2. `customer-{1,2,4,7}` → `st` (four pipelines collapse to one) - -Single-region groups (`de`, `us`, `control`, `snty-tools`) keep existing names. - -### Repos that need updates - -| Repo | Severity | What breaks | -|------|----------|-------------| -| gocd-deployment-visualizer | Critical | `isMultiregion()`, `isPipedreamPipeline()`, `findPrimaryTrigger()`, `PIPEDREAM_SUFFIX_CUSTOMER_MAP`, `constructTree()` | -| eng-pipes | Critical | Slack feed filters, `getFormattedRegion()` Datadog tagging | -| sentry-feature-scoring | High | Hardcoded `deploy-getsentry-backend-s4s2` exclusion, `deploy-getsentry-backend-customer-7` completeness check | -| sentry-feature-scoring-frontend | High | `BACKEND_REGIONS` array, pipeline name construction | -| devinfra-pause-metrics | Medium | Region parsing from pipeline suffix | -| ops | Medium | `dd-event.py` `REGIONS_MAP` | - -### Double-suffixed job names - -Services that embed region/pop in job names (uptime-checker, relay-pop) will get -`deploy-primary-de-de` style names. Functionally correct but ugly. Only affects -the `st` group in practice since `s4s` is now single-region. - ---- - -## s4s/s4s2 Alignment - -Updated `getsentry.libsonnet` so the `s4s` group contains only `['s4s2']`, -matching main where s4s2 is the prod region (commit `3b7e3b1`). - -**Remaining TODO:** `test_groups` is empty. On main, `s4s` is a test region. -Consider adding it as a test group. - ---- - -## Pilot Testing — devinfra-example-service (DI-1607) - -Bumped `devinfra-example-service` to `v3.0.0-rc.1`. -PR: https://github.com/getsentry/devinfra-example-service/pull/24 - -Generated pipelines verified locally (`make gocd`): - -- `deploy-devinfra-example-service-s4s` — single job (`s4s2`), `SENTRY_REGION` at stage level -- `deploy-devinfra-example-service-de` — single job, chained after s4s -- `deploy-devinfra-example-service-us` — single job, chained after de -- `deploy-devinfra-example-service-st` — 4 parallel jobs (`customer-1`, `-2`, `-4`, `-7`), `SENTRY_REGION` at job level -- `rollback-devinfra-example-service` — watches st, references all 4 group pipelines -- `control`/`snty-tools` correctly excluded by default - ---- - -## Issues Found During Pilot Review - -### 1. Silent stage loss when stage objects have multiple keys - -`demo.libsonnet` has a missing comma between two stage objects (lines 30–31), -causing jsonnet to merge them via implicit `+`. The resulting single object -`{first: {...}, 'deploy-primary': {...}}` is treated as one stage, and -`get_stage_name()` (which calls `std.objectFields(stage)[0]`) only sees -`deploy-primary` (alphabetical). The `first` stage silently disappears. - -This is a pre-existing bug in the example service, not in gocd-jsonnet. But it -reveals a footgun: **any service with a missing comma between stages would -silently lose a stage.** Should add a build-time assertion in `transform_stage` -that each stage object has exactly one key. - -### 2. Stage property conflict validation (still TODO) - -`get_stage_props` takes the first region's properties and ignores the rest. -If `customer-1` defines `approval: manual` and `customer-7` defines -`approval: success` on the same stage in the `st` group, the first region wins -silently. Need ~10 lines of validation to error at build time when regions in -the same group define conflicting stage props. (Also noted in "Stage -Properties" section above.) - -### 3. No `exclude_groups` config option - -`exclude_regions` only operates on region names. To exclude the entire `st` -group you'd need: `exclude_regions: ['customer-1', 'customer-2', 'customer-4', -'customer-7']`. An `exclude_groups: ['st']` option would be cleaner and -wouldn't break when new regions are added to a group. Low priority but nice -to have. - ---- - -## Code Changes on Branch - -- `libs/pipedream.libsonnet`: env var optimization (common vars at stage level), - `pipeline_fn` result caching, extracted `get_matching_stage` helper -- `libs/getsentry.libsonnet`: s4s group → `['s4s2']` only -- `test/pipedream.js`: updated s4s group assertion -- Test fixtures updated to use `st` group for multi-region tests - (`env-vars-precedence`, `stage-props`, `different-stages-per-region`) -- Added `rollback-final-stage-override.jsonnet` fixture -- All golden files regenerated -- Pre-release `v3.0.0-rc.1` cut from `iw/grouped-pipedream` branch diff --git a/status.md b/status.md deleted file mode 100644 index 1dd32f4..0000000 --- a/status.md +++ /dev/null @@ -1,266 +0,0 @@ -# Pipedream Cellularization — Comprehensive Status - -> Updated 2026-04-09. Working directory: `/Users/mingchen/Desktop/sentry/gocd-jsonnet` -> Branch: `iw/grouped-pipedream` (6 commits ahead of main) - ---- - -## 1. What This Project Does - -Changes the Pipedream deployment model from **one GoCD pipeline per region** to **one pipeline per group**, with regions within each group running as parallel jobs. - -**Before (main):** Sequential pipeline per region: -``` -deploy-example-s4s2 → deploy-example-de → deploy-example-us → deploy-example-customer-1 → ... → deploy-example-customer-7 -``` - -**After (this branch):** Sequential pipeline per group, parallel jobs within: -``` -deploy-example-s4s → deploy-example-de → deploy-example-us → deploy-example-st - (customer-1, -2, -4, -7 as parallel jobs) -``` - -The groups are defined in `libs/getsentry.libsonnet`: -```jsonnet -{ - s4s: ['s4s2'], // single region - de: ['de'], // single region - us: ['us'], // single region - control: ['control'], // default-excluded - 'snty-tools': ['snty-tools'], // default-excluded - st: ['customer-1', 'customer-2', 'customer-4', 'customer-7'], // multi-region -} -``` - -### Pipeline Name Changes (only 2 actual renames) - -| Old Name | New Name | Reason | -|----------|----------|--------| -| `deploy-{service}-s4s2` | `deploy-{service}-s4s` | Group name differs from region | -| `deploy-{service}-customer-{1,2,4,7}` (4 pipelines) | `deploy-{service}-st` (1 pipeline) | Collapsed into group | -| `deploy-{service}-de` | unchanged | Single-region group | -| `deploy-{service}-us` | unchanged | Single-region group | -| `deploy-{service}-control` | unchanged | Single-region group | -| `deploy-{service}-snty-tools` | unchanged | Single-region group | - ---- - -## 2. Branch State - -### Commits on `iw/grouped-pipedream` (6 ahead of main) - -1. `0b17dc0` — ref(pipedream): Adding grouping support -2. `1f4b769` — cascade pipeline and stage level environment variables down to the job level -3. `3bdb415` — ref(pipedream): env var optimization, s4s2 alignment, and test coverage -4. `4394b71` — ref(pipedream): add build-time validation for conflicting stage properties -5. `8403a80` — ref(pipedream): add single-key stage object assertion -6. `cda941a` — update repo readme to reflect grouping (cherry-picked from 98abdde) - -### Uncommitted Changes - -None — all work committed. - -### All 49 tests pass - -47 original + 2 new (stage property conflict detection + merged stage detection). - -### Stashes (all superseded — safe to drop) - -- `stash@{0}` and `stash@{1}`: On `mingchen/di-1685-gocd-static-agent-blocking-for-pipeline-complete`. Older version of env var/s4s2 work — already incorporated and improved upon in commit `3bdb415`. -- `stash@{2}`: On `iw/grouped-pipedream` at older commit `17530cd`. Subset, also superseded. - ---- - -## 3. Key Files Changed - -### `libs/pipedream.libsonnet` (core — ~500 lines) -Complete rewrite of pipeline generation: -- `generate_group_pipeline()` — creates one pipeline per group, aggregates jobs from all regions -- `transform_stage()` — merges jobs from regions, optimizes env vars (common at stage, region-specific at job) -- `get_service_pipelines()` — generates pipelines for all groups -- Caches `pipeline_fn` results per region to avoid redundant calls -- `get_matching_stage()` helper for stage lookup -- **Build-time assertion:** Stage property conflict validation across regions -- **Build-time assertion:** Single-key stage object validation (catches missing commas) - -### `libs/getsentry.libsonnet` (config — 27 lines) -Changed from flat `prod_regions`/`test_regions` arrays to `pipeline_groups` object with `group_order`. Added `group_names`, `test_group_names`, `get_targets(group)` accessors. - -### `README.md` -Updated with grouping docs, typo fixes (cherry-picked from 98abdde). - -### Test fixtures (16 fixtures) -- `basic-autodeploy.jsonnet`, `basic-manual.jsonnet` — core flows -- `multi-region-group.jsonnet` — st group with 4 regions -- `parallel-mode.jsonnet` — fan-out mode -- `exclude-region.jsonnet`, `exclude-entire-group.jsonnet` — exclusion patterns -- `include-default-excluded.jsonnet` — control/snty-tools inclusion -- `env-vars-precedence.jsonnet` — pipeline/stage/job cascading -- `different-stages-per-region.jsonnet` — conditional stages -- `stage-props.jsonnet` — consistent stage properties across regions -- `stage-props-conflict.failing.jsonnet` — conflict detection -- `merged-stages.failing.jsonnet` — single-key assertion -- `rollback.jsonnet`, `rollback-final-stage-override.jsonnet` — rollback config -- `rollback-bad-stage.failing.jsonnet`, `rollback-bad-final-stage.failing.jsonnet` — error cases - ---- - -## 4. Remaining TODOs in gocd-jsonnet - -### Done This Session -- [x] Stage property conflict validation (~18 lines in `transform_stage`) -- [x] Failing fixture + test for conflict detection -- [x] Single-key stage object assertion (~18 lines in `generate_group_pipeline`) -- [x] Failing fixture + test for merged stages -- [x] Cherry-picked README update from commit `98abdde` - -### Done Previously -- [x] Group-based pipeline generation -- [x] Env var cascade optimization -- [x] s4s/s4s2 alignment -- [x] All test fixtures and golden files - -### Still TODO -- [ ] **`test_groups` decision** — On main, `s4s` is a test region (in `test_regions`). Currently `test_groups` is empty. Should `s4s` be added as a test group? -- [ ] **Tag v3.0.0** — No release tag exists yet. Ready to tag once branch is merged or at Ming's discretion. -- [ ] **Push latest commits** — 3 new commits (4394b71, 8403a80, cda941a) need pushing to origin. - ---- - -## 5. devinfra-example-service (Pilot) - -### Comprehensive Pipeline — DONE - -- Repo: `/Users/mingchen/Desktop/sentry/devinfra-example-service/` -- Branch: `mingchen/comprehensive-pipedream-pilot` (1 commit ahead of main) -- Commit: `a3e1d52` — ref(pipedream): comprehensive pilot pipeline exercising all features - -**Features exercised:** - -| Feature | How | -|---------|-----| -| Manual deploy | `auto_deploy: false` → trigger pipeline with manual approval | -| Region-conditional stages | checks (us/de), canary (us/de/control), soak-time (us only) | -| Region-conditional jobs | health-check only in us deploy-primary | -| Pipeline env var cascade | SENTRY_REGION, SENTRY_DEPLOY_ENV differ per region → cascade to jobs | -| Stage env vars | DEPLOY_TIMEOUT common across regions → stays at stage level | -| Job env vars | LABEL_SELECTOR at job level | -| Stage properties | fetch_materials: true, approval: { type: 'success' } | -| Include default-excluded | control region included via `include_regions` | -| Multi-region parallel jobs | st group: customer-1,-2,-4,-7 as parallel jobs | -| Rollback + final_stage override | Rollback watches deploy-primary instead of pipeline-complete | - -**Generated pipelines (7 files):** -``` -deploy-devinfra-example-service.yaml (trigger) -deploy-devinfra-example-service-s4s.yaml (s4s2 — deploy-primary only) -deploy-devinfra-example-service-de.yaml (de — checks, canary, primary) -deploy-devinfra-example-service-us.yaml (us — checks, canary, primary, soak-time + health-check) -deploy-devinfra-example-service-control.yaml (control — canary, primary) -deploy-devinfra-example-service-st.yaml (4 parallel jobs — primary only) -rollback-devinfra-example-service.yaml (rollback with final_stage=deploy-primary) -``` - -### Still TODO for pilot -- [ ] Update `jsonnetfile.json` to point at `v3.0.0` once tagged -- [ ] Run `make gocd` with official version (currently tested with locally copied vendor libs) -- [ ] Push branch and create PR -- [ ] Verify GoCD loads the config correctly - ---- - -## 6. Downstream Impact — Full Inventory - -### Version Pinning -All 71 downstream repos pin gocd-jsonnet at `v2.19.0` via `jsonnetfile.json`. Rollout = bump to `v3.0.0`. - -### Downstream Tool Updates - -| Repo | Severity | Status | What Needs Changing | -|------|----------|--------|---------------------| -| **gocd-deployment-visualizer** | Critical | **DONE** (branch exists) | Branch: `mingchen/di-1670-update-for-grouped-pipedream` (commit `af9b359`). Updated `isPipedreamPipeline()`, `findPrimaryTrigger()`, `PIPEDREAM_SUFFIX_CUSTOMER_MAP`, `isMultiregion()`. 6 files changed. **Needs merge to main.** | -| **eng-pipes** | Critical | **NOT STARTED** | 60+ hardcoded pipeline names in Slack feed filters (`gocdSlackFeeds/index.ts`), `getFormattedRegion()` in `deployDatadogEvents.ts` (parses region from pipeline name suffix for Datadog tags), paused pipeline reminders config, no-deploys alert config, consecutive unsuccessful alert config. **Files:** `src/config/index.ts`, `src/brain/gocd/gocdSlackFeeds/index.ts`, `src/brain/gocd/gocdDataDog/deployDatadogEvents.ts`, `src/brain/gocd/gocdNoDeploysAlert/index.ts`, `src/brain/gocd/gocdConsecutiveUnsuccessfulAlert/index.ts` | -| **sentry-feature-scoring** | High | **NOT STARTED** | Hardcoded `deploy-getsentry-backend-s4s2` in `src/consts.py:28`. `deploy-getsentry-backend-customer-7` completeness check in `src/productivity/metrics/deploy_failure_rate.py:58,131`. Deploy rate/duration queries reference `deploy-getsentry-backend-us` (unchanged, but verify). | -| **sentry-feature-scoring-frontend** | High | **NOT STARTED** | `BACKEND_REGIONS` array, pipeline name construction via template literals. | -| **devinfra-pause-metrics** | Medium | **NOT STARTED** | Region parsing from pipeline suffix, hardcoded region buckets. Repo not found locally. | -| **ops** | Medium | **NOT STARTED** | `dd-event.py` has `REGIONS_MAP` with `customer-*` keys. | - -### Service Repos (71 consumers — just version bump) - -**15 service repos** with their own pipedream templates: -getsentry-backend, relay-pop, relay-processing, snuba-py, snuba-rs, symbolicator, chartcuterie, taskbroker, conduit, vroom, uptime-checker, super-big-consumers, seer, seer-gpu, launchpad, objectstore, tempest, sentry-scripts - -**50+ ops k8s templates** in `/Users/mingchen/Desktop/sentry/ops/gocd/templates/` - -All just need `jsonnetfile.json` bumped from `v2.19.0` → `v3.0.0`. - ---- - -## 7. Linear Project Status - -**Project:** [Pipedream Cellularization](https://linear.app/getsentry/project/pipedream-cellularization-97a9f328cfeb/overview) -- **Lead:** Ming Chen -- **Team:** DevInfra (DI) -- **Target date:** 2026-04-17 -- **Priority:** High -- **Initiative:** CI/CD allows gradual rollouts and easy rollbacks - -**Milestones:** -1. Finalize Implementation (due Mar 6, 100%) -2. deploy-tools Updates (due Mar 27, 100%) -3. Pilot Testing (due Mar 27, 100%) -4. **Release and Rollout** (due Apr 10, 0%) -5. **Rollout Completion and Documentation** (due Apr 16, 0%) - ---- - -## 8. Proposed Rollout Order - -### Phase 0 — Finish gocd-jsonnet — NEARLY DONE -1. ~~Commit stage prop validation + tests~~ DONE -2. ~~Add single-key stage assertion~~ DONE -3. ~~Cherry-pick README update~~ DONE -4. Push 3 new commits to origin -5. Tag `v3.0.0` (after merge or at Ming's discretion) - -### Phase 1 — Update downstream tools (BEFORE any service cuts over) -1. Merge gocd-deployment-visualizer branch (already done, just merge) -2. Update eng-pipes (critical — Slack feeds, Datadog tagging, pipeline filters) -3. Update sentry-feature-scoring + frontend -4. Update devinfra-pause-metrics -5. Update ops dd-event.py -6. During transition, tools should ideally handle BOTH old and new naming - -### Phase 2 — Pilot with devinfra-example-service — PIPELINE READY -1. ~~Build comprehensive pipeline_fn~~ DONE (branch `mingchen/comprehensive-pipedream-pilot`) -2. Update jsonnetfile.json to v3.0.0 (once tagged) -3. Run `make gocd` with official version -4. Push and create PR -5. Verify GoCD loads config, visualizer works, eng-pipes processes events - -### Phase 3 — Production services (progressive) -- **Wave 1 (low-risk):** chartcuterie, tempest, launchpad, devinfra-example-service -- **Wave 2 (medium):** conduit, vroom, uptime-checker, seer, taskbroker, objectstore -- **Wave 3 (critical):** snuba-py, snuba-rs, relay-pop, relay-processing, symbolicator -- **Wave 4 (core):** getsentry-backend, sentry-scripts, super-big-consumers -- Each wave: bump `jsonnetfile.json`, run `make gocd`, merge, verify in GoCD - -### Phase 4 — Cleanup -- Remove backward-compat code from downstream tools -- Clean up orphaned per-region pipeline configs in GoCD -- Update Notion/docs -- Close Linear milestones - ---- - -## 9. Reference: Downstream Repo Locations - -| Repo | Local Path | -|------|-----------| -| gocd-jsonnet | `/Users/mingchen/Desktop/sentry/gocd-jsonnet` | -| gocd-deployment-visualizer | `/Users/mingchen/Desktop/sentry/gocd-deployment-visualizer` | -| eng-pipes | `/Users/mingchen/Desktop/sentry/eng-pipes` (helios) | -| sentry-feature-scoring | `/Users/mingchen/Desktop/sentry/sentry-feature-scoring` | -| devinfra-example-service | `/Users/mingchen/Desktop/sentry/devinfra-example-service` | -| ops | `/Users/mingchen/Desktop/sentry/ops` | -| getsentry | `/Users/mingchen/Desktop/sentry/getsentry` | From 5954fc926a4bc2dd65db8c93921e131502ba4dc6 Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 15 Apr 2026 11:33:31 -0700 Subject: [PATCH 10/16] fix(pipedream): restore fetch_materials on pipeline-complete stages, fix README and cleanup - Restore fetch_materials: false on both pipeline-complete stages (trigger and group pipelines) that was accidentally dropped during the rewrite - Fix README example to use s4s2 group name instead of s4s - Remove working notes from .gitignore (files already deleted) - Fix misleading comment in env-vars-precedence test fixture - Regenerate golden files --- .gitignore | 5 ----- README.md | 2 +- libs/pipedream.libsonnet | 4 ++-- test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet | 2 +- .../pipedream/basic-autodeploy.jsonnet_output-files.golden | 4 ++++ .../pipedream/basic-autodeploy.jsonnet_single-file.golden | 4 ++++ .../pipedream/basic-manual.jsonnet_output-files.golden | 5 +++++ .../pipedream/basic-manual.jsonnet_single-file.golden | 5 +++++ .../different-stages-per-region.jsonnet_output-files.golden | 4 ++++ .../different-stages-per-region.jsonnet_single-file.golden | 4 ++++ .../env-vars-precedence.jsonnet_output-files.golden | 2 ++ .../pipedream/env-vars-precedence.jsonnet_single-file.golden | 2 ++ .../exclude-entire-group.jsonnet_output-files.golden | 3 +++ .../exclude-entire-group.jsonnet_single-file.golden | 3 +++ .../pipedream/exclude-region.jsonnet_output-files.golden | 4 ++++ .../pipedream/exclude-region.jsonnet_single-file.golden | 4 ++++ .../include-default-excluded.jsonnet_output-files.golden | 5 +++++ .../include-default-excluded.jsonnet_single-file.golden | 5 +++++ .../pipedream/multi-region-group.jsonnet_output-files.golden | 2 ++ .../pipedream/multi-region-group.jsonnet_single-file.golden | 2 ++ .../pipedream/parallel-mode.jsonnet_output-files.golden | 5 +++++ .../pipedream/parallel-mode.jsonnet_single-file.golden | 5 +++++ ...rollback-final-stage-override.jsonnet_output-files.golden | 4 ++++ .../rollback-final-stage-override.jsonnet_single-file.golden | 4 ++++ .../goldens/pipedream/rollback.jsonnet_output-files.golden | 4 ++++ .../goldens/pipedream/rollback.jsonnet_single-file.golden | 4 ++++ .../pipedream/stage-props.jsonnet_output-files.golden | 2 ++ .../goldens/pipedream/stage-props.jsonnet_single-file.golden | 2 ++ 28 files changed, 92 insertions(+), 9 deletions(-) diff --git a/.gitignore b/.gitignore index 13fcb3c..1f65af1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,8 +8,3 @@ vendor/ trash/ *.bak *.log - -# Working notes (do not commit) -PIPEDREAM_CELLULARIZATION_NOTES.md -PIPEDREAM_USE_CASES.md -notes.md diff --git a/README.md b/README.md index 537eb8e..402b703 100644 --- a/README.md +++ b/README.md @@ -93,7 +93,7 @@ parallel jobs. Groups are chained sequentially by default, or can fan out in parallel via `pipedream.render(config, pipeline_fn, parallel=true)`. The end result is a trigger pipeline `deploy-` and a pipeline per -group (e.g. `deploy-example-s4s`, `deploy-example-st`). +group (e.g. `deploy-example-s4s2`, `deploy-example-st`). `control` and `snty-tools` are excluded by default; use `include_regions` to opt in. diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index 0166d9c..53cb438 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -95,7 +95,7 @@ local pipedream_trigger_pipeline(pipedream_config) = materials: materials, lock_behavior: 'unlockWhenFinished', stages: [ - gocd_stages.basic('pipeline-complete', [gocd_tasks.noop], { approval: 'manual' }), + gocd_stages.basic('pipeline-complete', [gocd_tasks.noop], { approval: 'manual', fetch_materials: false }), ], }, }; @@ -415,7 +415,7 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde // The de final stage completes and causes the US pipeline to // re-run. With `pipeline-complete` as the final stage, it isn't // re-run by a rollback, preventing this domino effect. - gocd_stages.basic('pipeline-complete', [gocd_tasks.noop]), + gocd_stages.basic('pipeline-complete', [gocd_tasks.noop], { fetch_materials: false }), ], }; diff --git a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet index e7561a3..c7b5a0c 100644 --- a/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet +++ b/test/testdata/fixtures/pipedream/env-vars-precedence.jsonnet @@ -14,7 +14,7 @@ local pipeline_fn(region) = { environment_variables: { PIPELINE_VAR: 'pipeline-' + region, // Should cascade down to becoming a job level var SHARED_VAR_JOB: 'from-pipeline', // Should be overwritten by stage, then job - SHARED_VAR_STAGE: 'from-pipeline', // This should win + SHARED_VAR_STAGE: 'from-pipeline', // Overridden by stage }, materials: { example_repo: { diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden index cacfda1..b63c2bf 100644 --- a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_output-files.golden @@ -33,6 +33,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -80,6 +81,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -155,6 +157,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -206,6 +209,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden index fca2a4c..fdcb998 100644 --- a/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/basic-autodeploy.jsonnet_single-file.golden @@ -32,6 +32,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -74,6 +75,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -144,6 +146,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -190,6 +193,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden index 3c47f8c..4d3835a 100644 --- a/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_output-files.golden @@ -33,6 +33,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -84,6 +85,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -159,6 +161,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -210,6 +213,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -247,6 +251,7 @@ "approval": { "type": "manual" }, + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden index be9d566..0d29336 100644 --- a/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/basic-manual.jsonnet_single-file.golden @@ -18,6 +18,7 @@ "approval": { "type": "manual" }, + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -64,6 +65,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -110,6 +112,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -180,6 +183,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -226,6 +230,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden index 9620b01..f167dda 100644 --- a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_output-files.golden @@ -54,6 +54,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -122,6 +123,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -254,6 +256,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -326,6 +329,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden index 5d87224..08c0c04 100644 --- a/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/different-stages-per-region.jsonnet_single-file.golden @@ -53,6 +53,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -116,6 +117,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -243,6 +245,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -310,6 +313,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden index d6ad891..376daea 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_output-files.golden @@ -37,6 +37,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -135,6 +136,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden index 1a59c4b..aa65b0d 100644 --- a/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/env-vars-precedence.jsonnet_single-file.golden @@ -36,6 +36,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -129,6 +130,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden index 295d8e4..49eebbf 100644 --- a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_output-files.golden @@ -33,6 +33,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -80,6 +81,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -131,6 +133,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden index aba71ee..6870b69 100644 --- a/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/exclude-entire-group.jsonnet_single-file.golden @@ -32,6 +32,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -74,6 +75,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -120,6 +122,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden index 6b3100b..89c53b3 100644 --- a/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_output-files.golden @@ -33,6 +33,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -80,6 +81,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -147,6 +149,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -198,6 +201,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden index b468a3c..fd21359 100644 --- a/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/exclude-region.jsonnet_single-file.golden @@ -32,6 +32,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -74,6 +75,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -136,6 +138,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -182,6 +185,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden index 1b0e02f..8202a68 100644 --- a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_output-files.golden @@ -33,6 +33,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -84,6 +85,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -131,6 +133,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -206,6 +209,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -257,6 +261,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden index d51268d..3f7973e 100644 --- a/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/include-default-excluded.jsonnet_single-file.golden @@ -32,6 +32,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -78,6 +79,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -120,6 +122,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -190,6 +193,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -236,6 +240,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden index 9588544..eb0cada 100644 --- a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_output-files.golden @@ -29,6 +29,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -104,6 +105,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden index c0fedff..3fa463f 100644 --- a/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/multi-region-group.jsonnet_single-file.golden @@ -28,6 +28,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -98,6 +99,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden index 24b5c30..082b1ac 100644 --- a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_output-files.golden @@ -33,6 +33,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -84,6 +85,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -159,6 +161,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -210,6 +213,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -247,6 +251,7 @@ "approval": { "type": "manual" }, + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden index 84fc072..f1a07ed 100644 --- a/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/parallel-mode.jsonnet_single-file.golden @@ -18,6 +18,7 @@ "approval": { "type": "manual" }, + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -64,6 +65,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -110,6 +112,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -180,6 +183,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -226,6 +230,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden index 3861339..61f3ffb 100644 --- a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_output-files.golden @@ -33,6 +33,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -80,6 +81,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -155,6 +157,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -206,6 +209,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden index 6a2cec5..3fae80f 100644 --- a/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/rollback-final-stage-override.jsonnet_single-file.golden @@ -32,6 +32,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -74,6 +75,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -144,6 +146,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -190,6 +193,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden index 329cbe3..3d98a09 100644 --- a/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_output-files.golden @@ -33,6 +33,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -80,6 +81,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -155,6 +157,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -206,6 +209,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden index 1fef74f..db31f0e 100644 --- a/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/rollback.jsonnet_single-file.golden @@ -32,6 +32,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -74,6 +75,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -144,6 +146,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -190,6 +193,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden b/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden index 2004ab4..064f382 100644 --- a/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden +++ b/test/testdata/goldens/pipedream/stage-props.jsonnet_output-files.golden @@ -31,6 +31,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -105,6 +106,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ diff --git a/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden b/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden index 18c7705..b7bffb8 100644 --- a/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden +++ b/test/testdata/goldens/pipedream/stage-props.jsonnet_single-file.golden @@ -30,6 +30,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ @@ -99,6 +100,7 @@ }, { "pipeline-complete": { + "fetch_materials": false, "jobs": { "pipeline-complete": { "tasks": [ From 395e72bd5d627a085933dad97dcf166d97375100 Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 15 Apr 2026 11:37:26 -0700 Subject: [PATCH 11/16] chore: fix prettier formatting in test/pipedream.js --- test/pipedream.js | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/test/pipedream.js b/test/pipedream.js index 397b924..11cfc22 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -162,14 +162,9 @@ test("conflicting stage properties across regions errors", (t) => { test("merged stage objects (missing comma) errors", (t) => { const err = t.throws(() => - get_fixture_content( - "pipedream/merged-stages.failing.jsonnet", - false, - ), - ); - t.true( - err.message.includes("each stage must have exactly one key"), + get_fixture_content("pipedream/merged-stages.failing.jsonnet", false), ); + t.true(err.message.includes("each stage must have exactly one key")); }); test("all pipelines end with pipeline-complete stage", async (t) => { From c1cd572d20e72a57173bfc80b833db1722d1ff8e Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 15 Apr 2026 11:51:00 -0700 Subject: [PATCH 12/16] fix(getsentry): update is_st to match s4s2 and fix jsonnetfmt formatting s4s was removed as a region (PR #91) and replaced by s4s2. Update is_st() to check for s4s2 instead of the defunct s4s region. Add s4s2 to the is-st test fixture. Also fix jsonnetfmt formatting in pipedream.libsonnet (indentation and quote style in assertion messages). --- libs/getsentry.libsonnet | 2 +- libs/pipedream.libsonnet | 12 ++++++------ test/testdata/fixtures/getsentry/is-st.jsonnet | 1 + .../getsentry/is-st.jsonnet_output-files.golden | 1 + 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index b673eed..9238f78 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -23,5 +23,5 @@ get_targets(group):: if std.objectHas(self.pipeline_groups, group) then self.pipeline_groups[group] else self.test_groups[group], - is_st(region):: (region == 's4s' || std.startsWith(region, 'customer-')), + is_st(region):: (region == 's4s2' || std.startsWith(region, 'customer-')), } diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index 53cb438..a248c34 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -246,9 +246,9 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde function(acc2, stage) local keys = std.objectFields(stage); assert std.length(keys) == 1 : - "Stage object has %d keys (%s) — each stage must have exactly one key. " - % [std.length(keys), std.join(', ', keys)] - + "This usually means a missing comma between stage definitions."; + 'Stage object has %d keys (%s) — each stage must have exactly one key. ' + % [std.length(keys), std.join(', ', keys)] + + 'This usually means a missing comma between stage definitions.'; true, stages, true @@ -298,9 +298,9 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde local rs = get_matching_stage(p, stage_name); local props = if rs != null then get_stage_props(rs) else stage_props; assert props == stage_props : - "Stage '%s': conflicting properties across regions in group. " - % [stage_name] - + "Region '%s' differs from '%s'." % [r, regions[0]]; + "Stage '%s': conflicting properties across regions in group. " + % [stage_name] + + "Region '%s' differs from '%s'." % [r, regions[0]]; true, regions[1:], true diff --git a/test/testdata/fixtures/getsentry/is-st.jsonnet b/test/testdata/fixtures/getsentry/is-st.jsonnet index 4feba98..e0b2fcc 100644 --- a/test/testdata/fixtures/getsentry/is-st.jsonnet +++ b/test/testdata/fixtures/getsentry/is-st.jsonnet @@ -1,6 +1,7 @@ local getsentry = import '../../../../libs/getsentry.libsonnet'; { + s4s2: getsentry.is_st('s4s2'), us: getsentry.is_st('us'), eu: getsentry.is_st('eu'), 'customer-1': getsentry.is_st('customer-1'), diff --git a/test/testdata/goldens/getsentry/is-st.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/is-st.jsonnet_output-files.golden index 5fc540a..d454337 100644 --- a/test/testdata/goldens/getsentry/is-st.jsonnet_output-files.golden +++ b/test/testdata/goldens/getsentry/is-st.jsonnet_output-files.golden @@ -3,5 +3,6 @@ "customer-a": true, "demo-customer": false, "eu": false, + "s4s2": true, "us": false } From ecff2a629d27e32d05391b9a89180f22e5438492 Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 15 Apr 2026 11:59:41 -0700 Subject: [PATCH 13/16] fix(getsentry): remove dead s4s check from is_st s4s no longer exists as a region, and s4s2 uses the SaaS-style multi-container deploy (not ST), so the s4s check was dead code. Simplify is_st to only match customer-* regions. --- libs/getsentry.libsonnet | 2 +- test/testdata/fixtures/getsentry/is-st.jsonnet | 1 - .../goldens/getsentry/is-st.jsonnet_output-files.golden | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index 9238f78..25dcc02 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -23,5 +23,5 @@ get_targets(group):: if std.objectHas(self.pipeline_groups, group) then self.pipeline_groups[group] else self.test_groups[group], - is_st(region):: (region == 's4s2' || std.startsWith(region, 'customer-')), + is_st(region):: std.startsWith(region, 'customer-'), } diff --git a/test/testdata/fixtures/getsentry/is-st.jsonnet b/test/testdata/fixtures/getsentry/is-st.jsonnet index e0b2fcc..4feba98 100644 --- a/test/testdata/fixtures/getsentry/is-st.jsonnet +++ b/test/testdata/fixtures/getsentry/is-st.jsonnet @@ -1,7 +1,6 @@ local getsentry = import '../../../../libs/getsentry.libsonnet'; { - s4s2: getsentry.is_st('s4s2'), us: getsentry.is_st('us'), eu: getsentry.is_st('eu'), 'customer-1': getsentry.is_st('customer-1'), diff --git a/test/testdata/goldens/getsentry/is-st.jsonnet_output-files.golden b/test/testdata/goldens/getsentry/is-st.jsonnet_output-files.golden index d454337..5fc540a 100644 --- a/test/testdata/goldens/getsentry/is-st.jsonnet_output-files.golden +++ b/test/testdata/goldens/getsentry/is-st.jsonnet_output-files.golden @@ -3,6 +3,5 @@ "customer-a": true, "demo-customer": false, "eu": false, - "s4s2": true, "us": false } From 60e6f402e11c931b7a98ba1a57dae9efe5e9f9ea Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 15 Apr 2026 12:14:18 -0700 Subject: [PATCH 14/16] fix(pipedream): improve stage props conflict error message Remove reference to specific baseline region since the baseline may not be regions[0] when regions have different stage sets. --- libs/pipedream.libsonnet | 4 ++-- test/pipedream.js | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index a248c34..1ccff03 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -298,9 +298,9 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde local rs = get_matching_stage(p, stage_name); local props = if rs != null then get_stage_props(rs) else stage_props; assert props == stage_props : - "Stage '%s': conflicting properties across regions in group. " + "Stage '%s': conflicting stage properties across regions in group. " % [stage_name] - + "Region '%s' differs from '%s'." % [r, regions[0]]; + + "Region '%s' differs." % [r]; true, regions[1:], true diff --git a/test/pipedream.js b/test/pipedream.js index 11cfc22..1b79877 100644 --- a/test/pipedream.js +++ b/test/pipedream.js @@ -156,7 +156,9 @@ test("conflicting stage properties across regions errors", (t) => { ), ); t.true( - err.message.includes("conflicting properties across regions in group"), + err.message.includes( + "conflicting stage properties across regions in group", + ), ); }); From f730e65daff7e9742e63ac68743699cb951b107f Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 15 Apr 2026 12:24:09 -0700 Subject: [PATCH 15/16] chore: add clarifying comments for test_groups and stage collection --- libs/getsentry.libsonnet | 1 + libs/pipedream.libsonnet | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libs/getsentry.libsonnet b/libs/getsentry.libsonnet index 25dcc02..d4ae4c3 100644 --- a/libs/getsentry.libsonnet +++ b/libs/getsentry.libsonnet @@ -4,6 +4,7 @@ { group_order: ['s4s2', 'de', 'us', 'control', 'snty-tools', 'st'], + // Empty for now — add future test groups here test_group_order: [], // These groupings consist of user facing deployments pipeline_groups: { diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index 1ccff03..7676274 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -260,7 +260,9 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde local template_pipeline = region_pipelines[regions[0]]; - // Collect all unique stages across all regions in the group + // Collect all unique stages across all regions in the group. + // If a region doesn't define a stage that another region has, + // it simply contributes no jobs to that stage. local all_stages = std.foldl( function(acc, region) local p = region_pipelines[region]; From 3465d94bb68e41c24d93a80d0b3f0ab8168d26c6 Mon Sep 17 00:00:00 2001 From: mchen-sentry Date: Wed, 15 Apr 2026 12:25:23 -0700 Subject: [PATCH 16/16] ref(pipedream): add pipeline-level attribute validation across regions Assert that materials, lock_behavior, and other pipeline-level attributes are consistent across all regions in a group, matching the existing stage-level property validation. --- libs/pipedream.libsonnet | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/libs/pipedream.libsonnet b/libs/pipedream.libsonnet index 7676274..63e36bc 100644 --- a/libs/pipedream.libsonnet +++ b/libs/pipedream.libsonnet @@ -260,6 +260,22 @@ local generate_group_pipeline(pipedream_config, pipeline_fn, group, display_orde local template_pipeline = region_pipelines[regions[0]]; + // Validate pipeline-level attributes (materials, lock_behavior, etc.) + // are consistent across all regions in the group. + local pipeline_props(p) = + { [k]: p[k] for k in std.objectFields(p) if k != 'stages' && k != 'environment_variables' }; + local template_props = pipeline_props(template_pipeline); + assert std.foldl( + function(acc, r) + local props = pipeline_props(region_pipelines[r]); + assert props == template_props : + 'Pipeline-level attributes differ across regions in group. ' + + "Region '%s' differs." % [r]; + true, + regions[1:], + true + ); + // Collect all unique stages across all regions in the group. // If a region doesn't define a stage that another region has, // it simply contributes no jobs to that stage.