Skip to content

Commit 2c5b26e

Browse files
shashjarshashjar
authored
Merge branch 'master' into shashjar/eventstore-get-events-eap-query
2 parents fb215d0 + 64c7b9b commit 2c5b26e

File tree

1,047 files changed

+10025
-3824
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,047 files changed

+10025
-3824
lines changed

‎.github/CODEOWNERS‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -692,7 +692,7 @@ tests/sentry/api/endpoints/test_organization_attribute_mappings.py @get
692692
/static/app/components/events/highlights/ @getsentry/issue-workflow
693693
/static/app/components/issues/ @getsentry/issue-workflow
694694
/static/app/components/stackTrace/ @getsentry/issue-workflow
695-
/static/app/components/stream/supergroupRow.tsx @getsentry/issue-detection-frontend
695+
/static/app/components/stream/supergroups/ @getsentry/issue-detection-frontend
696696
/static/app/views/issueList/ @getsentry/issue-workflow
697697
/static/app/views/issueList/issueListSeerComboBox.tsx @getsentry/issue-workflow @getsentry/machine-learning-ai
698698
/static/app/views/issueList/pages/supergroups.tsx @getsentry/issue-detection-frontend
@@ -856,6 +856,8 @@ tests/sentry/api/endpoints/test_organization_attribute_mappings.py @get
856856
/.agents/skills/lint-fix/ @getsentry/design-engineering
857857
/.agents/skills/lint-new/ @getsentry/design-engineering
858858
/.agents/skills/react-component-documentation/ @getsentry/design-engineering
859+
# Orphaned files we had to touch to fix errors
860+
/static/gsAdmin/views/sentryAppDetails.spec.tsx @getsentry/design-engineering
859861
## End of Frontend Platform
860862

861863
# Coding Workflows

‎.github/actions/setup-devservices/action.yml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
name: 'Early Devservices'
2+
description: 'Starts devservices in the background so image pulls overlap with venv setup'
3+
inputs:
4+
mode:
5+
description: 'devservices mode (must match the mode passed to setup-sentry)'
6+
required: true
7+
timeout-minutes:
8+
description: 'Maximum minutes for devservices up'
9+
required: false
10+
default: '10'
11+
12+
runs:
13+
using: 'composite'
14+
steps:
15+
- uses: astral-sh/setup-uv@884ad927a57e558e7a70b92f2bccf9198a4be546 # v6
16+
with:
17+
version: '0.9.28'
18+
enable-cache: false
19+
20+
- name: Start devservices in background
21+
shell: bash --noprofile --norc -euo pipefail {0}
22+
run: |
23+
DS_VERSION=$(python3 -c "
24+
import tomllib
25+
with open('uv.lock', 'rb') as f:
26+
lock = tomllib.load(f)
27+
for pkg in lock['package']:
28+
if pkg['name'] == 'devservices':
29+
print(pkg['version'])
30+
break
31+
")
32+
echo "Installing devservices==${DS_VERSION}"
33+
uv venv /tmp/ds-venv --python python3 -q
34+
uv pip install --python /tmp/ds-venv/bin/python -q \
35+
--index-url https://pypi.devinfra.sentry.io/simple \
36+
"devservices==${DS_VERSION}"
37+
(set +e; timeout ${{ inputs.timeout-minutes }}m /tmp/ds-venv/bin/devservices up --mode ${{ inputs.mode }}; echo $? > /tmp/ds-exit) \
38+
> /tmp/ds.log 2>&1 &

‎.github/actions/setup-devservices/bootstrap-snuba.py‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
#!/usr/bin/env python3
2+
"""Bootstrap per-worker Snuba instances for CI.
3+
4+
Overlaps the expensive ClickHouse table setup with the devservices
5+
health-check wait.
6+
7+
Phase 1 (early): As soon as ClickHouse is accepting queries, create
8+
per-worker databases and run ``snuba bootstrap --force``.
9+
Phase 2 (after devservices): Stop snuba-snuba-1 and start per-worker
10+
API containers. We must wait for devservices to finish first —
11+
stopping the container while devservices is health-checking it would
12+
cause a timeout.
13+
14+
Requires: XDIST_WORKERS env var
15+
Reads: /tmp/ds-exit (written by setup-devservices/wait.sh)
16+
Writes: /tmp/snuba-bootstrap-exit
17+
"""
18+
19+
from __future__ import annotations
20+
21+
import os
22+
import subprocess
23+
import sys
24+
import time
25+
from concurrent.futures import ThreadPoolExecutor, as_completed
26+
from functools import partial
27+
from pathlib import Path
28+
from typing import Any, Callable
29+
from urllib.error import URLError
30+
from urllib.request import urlopen
31+
32+
DS_EXIT = Path("/tmp/ds-exit")
33+
SNUBA_EXIT = Path("/tmp/snuba-bootstrap-exit")
34+
35+
SNUBA_ENV = {
36+
"CLICKHOUSE_HOST": "clickhouse",
37+
"CLICKHOUSE_PORT": "9000",
38+
"CLICKHOUSE_HTTP_PORT": "8123",
39+
"DEFAULT_BROKERS": "kafka:9093",
40+
"REDIS_HOST": "redis",
41+
"REDIS_PORT": "6379",
42+
"REDIS_DB": "1",
43+
"SNUBA_SETTINGS": "docker",
44+
}
45+
46+
ENV_ARGS = [flag for k, v in SNUBA_ENV.items() for flag in ("-e", f"{k}={v}")]
47+
48+
49+
def retry(
50+
fn: Callable[[], Any], *, attempts: int = 3, delay: int = 5, label: str = "operation"
51+
) -> Any:
52+
for i in range(attempts):
53+
try:
54+
return fn()
55+
except Exception:
56+
if i == attempts - 1:
57+
raise
58+
log(f"{label} failed (attempt {i + 1}/{attempts}), retrying in {delay}s...")
59+
time.sleep(delay)
60+
61+
62+
def log(msg: str) -> None:
63+
print(msg, flush=True)
64+
65+
66+
def fail(msg: str) -> None:
67+
log(f"::error::{msg}")
68+
SNUBA_EXIT.write_text("1")
69+
sys.exit(1)
70+
71+
72+
def http_ok(url: str) -> bool:
73+
try:
74+
with urlopen(url, timeout=3):
75+
return True
76+
except (URLError, OSError):
77+
return False
78+
79+
80+
def docker(
81+
*args: str, check: bool = False, timeout: int | None = None
82+
) -> subprocess.CompletedProcess[str]:
83+
return subprocess.run(
84+
["docker", *args], capture_output=True, text=True, check=check, timeout=timeout
85+
)
86+
87+
88+
def docker_inspect(container: str, fmt: str) -> str:
89+
r = docker("inspect", container, "--format", fmt)
90+
return r.stdout.strip() if r.returncode == 0 else ""
91+
92+
93+
def inspect_snuba_container() -> tuple[str, str]:
94+
image = docker_inspect("snuba-snuba-1", "{{.Config.Image}}")
95+
network = docker_inspect(
96+
"snuba-snuba-1",
97+
"{{range $k, $v := .NetworkSettings.Networks}}{{$k}}{{end}}",
98+
)
99+
if not image or not network:
100+
fail("Could not inspect snuba-snuba-1 container")
101+
return image, network
102+
103+
104+
def run_parallel(fn: Callable[[int], Any], workers: range, *, fail_fast: bool = True) -> int:
105+
"""Run fn(i) in parallel for each i in workers. Returns 0 on full success."""
106+
rc = 0
107+
with ThreadPoolExecutor(max_workers=len(workers)) as pool:
108+
futs = {pool.submit(fn, i): i for i in workers}
109+
for fut in as_completed(futs):
110+
try:
111+
fut.result()
112+
except Exception as e:
113+
if fail_fast:
114+
fail(str(e))
115+
log(f"ERROR: {e}")
116+
rc = 1
117+
return rc
118+
119+
120+
def wait_for_prerequisites(timeout: int = 300) -> None:
121+
log("Waiting for ClickHouse and Snuba container...")
122+
start = time.monotonic()
123+
while True:
124+
if time.monotonic() - start > timeout:
125+
fail("Timed out waiting for Snuba bootstrap prerequisites")
126+
if http_ok("http://localhost:8123/") and docker_inspect("snuba-snuba-1", "{{.Id}}"):
127+
break
128+
time.sleep(2)
129+
log(f"Prerequisites ready ({time.monotonic() - start:.0f}s)")
130+
131+
132+
def wait_for_devservices(timeout: int = 300) -> None:
133+
start = time.monotonic()
134+
while not DS_EXIT.exists():
135+
if time.monotonic() - start > timeout:
136+
fail("Timed out waiting for devservices to finish")
137+
time.sleep(1)
138+
rc = int(DS_EXIT.read_text().strip())
139+
if rc != 0:
140+
fail(f"devservices failed (exit {rc}), skipping Phase 2")
141+
142+
143+
def bootstrap_worker(worker_id: int, *, image: str, network: str) -> None:
144+
"""Create a ClickHouse database and run snuba bootstrap."""
145+
db = f"default_gw{worker_id}"
146+
147+
def create_db() -> None:
148+
with urlopen(
149+
"http://localhost:8123/", f"CREATE DATABASE IF NOT EXISTS {db}".encode(), timeout=30
150+
):
151+
pass
152+
153+
retry(create_db, label=f"CREATE DATABASE {db}")
154+
155+
def run_bootstrap() -> None:
156+
r = docker(
157+
"run",
158+
"--rm",
159+
"--network",
160+
network,
161+
"-e",
162+
f"CLICKHOUSE_DATABASE={db}",
163+
*ENV_ARGS,
164+
image,
165+
"bootstrap",
166+
"--force",
167+
)
168+
for line in (r.stdout + r.stderr).strip().splitlines()[-3:]:
169+
log(line)
170+
if r.returncode != 0:
171+
raise RuntimeError(f"snuba bootstrap failed for worker {worker_id}")
172+
173+
retry(run_bootstrap, label=f"snuba bootstrap gw{worker_id}")
174+
175+
176+
def start_worker_container(worker_id: int, *, image: str, network: str) -> None:
177+
"""Start a per-worker Snuba API container and wait for health."""
178+
db = f"default_gw{worker_id}"
179+
port = 1230 + worker_id
180+
name = f"snuba-gw{worker_id}"
181+
182+
docker("rm", "-f", name)
183+
184+
r = docker(
185+
"run",
186+
"-d",
187+
"--name",
188+
name,
189+
"--network",
190+
network,
191+
"-p",
192+
f"{port}:1218",
193+
"-e",
194+
f"CLICKHOUSE_DATABASE={db}",
195+
*ENV_ARGS,
196+
"-e",
197+
"DEBUG=1",
198+
image,
199+
"api",
200+
)
201+
if r.returncode != 0:
202+
raise RuntimeError(f"docker run {name} failed: {r.stderr.strip()}")
203+
204+
for attempt in range(1, 31):
205+
if http_ok(f"http://127.0.0.1:{port}/health"):
206+
log(f"{name} healthy on port {port}")
207+
return
208+
if attempt == 30:
209+
r = docker("logs", name)
210+
for line in (r.stdout + r.stderr).strip().splitlines()[-20:]:
211+
log(line)
212+
raise RuntimeError(f"{name} failed health check after 30 attempts")
213+
time.sleep(2)
214+
215+
216+
def main() -> None:
217+
workers_str = os.environ.get("XDIST_WORKERS")
218+
if not workers_str:
219+
fail("XDIST_WORKERS must be set")
220+
workers = range(int(workers_str))
221+
start = time.monotonic()
222+
223+
wait_for_prerequisites()
224+
image, network = inspect_snuba_container()
225+
226+
log("Phase 1: bootstrapping ClickHouse databases")
227+
run_parallel(partial(bootstrap_worker, image=image, network=network), workers)
228+
log(f"Phase 1 done ({time.monotonic() - start:.0f}s)")
229+
230+
wait_for_devservices()
231+
try:
232+
docker("stop", "snuba-snuba-1", timeout=30)
233+
except subprocess.TimeoutExpired:
234+
log("WARNING: docker stop snuba-snuba-1 timed out, killing")
235+
docker("kill", "snuba-snuba-1")
236+
237+
log("Phase 2: starting per-worker Snuba API containers")
238+
rc = run_parallel(
239+
partial(start_worker_container, image=image, network=network),
240+
workers,
241+
fail_fast=False,
242+
)
243+
244+
log(f"Snuba bootstrap complete ({time.monotonic() - start:.0f}s total)")
245+
SNUBA_EXIT.write_text(str(rc))
246+
sys.exit(rc)
247+
248+
249+
if __name__ == "__main__":
250+
main()

‎.github/actions/setup-devservices/wait.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
# Wait for the background devservices process started by the setup-devservices action.
5+
# Usage: wait.sh [timeout_seconds]
6+
TIMEOUT=${1:-600}
7+
8+
SECONDS=0
9+
while [ ! -f /tmp/ds-exit ]; do
10+
if [ $SECONDS -gt "$TIMEOUT" ]; then
11+
echo "::error::Timed out waiting for devservices after ${TIMEOUT}s"
12+
cat /tmp/ds.log
13+
exit 1
14+
fi
15+
sleep 2
16+
done
17+
18+
DS_RC=$(< /tmp/ds-exit)
19+
if [ "$DS_RC" -ne 0 ]; then
20+
echo "::error::devservices up failed (exit $DS_RC)"
21+
cat /tmp/ds.log
22+
exit 1
23+
fi
24+
25+
echo "DJANGO_LIVE_TEST_SERVER_ADDRESS=$(docker network inspect bridge --format='{{(index .IPAM.Config 0).Gateway}}')" >> "$GITHUB_ENV"
26+
docker ps -a

‎.github/workflows/acceptance.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,12 @@ jobs:
6868
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
6969
name: Checkout sentry
7070

71+
- run: mkdir -p config/chartcuterie
72+
73+
- uses: ./.github/actions/setup-devservices
74+
with:
75+
mode: acceptance-ci
76+
7177
- name: Step configurations
7278
id: config
7379
run: |
@@ -98,13 +104,24 @@ jobs:
98104
id: setup
99105
with:
100106
mode: acceptance-ci
107+
skip-devservices: 'true'
108+
109+
- name: Wait for devservices
110+
run: |
111+
sentry init
112+
./.github/actions/setup-devservices/wait.sh
101113
102114
- name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
103115
run: make run-acceptance
104116

105117
- name: Inspect failure
106118
if: failure()
107119
run: |
120+
if [ -f /tmp/ds.log ]; then
121+
echo "--- devservices startup log ---"
122+
cat /tmp/ds.log
123+
fi
124+
108125
if command -v devservices; then
109126
devservices logs
110127
fi

0 commit comments

Comments
 (0)