Skip to content

Commit c394938

Browse files
haroldsphinxharoldsphinx
authored
Merge branch 'polytope-labs:staging' into staging
2 parents 358c4be + 0ab6a9d commit c394938

File tree

72 files changed

+1914
-362
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+1914
-362
lines changed

‎.github/workflows/migration-test.yml‎

Lines changed: 304 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,304 @@
1+
name: Schema Migration Test (Indexer)
2+
3+
# Cancel running workflows from the same PR
4+
concurrency:
5+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
6+
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
7+
8+
on:
9+
push:
10+
branches: [staging]
11+
paths:
12+
- "packages/indexer/**"
13+
- ".github/workflows/migration-test.yml"
14+
workflow_dispatch:
15+
16+
env:
17+
OLD_COMMIT_REF: "fcee4b2cdd8224ccab618f8afd1f7fc09e5060e8"
18+
19+
jobs:
20+
schema-migration-test:
21+
runs-on: ubuntu-latest
22+
timeout-minutes: 180
23+
24+
steps:
25+
- name: Checkout repository
26+
uses: actions/checkout@v4
27+
with:
28+
fetch-depth: 0 # Fetch all history for git operations
29+
30+
- name: Setup Node.js
31+
uses: actions/setup-node@v4
32+
with:
33+
node-version: "22"
34+
35+
- name: Setup pnpm
36+
uses: pnpm/action-setup@v2
37+
with:
38+
version: "7"
39+
40+
- name: Install system dependencies
41+
run: |
42+
sudo apt-get update
43+
sudo apt-get install -y netcat-openbsd curl jq unzip postgresql-client
44+
45+
curl -SL https://github.com/docker/compose/releases/download/v2.37.3/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
46+
chmod +x /usr/local/bin/docker-compose
47+
sudo ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose
48+
49+
docker-compose version
50+
51+
- name: Set up environment variables
52+
run: |
53+
cat > .env.local << EOF
54+
BSC_CHAPEL=${{ secrets.BSC_CHAPEL }}
55+
GNOSIS_CHIADO=${{ secrets.GNOSIS_CHIADO }}
56+
HYPERBRIDGE_GARGANTUA=${{ secrets.HYPERBRIDGE_GARGANTUA }}
57+
PASEO_RPC_URL=${{ secrets.PASEO_RPC_URL }}
58+
BIFROST_PASEO=${{ secrets.BIFROST_PASEO }}
59+
CERE_LOCAL=${{ secrets.CERE_RPC_URL }}
60+
INDEXER_URL=${{ secrets.INDEXER_URL }}
61+
PRIVATE_KEY=${{ secrets.PRIVATE_KEY }}
62+
SECRET_PHRASE="${{ secrets.SECRET_PHRASE }}"
63+
PING_MODULE_ADDRESS="0xFE9f23F0F2fE83b8B9576d3FC94e9a7458DdDD35"
64+
TOKEN_GATEWAY_ADDRESS="0xFcDa26cA021d5535C3059547390E6cCd8De7acA6"
65+
DB_USER=postgres
66+
DB_PASS=postgres
67+
DB_DATABASE=postgres
68+
DB_HOST=localhost
69+
DB_PORT=5432
70+
DB_PATH="./local/db/"
71+
EOF
72+
73+
# Step 1: Checkout old schema commit and start indexer
74+
- name: Checkout old schema commit
75+
run: |
76+
git stash push -m "Stash current changes"
77+
git checkout $OLD_COMMIT_REF
78+
git log --oneline -1
79+
80+
- name: Install dependencies and Build old schema version
81+
env:
82+
ENV: local
83+
run: |
84+
pnpm install
85+
pnpm build
86+
87+
# Step 2: Publish old schema manifest
88+
- name: Publish old schema manifest
89+
env:
90+
ENV: local
91+
SUBQL_ACCESS_TOKEN: ${{ secrets.SUBQL_ACCESS_TOKEN }}
92+
run: |
93+
cd packages/indexer
94+
95+
./node_modules/.bin/subql publish
96+
97+
pnpm build
98+
99+
# Step 3: Start indexer with old schema
100+
- name: Start indexer with old schema
101+
run: |
102+
cd packages/indexer
103+
104+
docker-compose -f docker/docker-compose.local.yml --env-file ../../.env.local down --remove-orphans || true
105+
106+
nohup docker-compose -f docker/docker-compose.local.yml --env-file ../../.env.local up --force-recreate --remove-orphans > indexer_old_schema.log 2>&1 &
107+
echo $! > indexer_pid.txt
108+
echo "Indexer started with PID: $(cat indexer_pid.txt)"
109+
110+
- name: Wait for GraphQL server and verify old schema indexing
111+
run: |
112+
echo "Waiting for GraphQL server and verifying old schema is indexing data..."
113+
timeout=300 # 5 minutes timeout
114+
elapsed=0
115+
interval=10
116+
117+
while true; do
118+
if [ "$elapsed" -ge "$timeout" ]; then
119+
echo "❌ Timed out waiting for GraphQL server and old schema indexing"
120+
echo "=== Database status ==="
121+
PGPASSWORD=${{ env.DB_PASS }} psql -h ${{ env.DB_HOST }} -p 5432 -U ${{ env.DB_USER }} -d ${{ env.DB_DATABASE }} -c "\dt" || echo "Could not connect to database"
122+
echo "=== Indexer logs (old schema) ==="
123+
cat packages/indexer/indexer_old_schema.log || echo "No log file found"
124+
exit 1
125+
fi
126+
127+
echo "Attempting to query StateMachineUpdateEvents (elapsed: ${elapsed}s)..."
128+
129+
# Query StateMachineUpdateEvents
130+
state_machine_result=$(curl -s -X POST http://localhost:3100/graphql \
131+
-H "Content-Type: application/json" \
132+
-d '{"query": "query StateMachineUpdateEvents { stateMachineUpdateEvents { totalCount } }"}' 2>/dev/null || echo "failed")
133+
134+
if [[ "$state_machine_result" == *"stateMachineUpdateEvents"* ]]; then
135+
echo "GraphQL server is responding with valid schema structure"
136+
echo "StateMachineUpdateEvents Response: $state_machine_result"
137+
138+
total_count=$(echo "$state_machine_result" | jq -r '.data.stateMachineUpdateEvents.totalCount // 0' 2>/dev/null)
139+
140+
if [[ "$total_count" -gt 1 ]]; then
141+
142+
# Query Metadata
143+
metadata_result=$(curl -s -X POST http://localhost:3100/graphql \
144+
-H "Content-Type: application/json" \
145+
-d '{"query": "query Metadata { _metadatas { totalCount nodes { chain deployments lastProcessedHeight startHeight targetHeight } } }"}' 2>/dev/null || echo "failed")
146+
147+
echo "Metadata Response: $metadata_result"
148+
echo "Old schema indexing successful: Found $total_count StateMachineUpdateEvents"
149+
150+
break
151+
fi
152+
else
153+
http_code=$(curl -s -o /dev/null -w "%{http_code}" http://localhost:3100/graphql 2>/dev/null || echo "000")
154+
155+
if [[ "$http_code" == "200" ]]; then
156+
echo "GraphQL server responding but schema not ready yet..."
157+
elif [[ "$http_code" == "000" ]]; then
158+
echo "GraphQL server not yet available (connection refused)"
159+
else
160+
echo "GraphQL server responding with HTTP $http_code"
161+
fi
162+
163+
echo "StateMachineUpdateEvents Response: $state_machine_result"
164+
fi
165+
166+
sleep $interval
167+
elapsed=$((elapsed + interval))
168+
done
169+
170+
echo "Old schema verification completed successfully!"
171+
echo "Final verification details:"
172+
echo "$state_machine_result" | jq '.' 2>/dev/null || echo "$state_machine_result"
173+
174+
- name: Keep database running for schema update
175+
run: |
176+
cd packages/indexer
177+
178+
docker-compose -f docker/docker-compose.local.yml --env-file ../../.env.local down --remove-orphans subquery-node-hyperbridge-gargantua-local subquery-node-bsc-chapel-local subquery-node-gnosis-chiado-local || true
179+
180+
# Step 4: Checkout new schema - NOW WE HAVE ACCESS TO NEW SCRIPTS
181+
- name: Checkout new schema and get latest scripts
182+
run: |
183+
git stash push -m "Stash old schema changes" || true
184+
git checkout ${{ github.sha }}
185+
git log --oneline -1
186+
echo "Now using latest commit with updated scripts"
187+
188+
- name: Install dependencies for new schema
189+
env:
190+
ENV: local
191+
run: |
192+
pnpm install
193+
pnpm build
194+
195+
# Step 5: Use migration-deployments.sh script for the actual migration
196+
- name: Update _metadata_*.deployments using update-deployments.sh
197+
env:
198+
ENV: local
199+
DB_USER: postgres
200+
DB_PASS: postgres
201+
DB_DATABASE: postgres
202+
DB_PORT: 5432
203+
run: |
204+
cd packages/indexer
205+
206+
if pnpm migration:update; then
207+
echo "Deployments updated successfully"
208+
else
209+
echo "❌ update-deployments.sh script failed"
210+
exit 1
211+
fi
212+
213+
# Step 6: Construct chain configuration & rebuild manifests
214+
- name: Build configuration
215+
env:
216+
ENV: local
217+
SUBQL_ACCESS_TOKEN: ${{ secrets.SUBQL_ACCESS_TOKEN }}
218+
run: |
219+
cd packages/indexer
220+
221+
pnpm migration:build
222+
223+
# Step 7: Restart Indexer
224+
- name: Restart Indexer for Migration
225+
run: |
226+
cd packages/indexer
227+
228+
docker-compose -f docker/docker-compose.local.yml --env-file ../../.env.local down --remove-orphans || true
229+
230+
nohup docker-compose -f docker/docker-compose.local.yml --env-file ../../.env.local up --force-recreate --remove-orphans > indexer_migration_final.log 2>&1 &
231+
232+
- name: Wait for GraphQL Server and Verify migration finality
233+
env:
234+
ENV: local
235+
run: |
236+
cd packages/indexer
237+
238+
if pnpm migration:wait; then
239+
echo "Waiting migration finality"
240+
else
241+
echo "❌ migration failed to finalize"
242+
exit 1
243+
fi
244+
245+
- name: Rebuild and Restart
246+
env:
247+
ENV: local
248+
run: |
249+
cd packages/indexer
250+
251+
docker-compose -f docker/docker-compose.local.yml --env-file ../../.env.local down --remove-orphans || true
252+
253+
pnpm build
254+
255+
nohup docker-compose -f docker/docker-compose.local.yml --env-file ../../.env.local up --force-recreate --remove-orphans >> indexer_migration_final.log 2>&1 &
256+
257+
# Step 8: Run test suites
258+
- name: Run SDK tests
259+
run: pnpm --filter="hyperbridge-sdk" test
260+
261+
# - name: Run Intent Filler tests
262+
# run: pnpm --filter="filler" test
263+
264+
- name: Show indexer logs on failure
265+
if: failure()
266+
run: |
267+
cd packages/indexer
268+
269+
echo "=== Schema Migration Test Failed ==="
270+
echo "--- Old schema logs ---"
271+
cat indexer_old_schema.log 2>/dev/null || echo "No old schema log file found"
272+
echo "\n\n"
273+
echo "--- Migration logs ---"
274+
cat indexer_migration_final.log 2>/dev/null || echo "No migration log file found"
275+
276+
- name: Cleanup and prepare artifacts
277+
if: always()
278+
run: |
279+
cd packages/indexer
280+
281+
if [ -f indexer_pid.txt ]; then
282+
OLD_PID=$(cat indexer_pid.txt)
283+
kill $OLD_PID 2>/dev/null || echo "Old indexer process already stopped"
284+
fi
285+
286+
if [ -f indexer_migration_pid.txt ]; then
287+
MIGRATION_PID=$(cat indexer_migration_pid.txt)
288+
kill $MIGRATION_PID 2>/dev/null || echo "Migration indexer process already stopped"
289+
fi
290+
291+
rm -f indexer_pid.txt indexer_migration_pid.txt
292+
docker-compose -f docker/docker-compose.local.yml down --remove-orphans || true
293+
294+
mkdir -p migration-logs
295+
cp indexer_old_schema.log migration-logs/ || true
296+
cp indexer_migration_final.log migration-logs/ || true
297+
298+
- name: Upload logs as artifacts
299+
if: always()
300+
uses: actions/upload-artifact@v4
301+
with:
302+
name: migration-test-logs-${{ github.run_id }}
303+
path: packages/indexer/migration-logs/
304+
retention-days: 7

‎.github/workflows/test-sdk.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,8 @@ jobs:
105105
- name: Run SDK tests
106106
run: pnpm --filter="hyperbridge-sdk" test
107107

108-
- name: Run Intent Filler tests
109-
run: pnpm --filter="filler" test
108+
# - name: Run Intent Filler tests
109+
# run: pnpm --filter="filler" test
110110

111111
- name: Clean up
112112
if: always()

‎.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,11 @@ package-lock.json
1414

1515
packages/indexer/configs/*.yaml
1616
packages/indexer/src/configs/*.yaml
17+
packages/indexer/src/types/*
18+
!packages/indexer/src/types/global.d.ts
19+
!packages/indexer/src/types/network.types.ts
20+
packages/indexer/*.yaml
21+
!packages/indexer/pnpm-lock.yaml
1722
packages/indexer/configs/src/types*
1823
packages/indexer/src/configs/src/types*
1924
packages/indexer/docker/testnet/
@@ -76,10 +81,14 @@ Thumbs.db
7681
.env*
7782
.tsconfig
7883
.tsbuildinfo
84+
.*-cid
7985

8086
evm-ws.json
8187
substrate-ws.json
8288
chain-ids-by-genesis.ts
8389
chains-by-ismp-host.ts
90+
chains-block-number.*
8491

8592
packages/filler/src/config/chain.ts
93+
94+
artifact.zip

‎packages/indexer/chaintypes/hyperbridge.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"typesBundle": {
3+
"spec": {
4+
"gargantua": {
5+
"hasher": "keccakAsU8a"
6+
},
7+
"nexus": {
8+
"hasher": "keccakAsU8a"
9+
}
10+
}
11+
}
12+
}

0 commit comments

Comments
 (0)