@@ -299,15 +299,76 @@ def wait_for_flags_visible(timeout=30, interval=2):
299299def postgres_container ():
300300 """Create and start PostgreSQL container."""
301301 with PostgresContainer ("postgres:15" , driver = None ) as postgres :
302- postgres .start ()
303302 postgres_url = postgres .get_connection_url ()
304303 logger .info (f"PostgreSQL started at: { postgres_url } " )
305304
306305 yield postgres
307306
308307
308+ def _wait_for_healthy (container , max_wait_time = 120 ):
309+ """Poll the Unleash container until its /health endpoint returns 200.
310+
311+ Returns the base URL on success, raises on timeout or container death.
312+ """
313+ start_time = time .time ()
314+
315+ while time .time () - start_time < max_wait_time :
316+ try :
317+ try :
318+ exposed_port = container .get_exposed_port (4242 )
319+ unleash_url = f"http://localhost:{ exposed_port } "
320+ logger .info (f"Trying health check at: { unleash_url } " )
321+ except Exception as port_error :
322+ # if the container exited, fail fast with its logs
323+ docker_container = container .get_wrapped_container ()
324+ if docker_container :
325+ docker_container .reload ()
326+ if docker_container .status in ("exited" , "dead" ):
327+ logs = docker_container .logs ().decode (errors = "replace" )
328+ raise RuntimeError (
329+ f"Unleash container died ({ docker_container .status } ).\n "
330+ f"Logs:\n { logs } "
331+ ) from port_error
332+ logger .error (f"Port not ready yet: { port_error } " )
333+ time .sleep (2 )
334+ continue
335+
336+ response = requests .get (f"{ unleash_url } /health" , timeout = 5 )
337+ if response .status_code == 200 :
338+ logger .info ("Unleash container is healthy!" )
339+ return unleash_url
340+
341+ logger .error (f"Health check failed, status: { response .status_code } " )
342+ time .sleep (2 )
343+
344+ except RuntimeError :
345+ raise
346+ except Exception as e :
347+ logger .error (f"Health check error: { e } " )
348+ time .sleep (2 )
349+
350+ # timeout; dump container logs for debugging
351+ try :
352+ docker_container = container .get_wrapped_container ()
353+ if docker_container :
354+ docker_container .reload ()
355+ logs = docker_container .logs ().decode (errors = "replace" )
356+ logger .error (
357+ f"Unleash container status: { docker_container .status } \n Logs:\n { logs } "
358+ )
359+ except Exception :
360+ logger .exception ("Failed to retrieve container logs" )
361+ raise RuntimeError ("Unleash container did not become healthy within timeout" )
362+
363+
364+ # Unleash's migration runner can hit a pg_class_relname_nsp_index race
365+ # condition that kills the process on first start. Retrying is safe because
366+ # the partially-created objects already exist on the second attempt.
367+ MAX_UNLEASH_ATTEMPTS = 3
368+
369+
309370@pytest .fixture (scope = "session" )
310- def unleash_container (postgres_container ): # noqa: PLR0915
371+ def unleash_container (postgres_container ):
311372 """Create and start Unleash container with PostgreSQL dependency."""
312373 global UNLEASH_URL
313374
@@ -322,75 +383,40 @@ def unleash_container(postgres_container): # noqa: PLR0915
322383 f":{ exposed_port } " , ":5432"
323384 )
324385
325- unleash = UnleashContainer ( internal_url )
386+ last_error = None
326387
327- with unleash as container :
328- logger .info ("Starting Unleash container..." )
329- container .start ()
330- logger .info ("Unleash container started" )
388+ for attempt in range (1 , MAX_UNLEASH_ATTEMPTS + 1 ):
389+ unleash = UnleashContainer (internal_url )
331390
332- # Wait for health check to pass
333- logger .info ("Waiting for Unleash container to be healthy ..." )
334- max_wait_time = 120 # 2 minutes; Unleash DB migrations can be slow in CI
335- start_time = time . time ( )
391+ with unleash as container :
392+ logger .info (f"Starting Unleash container (attempt { attempt } ) ..." )
393+ container . start ()
394+ logger . info ( "Unleash container started" )
336395
337- while time .time () - start_time < max_wait_time :
338396 try :
339- try :
340- exposed_port = container .get_exposed_port (4242 )
341- unleash_url = f"http://localhost:{ exposed_port } "
342- logger .info (f"Trying health check at: { unleash_url } " )
343- except Exception as port_error :
344- # if the container exited, fail fast with its logs
345- docker_container = container .get_wrapped_container ()
346- if docker_container :
347- docker_container .reload ()
348- if docker_container .status in ("exited" , "dead" ):
349- logs = docker_container .logs ().decode (errors = "replace" )
350- raise RuntimeError (
351- f"Unleash container died ({ docker_container .status } ).\n "
352- f"Logs:\n { logs } "
353- ) from port_error
354- logger .error (f"Port not ready yet: { port_error } " )
355- time .sleep (2 )
397+ unleash_url = _wait_for_healthy (container )
398+ except RuntimeError as exc :
399+ last_error = exc
400+ if "pg_class_relname_nsp_index" in str (exc ) or "died" in str (exc ):
401+ logger .warning (
402+ f"Unleash failed on attempt { attempt } "
403+ f"(likely migration race); retrying..."
404+ )
356405 continue
357-
358- response = requests .get (f"{ unleash_url } /health" , timeout = 5 )
359- if response .status_code == 200 :
360- logger .info ("Unleash container is healthy!" )
361- break
362-
363- logger .error (f"Health check failed, status: { response .status_code } " )
364- time .sleep (2 )
365-
366- except RuntimeError :
367406 raise
368- except Exception as e :
369- logger .error (f"Health check error: { e } " )
370- time .sleep (2 )
371- else :
372- # timeout; dump container logs for debugging
373- try :
374- docker_container = container .get_wrapped_container ()
375- if docker_container :
376- docker_container .reload ()
377- logs = docker_container .logs ().decode (errors = "replace" )
378- logger .error (
379- f"Unleash container status: { docker_container .status } \n "
380- f"Logs:\n { logs } "
381- )
382- except Exception :
383- logger .exception ("Failed to retrieve container logs" )
384- raise Exception ("Unleash container did not become healthy within timeout" )
385407
386- # Get the exposed port and set global URL
387- UNLEASH_URL = f"http://localhost:{ container .get_exposed_port (4242 )} "
388- logger .info (f"Unleash started at: { unleash_url } " )
408+ UNLEASH_URL = unleash_url
409+ logger .info (f"Unleash started at: { unleash_url } " )
410+
411+ insert_admin_token (postgres_container )
412+ logger .info ("Admin token inserted into database" )
389413
390- insert_admin_token ( postgres_container )
391- logger . info ( "Admin token inserted into database" )
414+ yield container , unleash_url
415+ return
392416
393- yield container , unleash_url
417+ raise RuntimeError (
418+ f"Unleash failed to start after { MAX_UNLEASH_ATTEMPTS } attempts"
419+ ) from last_error
394420
395421
396422@pytest .fixture (scope = "session" , autouse = True )
0 commit comments