From c872c7d43854e7ddf026125099f1b8653fc0cf84 Mon Sep 17 00:00:00 2001 From: vlady Date: Fri, 18 Jul 2025 18:23:15 +0000 Subject: [PATCH 1/2] Enhance install_python_packages.sh with usage instructions and dependency group support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-commit checks: All checks passed ✅ --- devops/docker_build/install_python_packages.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/devops/docker_build/install_python_packages.sh b/devops/docker_build/install_python_packages.sh index 44ed3bff4..7a9b5b198 100755 --- a/devops/docker_build/install_python_packages.sh +++ b/devops/docker_build/install_python_packages.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash # # Install Python packages. +# +# Usage: +# ./install_python_packages.sh [dependency_groups] +# dependency_groups: Comma-separated list of dependency groups to install. # echo "#############################################################################" @@ -42,7 +46,13 @@ if [[ 1 == 1 ]]; then python3 -m ${ENV_NAME} /${ENV_NAME} source /${ENV_NAME}/bin/activate #pip3 install wheel - poetry install --no-root + # Install only specified dependency groups. + # e.g.: `poetry install --no-root --with dev,docs`. + if [[ -n "$1" ]]; then + poetry install --no-root --with "$1" + else + poetry install --no-root + fi poetry env list # Clean up. if [[ $CLEAN_UP_INSTALLATION ]]; then From f8eb67f7321d4233e0ab16e57655c4c3af24cbc7 Mon Sep 17 00:00:00 2001 From: vlady Date: Mon, 21 Jul 2025 12:32:59 +0000 Subject: [PATCH 2/2] Add support for base image tagging and pushing in Docker tasks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pre-commit checks: All checks passed ✅ --- helpers/lib_tasks_docker.py | 4 +- helpers/lib_tasks_docker_release.py | 118 +++++++++++++++++++++++++--- tasks.py | 2 + 3 files changed, 112 insertions(+), 12 deletions(-) diff --git a/helpers/lib_tasks_docker.py b/helpers/lib_tasks_docker.py index d7760b6a2..ebe7d9e07 100644 --- a/helpers/lib_tasks_docker.py +++ b/helpers/lib_tasks_docker.py @@ -918,7 +918,7 @@ def dassert_is_subsequent_version( _IMAGE_USER_RE = r"[a-z0-9_-]+" # For candidate prod images which have added hash for easy identification. _IMAGE_HASH_RE = r"[a-z0-9]{9}" -_IMAGE_STAGE_RE = rf"(local(?:-{_IMAGE_USER_RE})?|dev|prod|prod(?:-{_IMAGE_USER_RE})(?:-{_IMAGE_HASH_RE})?|prod(?:-{_IMAGE_HASH_RE})?)" +_IMAGE_STAGE_RE = rf"(local(?:-{_IMAGE_USER_RE})?|base|dev|prod|prod(?:-{_IMAGE_USER_RE})(?:-{_IMAGE_HASH_RE})?|prod(?:-{_IMAGE_HASH_RE})?)" # TODO(Grisha): call `_dassert_is_base_image_name_valid()` and a separate @@ -1028,7 +1028,7 @@ def get_image( """ # Docker refers the default image as "latest", although in our stage # nomenclature we call it "dev". - hdbg.dassert_in(stage, "local dev prod".split()) + hdbg.dassert_in(stage, "local base dev prod".split()) # Get the base image. base_image = _get_base_image(base_image) _dassert_is_base_image_name_valid(base_image) diff --git a/helpers/lib_tasks_docker_release.py b/helpers/lib_tasks_docker_release.py index 06d9c06af..c7dfc33a4 100644 --- a/helpers/lib_tasks_docker_release.py +++ b/helpers/lib_tasks_docker_release.py @@ -97,6 +97,7 @@ def _build_multi_arch_image( build_args: str, build_image: str, dockerfile: str, + target_opts: str, ) -> None: """ Build a multi-architecture Docker image in a remote Docker registry. @@ -108,6 +109,7 @@ def _build_multi_arch_image( :param build_args: build arguments for the Docker build command :param build_image: name of the image to build :param dockerfile: path to the Dockerfile to use for building + :param target_opts: Docker target stage, e.g., `--target builder` """ # Build the multi-arch image. # Compress the current directory (in order to dereference symbolic @@ -121,6 +123,7 @@ def _build_multi_arch_image( --push \ --platform {multi_arch} \ {build_args} \ + {target_opts} \ --tag {build_image} \ --file {dockerfile} \ - @@ -341,6 +344,7 @@ def docker_build_local_image( # type: ignore just_do_it=False, multi_arch="", cleanup_installation=True, + target_stage="", ): """ Build a local image, i.e., a release candidate "dev" image. @@ -366,6 +370,7 @@ def docker_build_local_image( # type: ignore `linux/amd64,linux/arm64` :param cleanup_installation: force clean up Docker installation. This can be disabled to speed up the build process + :param target_stage: target stage for the Docker image """ hlitauti.report_task(container_dir_name=container_dir_name) # For poetry_mode="update", the `poetry.lock` file is updated and saved as @@ -373,6 +378,7 @@ def docker_build_local_image( # type: ignore # For poetry_mode="no_update", the `poetry.lock` file from the repo is used, # and it's passed as `/install/poetry.lock.in` to the container. hdbg.dassert_in(poetry_mode, ("update", "no_update")) + hdbg.dassert_in(target_stage, ("", "base", "dev")) if just_do_it: _LOG.warning("Skipping subsequent version check") else: @@ -392,6 +398,7 @@ def docker_build_local_image( # type: ignore # files inside the tar stream and avoids file not found errors. # dockerfile = _to_abs_path(dockerfile) opts = "--no-cache" if not cache else "" + target_stage_opts = f"--target {target_stage}" if target_stage else "" build_args = [ ("AM_CONTAINER_VERSION", dev_version), ("INSTALL_DIND", True), @@ -406,7 +413,13 @@ def docker_build_local_image( # type: ignore hlitadoc.docker_login(ctx) _create_multiarch_builder(ctx) _build_multi_arch_image( - ctx, opts, multi_arch, build_args, image_local, dockerfile + ctx, + opts, + multi_arch, + build_args, + image_local, + dockerfile, + target_stage_opts ) # TODO(sandeep): If possible, switch to using hlitadoc._docker_pull(). # Pull the image from registry after building. @@ -423,6 +436,7 @@ def docker_build_local_image( # type: ignore docker build \ {opts} \ {build_args} \ + {target_stage_opts} \ --tag {image_local} \ --file {dockerfile} \ - @@ -454,10 +468,10 @@ def docker_build_local_image( # type: ignore _list_image(ctx, image_local) -@task -def docker_tag_local_image_as_dev( # type: ignore +def _docker_tag_local_image( # type: ignore ctx, version, + target_tag, base_image="", container_dir_name=".", ): @@ -466,36 +480,78 @@ def docker_tag_local_image_as_dev( # type: ignore :param ctx: invoke context :param version: version to tag the image and code with + :param target_tag: tag to apply to the local image (e.g., "dev") :param base_image: e.g., *****.dkr.ecr.us-east-1.amazonaws.com/amp :param container_dir_name: directory where the Dockerfile is located """ hlitauti.report_task(container_dir_name=container_dir_name) # Get the version. dev_version = _get_dev_version(version, container_dir_name) - # Tag local image as versioned dev image (e.g., `dev-1.0.0`). + # Tag local image as versioned image (e.g., `dev-1.0.0`). image_versioned_local = hlitadoc.get_image(base_image, "local", dev_version) - image_versioned_dev = hlitadoc.get_image(base_image, "dev", dev_version) + image_versioned_dev = hlitadoc.get_image(base_image, target_tag, dev_version) cmd = f"docker tag {image_versioned_local} {image_versioned_dev}" hlitauti.run(ctx, cmd) - # Tag local image as dev image. + # Tag local image. latest_version = None - image_dev = hlitadoc.get_image(base_image, "dev", latest_version) + image_dev = hlitadoc.get_image(base_image, target_tag, latest_version) cmd = f"docker tag {image_versioned_local} {image_dev}" hlitauti.run(ctx, cmd) @task -def docker_push_dev_image( # type: ignore +def docker_tag_local_image_as_dev( # type: ignore ctx, version, base_image="", container_dir_name=".", +): + """ + Mark the "local" image as "dev". + + See _docker_tag_local_image() for details. + """ + _docker_tag_local_image( + ctx, + version, + target_tag="dev", + base_image=base_image, + container_dir_name=container_dir_name, + ) + +@task +def docker_tag_local_image_as_base( # type: ignore + ctx, + version, + base_image="", + container_dir_name=".", +): + """ + Mark the "local" image as "base". + + See _docker_tag_local_image() for details. + """ + _docker_tag_local_image( + ctx, + version, + target_tag="base", + base_image=base_image, + container_dir_name=container_dir_name, + ) + +def _docker_push_image( # type: ignore + ctx, + version, + target_tag, + base_image="", + container_dir_name=".", ): """ Push the "dev" image to ECR. :param ctx: invoke context :param version: version to tag the image and code with + :param target_tag: tag to apply to the local image (e.g., "dev") :param base_image: e.g., *****.dkr.ecr.us-east-1.amazonaws.com/amp :param container_dir_name: directory where the Dockerfile is located """ @@ -505,16 +561,58 @@ def docker_push_dev_image( # type: ignore # hlitadoc.docker_login(ctx) # Push Docker versioned tag. - image_versioned_dev = hlitadoc.get_image(base_image, "dev", dev_version) + image_versioned_dev = hlitadoc.get_image(base_image, target_tag, dev_version) cmd = f"docker push {image_versioned_dev}" hlitauti.run(ctx, cmd, pty=True) # Push Docker tag. latest_version = None - image_dev = hlitadoc.get_image(base_image, "dev", latest_version) + image_dev = hlitadoc.get_image(base_image, target_tag, latest_version) cmd = f"docker push {image_dev}" hlitauti.run(ctx, cmd, pty=True) +@task +def docker_push_dev_image( # type: ignore + ctx, + version, + base_image="", + container_dir_name=".", +): + """ + Push the "dev" image to ECR. + + See _docker_push_image() for details. + """ + _docker_push_image( + ctx, + version, + target_tag="dev", + base_image=base_image, + container_dir_name=container_dir_name, + ) + + +@task +def docker_push_base_image( # type: ignore + ctx, + version, + base_image="", + container_dir_name=".", +): + """ + Push the "dev" image to ECR. + + See _docker_push_image() for details. + """ + _docker_push_image( + ctx, + version, + target_tag="base", + base_image=base_image, + container_dir_name=container_dir_name, + ) + + @task def docker_release_dev_image( # type: ignore ctx, diff --git a/tasks.py b/tasks.py index 20c15a4b9..1ff22798c 100644 --- a/tasks.py +++ b/tasks.py @@ -35,6 +35,7 @@ docker_pull, docker_pull_helpers, docker_push_dev_image, + docker_push_base_image, docker_push_prod_candidate_image, docker_push_prod_image, docker_release_dev_image, @@ -43,6 +44,7 @@ docker_rollback_prod_image, docker_stats, docker_tag_local_image_as_dev, # TODO(gp): -> docker_release_... + docker_tag_local_image_as_base, docker_tag_push_multi_arch_prod_image, docker_update_prod_task_definition, #