CmampTask12784_Slim_docker_image_strategy

devops/docker_build/install_python_packages.sh

@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 #
 # Install Python packages.
+# 
+# Usage:
+#   ./install_python_packages.sh [dependency_groups]
+#   dependency_groups: Comma-separated list of dependency groups to install.
 #
 
 echo "#############################################################################"
@@ -42,7 +46,13 @@ if [[ 1 == 1 ]]; then
   python3 -m ${ENV_NAME} /${ENV_NAME}
   source /${ENV_NAME}/bin/activate
   #pip3 install wheel
-  poetry install --no-root
+  # Install only specified dependency groups.
+  # e.g.: `poetry install --no-root --with dev,docs`.
+  if [[ -n "$1" ]]; then
+    poetry install --no-root --with "$1"
+  else
+    poetry install --no-root
+  fi
   poetry env list
   # Clean up.
   if [[ $CLEAN_UP_INSTALLATION ]]; then

helpers/lib_tasks_docker.py

@@ -918,7 +918,7 @@ def dassert_is_subsequent_version(
 _IMAGE_USER_RE = r"[a-z0-9_-]+"
 # For candidate prod images which have added hash for easy identification.
 _IMAGE_HASH_RE = r"[a-z0-9]{9}"
-_IMAGE_STAGE_RE = rf"(local(?:-{_IMAGE_USER_RE})?|dev|prod|prod(?:-{_IMAGE_USER_RE})(?:-{_IMAGE_HASH_RE})?|prod(?:-{_IMAGE_HASH_RE})?)"
+_IMAGE_STAGE_RE = rf"(local(?:-{_IMAGE_USER_RE})?|base|dev|prod|prod(?:-{_IMAGE_USER_RE})(?:-{_IMAGE_HASH_RE})?|prod(?:-{_IMAGE_HASH_RE})?)"
 
 
 # TODO(Grisha): call `_dassert_is_base_image_name_valid()` and a separate
@@ -1028,7 +1028,7 @@ def get_image(
     """
     # Docker refers the default image as "latest", although in our stage
     # nomenclature we call it "dev".
-    hdbg.dassert_in(stage, "local dev prod".split())
+    hdbg.dassert_in(stage, "local base dev prod".split())
     # Get the base image.
     base_image = _get_base_image(base_image)
     _dassert_is_base_image_name_valid(base_image)

helpers/lib_tasks_docker_release.py

@@ -97,6 +97,7 @@ def _build_multi_arch_image(
     build_args: str,
     build_image: str,
     dockerfile: str,
+    target_opts: str,
 ) -> None:
     """
     Build a multi-architecture Docker image in a remote Docker registry.
@@ -108,6 +109,7 @@ def _build_multi_arch_image(
     :param build_args: build arguments for the Docker build command
     :param build_image: name of the image to build
     :param dockerfile: path to the Dockerfile to use for building
+    :param target_opts: Docker target stage, e.g., `--target builder`
     """
     # Build the multi-arch image.
     # Compress the current directory (in order to dereference symbolic
@@ -121,6 +123,7 @@ def _build_multi_arch_image(
         --push \
         --platform {multi_arch} \
         {build_args} \
+        {target_opts} \
         --tag {build_image} \
         --file {dockerfile} \
         -
@@ -341,6 +344,7 @@ def docker_build_local_image(  # type: ignore
     just_do_it=False,
     multi_arch="",
     cleanup_installation=True,
+    target_stage="",
 ):
     """
     Build a local image, i.e., a release candidate "dev" image.
@@ -366,13 +370,15 @@ def docker_build_local_image(  # type: ignore
           `linux/amd64,linux/arm64`
     :param cleanup_installation: force clean up Docker installation. This can
         be disabled to speed up the build process
+    :param target_stage: target stage for the Docker image
     """
     hlitauti.report_task(container_dir_name=container_dir_name)
     # For poetry_mode="update", the `poetry.lock` file is updated and saved as
     # `/install/poetry.lock.out` to the container.
     # For poetry_mode="no_update", the `poetry.lock` file from the repo is used,
     # and it's passed as `/install/poetry.lock.in` to the container.
     hdbg.dassert_in(poetry_mode, ("update", "no_update"))
+    hdbg.dassert_in(target_stage, ("", "base", "dev"))
     if just_do_it:
         _LOG.warning("Skipping subsequent version check")
     else:
@@ -392,6 +398,7 @@ def docker_build_local_image(  # type: ignore
     # files inside the tar stream and avoids file not found errors.
     # dockerfile = _to_abs_path(dockerfile)
     opts = "--no-cache" if not cache else ""
+    target_stage_opts = f"--target {target_stage}" if target_stage else ""
     build_args = [
         ("AM_CONTAINER_VERSION", dev_version),
         ("INSTALL_DIND", True),
@@ -406,7 +413,13 @@ def docker_build_local_image(  # type: ignore
         hlitadoc.docker_login(ctx)
         _create_multiarch_builder(ctx)
         _build_multi_arch_image(
-            ctx, opts, multi_arch, build_args, image_local, dockerfile
+            ctx,
+            opts,
+            multi_arch,
+            build_args,
+            image_local,
+            dockerfile,
+            target_stage_opts
         )
         # TODO(sandeep): If possible, switch to using hlitadoc._docker_pull().
         # Pull the image from registry after building.
@@ -423,6 +436,7 @@ def docker_build_local_image(  # type: ignore
             docker build \
             {opts} \
             {build_args} \
+            {target_stage_opts} \
             --tag {image_local} \
             --file {dockerfile} \
             -
@@ -454,10 +468,10 @@ def docker_build_local_image(  # type: ignore
     _list_image(ctx, image_local)
 
 
-@task
-def docker_tag_local_image_as_dev(  # type: ignore
+def _docker_tag_local_image(  # type: ignore
     ctx,
     version,
+    target_tag,
     base_image="",
     container_dir_name=".",
 ):
@@ -466,36 +480,78 @@ def docker_tag_local_image_as_dev(  # type: ignore
 
     :param ctx: invoke context
     :param version: version to tag the image and code with
+    :param target_tag: tag to apply to the local image (e.g., "dev")
     :param base_image: e.g., *****.dkr.ecr.us-east-1.amazonaws.com/amp
     :param container_dir_name: directory where the Dockerfile is located
     """
     hlitauti.report_task(container_dir_name=container_dir_name)
     # Get the version.
     dev_version = _get_dev_version(version, container_dir_name)
-    # Tag local image as versioned dev image (e.g., `dev-1.0.0`).
+    # Tag local image as versioned image (e.g., `dev-1.0.0`).
     image_versioned_local = hlitadoc.get_image(base_image, "local", dev_version)
-    image_versioned_dev = hlitadoc.get_image(base_image, "dev", dev_version)
+    image_versioned_dev = hlitadoc.get_image(base_image, target_tag, dev_version)
     cmd = f"docker tag {image_versioned_local} {image_versioned_dev}"
     hlitauti.run(ctx, cmd)
-    # Tag local image as dev image.
+    # Tag local image.
     latest_version = None
-    image_dev = hlitadoc.get_image(base_image, "dev", latest_version)
+    image_dev = hlitadoc.get_image(base_image, target_tag, latest_version)
     cmd = f"docker tag {image_versioned_local} {image_dev}"
     hlitauti.run(ctx, cmd)
 
 
 @task
-def docker_push_dev_image(  # type: ignore
+def docker_tag_local_image_as_dev(  # type: ignore
     ctx,
     version,
     base_image="",
     container_dir_name=".",
+):
+    """
+    Mark the "local" image as "dev".
+
+    See _docker_tag_local_image() for details.
+    """
+    _docker_tag_local_image(
+        ctx,
+        version,
+        target_tag="dev",
+        base_image=base_image,
+        container_dir_name=container_dir_name,
+    )
+
+@task
+def docker_tag_local_image_as_base(  # type: ignore
+    ctx,
+    version,
+    base_image="",
+    container_dir_name=".",
+):
+    """
+    Mark the "local" image as "base".
+
+    See _docker_tag_local_image() for details.
+    """
+    _docker_tag_local_image(
+        ctx,
+        version,
+        target_tag="base",
+        base_image=base_image,
+        container_dir_name=container_dir_name,
+    )    
+
+def _docker_push_image(  # type: ignore
+    ctx,
+    version,
+    target_tag,
+    base_image="",
+    container_dir_name=".",
 ):
     """
     Push the "dev" image to ECR.
 
     :param ctx: invoke context
     :param version: version to tag the image and code with
+    :param target_tag: tag to apply to the local image (e.g., "dev")
     :param base_image: e.g., *****.dkr.ecr.us-east-1.amazonaws.com/amp
     :param container_dir_name: directory where the Dockerfile is located
     """
@@ -505,16 +561,58 @@ def docker_push_dev_image(  # type: ignore
     #
     hlitadoc.docker_login(ctx)
     # Push Docker versioned tag.
-    image_versioned_dev = hlitadoc.get_image(base_image, "dev", dev_version)
+    image_versioned_dev = hlitadoc.get_image(base_image, target_tag, dev_version)
     cmd = f"docker push {image_versioned_dev}"
     hlitauti.run(ctx, cmd, pty=True)
     # Push Docker tag.
     latest_version = None
-    image_dev = hlitadoc.get_image(base_image, "dev", latest_version)
+    image_dev = hlitadoc.get_image(base_image, target_tag, latest_version)
     cmd = f"docker push {image_dev}"
     hlitauti.run(ctx, cmd, pty=True)
 
 
+@task
+def docker_push_dev_image(  # type: ignore
+    ctx,
+    version,
+    base_image="",
+    container_dir_name=".",
+):
+    """
+    Push the "dev" image to ECR.
+
+    See _docker_push_image() for details.
+    """
+    _docker_push_image(
+        ctx,
+        version,
+        target_tag="dev",
+        base_image=base_image,
+        container_dir_name=container_dir_name,
+    )
+
+
+@task
+def docker_push_base_image(  # type: ignore
+    ctx,
+    version,
+    base_image="",
+    container_dir_name=".",
+):
+    """
+    Push the "dev" image to ECR.
+
+    See _docker_push_image() for details.
+    """
+    _docker_push_image(
+        ctx,
+        version,
+        target_tag="base",
+        base_image=base_image,
+        container_dir_name=container_dir_name,
+    )
+
+
 @task
 def docker_release_dev_image(  # type: ignore
     ctx,

tasks.py

@@ -35,6 +35,7 @@
     docker_pull,
     docker_pull_helpers,
     docker_push_dev_image,
+    docker_push_base_image,
     docker_push_prod_candidate_image,
     docker_push_prod_image,
     docker_release_dev_image,
@@ -43,6 +44,7 @@
     docker_rollback_prod_image,
     docker_stats,
     docker_tag_local_image_as_dev,  # TODO(gp): -> docker_release_...
+    docker_tag_local_image_as_base,  
     docker_tag_push_multi_arch_prod_image,
     docker_update_prod_task_definition,
     #