From 411102c4c7c7147fd9ec1eace90e7c25dbc8d352 Mon Sep 17 00:00:00 2001 From: cxnt Date: Wed, 26 Nov 2025 20:23:21 +0400 Subject: [PATCH 01/15] Refactor Dockerfile and agent code to remove torch dependency and optimise docker image size --- Dockerfile | 44 +++++------- agent/worker/agent.py | 2 - agent/worker/system_info.py | 138 ++++++++++++++++++------------------ build_dev.sh | 4 ++ requirements.txt | 4 -- 5 files changed, 91 insertions(+), 101 deletions(-) create mode 100755 build_dev.sh diff --git a/Dockerfile b/Dockerfile index 0ef2e80..ac9d4d3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,3 @@ -# FROM ubuntu:24.04 # No GPU support FROM nvidia/cuda:12.8.1-cudnn-runtime-ubuntu24.04 ARG LABEL_VERSION @@ -21,8 +20,11 @@ ENV \ PIP_BREAK_SYSTEM_PACKAGES=1 \ PATH=/root/.local/bin:$PATH -RUN apt-get update \ - && apt-get install -y --no-install-recommends \ +COPY requirements.txt /workdir/requirements.txt + +RUN set -eux; \ + apt-get update; \ + apt-get install -y --no-install-recommends \ build-essential \ python3.12 \ python3.12-venv \ @@ -59,31 +61,17 @@ RUN apt-get update \ html2text \ htop \ tree \ - && ln -sf /usr/bin/python3.12 /usr/bin/python \ - && ln -sf /usr/bin/pip3 /usr/bin/pip \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log - -RUN python -m pip install --ignore-installed --upgrade pip setuptools wheel - -RUN python -m pip install torch==2.9.1 torchvision==0.24.1 --index-url https://download.pytorch.org/whl/cu128 - -# Install runtime dependencies that must stay -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ libmagic-dev \ openssh-server \ ffmpeg \ fonts-noto \ - && mkdir -p /var/run/sshd \ - && apt-get -qq -y autoremove \ - && apt-get autoclean \ - && rm -rf /var/lib/apt/lists/* - -COPY requirements.txt /workdir/requirements.txt -RUN python -m pip install --no-cache-dir -r /workdir/requirements.txt - -RUN apt-get purge -y --auto-remove \ + ; \ + mkdir -p /var/run/sshd; \ + ln -sf /usr/bin/python3.12 /usr/bin/python; \ + ln -sf /usr/bin/pip3 /usr/bin/pip; \ + python -m pip install --ignore-installed --upgrade pip setuptools wheel; \ + python -m pip install --no-cache-dir -r /workdir/requirements.txt; \ + apt-get purge -y --auto-remove \ build-essential \ python3.12-dev \ libexiv2-dev \ @@ -104,11 +92,15 @@ RUN apt-get purge -y --auto-remove \ libatlas-base-dev \ gfortran \ pkg-config \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* /var/log/dpkg.log + ; \ + apt-get -qq -y autoremove; \ + apt-get autoclean; \ + rm -rf /var/lib/apt/lists/* /var/log/dpkg.log /root/.cache/pip COPY agent /workdir/agent WORKDIR /workdir/agent ENTRYPOINT ["python", "-u", "/workdir/agent/main.py"] + + diff --git a/agent/worker/agent.py b/agent/worker/agent.py index dc6598f..6c0beb8 100644 --- a/agent/worker/agent.py +++ b/agent/worker/agent.py @@ -28,8 +28,6 @@ warnings.filterwarnings(action="ignore", category=UserWarning) -import torch # pylint: disable=import-error - from worker import constants from worker import agent_utils from worker import docker_utils diff --git a/agent/worker/system_info.py b/agent/worker/system_info.py index 4f1e627..d2e7fb3 100644 --- a/agent/worker/system_info.py +++ b/agent/worker/system_info.py @@ -13,8 +13,6 @@ warnings.filterwarnings(action="ignore", category=UserWarning) -import torch # pylint: disable=import-error - import supervisely_lib as sly from worker import constants @@ -228,78 +226,80 @@ def get_self_docker_image_digest(): return sly.catch_silently(_get_self_docker_image_digest) -def get_gpu_info_with_torch(logger): - torch.cuda.init() - gpu_info = None - try: - gpu_info = {} - gpu_info["is_available"] = torch.cuda.is_available() - if gpu_info["is_available"]: - gpu_info["device_count"] = torch.cuda.device_count() - gpu_info["device_names"] = [] - gpu_info["device_memory"] = [] - for idx in range(gpu_info["device_count"]): - gpu_info["device_names"].append(torch.cuda.get_device_name(idx)) - mem = {} - try: - device_props = torch.cuda.get_device_properties(idx) - t = device_props.total_memory - r = torch.cuda.memory_reserved(idx) - a = torch.cuda.memory_allocated(idx) - mem = { - "total": t, - "reserved": r, - "allocated": a, - "free": t - r, - } - except Exception as e: - logger.debug(repr(e)) - finally: - gpu_info["device_memory"].append(mem) - - except Exception as e: - logger.warning(repr(e)) - return gpu_info +def get_gpu_info(logger): + """ + Collect GPU information using NVML. + """ + + gpu_info = { + "is_available": False, + "device_count": 0, + "device_names": [], + "device_memory": [], + "device_capability": [], + } + try: + smi.nvmlInit() + except Exception as e: # pylint: disable=broad-except + logger.warning("Failed to initialize NVML: %s", repr(e)) + return gpu_info -def get_gpu_info(logger): - gpu_info = None try: - gpu_info = {} - gpu_info["is_available"] = torch.cuda.is_available() - if gpu_info["is_available"]: - smi.nvmlInit() - gpu_info["device_count"] = smi.nvmlDeviceGetCount() - gpu_info["device_names"] = [] - gpu_info["device_memory"] = [] - gpu_info["device_capability"] = [] - for idx in range(gpu_info["device_count"]): - handle = smi.nvmlDeviceGetHandleByIndex(idx) - capability = smi.nvmlDeviceGetCudaComputeCapability(handle) - capability = "{major}.{minor}".format(major=capability[0], minor=capability[1]) - gpu_info["device_names"].append(smi.nvmlDeviceGetName(handle)) - gpu_info["device_capability"].append( - { - "device": f"GPU {idx}", - "compute_capability": capability, - } - ) - mem = {} - try: - device_props = smi.nvmlDeviceGetMemoryInfo(handle) - mem = { - "total": device_props.total, - "reserved": device_props.used, - "available": device_props.free, - } - except Exception as e: - logger.debug(repr(e)) - finally: - gpu_info["device_memory"].append(mem) + try: + device_count = smi.nvmlDeviceGetCount() + except Exception as e: # pylint: disable=broad-except + logger.warning("Failed to get GPU count via NVML: %s", repr(e)) + return gpu_info + + gpu_info["device_count"] = device_count + gpu_info["is_available"] = device_count > 0 + + if not gpu_info["is_available"]: + return gpu_info + + for idx in range(device_count): + handle = smi.nvmlDeviceGetHandleByIndex(idx) + capability = smi.nvmlDeviceGetCudaComputeCapability(handle) + capability_str = "{major}.{minor}".format(major=capability[0], minor=capability[1]) + gpu_info["device_names"].append(smi.nvmlDeviceGetName(handle)) + gpu_info["device_capability"].append( + { + "device": f"GPU {idx}", + "compute_capability": capability_str, + } + ) + + mem = {} + try: + device_props = smi.nvmlDeviceGetMemoryInfo(handle) + mem = { + "total": device_props.total, + "reserved": device_props.used, + "available": device_props.free, + } + except Exception as e: # pylint: disable=broad-except + logger.debug("Failed to collect GPU memory info: %s", repr(e)) + finally: + gpu_info["device_memory"].append(mem) + + try: gpu_info["driver_version"] = smi.nvmlSystemGetDriverVersion() + except Exception as e: # pylint: disable=broad-except + logger.debug("Failed to get NVIDIA driver version: %s", repr(e)) + + try: gpu_info["cuda_version"] = smi.nvmlSystemGetCudaDriverVersion() + except Exception as e: # pylint: disable=broad-except + logger.debug("Failed to get CUDA driver version: %s", repr(e)) + + except Exception as e: # pylint: disable=broad-except + logger.warning("Failed to collect GPU info via NVML: %s", repr(e)) + finally: + try: smi.nvmlShutdown() + except Exception: # pylint: disable=broad-except + # Ignore shutdown errors + pass - except Exception as e: - logger.warning(repr(e)) return gpu_info diff --git a/build_dev.sh b/build_dev.sh new file mode 100755 index 0000000..f0043b8 --- /dev/null +++ b/build_dev.sh @@ -0,0 +1,4 @@ +docker build -t supervisely/agent:dev \ + --build-arg LABEL_VERSION=agent:6.999.0 \ + . && \ +docker push supervisely/agent:dev \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 0d6a023..aafd0a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,10 +15,6 @@ nvidia-ml-py==12.535.77 httpx>=0.26.0 filelock==3.13.1 -# Installed inside Dockerfile -# torch==2.9.1+cu128 -# torchvision==0.24.1+cu128 - supervisely==6.73.474 supervisely[agent]==6.73.474 # for development From 40b34336ab31bb8386aafcb1b3a6bfff133ebd1a Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Wed, 26 Nov 2025 18:54:16 +0100 Subject: [PATCH 02/15] Remove unnecessary dependencies from Dockerfile to streamline image size --- Dockerfile | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index ac9d4d3..f8be522 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,9 +31,7 @@ RUN set -eux; \ python3.12-dev \ python3-pip \ python3-grpcio \ - libexiv2-27 \ libexiv2-dev \ - libboost-all-dev \ libgeos-dev \ libsm6 \ libxext6 \ @@ -45,9 +43,6 @@ RUN set -eux; \ libavcodec-dev \ libavformat-dev \ libswscale-dev \ - libv4l-dev \ - libxvidcore-dev \ - libx264-dev \ libjpeg-dev \ libpng-dev \ libtiff-dev \ @@ -75,7 +70,6 @@ RUN set -eux; \ build-essential \ python3.12-dev \ libexiv2-dev \ - libboost-all-dev \ libgeos-dev \ libxrender-dev \ libgl1-mesa-dev \ @@ -83,14 +77,12 @@ RUN set -eux; \ libavcodec-dev \ libavformat-dev \ libswscale-dev \ - libv4l-dev \ - libxvidcore-dev \ - libx264-dev \ libjpeg-dev \ libpng-dev \ libtiff-dev \ libatlas-base-dev \ gfortran \ + libmagic-dev \ pkg-config \ ; \ apt-get -qq -y autoremove; \ From 905bbc95a2285a91bdfcc1d536822e9a981b8e7c Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Wed, 26 Nov 2025 19:21:50 +0100 Subject: [PATCH 03/15] Add missing library dependencies to Dockerfile for improved functionality --- Dockerfile | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Dockerfile b/Dockerfile index f8be522..c6c379e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,7 @@ RUN set -eux; \ python3.12-dev \ python3-pip \ python3-grpcio \ + libexiv2-27 \ libexiv2-dev \ libgeos-dev \ libsm6 \ @@ -43,6 +44,9 @@ RUN set -eux; \ libavcodec-dev \ libavformat-dev \ libswscale-dev \ + libv4l-dev \ + libxvidcore-dev \ + libx264-dev \ libjpeg-dev \ libpng-dev \ libtiff-dev \ @@ -77,6 +81,9 @@ RUN set -eux; \ libavcodec-dev \ libavformat-dev \ libswscale-dev \ + libv4l-dev \ + libxvidcore-dev \ + libx264-dev \ libjpeg-dev \ libpng-dev \ libtiff-dev \ From 71d5bd0ba543d3a8a94fa80518ef227949f10030 Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Wed, 26 Nov 2025 19:46:09 +0100 Subject: [PATCH 04/15] Update GitHub workflows to use gha-runner-supervisely and upgrade action versions --- .github/workflows/app-release.yml | 14 +- .github/workflows/build-push-dev.yml | 48 +++---- .github/workflows/build-push-release.yml | 68 +++------- .../workflows/manual-build-push-release.yml | 60 +++------ .github/workflows/pr-check-with-pylint.yml | 123 +++++++++--------- Dockerfile | 2 + 6 files changed, 128 insertions(+), 187 deletions(-) diff --git a/.github/workflows/app-release.yml b/.github/workflows/app-release.yml index aa9a43e..e89aeb0 100644 --- a/.github/workflows/app-release.yml +++ b/.github/workflows/app-release.yml @@ -31,19 +31,21 @@ permissions: jobs: Update-App-Config: - runs-on: ubuntu-latest + runs-on: gha-runner-supervisely outputs: RELEASE_TYPE: ${{ steps.update_app_config.outputs.RELEASE_TYPE }} COMMIT_HASH: ${{ steps.update_app_config.outputs.COMMIT_HASH }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 with: token: ${{ secrets.GH_ACCESS_TOKEN }} ref: master + - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: - python-version: 3.8 + python-version: "3.12" + - name: Update App Config env: GITHUB_TOKEN: ${{ secrets.GH_ACCESS_TOKEN }} @@ -54,7 +56,7 @@ jobs: VERSION="${VERSION:1}" fi echo "DOCKER_IMAGE=$VERSION" >> $GITHUB_ENV - + SEMVER_REGEX="^[0-9]+\.[0-9]+\.[0-9]+$" if [[ ! $VERSION =~ $SEMVER_REGEX ]]; then RELEASE_TYPE="release-branch" @@ -63,7 +65,7 @@ jobs: fi echo "RELEASE_TYPE=$RELEASE_TYPE" >> $GITHUB_OUTPUT echo "RELEASE_TYPE=$RELEASE_TYPE" - + python app/update_config.py $VERSION git add app/config.json diff --git a/.github/workflows/build-push-dev.yml b/.github/workflows/build-push-dev.yml index 8f7885d..5677baf 100644 --- a/.github/workflows/build-push-dev.yml +++ b/.github/workflows/build-push-dev.yml @@ -5,41 +5,29 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: gha-runner-supervisely steps: - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Write Tag to ENV variable run: echo "BRANCH_NAME=${{ github.event.release.tag_name }}" >> $GITHUB_ENV - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME_COMMUNITY }} - password: ${{ secrets.DOCKER_TOKEN_COMMUNITY }} + - name: Get Docker Labels from python script run: python .github/workflows/docker_labels.py - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: v0.9.1 + - name: Build and push - uses: docker/build-push-action@v5 - with: - context: ./ - file: ./Dockerfile - provenance: false - builder: ${{ steps.buildx.outputs.name }} - push: true - tags: | - supervisely/agent:dev - build-args: | - LABEL_VERSION=agent:6.999.0 - LABEL_INFO=${{ env.LABEL_INFO }} - LABEL_MODES=${{ env.LABEL_MODES }} - LABEL_README=${{ env.LABEL_README }} - LABEL_BUILT_AT=${{ env.LABEL_BUILT_AT }} - cache-from: type=registry,ref=supervisely/agent:cache - cache-to: type=registry,ref=supervisely/agent:cache,mode=max - # cache-from: type=gha - # cache-to: type=gha,mode=max + run: | + buildctl build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=. \ + --output type=image,name=supervisely/agent:dev,push=true \ + --opt provenance=false \ + --opt build-arg=LABEL_VERSION=6.999.0 \ + --opt build-arg=LABEL_INFO=${{ env.LABEL_INFO }} \ + --opt build-arg=LABEL_MODES=${{ env.LABEL_MODES }} \ + --opt build-arg=LABEL_README=${{ env.LABEL_README }} \ + --opt build-arg=LABEL_BUILT_AT=${{ env.LABEL_BUILT_AT }} \ + --import-cache type=registry,ref=supervisely/agent:dev-cache \ + --export-cache type=registry,ref=supervisely/agent:dev-cache,mode=max diff --git a/.github/workflows/build-push-release.yml b/.github/workflows/build-push-release.yml index 0dfe4a6..c2b93aa 100644 --- a/.github/workflows/build-push-release.yml +++ b/.github/workflows/build-push-release.yml @@ -6,73 +6,46 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: gha-runner-supervisely steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - tool-cache: false - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: true - - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Echo ${{ github.event.release.tag_name }} run: echo ${{ github.event.release.tag_name }} - name: Write version to ENV variable run: | - TAG_NAME=${{ github.event.release.tag_name }} - echo "LABEL_VERSION=${TAG_NAME:1}" >> $GITHUB_ENV - - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME_COMMUNITY }} - password: ${{ secrets.DOCKER_TOKEN_COMMUNITY }} + TAG_NAME=${{ github.event.release.tag_name }} + echo "LABEL_VERSION=${TAG_NAME:1}" >> $GITHUB_ENV - name: Get Docker Labels from python script run: python .github/workflows/docker_labels.py - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: v0.9.1 - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: ./ - file: ./Dockerfile - provenance: false - builder: ${{ steps.buildx.outputs.name }} - push: true - tags: | - supervisely/agent:${{ env.LABEL_VERSION }} - build-args: | - LABEL_VERSION=agent:${{ env.LABEL_VERSION }} - LABEL_INFO=${{ env.LABEL_INFO }} - LABEL_MODES=${{ env.LABEL_MODES }} - LABEL_README=${{ env.LABEL_README }} - LABEL_BUILT_AT=${{ env.LABEL_BUILT_AT }} - cache-from: type=registry,ref=supervisely/agent:cache - cache-to: type=registry,ref=supervisely/agent:cache,mode=max - # cache-from: type=gha - # cache-to: type=gha,mode=max - + run: | + buildctl build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=. \ + --output type=image,name=supervisely/agent:${{ env.LABEL_VERSION }},push=true \ + --opt provenance=false \ + --opt build-arg=LABEL_VERSION=agent:${{ env.LABEL_VERSION }} \ + --opt build-arg=LABEL_INFO=${{ env.LABEL_INFO }} \ + --opt build-arg=LABEL_MODES=${{ env.LABEL_MODES }} \ + --opt build-arg=LABEL_README=${{ env.LABEL_README }} \ + --opt build-arg=LABEL_BUILT_AT=${{ env.LABEL_BUILT_AT }} \ + --import-cache type=registry,ref=supervisely/agent:cache \ + --export-cache type=registry,ref=supervisely/agent:cache,mode=max + app-release: needs: build permissions: contents: write actions: write uses: supervisely/agent/.github/workflows/app-release.yml@app-release - secrets: + secrets: SUPERVISELY_DEV_API_TOKEN: "${{ secrets.SUPERVISELY_DEV_API_TOKEN }}" SUPERVISELY_PRIVATE_DEV_API_TOKEN: "${{ secrets.SUPERVISELY_PRIVATE_DEV_API_TOKEN }}" SUPERVISELY_PROD_API_TOKEN: "${{ secrets.SUPERVISELY_PROD_API_TOKEN }}" @@ -82,4 +55,3 @@ jobs: SUPERVISELY_PROD_SERVER_ADDRESS: "${{ vars.SUPERVISELY_PROD_SERVER_ADDRESS }}" RELEASE_VERSION: "${{ github.event.release.tag_name }}" RELEASE_DESCRIPTION: "${{ github.event.release.name }}" - diff --git a/.github/workflows/manual-build-push-release.yml b/.github/workflows/manual-build-push-release.yml index 311b5e1..d98dcb3 100644 --- a/.github/workflows/manual-build-push-release.yml +++ b/.github/workflows/manual-build-push-release.yml @@ -14,55 +14,36 @@ on: jobs: build: - runs-on: ubuntu-latest + runs-on: gha-runner-supervisely steps: - - name: Free Disk Space (Ubuntu) - uses: jlumbroso/free-disk-space@main - with: - tool-cache: false - android: true - dotnet: true - haskell: true - large-packages: true - docker-images: true - swap-storage: true - name: Checkout - uses: actions/checkout@v4 + uses: actions/checkout@v6 + - name: Tag Version run: echo ${{ github.event.inputs.tag_version }} + - name: Write Tag to ENV variable run: echo "LABEL_VERSION=${{ github.event.inputs.tag_version }}" >> $GITHUB_ENV - - name: Login to DockerHub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_USERNAME_COMMUNITY }} - password: ${{ secrets.DOCKER_TOKEN_COMMUNITY }} + - name: Get Docker Labels from python script run: python .github/workflows/docker_labels.py - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - with: - version: v0.9.1 - - name: Build and push - uses: docker/build-push-action@v5 - with: - context: ./ - file: ./Dockerfile - provenance: false - builder: ${{ steps.buildx.outputs.name }} - push: true - tags: | - supervisely/agent:${{ env.LABEL_VERSION }} - build-args: | - LABEL_VERSION=agent:${{ env.LABEL_VERSION }} - LABEL_INFO=${{ env.LABEL_INFO }} - LABEL_MODES=${{ env.LABEL_MODES }} - LABEL_README=${{ env.LABEL_README }} - LABEL_BUILT_AT=${{ env.LABEL_BUILT_AT }} - cache-from: type=registry,ref=supervisely/agent:cache - cache-to: type=registry,ref=supervisely/agent:cache,mode=max + - name: Build and push + run: | + buildctl build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=. \ + --output type=image,name=supervisely/agent:${{ env.LABEL_VERSION }},push=true \ + --opt provenance=false \ + --opt build-arg=LABEL_VERSION=agent:${{ env.LABEL_VERSION }} \ + --opt build-arg=LABEL_INFO=${{ env.LABEL_INFO }} \ + --opt build-arg=LABEL_MODES=${{ env.LABEL_MODES }} \ + --opt build-arg=LABEL_README=${{ env.LABEL_README }} \ + --opt build-arg=LABEL_BUILT_AT=${{ env.LABEL_BUILT_AT }} \ + --import-cache type=registry,ref=supervisely/agent:cache \ + --export-cache type=registry,ref=supervisely/agent:cache,mode=max app-release: needs: build @@ -80,4 +61,3 @@ jobs: SUPERVISELY_PROD_SERVER_ADDRESS: "${{ vars.SUPERVISELY_PROD_SERVER_ADDRESS }}" RELEASE_VERSION: "${{ inputs.tag_version }}" RELEASE_DESCRIPTION: "${{ inputs.release_description }}" - diff --git a/.github/workflows/pr-check-with-pylint.yml b/.github/workflows/pr-check-with-pylint.yml index 57e6c08..d5d4dc4 100644 --- a/.github/workflows/pr-check-with-pylint.yml +++ b/.github/workflows/pr-check-with-pylint.yml @@ -7,67 +7,65 @@ on: jobs: pylint: - runs-on: ubuntu-22.04 + runs-on: gha-runner-supervisely env: - ISSUES_URL: "https://api.github.com/repos/supervisely/issues/issues" - PROJECT_NUMBER: "2" - TODO_NAME: "🚀 Todo (now!)" - STATUS_FIELD_ID: "" - STATUS_ID: "" - ISSUE_NODE_ID: "" - ITEM_TO_MOVE: "" - ORG_PROJECT_ID: "" - ORG_LOGIN: "supervisely" + # ISSUES_URL: "https://api.github.com/repos/supervisely/issues/issues" + # PROJECT_NUMBER: "2" + # TODO_NAME: "🚀 Todo (now!)" + # STATUS_FIELD_ID: "" + # STATUS_ID: "" + # ISSUE_NODE_ID: "" + # ITEM_TO_MOVE: "" + # ORG_PROJECT_ID: "" + # ORG_LOGIN: "supervisely" ERRORS_DETECTED: false steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - repository: ${{ github.repository }} - token: ${{ secrets.PYLINT_TOKEN }} - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.8 - - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install python3-all-dev libboost-python-dev libexiv2-dev - sudo ln -s /usr/lib/x86_64-linux-gnu/libboost_python310.so /usr/lib/x86_64-linux-gnu/libboost_python38.so - pip install --upgrade pip - pip install pylint - pip install -r requirements.txt - - - - name: Run check with pylint - run: | - export PYTHONPATH=$PYTHONPATH:$PWD/agent/ - pylint_output=$(pylint --ignore-patterns=".*\.json$|.*\.gitignore$" "agent") || true - if [[ $pylint_output == *"E"* ]] || [[ $pylint_output == *"F"* ]]; then - # Save pylint output to a file - echo "$pylint_output" > pylint_errors.txt - echo "ERRORS_DETECTED=true" >> $GITHUB_ENV - else - echo "ERRORS_DETECTED=false" >> $GITHUB_ENV - fi - id: pylint - - - name: Fail if pylint errors detected - if: ${{ env.ERRORS_DETECTED == 'true' }} - run: | - issue_body=$(cat pylint_errors.txt) - echo "Pylint Errors: $issue_body" - echo "RESULT=failure" >> $GITHUB_ENV - exit 1 + - name: Checkout repository + uses: actions/checkout@v6 + with: + repository: ${{ github.repository }} + token: ${{ secrets.PYLINT_TOKEN }} + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install python3-all-dev libboost-python-dev libexiv2-dev + pip install --upgrade pip + pip install pylint + pip install -r requirements.txt + + - name: Run check with pylint + run: | + export PYTHONPATH=$PYTHONPATH:$PWD/agent/ + pylint_output=$(pylint --ignore-patterns=".*\.json$|.*\.gitignore$" "agent") || true + if [[ $pylint_output == *"E"* ]] || [[ $pylint_output == *"F"* ]]; then + # Save pylint output to a file + echo "$pylint_output" > pylint_errors.txt + echo "ERRORS_DETECTED=true" >> $GITHUB_ENV + else + echo "ERRORS_DETECTED=false" >> $GITHUB_ENV + fi + id: pylint + + - name: Fail if pylint errors detected + if: ${{ env.ERRORS_DETECTED == 'true' }} + run: | + issue_body=$(cat pylint_errors.txt) + echo "Pylint Errors: $issue_body" + echo "RESULT=failure" >> $GITHUB_ENV + exit 1 # - name: Create GitHub issue # run: | # if [[ "${{ env.ERRORS_DETECTED }}" == "true" ]]; then # issue_body=$(cat pylint_errors.txt) # echo "Issue body: $issue_body" - + # json=$(jq -n \ # --arg title "Pylint Errors for ${{ github.event_name }} #${{ github.event.release.tag_name }} at $(date -u +'%Y-%m-%d %H:%M') UTC+0" \ # --arg body "$issue_body" \ @@ -87,13 +85,12 @@ jobs: # echo "issue_url=$issue_url" >> $GITHUB_ENV # issue_node_id=$(echo "$issue_response" | jq -r '.node_id') # echo "ISSUE_NODE_ID=$issue_node_id" >> $GITHUB_ENV - + # else # echo "No pylint errors detected." # fi - - - # - name: Get Project ID + + # - name: Get Project ID # if: ${{ env.ERRORS_DETECTED == 'true' }} # run: | # org_login=${{ env.ORG_LOGIN }} @@ -112,10 +109,10 @@ jobs: # https://api.github.com/graphql) # echo "Response from GitHub API: $response" # project_id=$(echo "$response" | jq -r '.data.organization.projectV2.id') - + # echo "Organization Project ID: $project_id" # echo "ORG_PROJECT_ID=$project_id" >> $GITHUB_ENV - + # - name: Get Status Field ID and Status ID # if: ${{ env.ERRORS_DETECTED == 'true' }} # run: | @@ -130,10 +127,10 @@ jobs: # }' \ # https://api.github.com/graphql) # echo "Response from GitHub API: $response" - # status_field_id=$(echo "$response" | jq -r '.data.node.fields.nodes[] | select(.name == "Status") | .id') + # status_field_id=$(echo "$response" | jq -r '.data.node.fields.nodes[] | select(.name == "Status") | .id') # status_id=$(echo "$response" | jq -r '.data.node.fields.nodes[] | select(.name == "Status") | .options[] | select(.name == "${{ env.TODO_NAME }}") | .id') # echo "STATUS_FIELD_ID=$status_field_id" >> $GITHUB_ENV - # echo "STATUS_ID=$status_id" >> $GITHUB_ENV + # echo "STATUS_ID=$status_id" >> $GITHUB_ENV # echo "Status Field ID: $status_id" # echo "Todo ID: $todo_id" @@ -142,7 +139,7 @@ jobs: # run: | # issue_id=${{ env.ISSUE_NODE_ID }} # project_id=${{ env.ORG_PROJECT_ID }} - + # item_id=$(curl -X POST -H "Authorization: Bearer ${{ secrets.PYLINT_TOKEN }}" \ # -H "Accept: application/vnd.github.v3+json" \ # https://api.github.com/graphql \ @@ -164,10 +161,10 @@ jobs: # } # EOF # ) - + # echo "Item ID: $item_id" # echo "ITEM_TO_MOVE=$item_id" >> $GITHUB_ENV - + # - name: Move issue to Todo column # if: ${{ env.ERRORS_DETECTED == 'true' }} # run: | diff --git a/Dockerfile b/Dockerfile index c6c379e..f4136a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -33,6 +33,7 @@ RUN set -eux; \ python3-grpcio \ libexiv2-27 \ libexiv2-dev \ + libboost-python-dev \ libgeos-dev \ libsm6 \ libxext6 \ @@ -74,6 +75,7 @@ RUN set -eux; \ build-essential \ python3.12-dev \ libexiv2-dev \ + libboost-python-dev \ libgeos-dev \ libxrender-dev \ libgl1-mesa-dev \ From 5384016ff274ce773a0fb66e6969b13056bae5ac Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Wed, 26 Nov 2025 20:20:01 +0100 Subject: [PATCH 05/15] Enhance Dockerfile cleanup process to reduce image size by removing additional temporary files and cache --- Dockerfile | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f4136a3..1174464 100644 --- a/Dockerfile +++ b/Dockerfile @@ -95,8 +95,15 @@ RUN set -eux; \ pkg-config \ ; \ apt-get -qq -y autoremove; \ - apt-get autoclean; \ - rm -rf /var/lib/apt/lists/* /var/log/dpkg.log /root/.cache/pip + apt-get autoclean && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* \ + /var/cache/apt/* \ + /var/log/dpkg.log \ + /var/log/apt/* \ + /root/.cache/pip \ + /tmp/* \ + /var/tmp/* COPY agent /workdir/agent From c2a749f58ae6a0ed0119a9edfbe19f8d1bf925e4 Mon Sep 17 00:00:00 2001 From: GoldenAnpu Date: Wed, 26 Nov 2025 20:31:55 +0100 Subject: [PATCH 06/15] Add libmagic1 to Dockerfile for improved functionality --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 1174464..ed10378 100644 --- a/Dockerfile +++ b/Dockerfile @@ -62,6 +62,7 @@ RUN set -eux; \ htop \ tree \ libmagic-dev \ + libmagic1 \ openssh-server \ ffmpeg \ fonts-noto \ From d43f14df32be2f5db891f579863f0986bdcfeee6 Mon Sep 17 00:00:00 2001 From: Sergey <57998637+GoldenAnpu@users.noreply.github.com> Date: Thu, 27 Nov 2025 10:53:38 +0100 Subject: [PATCH 07/15] Add '-y' flag to apt-get install command --- .github/workflows/pr-check-with-pylint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-check-with-pylint.yml b/.github/workflows/pr-check-with-pylint.yml index d5d4dc4..d907fa8 100644 --- a/.github/workflows/pr-check-with-pylint.yml +++ b/.github/workflows/pr-check-with-pylint.yml @@ -34,7 +34,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install python3-all-dev libboost-python-dev libexiv2-dev + sudo apt-get install -y python3-all-dev libboost-python-dev libexiv2-dev pip install --upgrade pip pip install pylint pip install -r requirements.txt From 51fee3356faa7c92049efd788ef05c5a8a4ae54d Mon Sep 17 00:00:00 2001 From: Sergey <57998637+GoldenAnpu@users.noreply.github.com> Date: Thu, 27 Nov 2025 11:01:35 +0100 Subject: [PATCH 08/15] Refactor dependency installation in CI workflow --- .github/workflows/pr-check-with-pylint.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pr-check-with-pylint.yml b/.github/workflows/pr-check-with-pylint.yml index d907fa8..9be6b51 100644 --- a/.github/workflows/pr-check-with-pylint.yml +++ b/.github/workflows/pr-check-with-pylint.yml @@ -26,15 +26,18 @@ jobs: repository: ${{ github.repository }} token: ${{ secrets.PYLINT_TOKEN }} + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-all-dev libboost-python-dev libexiv2-dev + - name: Set up Python uses: actions/setup-python@v6 with: python-version: "3.12" - - name: Install dependencies + - name: Install pylint and requirements run: | - sudo apt-get update - sudo apt-get install -y python3-all-dev libboost-python-dev libexiv2-dev pip install --upgrade pip pip install pylint pip install -r requirements.txt From bd3256e3ab653ab04c7ee5ef7510d692a9d1fe47 Mon Sep 17 00:00:00 2001 From: Sergey <57998637+GoldenAnpu@users.noreply.github.com> Date: Thu, 27 Nov 2025 11:05:16 +0100 Subject: [PATCH 09/15] Add g++ and python3-dev to dependencies installation --- .github/workflows/pr-check-with-pylint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pr-check-with-pylint.yml b/.github/workflows/pr-check-with-pylint.yml index 9be6b51..367030a 100644 --- a/.github/workflows/pr-check-with-pylint.yml +++ b/.github/workflows/pr-check-with-pylint.yml @@ -29,7 +29,7 @@ jobs: - name: Install dependencies run: | sudo apt-get update - sudo apt-get install -y python3-all-dev libboost-python-dev libexiv2-dev + sudo apt-get install -y python3-all-dev libboost-python-dev libexiv2-dev g++ python3-dev - name: Set up Python uses: actions/setup-python@v6 From c17bf33c3eeb1145919247a85b7be270a966d27e Mon Sep 17 00:00:00 2001 From: Sergey <57998637+GoldenAnpu@users.noreply.github.com> Date: Thu, 27 Nov 2025 11:19:06 +0100 Subject: [PATCH 10/15] Update PR workflow to include Python version check Added steps to get the system Python version and create a Boost Python symlink. --- .github/workflows/pr-check-with-pylint.yml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pr-check-with-pylint.yml b/.github/workflows/pr-check-with-pylint.yml index 367030a..f643dfb 100644 --- a/.github/workflows/pr-check-with-pylint.yml +++ b/.github/workflows/pr-check-with-pylint.yml @@ -25,16 +25,28 @@ jobs: with: repository: ${{ github.repository }} token: ${{ secrets.PYLINT_TOKEN }} - - - name: Install dependencies + + - name: Get system python version run: | - sudo apt-get update - sudo apt-get install -y python3-all-dev libboost-python-dev libexiv2-dev g++ python3-dev + PYVER=$(python3 -c "import sys; print(f'{sys.version_info.major}{sys.version_info.minor}')") + echo "Detected Python version: $PYVER" + echo "PYVER=$PYVER" >> $GITHUB_ENV - name: Set up Python uses: actions/setup-python@v6 with: python-version: "3.12" + + - name: Install dependencies + run: | + sudo apt-get update + sudo apt-get install -y python3-all-dev libboost-python-dev libexiv2-dev + + - name: Create Boost Python symlink + run: | + sudo ln -s \ + "/usr/lib/x86_64-linux-gnu/libboost_python312.so" \ + "/usr/lib/x86_64-linux-gnu/libboost_python${PYVER}.so" - name: Install pylint and requirements run: | From bffdfd9d85304cb540c09b1fbc51ed7a15f8780f Mon Sep 17 00:00:00 2001 From: Sergey <57998637+GoldenAnpu@users.noreply.github.com> Date: Thu, 27 Nov 2025 11:21:35 +0100 Subject: [PATCH 11/15] Fix Boost Python symlink creation order --- .github/workflows/pr-check-with-pylint.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr-check-with-pylint.yml b/.github/workflows/pr-check-with-pylint.yml index f643dfb..dd03544 100644 --- a/.github/workflows/pr-check-with-pylint.yml +++ b/.github/workflows/pr-check-with-pylint.yml @@ -45,8 +45,8 @@ jobs: - name: Create Boost Python symlink run: | sudo ln -s \ - "/usr/lib/x86_64-linux-gnu/libboost_python312.so" \ - "/usr/lib/x86_64-linux-gnu/libboost_python${PYVER}.so" + "/usr/lib/x86_64-linux-gnu/libboost_python${PYVER}.so" \ + "/usr/lib/x86_64-linux-gnu/libboost_python312.so" - name: Install pylint and requirements run: | From 60e4223bb54c713d4e51dc072232caea5b5b03a6 Mon Sep 17 00:00:00 2001 From: cxnt Date: Thu, 27 Nov 2025 15:16:34 +0400 Subject: [PATCH 12/15] Update requirements.txt to upgrade psutil and py3exiv2 versions, ensuring compatibility with Python 3.12 and resolving previous version conflicts. --- requirements.txt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/requirements.txt b/requirements.txt index aafd0a8..bfb072f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,13 +1,9 @@ docker==6.0.1 -psutil==5.9.0 urllib3==1.26.15 requests==2.28.1 requests-toolbelt>=1.0.0 hurry.filesize==0.9 scandir==1.10.0 -# grpcio installed from system packages (python3-grpcio) -# grpcio-tools removed due to protobuf version conflict with supervisely[agent] -py3exiv2==0.9.3 packaging==21.2 version-parser==1.0.1 python-slugify==6.1.2 @@ -15,7 +11,17 @@ nvidia-ml-py==12.535.77 httpx>=0.26.0 filelock==3.13.1 +# Upgraded to a version that provides wheels for Python 3.12 +psutil==7.1.3 +py3exiv2==0.12.0 + +# grpcio installed from system packages (python3-grpcio) +# grpcio-tools removed due to protobuf version conflict with supervisely[agent] + supervisely==6.73.474 supervisely[agent]==6.73.474 + + + # for development # git+https://github.com/supervisely/supervisely.git@dev-branch From 0783707cda0909df13297f8ecfcacaeaa781d801 Mon Sep 17 00:00:00 2001 From: cxnt Date: Thu, 27 Nov 2025 15:41:03 +0400 Subject: [PATCH 13/15] Revert psutil version to 5.9.8 in requirements.txt to maintain compatibility with existing dependencies. --- requirements.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index bfb072f..bed4bc2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,9 +10,7 @@ python-slugify==6.1.2 nvidia-ml-py==12.535.77 httpx>=0.26.0 filelock==3.13.1 - -# Upgraded to a version that provides wheels for Python 3.12 -psutil==7.1.3 +psutil==5.9.8 py3exiv2==0.12.0 # grpcio installed from system packages (python3-grpcio) @@ -21,7 +19,5 @@ py3exiv2==0.12.0 supervisely==6.73.474 supervisely[agent]==6.73.474 - - # for development # git+https://github.com/supervisely/supervisely.git@dev-branch From 99cc5601cac3067b0837feb7b931095e190876fa Mon Sep 17 00:00:00 2001 From: cxnt Date: Thu, 27 Nov 2025 15:50:12 +0400 Subject: [PATCH 14/15] Update requirements.txt to include pyexiv2 version 2.15.5 and revert py3exiv2 to version 0.9.3, while maintaining psutil at 5.9.8 for compatibility with existing dependencies. --- requirements.txt | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/requirements.txt b/requirements.txt index bed4bc2..b2873bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,20 +4,22 @@ requests==2.28.1 requests-toolbelt>=1.0.0 hurry.filesize==0.9 scandir==1.10.0 +# grpcio installed from system packages (python3-grpcio) +# grpcio-tools removed due to protobuf version conflict with supervisely[agent] packaging==21.2 version-parser==1.0.1 python-slugify==6.1.2 nvidia-ml-py==12.535.77 httpx>=0.26.0 filelock==3.13.1 -psutil==5.9.8 -py3exiv2==0.12.0 -# grpcio installed from system packages (python3-grpcio) -# grpcio-tools removed due to protobuf version conflict with supervisely[agent] +# py3exiv2==0.9.3 +# psutil==5.9.0 + +pyexiv2==2.15.5 +psutil==5.9.8 supervisely==6.73.474 supervisely[agent]==6.73.474 - # for development # git+https://github.com/supervisely/supervisely.git@dev-branch From 6e05910d4d82449a078a755ebbed14d3a5f2674c Mon Sep 17 00:00:00 2001 From: cxnt Date: Thu, 27 Nov 2025 15:57:49 +0400 Subject: [PATCH 15/15] Update requirements.txt to remove deprecated comments and clarify the installation of grpcio from system packages, while upgrading pyexiv2 and psutil to support Python 3.12. --- requirements.txt | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index b2873bc..c7e0778 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,6 @@ requests==2.28.1 requests-toolbelt>=1.0.0 hurry.filesize==0.9 scandir==1.10.0 -# grpcio installed from system packages (python3-grpcio) -# grpcio-tools removed due to protobuf version conflict with supervisely[agent] packaging==21.2 version-parser==1.0.1 python-slugify==6.1.2 @@ -13,11 +11,10 @@ nvidia-ml-py==12.535.77 httpx>=0.26.0 filelock==3.13.1 -# py3exiv2==0.9.3 -# psutil==5.9.0 - -pyexiv2==2.15.5 -psutil==5.9.8 +# grpcio installed from system packages (python3-grpcio) +# grpcio-tools removed due to protobuf version conflict with supervisely[agent] +pyexiv2==2.15.5 # Upgraded from py3exiv2==0.9.3 to support Python 3.12 +psutil==5.9.8 # Upgraded from psutil==5.9.0 to support Python 3.12 supervisely==6.73.474 supervisely[agent]==6.73.474