From 80e74c11e79bba3df21ad575755aee16f9b701d0 Mon Sep 17 00:00:00 2001 From: sosahi Date: Tue, 13 Jan 2026 07:05:37 -0800 Subject: [PATCH 01/11] Sohail/add 3rd party licenses (#1289) --- README.md | 3 +-- THIRD_PARTY_LICENSES.md | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 THIRD_PARTY_LICENSES.md diff --git a/README.md b/README.md index 4cad09192..4f1288976 100644 --- a/README.md +++ b/README.md @@ -383,8 +383,7 @@ https://pypi.org/project/pdfservices-sdk/ pdfservices-sdk before enabling this option. - **Built With Llama**: - **Description**: The NV-Ingest container comes with the `meta-llama/Llama-3.2-1B` tokenizer pre-downloaded so - that the split task can use it for token-based splitting without making a network request. Please review the - [license agreement](https://huggingface.co/meta-llama/Llama-3.2-1B) for Llama 3.2 materials. + that the split task can use it for token-based splitting without making a network request. The [Llama 3.2 Community License Agreement](https://huggingface.co/meta-llama/Llama-3.2-1B/blob/main/LICENSE.txt) governs your use of these Llama materials. If you're building the container yourself and want to pre-download this model, you'll first need to set `DOWNLOAD_LLAMA_TOKENIZER` to `True`. Because this is a gated model, you'll also need to diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md new file mode 100644 index 000000000..479a8a5ef --- /dev/null +++ b/THIRD_PARTY_LICENSES.md @@ -0,0 +1,8 @@ +# Third Party Licenses + +This project uses the following third-party components: + +## Llama 3.2 + +- **License**: [Llama 3.2 Community License Agreement](https://huggingface.co/meta-llama/Llama-3.2-1B/blob/main/LICENSE.txt) +- **Copyright**: © Meta Platforms, Inc. All Rights Reserved. From 2640917874535de3c5136e2556f9249620041efd Mon Sep 17 00:00:00 2001 From: Julio Perez <37191411+jperez999@users.noreply.github.com> Date: Tue, 13 Jan 2026 15:16:40 -0500 Subject: [PATCH 02/11] Rm llama core (#1291) --- src/pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/pyproject.toml b/src/pyproject.toml index 724458db7..e34cc3889 100644 --- a/src/pyproject.toml +++ b/src/pyproject.toml @@ -61,7 +61,6 @@ dependencies = [ "tqdm>=4.67.1", "uvicorn", "pip", - "llama-index-embeddings-nvidia", "opencv-python", # For some reason conda cant solve our req set with py-opencv so we need to use pip "pymilvus>=2.5.10", "pymilvus[bulk_writer, model]", From 20e2dca1f92c29c7bb2a422099c288a5cdecfdb9 Mon Sep 17 00:00:00 2001 From: Edward Kim <109497216+edknv@users.noreply.github.com> Date: Tue, 13 Jan 2026 12:56:48 -0800 Subject: [PATCH 03/11] Bump ffmpeg to 8.0.1 and add nasm build dependency (#1294) --- docker/scripts/install_ffmpeg.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/scripts/install_ffmpeg.sh b/docker/scripts/install_ffmpeg.sh index 31af22a95..78e593160 100755 --- a/docker/scripts/install_ffmpeg.sh +++ b/docker/scripts/install_ffmpeg.sh @@ -15,7 +15,7 @@ #!/bin/bash set -xeuo pipefail # Exit immediately if a command exits with a non-zero status -FFMPEG_VERSION=7.1.2 +FFMPEG_VERSION=8.0.1 for i in "$@"; do case $i in @@ -45,6 +45,7 @@ apt-get install -y \ libtool \ libmp3lame-dev \ libvpx-dev \ + nasm \ pkg-config \ vainfo \ yasm From 51b9f0d8f3e2e5aba944dbf6a023a66741187fc5 Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Tue, 13 Jan 2026 16:15:41 -0500 Subject: [PATCH 04/11] Prepare for 26.1.1 release --- README.md | 2 +- docker-compose.yaml | 2 +- docs/docs/extraction/quickstart-guide.md | 2 +- .../extraction/quickstart-library-mode.md | 2 +- examples/building_vdb_operator.ipynb | 2 +- helm/Chart.yaml | 2 +- helm/README.md | 48 +++++++------------ helm/README.md.gotmpl | 6 +-- helm/values.yaml | 2 +- 9 files changed, 28 insertions(+), 40 deletions(-) diff --git a/README.md b/README.md index 4f1288976..288bb7be4 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,7 @@ Create a fresh Python environment to install nv-ingest and dependencies. ```shell uv venv --python 3.12 nvingest && \ source nvingest/bin/activate && \ - uv pip install nv-ingest==25.9.0 nv-ingest-api==25.9.0 nv-ingest-client==25.9.0 milvus-lite==2.4.12 + uv pip install nv-ingest==26.1.1 nv-ingest-api==26.1.1 nv-ingest-client==26.1.1 milvus-lite==2.4.12 ``` Set your NVIDIA_API_KEY. If you don't have a key, you can get one on [build.nvidia.com](https://org.ngc.nvidia.com/setup/api-keys). For instructions, refer to [Generate Your NGC Keys](/docs/docs/extraction/ngc-api-key.md). diff --git a/docker-compose.yaml b/docker-compose.yaml index 45b19368e..6381c8a3b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -239,7 +239,7 @@ services: - audio nv-ingest-ms-runtime: - image: nvcr.io/nvidia/nemo-microservices/nv-ingest:latest + image: nvcr.io/nvidia/nemo-microservices/nv-ingest:26.1.1 shm_size: 40gb # Should be at minimum 30% of assigned memory per Ray documentation build: context: ${NV_INGEST_ROOT:-.} diff --git a/docs/docs/extraction/quickstart-guide.md b/docs/docs/extraction/quickstart-guide.md index 5ada080b4..69d0cd4cc 100644 --- a/docs/docs/extraction/quickstart-guide.md +++ b/docs/docs/extraction/quickstart-guide.md @@ -130,7 +130,7 @@ To interact from the host, you'll need a Python environment and install the clie # conda not required but makes it easy to create a fresh Python environment conda create --name nv-ingest-dev python=3.12.11 conda activate nv-ingest-dev -pip install nv-ingest==25.9.0 nv-ingest-api==25.9.0 nv-ingest-client==25.9.0 +pip install nv-ingest==26.1.1 nv-ingest-api==26.1.1 nv-ingest-client==26.1.1 ``` !!! tip diff --git a/docs/docs/extraction/quickstart-library-mode.md b/docs/docs/extraction/quickstart-library-mode.md index b3c0a2ea7..5bd9d5bff 100644 --- a/docs/docs/extraction/quickstart-library-mode.md +++ b/docs/docs/extraction/quickstart-library-mode.md @@ -34,7 +34,7 @@ Use the following procedure to prepare your environment. ``` uv venv --python 3.12 nvingest && \ source nvingest/bin/activate && \ - uv pip install nv-ingest==25.9.0 nv-ingest-api==25.9.0 nv-ingest-client==25.9.0 milvus-lite==2.4.12 + uv pip install nv-ingest==26.1.1 nv-ingest-api==26.1.1 nv-ingest-client==26.1.1 milvus-lite==2.4.12 ``` !!! tip diff --git a/examples/building_vdb_operator.ipynb b/examples/building_vdb_operator.ipynb index ca9983585..20ad1abf8 100644 --- a/examples/building_vdb_operator.ipynb +++ b/examples/building_vdb_operator.ipynb @@ -486,7 +486,7 @@ " self.write_to_index(records)\n", "```\n", "\n", - "This method is called by the NV-Ingest Ingestor class during the ingestion pipeline. For more information on how operators are integrated into NV-Ingest, refer to the [interface implementation](https://github.com/NVIDIA/nv-ingest/blob/release/25.6.2/client/src/nv_ingest_client/client/interface.py#L324).\n", + "This method is called by the NV-Ingest Ingestor class during the ingestion pipeline. For more information on how operators are integrated into NV-Ingest, refer to the [interface implementation](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.1/client/src/nv_ingest_client/client/interface.py#L324).\n", "\n", "The simplicity of this method belies its importance - it ensures that indexes are properly configured before data ingestion begins." ] diff --git a/helm/Chart.yaml b/helm/Chart.yaml index c8bb9fb71..ef5211d13 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: nv-ingest description: NV-Ingest Microservice type: application -version: 25.9.0 +version: 26.1.1 maintainers: - name: NVIDIA Corporation url: https://www.nvidia.com/ diff --git a/helm/README.md b/helm/README.md index b1280130a..ee2ac09cd 100644 --- a/helm/README.md +++ b/helm/README.md @@ -9,20 +9,8 @@ This documentation contains documentation for the NV-Ingest Helm charts. Before you install the Helm charts, be sure you meet the hardware and software prerequisites. Refer to the [supported configurations](https://github.com/NVIDIA/nv-ingest?tab=readme-ov-file#hardware). -> Starting with version 26.1.0, the [NVIDIA NIM Operator](https://docs.nvidia.com/nim-operator/latest/install.html) is required. All NIM services are now deployed by using NIM Operator CRDs (NIMCache and NIMService), not Helm subcharts. -> -> **Upgrading from 25.9.0:** -> 1. Install NIM Operator before upgrading -> 2. Update your values file with the new configuration keys: -> -> | 25.9.0 | 26.x | -> |--------|------| -> | `nim-vlm-image-captioning.deployed=true` | `nimOperator.nemotron_nano_12b_v2_vl.enabled=true` | -> | `paddleocr-nim.deployed=true` | `nimOperator.nemoretriever_ocr_v1.enabled=true` | -> | `riva-nim.deployed=true` | `nimOperator.audio.enabled=true` | -> | `nim-vlm-text-extraction.deployed=true` | `nimOperator.nemotron_parse.enabled=true` | - -The [Nvidia GPU Operator](https://docs.nvidia.com/datacenter/cloud-native/gpu-operator/latest/getting-started.html) must also be installed and configured in your cluster. +The [Nvidia nim-operator](https://docs.nvidia.com/nim-operator/latest/install.html) must also be installed and configured in your cluster to ensure that +the Nvidia NIMs are properly deployed. ## Initial Environment Setup @@ -57,7 +45,7 @@ To install or upgrade the Helm chart, run the following code. helm upgrade \ --install \ nv-ingest \ - https://helm.ngc.nvidia.com/nvidia/nemo-microservices/charts/nv-ingest-25.9.0.tgz \ + https://helm.ngc.nvidia.com/nvidia/nemo-microservices/charts/nv-ingest-26.1.1.tgz \ -n ${NAMESPACE} \ --username '$oauthtoken' \ --password "${NGC_API_KEY}" \ @@ -66,7 +54,7 @@ helm upgrade \ --set ngcApiSecret.create=true \ --set ngcApiSecret.password="${NGC_API_KEY}" \ --set image.repository="nvcr.io/nvidia/nemo-microservices/nv-ingest" \ - --set image.tag="25.9.0" + --set image.tag="26.1.1" ``` Optionally you can create your own versions of the `Secrets` if you do not want to use the creation via the helm chart. @@ -117,7 +105,7 @@ For more information, refer to [NV-Ingest-Client](https://github.com/NVIDIA/nv-i # Just to be cautious we remove any existing installation pip uninstall nv-ingest-client -pip install nv-ingest-client==25.9.0 +pip install nv-ingest-client==26.1.1 ``` #### Rest Endpoint Ingress @@ -359,7 +347,7 @@ You can also use NV-Ingest's Python client API to interact with the service runn | fullnameOverride | string | `""` | | | image.pullPolicy | string | `"IfNotPresent"` | | | image.repository | string | `"nvcr.io/nvidia/nemo-microservices/nv-ingest"` | | -| image.tag | string | `"25.9.0"` | | +| image.tag | string | `"26.1.1"` | | | imagePullSecrets[0].name | string | `"ngc-api"` | | | imagePullSecrets[1].name | string | `"ngc-secret"` | | | ingress.annotations | object | `{}` | | @@ -428,7 +416,7 @@ You can also use NV-Ingest's Python client API to interact with the service runn | nimOperator.audio.resources.limits."nvidia.com/gpu" | int | `1` | | | nimOperator.audio.storage.pvc.create | bool | `true` | | | nimOperator.audio.storage.pvc.size | string | `"25Gi"` | | -| nimOperator.audio.storage.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.audio.storage.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nimOperator.embedqa.authSecret | string | `"ngc-api"` | | | nimOperator.embedqa.enabled | bool | `true` | | | nimOperator.embedqa.env[0].name | string | `"NIM_HTTP_API_PORT"` | | @@ -445,12 +433,12 @@ You can also use NV-Ingest's Python client API to interact with the service runn | nimOperator.embedqa.image.pullPolicy | string | `"IfNotPresent"` | | | nimOperator.embedqa.image.pullSecrets[0] | string | `"ngc-secret"` | | | nimOperator.embedqa.image.repository | string | `"nvcr.io/nim/nvidia/llama-3.2-nv-embedqa-1b-v2"` | | -| nimOperator.embedqa.image.tag | string | `"1.10.0"` | | +| nimOperator.embedqa.image.tag | string | `"1.10.1"` | | | nimOperator.embedqa.replicas | int | `1` | | | nimOperator.embedqa.resources.limits."nvidia.com/gpu" | int | `1` | | | nimOperator.embedqa.storage.pvc.create | bool | `true` | | | nimOperator.embedqa.storage.pvc.size | string | `"50Gi"` | | -| nimOperator.embedqa.storage.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.embedqa.storage.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nimOperator.graphic_elements.authSecret | string | `"ngc-api"` | | | nimOperator.graphic_elements.enabled | bool | `true` | | | nimOperator.graphic_elements.env[0].name | string | `"NIM_HTTP_API_PORT"` | | @@ -476,7 +464,7 @@ You can also use NV-Ingest's Python client API to interact with the service runn | nimOperator.graphic_elements.resources.limits."nvidia.com/gpu" | int | `1` | | | nimOperator.graphic_elements.storage.pvc.create | bool | `true` | | | nimOperator.graphic_elements.storage.pvc.size | string | `"25Gi"` | | -| nimOperator.graphic_elements.storage.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.graphic_elements.storage.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nimOperator.llama_3_2_nv_rerankqa_1b_v2.authSecret | string | `"ngc-api"` | | | nimOperator.llama_3_2_nv_rerankqa_1b_v2.enabled | bool | `false` | | | nimOperator.llama_3_2_nv_rerankqa_1b_v2.env | list | `[]` | | @@ -491,7 +479,7 @@ You can also use NV-Ingest's Python client API to interact with the service runn | nimOperator.llama_3_2_nv_rerankqa_1b_v2.resources.limits."nvidia.com/gpu" | int | `1` | | | nimOperator.llama_3_2_nv_rerankqa_1b_v2.storage.pvc.create | bool | `true` | | | nimOperator.llama_3_2_nv_rerankqa_1b_v2.storage.pvc.size | string | `"50Gi"` | | -| nimOperator.llama_3_2_nv_rerankqa_1b_v2.storage.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.llama_3_2_nv_rerankqa_1b_v2.storage.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nimOperator.nemoretriever_ocr_v1.authSecret | string | `"ngc-api"` | | | nimOperator.nemoretriever_ocr_v1.enabled | bool | `true` | | | nimOperator.nemoretriever_ocr_v1.env[0].name | string | `"OMP_NUM_THREADS"` | | @@ -519,7 +507,7 @@ You can also use NV-Ingest's Python client API to interact with the service runn | nimOperator.nemoretriever_ocr_v1.resources.limits."nvidia.com/gpu" | int | `1` | | | nimOperator.nemoretriever_ocr_v1.storage.pvc.create | bool | `true` | | | nimOperator.nemoretriever_ocr_v1.storage.pvc.size | string | `"25Gi"` | | -| nimOperator.nemoretriever_ocr_v1.storage.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.nemoretriever_ocr_v1.storage.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nimOperator.nemotron_nano_12b_v2_vl.authSecret | string | `"ngc-api"` | | | nimOperator.nemotron_nano_12b_v2_vl.enabled | bool | `false` | | | nimOperator.nemotron_nano_12b_v2_vl.expose.service.grpcPort | int | `8001` | | @@ -533,7 +521,7 @@ You can also use NV-Ingest's Python client API to interact with the service runn | nimOperator.nemotron_nano_12b_v2_vl.resources.limits."nvidia.com/gpu" | int | `1` | | | nimOperator.nemotron_nano_12b_v2_vl.storage.pvc.create | bool | `true` | | | nimOperator.nemotron_nano_12b_v2_vl.storage.pvc.size | string | `"300Gi"` | | -| nimOperator.nemotron_nano_12b_v2_vl.storage.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.nemotron_nano_12b_v2_vl.storage.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nimOperator.nemotron_parse.authSecret | string | `"ngc-api"` | | | nimOperator.nemotron_parse.enabled | bool | `false` | | | nimOperator.nemotron_parse.env[0].name | string | `"NIM_HTTP_API_PORT"` | | @@ -551,11 +539,11 @@ You can also use NV-Ingest's Python client API to interact with the service runn | nimOperator.nemotron_parse.resources.limits."nvidia.com/gpu" | int | `1` | | | nimOperator.nemotron_parse.storage.pvc.create | bool | `true` | | | nimOperator.nemotron_parse.storage.pvc.size | string | `"100Gi"` | | -| nimOperator.nemotron_parse.storage.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.nemotron_parse.storage.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nimOperator.nimCache.pvc.create | bool | `true` | | | nimOperator.nimCache.pvc.size | string | `"25Gi"` | | | nimOperator.nimCache.pvc.storageClass | string | `"default"` | | -| nimOperator.nimCache.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.nimCache.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nimOperator.nimService.namespaces | list | `[]` | | | nimOperator.nimService.resources | object | `{}` | | | nimOperator.page_elements.authSecret | string | `"ngc-api"` | | @@ -603,7 +591,7 @@ You can also use NV-Ingest's Python client API to interact with the service runn | nimOperator.page_elements.resources.limits."nvidia.com/gpu" | int | `1` | | | nimOperator.page_elements.storage.pvc.create | bool | `true` | | | nimOperator.page_elements.storage.pvc.size | string | `"25Gi"` | | -| nimOperator.page_elements.storage.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.page_elements.storage.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nimOperator.table_structure.authSecret | string | `"ngc-api"` | | | nimOperator.table_structure.enabled | bool | `true` | | | nimOperator.table_structure.env[0].name | string | `"NIM_HTTP_API_PORT"` | | @@ -629,7 +617,7 @@ You can also use NV-Ingest's Python client API to interact with the service runn | nimOperator.table_structure.resources.limits."nvidia.com/gpu" | int | `1` | | | nimOperator.table_structure.storage.pvc.create | bool | `true` | | | nimOperator.table_structure.storage.pvc.size | string | `"25Gi"` | | -| nimOperator.table_structure.storage.pvc.volumeAccessMode | string | `"ReadWriteMany"` | | +| nimOperator.table_structure.storage.pvc.volumeAccessMode | string | `"ReadWriteOnce"` | | | nodeSelector | object | `{}` | | | opentelemetry-collector.config.exporters.debug.verbosity | string | `"detailed"` | | | opentelemetry-collector.config.exporters.zipkin.endpoint | string | `"http://nv-ingest-zipkin:9411/api/v2/spans"` | | @@ -682,7 +670,7 @@ You can also use NV-Ingest's Python client API to interact with the service runn | prometheus.alertmanager.enabled | bool | `false` | | | prometheus.enabled | bool | `false` | | | prometheus.server.enabled | bool | `false` | | -| readinessProbe.enabled | bool | `false` | | +| readinessProbe.enabled | bool | `true` | | | readinessProbe.failureThreshold | int | `220` | | | readinessProbe.httpGet.path | string | `"/v1/health/ready"` | | | readinessProbe.httpGet.port | string | `"http"` | | diff --git a/helm/README.md.gotmpl b/helm/README.md.gotmpl index 48e79c6b4..284d4991b 100644 --- a/helm/README.md.gotmpl +++ b/helm/README.md.gotmpl @@ -46,7 +46,7 @@ To install or upgrade the Helm chart, run the following code. helm upgrade \ --install \ nv-ingest \ - https://helm.ngc.nvidia.com/nvidia/nemo-microservices/charts/nv-ingest-25.9.0.tgz \ + https://helm.ngc.nvidia.com/nvidia/nemo-microservices/charts/nv-ingest-26.1.1.tgz \ -n ${NAMESPACE} \ --username '$oauthtoken' \ --password "${NGC_API_KEY}" \ @@ -55,7 +55,7 @@ helm upgrade \ --set ngcApiSecret.create=true \ --set ngcApiSecret.password="${NGC_API_KEY}" \ --set image.repository="nvcr.io/nvidia/nemo-microservices/nv-ingest" \ - --set image.tag="25.9.0" + --set image.tag="26.1.1" ``` Optionally you can create your own versions of the `Secrets` if you do not want to use the creation via the helm chart. @@ -107,7 +107,7 @@ For more information, refer to [NV-Ingest-Client](https://github.com/NVIDIA/nv-i # Just to be cautious we remove any existing installation pip uninstall nv-ingest-client -pip install nv-ingest-client==25.9.0 +pip install nv-ingest-client==26.1.1 ``` #### Rest Endpoint Ingress diff --git a/helm/values.yaml b/helm/values.yaml index f69c83fe6..670d362b7 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -28,7 +28,7 @@ nameOverride: "" image: pullPolicy: IfNotPresent repository: "nvcr.io/nvidia/nemo-microservices/nv-ingest" - tag: "25.9.0" + tag: "26.1.1" ## @section Pod Configuration ## @param podAnnotations [object] Sets additional annotations on the main deployment pods From f8097a676337396d8bf0767663d9ad59d6f2fb0b Mon Sep 17 00:00:00 2001 From: Jeremy Dyer Date: Wed, 14 Jan 2026 13:04:24 -0500 Subject: [PATCH 05/11] =?UTF-8?q?Remove=20huggingface=20access=20token=20a?= =?UTF-8?q?nd=20llama=20tokenizer=20download=20from=20con=E2=80=A6=20(#130?= =?UTF-8?q?4)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker-build-arm.yml | 2 +- .github/workflows/docker-build.yml | 2 +- .github/workflows/docker-nightly-publish.yml | 2 +- .github/workflows/docker-release-publish.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/docker-build-arm.yml b/.github/workflows/docker-build-arm.yml index ebfef4201..063b2d19d 100644 --- a/.github/workflows/docker-build-arm.yml +++ b/.github/workflows/docker-build-arm.yml @@ -33,7 +33,7 @@ jobs: - name: Build Docker image run: | docker buildx create --use - docker buildx build --platform linux/arm64 --load --target runtime --build-arg HF_ACCESS_TOKEN=${{ secrets.HF_ACCESS_TOKEN }} --build-arg DOWNLOAD_LLAMA_TOKENIZER=True --build-arg GIT_COMMIT=${GITHUB_SHA} --build-arg BASE_IMG=ubuntu --build-arg BASE_IMG_TAG=jammy-20250415.1 -t nv-ingest:latest . + docker buildx build --platform linux/arm64 --load --target runtime --build-arg GIT_COMMIT=${GITHUB_SHA} --build-arg BASE_IMG=ubuntu --build-arg BASE_IMG_TAG=jammy-20250415.1 -t nv-ingest:latest . # Only test a random subset (see --random-selection) because running pytest with QEMU emulation can be very slow. - name: Run Pytest inside Docker container diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index d63ac8b03..0c3eda14d 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -30,7 +30,7 @@ jobs: # base image and not the one hosted in nvcr.io - name: Build Docker image run: | - docker build --target runtime --build-arg HF_ACCESS_TOKEN=${{ secrets.HF_ACCESS_TOKEN }} --build-arg DOWNLOAD_LLAMA_TOKENIZER=True --build-arg GIT_COMMIT=${GITHUB_SHA} --build-arg BASE_IMG=ubuntu --build-arg BASE_IMG_TAG=jammy-20250415.1 -t nv-ingest:latest . + docker build --target runtime --build-arg GIT_COMMIT=${GITHUB_SHA} --build-arg BASE_IMG=ubuntu --build-arg BASE_IMG_TAG=jammy-20250415.1 -t nv-ingest:latest . - name: Run Pytest inside Docker container run: | diff --git a/.github/workflows/docker-nightly-publish.yml b/.github/workflows/docker-nightly-publish.yml index 219b004b7..f5060d503 100644 --- a/.github/workflows/docker-nightly-publish.yml +++ b/.github/workflows/docker-nightly-publish.yml @@ -40,4 +40,4 @@ jobs: - name: Build Docker image run: | docker buildx create --use - docker buildx build --platform linux/amd64,linux/arm64 --push --target runtime --build-arg HF_ACCESS_TOKEN=${{ secrets.HF_ACCESS_TOKEN }} --build-arg DOWNLOAD_LLAMA_TOKENIZER=True --build-arg GIT_COMMIT=${GITHUB_SHA} -t ${{ secrets.DOCKER_REGISTRY }}/nv-ingest:${{ env.CURRENT_DATE }} . + docker buildx build --platform linux/amd64,linux/arm64 --push --target runtime --build-arg GIT_COMMIT=${GITHUB_SHA} -t ${{ secrets.DOCKER_REGISTRY }}/nv-ingest:${{ env.CURRENT_DATE }} . diff --git a/.github/workflows/docker-release-publish.yml b/.github/workflows/docker-release-publish.yml index 070559e0e..82c8951bc 100644 --- a/.github/workflows/docker-release-publish.yml +++ b/.github/workflows/docker-release-publish.yml @@ -36,4 +36,4 @@ jobs: - name: Build Docker image run: | docker buildx create --use - docker buildx build --platform linux/amd64,linux/arm64 --push --target runtime --build-arg HF_ACCESS_TOKEN=${{ secrets.HF_ACCESS_TOKEN }} --build-arg DOWNLOAD_LLAMA_TOKENIZER=True --build-arg GIT_COMMIT=${GITHUB_SHA} -t ${{ secrets.DOCKER_REGISTRY }}/nv-ingest:${{ env.SHORT_BRANCH_NAME }} . + docker buildx build --platform linux/amd64,linux/arm64 --push --target runtime --build-arg GIT_COMMIT=${GITHUB_SHA} -t ${{ secrets.DOCKER_REGISTRY }}/nv-ingest:${{ env.SHORT_BRANCH_NAME }} . From a1f698911f010a78d4c19ad970d2c868a1432895 Mon Sep 17 00:00:00 2001 From: Julio Perez <37191411+jperez999@users.noreply.github.com> Date: Wed, 14 Jan 2026 13:06:46 -0500 Subject: [PATCH 06/11] remove conda llama package ref (#1302) --- conda/environments/nv_ingest_environment.base.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/conda/environments/nv_ingest_environment.base.yml b/conda/environments/nv_ingest_environment.base.yml index e51bdec80..70c74aea2 100644 --- a/conda/environments/nv_ingest_environment.base.yml +++ b/conda/environments/nv_ingest_environment.base.yml @@ -45,7 +45,6 @@ dependencies: - universal_pathlib - pip - pip: - - llama-index-embeddings-nvidia - milvus-lite==2.4.12 - opencv-python # For some reason conda cant solve our req set with py-opencv so we need to use pip - prometheus-client From a2fb0b65dd234ac6f816213af61469c72e1d4bdb Mon Sep 17 00:00:00 2001 From: Kurt Heiss Date: Wed, 14 Jan 2026 10:07:21 -0800 Subject: [PATCH 07/11] Kheiss/helm by default (#1303) --- docs/docs/extraction/quickstart-guide.md | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/docs/docs/extraction/quickstart-guide.md b/docs/docs/extraction/quickstart-guide.md index 69d0cd4cc..fabd054eb 100644 --- a/docs/docs/extraction/quickstart-guide.md +++ b/docs/docs/extraction/quickstart-guide.md @@ -43,23 +43,12 @@ If you prefer, you can run on Kubernetes by using [our Helm chart](https://githu NGC_API_KEY= NIM_NGC_API_KEY= ``` - - -5. (Optional) For faster OCR performance, you can use the [nemoretriever-ocr-v1](https://build.nvidia.com/nvidia/nemoretriever-ocr-v1) container. [Configure Helm](https://github.com/nvidia/nv-ingest/tree/main/helm) to deploy nemoretriever-ocr-v1 and then set these values in your .env file: - - ``` - OCR_IMAGE=nvcr.io/nvidia/nemo-microservices/nemoretriever-ocr-v1 - OCR_TAG=latest - OCR_MODEL_NAME=scene_text_ensemble - ``` - - Alternatively, you can modify the OCR service directly in your docker-compose.yaml file with these image tags. -6. Make sure NVIDIA is set as your default container runtime before running the docker compose command with the command: +5. Make sure NVIDIA is set as your default container runtime before running the docker compose command with the command: `sudo nvidia-ctk runtime configure --runtime=docker --set-as-default` -7. Start core services. This example uses the retrieval profile. For more information about other profiles, see [Profile Information](#profile-information). +6. Start core services. This example uses the retrieval profile. For more information about other profiles, see [Profile Information](#profile-information). `docker compose --profile retrieval up` @@ -82,7 +71,7 @@ If you prefer, you can run on Kubernetes by using [our Helm chart](https://githu --profile retrieval up ``` -8. When core services have fully started, `nvidia-smi` should show processes like the following: +7. When core services have fully started, `nvidia-smi` should show processes like the following: ``` # If it's taking > 1m for `nvidia-smi` to return, the bus will likely be busy setting up the models. @@ -100,7 +89,7 @@ If you prefer, you can run on Kubernetes by using [our Helm chart](https://githu +---------------------------------------------------------------------------------------+ ``` -9. Run the command `docker ps`. You should see output similar to the following. Confirm that the status of the containers is `Up`. +8. Run the command `docker ps`. You should see output similar to the following. Confirm that the status of the containers is `Up`. ``` CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES From 509e77b1979274fb0d061ef3b6e0c54d69087dc2 Mon Sep 17 00:00:00 2001 From: nkmcalli Date: Tue, 13 Jan 2026 14:50:19 -0800 Subject: [PATCH 08/11] Update version number (26.1.1) (#1298) Co-authored-by: Kurt Heiss --- docs/docs/extraction/content-metadata.md | 2 +- docs/docs/extraction/releasenotes-nv-ingest.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/docs/extraction/content-metadata.md b/docs/docs/extraction/content-metadata.md index b1634f4d9..617181d91 100644 --- a/docs/docs/extraction/content-metadata.md +++ b/docs/docs/extraction/content-metadata.md @@ -43,7 +43,7 @@ These fields apply to all content types including text, images, and tables. | Subtype | The type of the content for structured data types, such as table or chart. | — | | Content | Content extracted from the source. | Extracted | | Description | A text description of the content object. | Generated | -| Page \# | The page \# of the content in the source. Prior to 26.1.0, this field was 0-indexed. Beginning with 26.1.0, this field is 1-indexed. | Extracted | +| Page \# | The page \# of the content in the source. Prior to 26.1.1, this field was 0-indexed. Beginning with 26.1.1, this field is 1-indexed. | Extracted | | Hierarchy | The location or order of the content within the source. | Extracted | diff --git a/docs/docs/extraction/releasenotes-nv-ingest.md b/docs/docs/extraction/releasenotes-nv-ingest.md index 3d281c98b..ca4ce3906 100644 --- a/docs/docs/extraction/releasenotes-nv-ingest.md +++ b/docs/docs/extraction/releasenotes-nv-ingest.md @@ -8,11 +8,11 @@ This documentation contains the release notes for [NeMo Retriever extraction](ov -## Release 26.01 (26.1.0) +## Release 26.01 (26.1.1) The NeMo Retriever extraction 26.01 release adds new hardware and software support, and other improvements. -To upgrade the Helm Charts for this version, refer to [NV-Ingest Helm Charts](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.0/helm/README.md). +To upgrade the Helm Charts for this version, refer to [NV-Ingest Helm Charts](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.1/helm/README.md). ### Highlights @@ -49,8 +49,8 @@ This release contains the following key changes: The following are the known issues that are fixed in this version: -- A10G support is restored. To use A10G hardware, use release 26.1.0 or later. For details, refer to [Support Matrix](support-matrix.md). -- L40S support is restored. To use L40S hardware, use release 26.1.0 or later. For details, refer to [Support Matrix](support-matrix.md). +- A10G support is restored. To use A10G hardware, use release 26.1.1 or later. For details, refer to [Support Matrix](support-matrix.md). +- L40S support is restored. To use L40S hardware, use release 26.1.1 or later. For details, refer to [Support Matrix](support-matrix.md). - The page number field in the content metadata now starts at 1 instead of 0 so each page number is no longer off by one from what you would expect. For details, refer to [Content Metadata](content-metadata.md). - Support for audio files greater than approximately 400MB is restored. From bf5071bf069d643f9815aa744aa866083422ee34 Mon Sep 17 00:00:00 2001 From: nkmcalli Date: Tue, 13 Jan 2026 17:43:51 -0800 Subject: [PATCH 09/11] docs: update requirements file for doc build (26.1.1) (#1300) --- docs/requirements.txt | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index ee2324436..adad4c0a7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,19 +1,21 @@ -mkdocs-material -mkdocs-macros-plugin -mkdocs-minify-plugin -mkdocstrings[python] +autodoc_pydantic +make +mike +mkdocs mkdocs-gen-files -pymdown-extensions -mkdocs-jupyter mkdocs-include-dir-to-nav +mkdocs-jupyter mkdocs-literate-nav -mkdocs-site-urls +mkdocs-macros-plugin +mkdocs-material +mkdocs-minify-plugin mkdocs-redirects +mkdocs-site-urls +mkdocstrings[python] myst-parser nvidia-sphinx-theme +pymdown-extensions sphinx sphinx-markdown-builder sphinx-rtd-theme swagger-plugin-for-sphinx -mike -autodoc_pydantic From c3db0bfcb484cb3770cd8046746553b931852a3b Mon Sep 17 00:00:00 2001 From: sosahi Date: Fri, 16 Jan 2026 10:49:22 -0800 Subject: [PATCH 10/11] Update version references from 26.1.0 to 26.1.1 (#1310) --- CHANGELOG.md | 2 +- docs/docs/extraction/helm.md | 2 +- docs/docs/extraction/releasenotes-nv-ingest.md | 4 ++-- docs/docs/extraction/user-defined-functions.md | 2 +- src/nv_ingest/api/main.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7256980b3..e2d9b35eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# NVIDIA Ingest 26.1.0 +# NVIDIA Ingest 26.1.1 ## New Features diff --git a/docs/docs/extraction/helm.md b/docs/docs/extraction/helm.md index 5172885c1..522a9417c 100644 --- a/docs/docs/extraction/helm.md +++ b/docs/docs/extraction/helm.md @@ -3,4 +3,4 @@ To deploy [NeMo Retriever extraction](overview.md) by using Helm, -refer to [NV-Ingest Helm Charts](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.0/helm/README.md). +refer to [NV-Ingest Helm Charts](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.1/helm/README.md). diff --git a/docs/docs/extraction/releasenotes-nv-ingest.md b/docs/docs/extraction/releasenotes-nv-ingest.md index ca4ce3906..baf08c42f 100644 --- a/docs/docs/extraction/releasenotes-nv-ingest.md +++ b/docs/docs/extraction/releasenotes-nv-ingest.md @@ -20,7 +20,7 @@ To upgrade the Helm Charts for this version, refer to [NV-Ingest Helm Charts](ht This release contains the following key changes: - Added functional support for [H200 NVL](https://www.nvidia.com/en-us/data-center/h200/). For details, refer to [Support Matrix](support-matrix.md). -- All Helm deployments for Kubernetes now use [NVIDIA NIM Operator](https://docs.nvidia.com/nim-operator/latest/index.html). For details, refer to [NV-Ingest Helm Charts](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.0/helm/README.md). +- All Helm deployments for Kubernetes now use [NVIDIA NIM Operator](https://docs.nvidia.com/nim-operator/latest/index.html). For details, refer to [NV-Ingest Helm Charts](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.1/helm/README.md). - Updated RIVA NIM to version 1.4.0. For details, refer to [Extract Speech](audio.md). - Updated VLM NIM to [nemotron-nano-12b-v2-vl](https://build.nvidia.com/nvidia/nemotron-nano-12b-v2-vl/modelcard). For details, refer to [Extract Captions from Images](nv-ingest-python-api.md#extract-captions-from-images). - Added VLM caption prompt customization parameters, including reasoning control. For details, refer to [Caption Images and Control Reasoning](nv-ingest-python-api.md#caption-images-and-control-reasoning). @@ -33,7 +33,7 @@ This release contains the following key changes: - Large PDFs are now automatically split into chunks and processed in parallel, delivering faster ingestion for long documents. For details, refer to [PDF Pre-Splitting](v2-api-guide.md). - Issues maintaining extraction quality while processing very large files are now resolved with the V2 API. For details, refer to [V2 API Guide](v2-api-guide.md). - Updated the embedding task to support embedding on custom content fields like the results of summarization functions. For details, refer to [Use the Python API](nv-ingest-python-api.md). -- User-defined function summarization is now using `nemotron-mini-4b-instruct` which provides significant speed improvements. For details, refer to [User-defined Functions](user-defined-functions.md) and [NV-Ingest UDF Examples](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.0/examples/udfs/README.md). +- User-defined function summarization is now using `nemotron-mini-4b-instruct` which provides significant speed improvements. For details, refer to [User-defined Functions](user-defined-functions.md) and [NV-Ingest UDF Examples](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.1/examples/udfs/README.md). - In the `Ingestor.extract` method, the defaults for `extract_text` and `extract_images` are now set to `true` for consistency with `extract_tables` and `extract_charts`. For details, refer to [Use the Python API](nv-ingest-python-api.md). - The `table-structure` profile is no longer available. The table-structure profile is now part of the default profile. For details, refer to [Profile Information](quickstart-guide.md#profile-information). - New documentation [Why Throughput Is Dataset-Dependent](throughput-is-dataset-dependent.md). diff --git a/docs/docs/extraction/user-defined-functions.md b/docs/docs/extraction/user-defined-functions.md index 70402f010..39c4f46db 100644 --- a/docs/docs/extraction/user-defined-functions.md +++ b/docs/docs/extraction/user-defined-functions.md @@ -941,6 +941,6 @@ def debug_udf(control_message: IngestControlMessage) -> IngestControlMessage: ## Related Topics -- [NV-Ingest UDF Examples](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.0/examples/udfs/README.md) +- [NV-Ingest UDF Examples](https://github.com/NVIDIA/nv-ingest/blob/release/26.1.1/examples/udfs/README.md) - [User-Defined Stages for NeMo Retriever Extraction](user-defined-stages.md) - [NimClient Usage](nimclient.md) diff --git a/src/nv_ingest/api/main.py b/src/nv_ingest/api/main.py index eaf00dc7e..1995d7b64 100644 --- a/src/nv_ingest/api/main.py +++ b/src/nv_ingest/api/main.py @@ -23,7 +23,7 @@ app = FastAPI( title="NV-Ingest Microservice", description="Service for ingesting heterogenous datatypes", - version="26.1.0", + version="26.1.1", contact={ "name": "NVIDIA Corporation", "url": "https://nvidia.com", From 8b003b618c9b28a32d4bf92bde8c0084727331b3 Mon Sep 17 00:00:00 2001 From: sosahi Date: Fri, 27 Feb 2026 11:56:06 -0800 Subject: [PATCH 11/11] Fix GitHub Pages docs workflow and Dockerfile docs stage - build-docs.yml: add Pages source comment, verify generated site step, explicit github-pages artifact, always cleanup container - docs-deploy.yml: remove push trigger (manual only) to avoid duplicate deploys; add verify step and artifact name - Dockerfile: remove mamba clean -afy in docs stage (cache mount causes Device or resource busy) - Update ACTIONS_README, INDEX, WORKFLOWS_REFERENCE for build-docs as primary workflow and add troubleshooting Made-with: Cursor --- .github/ACTIONS_README.md | 223 ++++++++++ .github/INDEX.md | 236 +++++++++++ .github/WORKFLOWS_REFERENCE.md | 659 ++++++++++++++++++++++++++++++ .github/workflows/build-docs.yml | 18 +- .github/workflows/docs-deploy.yml | 15 +- Dockerfile | 11 + 6 files changed, 1157 insertions(+), 5 deletions(-) create mode 100644 .github/ACTIONS_README.md create mode 100644 .github/INDEX.md create mode 100644 .github/WORKFLOWS_REFERENCE.md diff --git a/.github/ACTIONS_README.md b/.github/ACTIONS_README.md new file mode 100644 index 000000000..eea4618c4 --- /dev/null +++ b/.github/ACTIONS_README.md @@ -0,0 +1,223 @@ +# GitHub Actions Structure + +This directory contains all GitHub Actions workflows, reusable components, and documentation for the nv-ingest CI/CD pipeline. + +## 📁 Directory Structure + +``` +.github/ +├── workflows/ # Workflows (including reusable workflows) +├── actions/ # Composite actions (3 actions) +├── ISSUE_TEMPLATE/ # Issue templates +├── CODEOWNERS # Code ownership +├── PULL_REQUEST_TEMPLATE.md +├── copy-pr-bot.yaml +│ +└── Documentation: + ├── README.md (this file) # Overview and quick reference + ├── WORKFLOWS_REFERENCE.md # Complete technical reference + ├── WORKFLOWS_QUICKSTART.md # Quick start guide + └── ARCHITECTURE.md # System architecture +``` + +## 🚀 Quick Start + +### For Developers +Read: [`WORKFLOWS_QUICKSTART.md`](./WORKFLOWS_QUICKSTART.md) + +### For Complete Reference +Read: [`WORKFLOWS_REFERENCE.md`](./WORKFLOWS_REFERENCE.md) + +### For Architecture Details +Read: [`ARCHITECTURE.md`](./ARCHITECTURE.md) + +## 🎯 Workflow Overview + +### Continuous Integration + +| Workflow | File | Trigger | Purpose | +|----------|------|---------|---------| +| PR Validation | `ci-pull-request.yml` | Pull requests | Pre-commit, build, test | +| Main CI | `ci-main.yml` | Push to main | Full validation + multi-platform | + +### Nightly & Scheduled + +| Workflow | File | Schedule | Purpose | +|----------|------|----------|---------| +| Nightly Builds | `scheduled-nightly.yml` | Daily 23:30 UTC | Docker + Conda + PyPI | + +### Release Management + +| Workflow | File | Trigger | Purpose | +|----------|------|---------|---------| +| Docker Release | `release-docker.yml` | release/* branch OR manual | Publish Docker images | +| Conda Release | `release-conda.yml` | release/* branch OR manual | Publish Conda packages | +| PyPI Release | `release-pypi.yml` | release/* branch OR manual | Publish Python wheels | + +### Documentation + +| Workflow | File | Trigger | Purpose | +|----------|------|---------|---------| +| Build & Deploy Docs | `build-docs.yml` | Push to main OR manual | Build docs in Docker, deploy to GitHub Pages (https://nvidia.github.io/nv-ingest/) | +| Docs Deploy (manual) | `docs-deploy.yml` | Manual only | Same as above; use for one-off deploys without pushing to main | + +## 🔧 Reusable Components + +### Workflows (in `workflows/`, prefixed with `reusable-`) + +- `reusable-docker-build.yml` - Flexible Docker image building +- `reusable-docker-test.yml` - Container-based testing +- `reusable-conda-build.yml` - Conda package building +- `reusable-conda-publish.yml` - Conda package publishing +- `reusable-pypi-build.yml` - Python wheel building +- `reusable-pypi-publish.yml` - PyPI publishing +- `reusable-integration-test.yml` - Library mode testing + +### Actions (in `actions/`) + +- `setup-docker-buildx/` - Docker Buildx + QEMU setup +- `docker-login-ngc/` - NGC registry authentication +- `determine-version/` - Smart version determination + +## 📊 Workflow Architecture + +``` +┌─────────────────────┐ +│ Main Workflows │ (Triggered by events) +│ - ci-pull-request │ +│ - ci-main │ +│ - scheduled-* │ +│ - release-* │ +└──────┬──────────────┘ + │ calls + ▼ +┌─────────────────────┐ +│ Reusable Workflows │ (Business logic) +│ - docker-build │ +│ - conda-publish │ +│ - pypi-build │ +└──────┬──────────────┘ + │ uses + ▼ +┌─────────────────────┐ +│ Composite Actions │ (Common operations) +│ - setup-buildx │ +│ - docker-login │ +│ - determine-ver │ +└─────────────────────┘ +``` + +## 🎯 Key Features + +### Reusable Workflows +- Docker build logic defined once, used everywhere +- Consistent patterns across all workflows +- Type-safe interfaces with validation + +### Flexible Configuration +- Reusable workflows accept inputs +- Composite actions are parameterized +- Easy to customize per use case + +### Clear Separation +- Main workflows = triggers + orchestration +- Reusable workflows = business logic +- Composite actions = common operations + +### ✅ Type Safety +- Inputs/outputs explicitly defined +- Required vs optional parameters +- Validation built-in + +### ✅ Better Testing +- Reusable components can be tested independently +- workflow_dispatch for manual testing +- Clear job dependencies + +## 🔐 Required Secrets + +### Docker/NGC +- `DOCKER_PASSWORD` - NGC API token +- `DOCKER_REGISTRY` - Registry URL (e.g., nvcr.io) +- `HF_ACCESS_TOKEN` - Hugging Face token + +### Conda +- `NVIDIA_CONDA_TOKEN` - Anaconda.org token + +### PyPI +- `ARTIFACTORY_URL` - PyPI repository URL +- `ARTIFACTORY_USERNAME` - Username +- `ARTIFACTORY_PASSWORD` - Password + +### Integration Tests +- `NGC_API_KEY` / `NVIDIA_API_KEY` +- `AUDIO_FUNCTION_ID` +- `EMBEDDING_NIM_MODEL_NAME` +- `NEMOTRON_PARSE_MODEL_NAME` +- `PADDLE_HTTP_ENDPOINT` +- `VLM_CAPTION_ENDPOINT` +- `VLM_CAPTION_MODEL_NAME` +- `YOLOX_*_HTTP_ENDPOINT` (multiple) + +## 📝 Common Tasks + +### Run PR checks locally +```bash +pre-commit run --all-files +docker build --target runtime -t nv-ingest:test . +docker run nv-ingest:test pytest -m "not integration" +``` + +### Trigger nightly build manually +``` +Actions → "Nightly Builds & Publishing" → Run workflow +``` + +### Create a release +```bash +# Automatic - All three artifact types (recommended) +git checkout -b release/25.4.0 +git push origin release/25.4.0 +# → Triggers Docker, Conda, AND PyPI releases automatically + +# Manual (for custom options) +Actions → Release - Docker/Conda/PyPI → Run workflow +``` + +### Debug workflows +``` +Actions → Select workflow → View logs → Expand steps +``` + +## 🐛 Troubleshooting + +| Issue | Solution | +|-------|----------| +| Workflow not found | Check path: `.github/workflows/` (reusable workflows are `reusable-*.yml`) | +| Secret not available | Verify in Settings → Secrets → Actions | +| Build timeout | Use `linux-large-disk` runner | +| Integration tests fail | Check NIM endpoints and credentials | + +## 📚 Documentation + +- **Quick Start**: [`WORKFLOWS_QUICKSTART.md`](./WORKFLOWS_QUICKSTART.md) +- **Complete Reference**: [`WORKFLOWS_REFERENCE.md`](./WORKFLOWS_REFERENCE.md) +- **Architecture**: [`ARCHITECTURE.md`](./ARCHITECTURE.md) + +## 🆘 Getting Help + +1. Check workflow logs in Actions tab +2. Review documentation in this folder +3. Search for similar issues +4. Contact DevOps team +5. Open an issue with details + +## 📞 Maintainers + +See [`CODEOWNERS`](./CODEOWNERS) for ownership information. + +--- + +**Architecture**: Reusable workflows + Composite actions +**Documentation**: 4 comprehensive guides +**Total Components**: Workflows + reusable workflows + composite actions diff --git a/.github/INDEX.md b/.github/INDEX.md new file mode 100644 index 000000000..578b94b0d --- /dev/null +++ b/.github/INDEX.md @@ -0,0 +1,236 @@ +# GitHub Actions Documentation Index + +**Complete CI/CD pipeline documentation for nv-ingest** + +--- + +## 📦 What's Included + +This documentation package contains **18 components** organized into a clean, maintainable structure: + +### 🎯 Main Workflows (7 files) +Located in: `.github/workflows/` + +1. **`ci-pull-request.yml`** - PR validation with pre-commit, Docker build/test, library mode +2. **`ci-main.yml`** - Main branch CI with multi-platform testing +3. **`scheduled-nightly.yml`** - Unified nightly builds (Docker + Conda + PyPI) +4. **`release-docker.yml`** - Docker release publishing +5. **`release-conda.yml`** - Conda package releases +6. **`release-pypi.yml`** - PyPI wheel releases +7. **`build-docs.yml`** - Build and deploy documentation to GitHub Pages (runs on push to main) +8. **`docs-deploy.yml`** - Documentation deploy (manual trigger only; primary workflow is build-docs.yml) + +### ♻️ Reusable Workflows (7 files) +Located in: `.github/workflows/` (prefixed with `reusable-`) + +1. **`reusable-docker-build.yml`** - Flexible Docker image building +2. **`reusable-docker-test.yml`** - Container-based testing +3. **`reusable-conda-build.yml`** - Conda package building +4. **`reusable-conda-publish.yml`** - Conda publishing to channels +5. **`reusable-pypi-build.yml`** - Python wheel building +6. **`reusable-pypi-publish.yml`** - PyPI publishing +7. **`reusable-integration-test.yml`** - Library mode integration tests + +### 🔧 Composite Actions (3 directories) +Located in: `.github/actions/` + +1. **`setup-docker-buildx/`** - Docker Buildx + QEMU setup +2. **`docker-login-ngc/`** - NGC registry authentication +3. **`determine-version/`** - Smart version determination + +### 📚 Documentation (4 files) +Located in: `.github/` + +1. **`README.md`** - Main entry point with directory overview +2. **`WORKFLOWS_QUICKSTART.md`** - Quick start guide for developers +3. **`WORKFLOWS_REFERENCE.md`** - Complete technical reference +4. **`ARCHITECTURE.md`** - Visual architecture diagrams +5. **`INDEX.md`** - This file + +--- + +## 🎓 How to Use This Documentation + +### For Different Audiences: + +#### 👨‍💻 **Developers** (Contributing code) +1. Read: **`WORKFLOWS_QUICKSTART.md`** +2. Reference: **`README.md`** for quick lookups + +#### 🔧 **Maintainers** (Managing releases) +1. Read: **`WORKFLOWS_QUICKSTART.md`** (Common tasks) +2. Reference: **`WORKFLOWS_REFERENCE.md`** (Complete details) + +#### 🏗️ **DevOps/SRE** (System maintenance) +1. Read: **`ARCHITECTURE.md`** (System design) +2. Read: **`WORKFLOWS_REFERENCE.md`** (Technical specs) +3. Reference: **`README.md`** (Overview) + +#### 📊 **Management** (Understanding scope) +1. Read: **`README.md`** (Executive summary) +2. Review: **`ARCHITECTURE.md`** (Visual diagrams) + +--- + +## 📊 System Overview + +### Workflow Components + +- **7** main trigger workflows +- **7** reusable workflow components +- **3** composite actions +- **4** documentation files + +### Key Capabilities + +- ✅ Automated PR validation +- ✅ Multi-platform Docker builds (amd64, arm64) +- ✅ Daily nightly builds (Docker, Conda, PyPI) +- ✅ Unified release process (one branch = all artifacts) +- ✅ Integration testing with conda environment +- ✅ Automatic documentation deployment + +--- + +## 🚀 Getting Started + +### Step 1: Understand the Structure +```bash +# Read the main README +cat .github/README.md + +# Review the architecture +cat .github/ARCHITECTURE.md +``` + +### Step 2: Choose Your Path + +**Quick Start (Developers):** +```bash +cat .github/WORKFLOWS_QUICKSTART.md +# Start contributing immediately +``` + +**Complete Reference (Advanced):** +```bash +cat .github/WORKFLOWS_REFERENCE.md +# Deep dive into every workflow +``` + +### Step 3: Start Using + +**For PRs:** +1. Create a pull request +2. Watch automated checks run +3. Address any failures + +**For Releases:** +```bash +git checkout -b release/25.4.0 +git push origin release/25.4.0 +# Automatically releases Docker, Conda, and PyPI +``` + +--- + +## 🔐 Security & Access + +### Required Secrets + +All secrets must be configured in repository settings: + +**Docker/NGC:** +- `DOCKER_PASSWORD` +- `DOCKER_REGISTRY` +- `HF_ACCESS_TOKEN` + +**Conda:** +- `NVIDIA_CONDA_TOKEN` + +**PyPI:** +- `ARTIFACTORY_URL` +- `ARTIFACTORY_USERNAME` +- `ARTIFACTORY_PASSWORD` + +**Integration Tests:** +- `NGC_API_KEY` / `NVIDIA_API_KEY` +- Multiple NIM endpoint secrets + +### Access Control +- External contributors require `ok-to-test` label +- `pull_request_target` used safely with access checks +- Secrets passed explicitly (no implicit access) +- Minimal permissions (least privilege) + +--- + +## 🎯 Quick Reference + +### Common Tasks + +| Task | Location | Action | +|------|----------|--------| +| View workflows | `.github/workflows/` | Browse main triggers | +| Understand logic | `.github/workflows/` | See reusable workflows (`reusable-*.yml`) | +| Check common operations | `.github/actions/` | Review composite actions | +| Quick help | `.github/WORKFLOWS_QUICKSTART.md` | Read guide | +| Complete reference | `.github/WORKFLOWS_REFERENCE.md` | Deep dive | + +### Workflow Triggers + +| Workflow | Automatic | Manual | Purpose | +|----------|-----------|--------|---------| +| PR Validation | PR events | ✓ | Validate changes | +| Main CI | Push to main | ✓ | Full validation | +| Nightly | Daily 23:30 UTC | ✓ | Build & publish | +| Docker Release | release/* branch | ✓ | Release Docker | +| Conda Release | release/* branch | ✓ | Release Conda | +| PyPI Release | release/* branch | ✓ | Release PyPI | +| Docs | Push to main | ✓ | Deploy docs | + +--- + +## 🐛 Troubleshooting + +### Common Issues & Solutions + +| Issue | Solution Document | Section | +|-------|------------------|---------| +| Workflow not triggering | `WORKFLOWS_QUICKSTART.md` | Troubleshooting | +| Reusable workflow not found | `WORKFLOWS_REFERENCE.md` | Reusable Workflows | +| Secret not available | `README.md` | Required Secrets | +| Build failing | `WORKFLOWS_REFERENCE.md` | Docker Build | +| Integration tests failing | `WORKFLOWS_QUICKSTART.md` | Troubleshooting | + +### Getting Help + +1. **Check logs**: Actions tab → Workflow run → Job → Step +2. **Review docs**: Search in `.github/` documentation +3. **Test locally**: Run pre-commit and Docker builds +4. **Ask team**: Contact DevOps or maintainers +5. **Open issue**: Include logs and context + +--- + +## 📚 File Index + +### Documentation Files +``` +.github/ +├── INDEX.md ← You are here +├── README.md ← Start here (overview) +├── ARCHITECTURE.md ← How it works (diagrams) +├── WORKFLOWS_QUICKSTART.md ← Quick reference (developers) +└── WORKFLOWS_REFERENCE.md ← Complete reference (advanced) +``` + +### Workflow Files +``` +.github/ +├── workflows/ ← Workflows (including `reusable-*.yml`) +└── actions/ ← Composite actions (3) +``` + +--- + +**For questions or issues, start with**: `.github/README.md` diff --git a/.github/WORKFLOWS_REFERENCE.md b/.github/WORKFLOWS_REFERENCE.md new file mode 100644 index 000000000..a36639b10 --- /dev/null +++ b/.github/WORKFLOWS_REFERENCE.md @@ -0,0 +1,659 @@ +# GitHub Actions Workflows Reference + +Complete reference documentation for nv-ingest GitHub Actions workflows. + +## Table of Contents +1. [Workflow Overview](#workflow-overview) +2. [Continuous Integration](#continuous-integration) +3. [Nightly Builds](#nightly-builds) +4. [Release Workflows](#release-workflows) +5. [Reusable Workflows](#reusable-workflows) +6. [Composite Actions](#composite-actions) + +--- + +## Workflow Overview + +### Trigger Summary + +| Workflow | PR | Main | Schedule | Manual | Branch Create | +|----------|-----|------|----------|--------|---------------| +| `ci-pull-request.yml` | ✓ | | | | | +| `ci-main.yml` | | ✓ | | | | +| `scheduled-nightly.yml` | | ✓ | 23:30 UTC | ✓ | | +| `release-docker.yml` | | | | ✓ | release/* | +| `release-conda.yml` | | | | ✓ | release/* | +| `release-pypi.yml` | | | | ✓ | release/* | +| `build-docs.yml` | | ✓ | | ✓ | | +| `docs-deploy.yml` | | | | ✓ | (manual only) | + +--- + +## Continuous Integration + +### `ci-pull-request.yml` + +**Purpose**: Validates pull requests before merge + +**Triggers**: +- Pull request (opened, synchronize, reopened) +- Pull request target (for external contributors) + +**Jobs**: + +#### 1. `pre-commit` +- **Runs**: Pre-commit hooks (linting, formatting) +- **Runner**: `ubuntu-latest` +- **Fast fail**: Yes (runs first) + +#### 2. `docker-build-test` +- **Runs**: Docker build for x86_64 +- **Platform**: `linux/amd64` +- **Image tag**: `nv-ingest:pr-{number}` +- **Push**: No (local only) +- **Base**: Ubuntu Jammy (public) + +#### 3. `docker-test` +- **Runs**: Full pytest suite +- **Coverage**: Enabled +- **Markers**: Excludes integration tests +- **Artifacts**: Coverage reports + +#### 4. `library-mode-build` + `library-mode-test` +- **Runs**: Integration tests (conda-based) +- **Requires**: Approval for external contributors +- **Access Control**: + - Auto-runs for MEMBER/COLLABORATOR/OWNER + - Requires `ok-to-test` label for others +- **Timeout**: 60 minutes +- **Dependencies**: Multiple NVIDIA NIMs + +**Status Checks**: +- Required: `pre-commit`, `docker-test` +- Optional: `library-mode-test` + +--- + +### `ci-main.yml` + +**Purpose**: Validates main branch commits and tests multi-platform builds + +**Triggers**: +- Push to `main` branch + +**Jobs**: + +#### 1. `pre-commit` +- Same as PR workflow + +#### 2. `docker-build` + `docker-test` +- **Platform**: `linux/amd64` +- **Full coverage**: Yes +- **Image tag**: `nv-ingest:main-{sha}` + +#### 3. `docker-build-arm` + `docker-test-arm` +- **Platform**: `linux/arm64` +- **Emulation**: QEMU +- **Testing**: Random 100 tests (faster) +- **Non-blocking**: Runs in parallel + +#### 4. `library-mode-build` + `library-mode-test` +- **Always runs**: No approval needed (trusted branch) +- Full integration test suite + +**Parallelization**: +- ARM and x86 builds run in parallel +- Tests run after respective builds complete + +--- + +## Nightly Builds + +### `scheduled-nightly.yml` + +**Purpose**: Automated nightly builds and publishing + +**Triggers**: +- Schedule: Daily at 23:30 UTC +- Push to `main` (optional) +- Manual dispatch with skip options + +**Manual Inputs**: +- `skip-docker`: Skip Docker build/publish +- `skip-conda`: Skip Conda build/publish +- `skip-pypi`: Skip PyPI build/publish + +**Jobs**: + +#### 1. `determine-version` +- Generates version from date: `YYYY.MM.DD` +- Used by all downstream jobs + +#### 2. `docker-build-publish` +- **Platforms**: `linux/amd64,linux/arm64` +- **Registry**: NGC (`nvcr.io`) +- **Tag**: `nv-ingest:YYYY.MM.DD` +- **Push**: Yes +- **Multi-platform**: Yes (buildx) + +#### 3. `conda-build` + `conda-publish` +- **Channel**: `dev` +- **Version**: Date-based (YYYY.MM.DD) +- **Packages**: All nv-ingest conda packages +- **Force upload**: Yes + +#### 4. `pypi-build` + `pypi-publish` +- **Release type**: `dev` +- **Version**: Date-based +- **Packages**: + - `nv-ingest-api` + - `nv-ingest-client` + - `nv-ingest` (service) +- **Repository**: Artifactory + +**Dependencies**: All jobs independent (run in parallel after version determination) + +--- + +## Release Workflows + +### `release-docker.yml` + +**Purpose**: Publish official Docker release images + +**Triggers**: +- Automatic: Branch creation matching `release/*` +- Manual: workflow_dispatch + +**Manual Inputs**: +- `version`: Version string (e.g., `25.4.0`) +- `source-ref`: Git ref to build from (default: `main`) + +**Version Determination**: +- **Automatic**: Extracted from branch name (`release/25.4.0` → `25.4.0`) +- **Manual**: Uses input version + +**Build Details**: +- **Platforms**: `linux/amd64,linux/arm64` +- **Registry**: NGC +- **Tag**: `nv-ingest:{version}` +- **Example**: `nv-ingest:25.4.0` + +**Usage Examples**: +```bash +# Automatic trigger +git checkout -b release/25.4.0 +git push origin release/25.4.0 + +# Manual trigger (GitHub UI) +Actions → Release - Docker → Run workflow + Version: 25.4.0 + Source ref: main +``` + +--- + +### `release-conda.yml` + +**Purpose**: Publish conda packages to RapidsAI channels + +**Triggers**: +- Automatic: Branch creation matching `release/*` +- Manual: workflow_dispatch + +**Optional Inputs** (manual dispatch): +- `version`: Version string (default: extracted from branch name) +- `channel`: Target channel (default: `main`, options: `dev` or `main`) +- `source-ref`: Git ref to build from (default: release branch or `main`) + +**Version Determination**: +- **Automatic**: Extracted from branch name (`release/25.4.0` → `25.4.0`) +- **Manual**: Uses input version or falls back to branch extraction + +**Build Details**: +- **Container**: `rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.12` +- **Packages**: All nv-ingest conda packages +- **Force upload**: Yes (overwrites existing) +- **Default channel**: `main` (for release branches) + +**Usage Examples**: +```bash +# Automatic trigger (recommended) +git checkout -b release/25.4.0 +git push origin release/25.4.0 +# → Publishes to main channel automatically + +# Manual trigger (for custom options) +Actions → Release - Conda → Run workflow + Version: 25.4.0 + Channel: dev (for testing) or main + Source ref: main +``` + +**Channels**: +- `dev`: Development/testing releases +- `main`: Production releases (default for release branches) + +--- + +### `release-pypi.yml` + +**Purpose**: Publish Python wheels to PyPI/Artifactory + +**Triggers**: +- Automatic: Branch creation matching `release/*` +- Manual: workflow_dispatch + +**Optional Inputs** (manual dispatch): +- `version`: Version string (default: extracted from branch name) +- `release-type`: Type (default: `release`, options: `dev` or `release`) +- `source-ref`: Git ref to build from (default: release branch or `main`) + +**Version Determination**: +- **Automatic**: Extracted from branch name (`release/25.4.0` → `25.4.0`) +- **Manual**: Uses input version or falls back to branch extraction + +**Build Details**: +- **Container**: `rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.12` +- **Packages built**: + - `nv-ingest-api` (from `api/`) + - `nv-ingest-client` (from `client/`) + - `nv-ingest` (from `src/`) +- **Artifacts**: Wheels (.whl) and source distributions (.tar.gz) +- **Default release type**: `release` (for release branches) + +**Usage Examples**: +```bash +# Automatic trigger (recommended) +git checkout -b release/25.4.0 +git push origin release/25.4.0 +# → Publishes as release type automatically + +# Manual trigger (for custom options) +Actions → Release - PyPI → Run workflow + Version: 25.4.0 + Release type: dev (for testing) or release + Source ref: main +``` + +**Release Types**: +- `dev`: Development releases (with dev suffix) +- `release`: Production releases (default for release branches) + +--- + +### `build-docs.yml` + +**Purpose**: Build and deploy documentation to GitHub Pages (primary docs workflow) + +**Triggers**: +- Push to `main` +- Manual dispatch + +**Process**: +1. Build docs Docker image (target: `docs`) +2. Run container to generate static site (`make docs` in `/workspace/docs`) +3. Copy generated site from container (`/workspace/docs/site` → `./generated-site`) +4. Verify `index.html` exists in generated site +5. Upload artifact (`github-pages`) and deploy via `deploy-pages` + +**Output**: https://nvidia.github.io/nv-ingest/ + +**Required repo setting**: Settings → Pages → Build and deployment → Source must be **"GitHub Actions"** (not "Deploy from a branch"), or the site will not update. + +**Permissions**: +- `contents: read` +- `pages: write` +- `id-token: write` + +**Concurrency**: Single deployment (no cancellation) + +**Troubleshooting (docs not updating)**: +1. **Pages source**: Repo **Settings → Pages → Build and deployment → Source** must be **"GitHub Actions"**. If it is "Deploy from a branch", the workflow uploads an artifact but GitHub will not deploy it to the live site. +2. **Workflow enabled**: In **Actions**, ensure "Build NV-Ingest Documentation" is present and not disabled. Workflows in `.github/workflows/` on the default branch are enabled by default. +3. **Build failures**: Check the **build** job logs; the "Verify generated site" step will fail with a clear error if the container did not produce `site/index.html`. Fix any `make docs` errors (e.g. missing Python deps, Sphinx/MkDocs errors) in the docs Docker stage. +4. **Deploy job**: The **deploy** job needs the `github-pages` artifact from the build job; it uses the `github-pages` environment. Ensure the environment exists (it is usually auto-created). + +--- + +### `docs-deploy.yml` + +**Purpose**: Same as `build-docs.yml`; manual trigger only (no push trigger to avoid duplicate deployments). + +**Triggers**: +- Manual dispatch only + +**Process**: Same as `build-docs.yml`. + +--- + +## Reusable Workflows + +### `reusable-docker-build.yml` + +**Purpose**: Reusable Docker image build logic + +**Inputs**: +- `platform`: Target platform(s) (default: `linux/amd64`) +- `target`: Docker build stage (default: `runtime`) +- `push`: Push to registry (default: `false`) +- `tags`: Image tags, comma-separated +- `base-image`: Base image name (default: `ubuntu`) +- `base-image-tag`: Base image tag +- `runner`: GitHub runner (default: `linux-large-disk`) +- `use-qemu`: Enable QEMU for cross-platform +- `registry`: Docker registry URL (optional) + +**Secrets**: +- `HF_ACCESS_TOKEN`: Hugging Face token +- `DOCKER_PASSWORD`: Registry password + +**Outputs**: +- `image-digest`: Built image digest + +**Features**: +- Automatic buildx setup for multi-platform +- Conditional QEMU setup +- Flexible tag support +- Registry login (if push enabled) + +--- + +### `reusable-docker-test.yml` + +**Purpose**: Run tests in Docker containers + +**Inputs**: +- `image-tag`: Docker image to test +- `platform`: Platform to test on +- `test-selection`: `full`, `random`, or marker-based +- `random-count`: Number of random tests +- `pytest-markers`: Pytest marker expression +- `coverage`: Enable coverage report +- `runner`: GitHub runner + +**Artifacts**: +- Coverage reports (if enabled) +- Test reports (always) + +**Usage Example**: +```yaml +test-arm: + uses: ./.github/workflows/reusable-docker-test.yml + with: + image-tag: 'nv-ingest:test' + platform: 'linux/arm64' + test-selection: 'random' + random-count: '100' +``` + +--- + +### `reusable-conda-build.yml` + +**Purpose**: Build conda packages + +**Inputs**: +- `version`: Explicit version (optional) +- `source-ref`: Git ref to build from +- `runner`: GitHub runner +- `upload-artifacts`: Upload build artifacts + +**Container**: `rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.12` + +**Outputs**: +- `package-path`: Path to built packages + +**Artifacts**: Conda packages (if upload enabled) + +--- + +### `reusable-conda-publish.yml` + +**Purpose**: Publish conda packages + +**Inputs**: +- `channel`: Target channel (`dev` or `main`) +- `package-path`: Path to packages +- `force-upload`: Overwrite existing packages + +**Secrets**: +- `NVIDIA_CONDA_TOKEN`: Anaconda authentication + +**Validation**: Ensures channel is `dev` or `main` + +--- + +### `reusable-pypi-build.yml` + +**Purpose**: Build Python wheels + +**Inputs**: +- `version`: Explicit version (optional, date if omitted) +- `release-type`: `dev` or `release` +- `source-ref`: Git ref to build from +- `runner`: GitHub runner + +**Outputs**: +- `version`: Version that was built + +**Artifacts**: Python wheels and source distributions + +**Process**: +1. Installs build dependencies +2. Builds all three packages (api, client, service) +3. Uploads artifacts for publishing + +--- + +### `reusable-pypi-publish.yml` + +**Purpose**: Publish Python wheels to Artifactory + +**Inputs**: +- `repository-url`: PyPI repository URL + +**Secrets**: +- `ARTIFACTORY_URL`: Repository URL +- `ARTIFACTORY_USERNAME`: Auth username +- `ARTIFACTORY_PASSWORD`: Auth password + +**Process**: +1. Downloads wheel artifacts +2. Installs twine +3. Publishes all packages + +--- + +### `reusable-integration-test.yml` + +**Purpose**: Run integration tests with conda environment + +**Inputs**: +- `runner`: GitHub runner +- `python-version`: Python version (default: `3.12.11`) +- `timeout-minutes`: Job timeout (default: 60) + +**Secrets**: Multiple NVIDIA NIM and service endpoints + +**Process**: +1. Download conda packages (from artifacts) +2. Setup Miniconda +3. Install packages and dependencies +4. Run integration tests + +**Dependencies**: +- NVIDIA NIMs (audio, VLM, OCR, YOLOX) +- Milvus +- Various Python packages + +--- + +## Composite Actions + +### `setup-docker-buildx` + +**Purpose**: Setup Docker Buildx with optional QEMU + +**Inputs**: +- `use-qemu`: Enable QEMU emulation (default: `false`) +- `platforms`: Supported platforms (default: `linux/amd64`) + +**Steps**: +1. Setup QEMU (if enabled) +2. Setup Docker Buildx + +**Usage**: +```yaml +- uses: ./.github/actions/setup-docker-buildx + with: + use-qemu: 'true' + platforms: 'linux/amd64,linux/arm64' +``` + +--- + +### `docker-login-ngc` + +**Purpose**: Authenticate with NGC registry + +**Inputs**: +- `registry`: Registry URL (default: `nvcr.io`) +- `password`: NGC API token (required) + +**Usage**: +```yaml +- uses: ./.github/actions/docker-login-ngc + with: + password: ${{ secrets.DOCKER_PASSWORD }} +``` + +--- + +### `determine-version` + +**Purpose**: Determine version from various sources + +**Inputs**: +- `version`: Explicit version (optional) +- `date-format`: Date format for auto-generation +- `branch-name`: Branch name to extract from + +**Outputs**: +- `version`: Determined version string + +**Priority**: +1. Explicit version input +2. Extract from branch name (release/*) +3. Generate from date + +**Usage**: +```yaml +- id: version + uses: ./.github/actions/determine-version + with: + date-format: '%Y.%m.%d' + +- run: echo "Version is ${{ steps.version.outputs.version }}" +``` + +--- + +## Quick Reference + +### Common Tasks + +#### Run PR validation locally +```bash +# Pre-commit checks +pre-commit run --all-files + +# Docker build +docker build --target runtime -t nv-ingest:test . + +# Run tests +docker run nv-ingest:test pytest -m "not integration" +``` + +#### Trigger nightly build manually +```bash +# GitHub UI +Actions → Nightly Builds & Publishing → Run workflow + Branch: main +``` + +#### Create a release +```bash +# Automatic - All three artifact types (recommended) +git checkout -b release/25.4.0 +git push origin release/25.4.0 +# → Automatically triggers: +# - Docker (multi-platform) +# - Conda (main channel) +# - PyPI (release type) + +# Manual - For custom options +Actions → Release - Docker/Conda/PyPI → Run workflow +``` + +#### Debug workflow issues +```bash +# View workflow runs +Actions → Select workflow → View runs + +# Download artifacts +Actions → Workflow run → Artifacts section + +# Re-run failed jobs +Actions → Workflow run → Re-run failed jobs +``` + +--- + +## Best Practices + +1. **Always test workflows locally** when possible +2. **Use manual dispatch** for testing workflow changes +3. **Check artifacts** for build outputs and logs +4. **Monitor first run** after workflow changes +5. **Use skip flags** in nightly builds during maintenance +6. **Label external PRs** with `ok-to-test` after review +7. **Create release branches** from tested commits +8. **Verify secrets** are available before running workflows + +--- + +## Maintenance + +### Updating Docker Base Images + +Edit base image references in workflows or create a workflow variable. + +Current: `ubuntu:jammy-20250415.1` + +### Updating Runner Types + +Change `runner:` inputs in workflow calls: +- `ubuntu-latest`: Small jobs, public images +- `linux-large-disk`: Large Docker builds + +### Updating Python/Conda Versions + +Edit container images in reusable workflows: +- Current: `rapidsai/ci-conda:cuda12.5.1-ubuntu22.04-py3.12` + +### Adding New Secrets + +1. Add to repository secrets (Settings → Secrets → Actions) +2. Add to workflow secrets declarations +3. Pass through reusable workflow calls + +--- + +## Support + +For issues or questions: +- Check workflow logs in Actions tab +- Review this documentation +- Check migration guide: `WORKFLOWS_MIGRATION.md` +- Contact DevOps team diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml index 173e96111..699205595 100644 --- a/.github/workflows/build-docs.yml +++ b/.github/workflows/build-docs.yml @@ -1,6 +1,9 @@ name: Build NV-Ingest Documentation -# Trigger for pull requests and pushing to main +# Publishes to https://nvidia.github.io/nv-ingest/ +# REQUIRED: In repo Settings → Pages → Build and deployment → Source, select "GitHub Actions" +# (not "Deploy from a branch"), or this workflow will not update the live site. + on: # Runs on pushes targeting the default branch push: @@ -52,13 +55,24 @@ jobs: - name: Copy generated docs site from the container run: docker cp $CONTAINER_ID:/workspace/docs/site ./generated-site + - name: Verify generated site + run: | + if [ ! -f ./generated-site/index.html ]; then + echo "::error::Generated site missing index.html. Build may have failed inside the container." + ls -la ./generated-site/ || true + exit 1 + fi + echo "Generated site contains $(find ./generated-site -type f | wc -l) files" + - name: Stop and remove the container - run: docker rm $CONTAINER_ID + if: always() + run: docker rm -f $CONTAINER_ID 2>/dev/null || true - name: Upload Site Artifacts uses: actions/upload-pages-artifact@v3 with: path: ./generated-site + name: github-pages deploy: needs: diff --git a/.github/workflows/docs-deploy.yml b/.github/workflows/docs-deploy.yml index 023859880..c0905ce93 100644 --- a/.github/workflows/docs-deploy.yml +++ b/.github/workflows/docs-deploy.yml @@ -1,9 +1,8 @@ name: Documentation Deploy +# NOTE: Docs are built and deployed by build-docs.yml on push to main. +# This workflow is kept for manual runs only; remove push trigger to avoid duplicate deployments. on: - push: - branches: - - main workflow_dispatch: permissions: @@ -50,6 +49,15 @@ jobs: - name: Copy generated docs from container run: docker cp $CONTAINER_ID:/workspace/docs/site ./generated-site + - name: Verify generated site + run: | + if [ ! -f ./generated-site/index.html ]; then + echo "::error::Generated site missing index.html. Build may have failed inside the container." + ls -la ./generated-site/ || true + exit 1 + fi + echo "Generated site contains $(find ./generated-site -type f | wc -l) files" + - name: Clean up container run: docker rm $CONTAINER_ID @@ -57,6 +65,7 @@ jobs: uses: actions/upload-pages-artifact@v3 with: path: ./generated-site + name: github-pages deploy: name: Deploy to GitHub Pages diff --git a/Dockerfile b/Dockerfile index a0db9803b..886924479 100644 --- a/Dockerfile +++ b/Dockerfile @@ -168,6 +168,17 @@ CMD ["/bin/bash"] FROM nv_ingest_install AS docs +# Install dependencies needed for docs generation +# +# NOTE: The nv_ingest_install base image may carry a broken apt/dpkg state +# (e.g., partially-installed libreoffice dependencies). Installing `make` +# via conda avoids apt entirely and is more reliable on CI runners. +# Do not run mamba clean -afy here: /opt/conda/pkgs is a cache mount and +# cannot be removed (Device or resource busy). +RUN --mount=type=cache,target=/opt/conda/pkgs \ + source activate nv_ingest_runtime \ + && mamba install -y make + COPY docs docs # Docs needs all the source code present so add it to the container