From ef5d297a63d3b317d9e4495fd5be77891f2297b5 Mon Sep 17 00:00:00 2001 From: CrimsonGlory Date: Mon, 26 Jan 2026 14:43:59 +0000 Subject: [PATCH 1/6] Implement immediate persistence of Sample metadata after subfile upload in Task class --- meta_extractor/redis/task.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/meta_extractor/redis/task.py b/meta_extractor/redis/task.py index c36972d6..2261cb40 100644 --- a/meta_extractor/redis/task.py +++ b/meta_extractor/redis/task.py @@ -7,6 +7,7 @@ # from .requeue import TaskRequeuer from .queue import TaskQueue from ..sample import Sample +from .. import api_connector class Task: @@ -60,4 +61,19 @@ def sample_found(self) -> bool: def split_into_subtasks_per_subfile(self) -> None: for subfile in self.sample.subfiles: subfile_seaweedfs_file_id = seaweedfs.upload_file(stream=subfile.content, name=subfile.file_name) + # Set the seaweedfs_file_id so it's included in metadata + subfile.seaweedfs_file_id = subfile_seaweedfs_file_id + # Immediately persist the Sample to Django + try: + response_data = { + 'force_reprocess': self.force_reprocess, + 'callback': self.callback, + 'sample': subfile.metadata + } + api_connector.send_result(self.api_url, response_data) + logging.info(f'Persisted Sample immediately after upload: sha1={subfile.sha1}, seaweedfs_file_id={subfile_seaweedfs_file_id}') + except Exception as e: + logging.error(f'Failed to persist Sample immediately after upload: {e}') + # Continue anyway - the task will be processed and Sample will be created later + # Then queue the task for processing TaskQueue().add_subfile_to_queue(self.settings, subfile_seaweedfs_file_id) From 495ae1caaf0c53edec6bbb08967caa23ce23b2f7 Mon Sep 17 00:00:00 2001 From: CrimsonGlory Date: Mon, 26 Jan 2026 20:43:43 +0000 Subject: [PATCH 2/6] Update CircleCI config to remove unnecessary Docker installation step --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c8205a04..88ee84b0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: working_directory: ~/daas steps: - checkout - - docker/install-docker + # Docker is pre-installed on machine executors, no need to install - docker/install-docker-compose - run: docker --version - run: docker compose version From 2f84e2a4dc959758ad78f083960c86c5c6801dae Mon Sep 17 00:00:00 2001 From: CrimsonGlory Date: Mon, 26 Jan 2026 21:09:03 +0000 Subject: [PATCH 3/6] Update certificates inside container --- peWorkerDockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/peWorkerDockerfile b/peWorkerDockerfile index 82471494..2515f5a9 100644 --- a/peWorkerDockerfile +++ b/peWorkerDockerfile @@ -15,7 +15,8 @@ ENV WINEARCH=win32 # Generic RUN apt-get update && \ apt-get install --no-install-recommends -y build-essential apt-transport-https && \ - apt-get install --no-install-recommends -y gnutls-bin \ + apt-get install --no-install-recommends -y ca-certificates \ + gnutls-bin \ host \ unzip \ xauth \ @@ -23,6 +24,7 @@ RUN apt-get update && \ zenity \ zlib1g \ zlib1g-dev && \ + update-ca-certificates && \ rm -rf /var/lib/apt/lists/* && \ apt-get clean From 246f08eb75d3e57a48ce93625f712adb99815fc9 Mon Sep 17 00:00:00 2001 From: CrimsonGlory Date: Tue, 27 Jan 2026 01:45:31 +0000 Subject: [PATCH 4/6] test --- peWorkerDockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/peWorkerDockerfile b/peWorkerDockerfile index 2515f5a9..29947257 100644 --- a/peWorkerDockerfile +++ b/peWorkerDockerfile @@ -60,7 +60,7 @@ COPY winetricks/corefonts/ /home/root/.cache/winetricks/corefonts/ # Dotnet45 should be installed on a different docker step. Otherwise, it will fail. RUN echo "Installing Dotnet45" && \ - winetricks -q dotnet45 corefonts && \ + winetricks -q --force dotnet45 corefonts && \ echo "Dotnet45 installed" RUN xvfb-run winetricks -q vcrun2010 && \ echo "vcrun2010 installed" From d9756be52c60e5de62255273e0d67ece8179b4db Mon Sep 17 00:00:00 2001 From: CrimsonGlory Date: Tue, 27 Jan 2026 18:56:17 +0000 Subject: [PATCH 5/6] try with a newer version of winetricks --- peWorkerDockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/peWorkerDockerfile b/peWorkerDockerfile index 29947257..6bf3c78c 100644 --- a/peWorkerDockerfile +++ b/peWorkerDockerfile @@ -47,9 +47,9 @@ RUN dpkg --add-architecture i386 && \ # C#: Winetricks, dotnet45, vcrun2010 RUN echo "Installing winetricks" && \ - wget -nc -nv https://github.com/Winetricks/winetricks/archive/20210825.zip -O /tmp/winetricks.zip && \ + wget -nc -nv https://github.com/Winetricks/winetricks/archive/20260125.zip -O /tmp/winetricks.zip && \ unzip /tmp/winetricks.zip -d /tmp/winetricks/ && \ - make -C /tmp/winetricks/winetricks-20210825 install && \ + make -C /tmp/winetricks/winetricks-20260125 install && \ rm -rf /tmp/winetricks.zip && \ rm -rf /tmp/winetricks && \ mkdir -p /home/root/.cache/winetricks/ && \ From 0429a379d451cd77c5610268881b2afdaff6a3fb Mon Sep 17 00:00:00 2001 From: CrimsonGlory Date: Tue, 27 Jan 2026 23:08:23 +0000 Subject: [PATCH 6/6] install microsoft cert --- peWorkerDockerfile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/peWorkerDockerfile b/peWorkerDockerfile index 6bf3c78c..f3e78ef9 100644 --- a/peWorkerDockerfile +++ b/peWorkerDockerfile @@ -16,6 +16,7 @@ ENV WINEARCH=win32 RUN apt-get update && \ apt-get install --no-install-recommends -y build-essential apt-transport-https && \ apt-get install --no-install-recommends -y ca-certificates \ + openssl \ gnutls-bin \ host \ unzip \ @@ -44,6 +45,13 @@ RUN dpkg --add-architecture i386 && \ rm -rf /var/lib/apt/lists/* && \ apt-get clean +# Add Microsoft TLS G2 ECC CA OCSP 02.crt to ca-certificates +# Without this wget https://download.microsoft.com fails due to cert error. +RUN wget --user-agent="Mozilla/5.0 (X11; Linux x86_64)" https://www.microsoft.com/pkiops/certs/Microsoft%20TLS%20G2%20ECC%20CA%20OCSP%2002.crt && \ + openssl x509 -inform DER -in "Microsoft TLS G2 ECC CA OCSP 02.crt" -out microsoft-tls-g2-ecc-ocsp-02.crt && \ + cp microsoft-tls-g2-ecc-ocsp-02.crt /usr/local/share/ca-certificates/ && \ + update-ca-certificates + # C#: Winetricks, dotnet45, vcrun2010 RUN echo "Installing winetricks" && \