feat: replace per-file HF downloads with bulk git clone snapshot

- Add snapshot_download_tasks() that clones entire HF dataset once (~6MB)
- Cache persists across evaluations (0 HTTP requests after first run)
- Fallback to parallel per-task download (5 concurrent) if snapshot fails
- Add git-lfs to Docker image
- Report download_errors in /evaluate response

Author: echobt

Dockerfile

@@ -13,7 +13,7 @@ FROM debian:bookworm-slim
 
 # Pre-install all common runtimes and tools at build time (as root)
 RUN apt-get update && apt-get install -y --no-install-recommends \
-    ca-certificates git curl wget unzip libssl3 libssl-dev pkg-config sudo openssh-client \
+    ca-certificates git git-lfs curl wget unzip libssl3 libssl-dev pkg-config sudo openssh-client \
     python3 python3-pip python3-venv python3-dev \
     build-essential gcc g++ make cmake autoconf automake libtool \
     default-jdk maven gradle \
@@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     imagemagick libmagickwand-dev \
     jq \
     && ln -sf /usr/bin/python3 /usr/bin/python \
+    && git lfs install \
     && rm -rf /var/lib/apt/lists/*
 
 # Install Go 1.23 (Debian bookworm ships 1.19 which is too old for most projects)

src/handlers.rs

@@ -1352,7 +1352,7 @@ async fn evaluate_with_stored_agent(
         })?
     };
 
-    // Download task files from HF repo (workspace.yaml, tests/*.sh, etc.)
+    // Download all task files from HF repo via snapshot (bulk git clone)
     let hf_client = crate::swe_forge::client::HuggingFaceClient::new().map_err(|e| {
         (
             StatusCode::INTERNAL_SERVER_ERROR,
@@ -1361,31 +1361,98 @@ async fn evaluate_with_stored_agent(
     })?;
 
     let dataset_id = "CortexLM/swe-forge";
-    let tasks_base = state.config.workspace_base.join("_hf_tasks");
-    let _ = tokio::fs::remove_dir_all(&tasks_base).await;
+    let snapshot_cache = state.config.workspace_base.join("_hf_dataset_cache");
 
+    // Bulk download the entire tasks/ folder (cached after first call)
+    hf_client
+        .snapshot_download_tasks(dataset_id, &snapshot_cache)
+        .await
+        .map_err(|e| {
+            tracing::warn!("Snapshot download failed: {}, falling back to per-task download", e);
+            (
+                StatusCode::BAD_GATEWAY,
+                Json(serde_json::json!({"error": format!("Failed to download HF dataset: {}", e)})),
+            )
+        })?;
+
+    // Resolve tasks from the cached snapshot directory.
+    // HF stores tasks as tasks/{org}/{repo-number}/ (nested dirs matching the task_id).
+    let tasks_dir = snapshot_cache.join("tasks");
     let mut hf_tasks: Vec<crate::task::SweForgeTask> = Vec::new();
     let mut errors: Vec<String> = Vec::new();
 
     for task_id in &task_ids {
-        let task_dir = tasks_base.join(task_id.replace('/', "__"));
-        match hf_client
-            .download_task_files(dataset_id, task_id, &task_dir)
-            .await
-        {
-            Ok(()) => match crate::task::parse_task(&task_dir) {
-                Ok(mut task) => {
-                    task.id = task_id.clone();
-                    hf_tasks.push(task);
-                }
-                Err(e) => {
-                    tracing::warn!("Failed to parse task {}: {}", task_id, e);
-                    errors.push(format!("{}: parse error: {}", task_id, e));
-                }
-            },
+        // task_id format: "org/repo-number" → HF path: tasks/org/repo-number/
+        let task_path = tasks_dir.join(task_id);
+
+        if !task_path.exists() || !task_path.is_dir() {
+            errors.push(format!("{}: not found in cached dataset", task_id));
+            continue;
+        }
+
+        match crate::task::parse_task(&task_path) {
+            Ok(mut task) => {
+                task.id = task_id.clone();
+                hf_tasks.push(task);
+            }
             Err(e) => {
-                tracing::warn!("Failed to download task {}: {}", task_id, e);
-                errors.push(format!("{}: download error: {}", task_id, e));
+                tracing::warn!("Failed to parse task {}: {}", task_id, e);
+                errors.push(format!("{}: parse error: {}", task_id, e));
+            }
+        }
+    }
+
+    // Fallback: if snapshot produced no matches, try per-task download for missing ones
+    if hf_tasks.is_empty() && !task_ids.is_empty() {
+        tracing::info!("No tasks found in snapshot cache, falling back to per-task download");
+        let hf_client = std::sync::Arc::new(
+            crate::swe_forge::client::HuggingFaceClient::new().map_err(|e| {
+                (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    Json(serde_json::json!({"error": format!("HF client error: {}", e)})),
+                )
+            })?,
+        );
+        let tasks_base = state.config.workspace_base.join("_hf_tasks");
+
+        use futures::stream::{self, StreamExt};
+
+        let download_results: Vec<_> = stream::iter(task_ids.clone().into_iter())
+            .map(|task_id| {
+                let hf = std::sync::Arc::clone(&hf_client);
+                let base = tasks_base.clone();
+                async move {
+                    let task_dir = base.join(task_id.replace('/', "__"));
+                    match hf
+                        .download_task_files(dataset_id, &task_id, &task_dir)
+                        .await
+                    {
+                        Ok(()) => match crate::task::parse_task(&task_dir) {
+                            Ok(mut task) => {
+                                task.id = task_id.clone();
+                                Ok(task)
+                            }
+                            Err(e) => {
+                                tracing::warn!("Failed to parse task {}: {}", task_id, e);
+                                Err(format!("{}: parse error: {}", task_id, e))
+                            }
+                        },
+                        Err(e) => {
+                            tracing::warn!("Failed to download task {}: {}", task_id, e);
+                            Err(format!("{}: download error: {}", task_id, e))
+                        }
+                    }
+                }
+            })
+            .buffer_unordered(5)
+            .collect()
+            .await;
+
+        errors.clear();
+        for result in download_results {
+            match result {
+                Ok(task) => hf_tasks.push(task),
+                Err(e) => errors.push(e),
             }
         }
     }
@@ -1444,6 +1511,7 @@ async fn evaluate_with_stored_agent(
             "batch_id": batch_id,
             "total_tasks": total_tasks,
             "matched_task_ids": task_ids,
+            "download_errors": errors,
         })),
     ))
 }

src/swe_forge/client.rs

@@ -8,6 +8,8 @@ const HF_DATASET_VIEWER_BASE: &str = "https://datasets-server.huggingface.co/row
 const HF_REPO_BASE: &str = "https://huggingface.co";
 const DEFAULT_TIMEOUT_SECS: u64 = 30;
 const MAX_PAGE_SIZE: usize = 100;
+/// Minimum number of task directories to consider the snapshot cache valid.
+const SNAPSHOT_MIN_DIRS: usize = 5;
 
 pub struct HuggingFaceClient {
     client: reqwest::Client,
@@ -136,6 +138,12 @@ impl HuggingFaceClient {
                 tokio::fs::create_dir_all(parent).await?;
             }
 
+            // Skip files that already exist with non-zero size (cached from previous run)
+            if local_path.exists() && local_path.metadata().map(|m| m.len() > 0).unwrap_or(false) {
+                debug!("Skipping cached file: {}", local_path.display());
+                continue;
+            }
+
             let download_url = format!(
                 "{}/datasets/{}/resolve/main/{}",
                 HF_REPO_BASE, dataset_id, file_path
@@ -167,6 +175,84 @@ impl HuggingFaceClient {
         Ok(())
     }
 
+    /// Download the entire `tasks/` folder from the HF dataset repo using a
+    /// shallow `git clone --depth 1`. This replaces hundreds of individual
+    /// HTTP API requests with a single bulk git operation.
+    ///
+    /// After this call, task files live at `<cache_dir>/tasks/{org}/{repo-number}/`.
+    pub async fn snapshot_download_tasks(
+        &self,
+        dataset_id: &str,
+        cache_dir: &std::path::Path,
+    ) -> Result<()> {
+        let tasks_dir = cache_dir.join("tasks");
+
+        // Check if already cached (has enough task directories)
+        if tasks_dir.exists() {
+            let mut entries = tokio::fs::read_dir(&tasks_dir).await?;
+            let mut count = 0usize;
+            while entries.next_entry().await?.is_some() {
+                count += 1;
+                if count >= SNAPSHOT_MIN_DIRS {
+                    info!(
+                        "HF tasks already cached at {:?} ({} top-level dirs)",
+                        tasks_dir, count
+                    );
+                    return Ok(());
+                }
+            }
+        }
+
+        info!(
+            "Downloading HF dataset via git clone --depth 1 to {:?}",
+            cache_dir
+        );
+
+        let repo_url = format!("https://huggingface.co/datasets/{}", dataset_id);
+
+        // Clean and recreate cache dir for a fresh clone
+        if cache_dir.exists() {
+            tokio::fs::remove_dir_all(cache_dir).await.ok();
+        }
+
+        // Shallow clone — the swe-forge dataset is small (~6 MB) so a full
+        // depth-1 clone is fast and avoids HF git server limitations with
+        // partial/sparse clones.
+        let output = tokio::process::Command::new("git")
+            .args(["clone", "--depth", "1"])
+            .arg(&repo_url)
+            .arg(cache_dir.as_os_str())
+            .stdout(std::process::Stdio::piped())
+            .stderr(std::process::Stdio::piped())
+            .output()
+            .await
+            .context("Failed to execute git clone")?;
+
+        if !output.status.success() {
+            let stderr = String::from_utf8_lossy(&output.stderr);
+            anyhow::bail!("git clone failed (exit {}): {}", output.status, stderr);
+        }
+
+        // Verify the download
+        let mut count = 0usize;
+        if tasks_dir.exists() {
+            let mut entries = tokio::fs::read_dir(&tasks_dir).await?;
+            while entries.next_entry().await?.is_some() {
+                count += 1;
+            }
+        }
+        info!("Downloaded {} top-level task directories from HF", count);
+
+        if count == 0 {
+            anyhow::bail!(
+                "Snapshot download produced no task directories at {:?}",
+                tasks_dir
+            );
+        }
+
+        Ok(())
+    }
+
     async fn list_tree_recursive(
         &self,
         dataset_id: &str,