feat: fetch task definitions from HF repo (workspace.yaml + tests/), remove auto_install hack

/evaluate now downloads workspace.yaml with install_config and test
scripts from the HF repo tasks/ directory instead of using parquet
rows with auto-generated install commands.

Author: echobt

src/handlers.rs

@@ -1150,7 +1150,7 @@ async fn evaluate_with_stored_agent(
         .get("task_ids")
         .and_then(|v| serde_json::from_value(v.clone()).ok())
         .unwrap_or_default();
-    let split = body
+    let _split = body
         .get("split")
         .and_then(|v| v.as_str())
         .unwrap_or("train")
@@ -1174,61 +1174,54 @@ async fn evaluate_with_stored_agent(
         })?
     };
 
-    // Fetch dataset from HuggingFace
+    // Download task files from HF repo (workspace.yaml, tests/*.sh, etc.)
     let hf_client = crate::swe_forge::client::HuggingFaceClient::new().map_err(|e| {
         (
             StatusCode::INTERNAL_SERVER_ERROR,
             Json(serde_json::json!({"error": format!("HF client error: {}", e)})),
         )
     })?;
 
-    let dataset_config = crate::swe_forge::types::DatasetConfig {
-        dataset_id: "CortexLM/swe-forge".to_string(),
-        split,
-        limit: 100,
-        offset: 0,
-    };
+    let dataset_id = "CortexLM/swe-forge";
+    let tasks_base = state.config.workspace_base.join("_hf_tasks");
+    let _ = tokio::fs::remove_dir_all(&tasks_base).await;
 
-    let dataset = hf_client
-        .fetch_dataset(&dataset_config)
-        .await
-        .map_err(|e| {
-            (
-                StatusCode::BAD_GATEWAY,
-                Json(serde_json::json!({"error": format!("Failed to fetch HF dataset: {}", e)})),
-            )
-        })?;
+    let mut hf_tasks: Vec<crate::task::SweForgeTask> = Vec::new();
+    let mut errors: Vec<String> = Vec::new();
 
-    let matched: Vec<&crate::swe_forge::types::DatasetEntry> = dataset
-        .entries
-        .iter()
-        .filter(|e| task_ids.contains(&e.instance_id))
-        .collect();
+    for task_id in &task_ids {
+        let task_dir = tasks_base.join(task_id.replace('/', "__"));
+        match hf_client
+            .download_task_files(dataset_id, task_id, &task_dir)
+            .await
+        {
+            Ok(()) => match crate::task::parse_task(&task_dir) {
+                Ok(mut task) => {
+                    task.id = task_id.clone();
+                    hf_tasks.push(task);
+                }
+                Err(e) => {
+                    tracing::warn!("Failed to parse task {}: {}", task_id, e);
+                    errors.push(format!("{}: parse error: {}", task_id, e));
+                }
+            },
+            Err(e) => {
+                tracing::warn!("Failed to download task {}: {}", task_id, e);
+                errors.push(format!("{}: download error: {}", task_id, e));
+            }
+        }
+    }
 
-    if matched.is_empty() {
+    if hf_tasks.is_empty() {
         return Err((
             StatusCode::NOT_FOUND,
-            Json(
-                serde_json::json!({"error": "No matching tasks found", "available": dataset.entries.len()}),
-            ),
+            Json(serde_json::json!({
+                "error": "No valid tasks found",
+                "details": errors,
+            })),
         ));
     }
 
-    // Build tasks from HF
-    let mut registry = crate::task::registry::TaskRegistry::new();
-    let hf_dataset = crate::swe_forge::types::HuggingFaceDataset {
-        dataset_id: dataset.dataset_id.clone(),
-        split: dataset.split.clone(),
-        entries: matched.into_iter().cloned().collect(),
-        total_count: dataset.total_count,
-    };
-    registry.load_from_huggingface(&hf_dataset).map_err(|e| {
-        (
-            StatusCode::INTERNAL_SERVER_ERROR,
-            Json(serde_json::json!({"error": format!("Failed to load tasks: {}", e)})),
-        )
-    })?;
-
     // Extract agent code only (no tasks/ required - we use HF tasks)
     let extract_dir = state.config.workspace_base.join("_extract_evaluate");
     let _ = tokio::fs::remove_dir_all(&extract_dir).await;
@@ -1243,7 +1236,6 @@ async fn evaluate_with_stored_agent(
             })?;
     let _ = tokio::fs::remove_dir_all(&extract_dir).await;
 
-    let hf_tasks: Vec<crate::task::SweForgeTask> = registry.get_tasks().to_vec();
     let final_archive = crate::task::ExtractedArchive {
         tasks: hf_tasks,
         agent_code,

src/swe_forge/client.rs

@@ -1,9 +1,11 @@
 use anyhow::{Context, Result};
-use tracing::{debug, info};
+use std::path::Path;
+use tracing::{debug, info, warn};
 
 use super::types::{DatasetConfig, DatasetEntry, HfRowsResponse, HuggingFaceDataset};
 
 const HF_DATASET_VIEWER_BASE: &str = "https://datasets-server.huggingface.co/rows";
+const HF_REPO_BASE: &str = "https://huggingface.co";
 const DEFAULT_TIMEOUT_SECS: u64 = 30;
 const MAX_PAGE_SIZE: usize = 100;
 
@@ -97,6 +99,121 @@ impl HuggingFaceClient {
             ))
     }
 
+    /// Download all task files for a given instance_id from the HF repo into a local directory.
+    /// The directory will have workspace.yaml, prompt.md, tests/*.sh, etc.
+    pub async fn download_task_files(
+        &self,
+        dataset_id: &str,
+        instance_id: &str,
+        dest_dir: &Path,
+    ) -> Result<()> {
+        let tree_url = format!(
+            "{}/api/datasets/{}/tree/main/tasks/{}",
+            HF_REPO_BASE, dataset_id, instance_id
+        );
+        info!("Listing HF task files: {}", tree_url);
+
+        // List all files (including subdirectories)
+        let files = self.list_tree_recursive(dataset_id, instance_id).await?;
+
+        if files.is_empty() {
+            anyhow::bail!(
+                "No files found for task {} in dataset {}",
+                instance_id,
+                dataset_id
+            );
+        }
+
+        tokio::fs::create_dir_all(dest_dir).await?;
+
+        for file_path in &files {
+            let relative = file_path
+                .strip_prefix(&format!("tasks/{}/", instance_id))
+                .unwrap_or(file_path);
+            let local_path = dest_dir.join(relative);
+
+            if let Some(parent) = local_path.parent() {
+                tokio::fs::create_dir_all(parent).await?;
+            }
+
+            let download_url = format!(
+                "{}/datasets/{}/resolve/main/{}",
+                HF_REPO_BASE, dataset_id, file_path
+            );
+            debug!("Downloading {} -> {}", download_url, local_path.display());
+
+            let resp = self
+                .client
+                .get(&download_url)
+                .send()
+                .await
+                .with_context(|| format!("Failed to download {}", download_url))?;
+
+            if !resp.status().is_success() {
+                warn!("Failed to download {}: HTTP {}", file_path, resp.status());
+                continue;
+            }
+
+            let bytes = resp.bytes().await?;
+            tokio::fs::write(&local_path, &bytes).await?;
+        }
+
+        info!(
+            "Downloaded {} files for task {} to {}",
+            files.len(),
+            instance_id,
+            dest_dir.display()
+        );
+        Ok(())
+    }
+
+    async fn list_tree_recursive(
+        &self,
+        dataset_id: &str,
+        instance_id: &str,
+    ) -> Result<Vec<String>> {
+        let mut all_files = Vec::new();
+        let mut dirs_to_visit = vec![format!("tasks/{}", instance_id)];
+
+        while let Some(dir_path) = dirs_to_visit.pop() {
+            let url = format!(
+                "{}/api/datasets/{}/tree/main/{}",
+                HF_REPO_BASE, dataset_id, dir_path
+            );
+            let resp = self
+                .client
+                .get(&url)
+                .send()
+                .await
+                .with_context(|| format!("Failed to list {}", url))?;
+
+            if !resp.status().is_success() {
+                warn!(
+                    "Failed to list directory {}: HTTP {}",
+                    dir_path,
+                    resp.status()
+                );
+                continue;
+            }
+
+            let entries: Vec<serde_json::Value> = resp.json().await?;
+            for entry in entries {
+                let entry_type = entry["type"].as_str().unwrap_or("");
+                let entry_path = entry["path"].as_str().unwrap_or("");
+                if entry_path.is_empty() {
+                    continue;
+                }
+                match entry_type {
+                    "file" => all_files.push(entry_path.to_string()),
+                    "directory" => dirs_to_visit.push(entry_path.to_string()),
+                    _ => {}
+                }
+            }
+        }
+
+        Ok(all_files)
+    }
+
     async fn fetch_page(
         &self,
         dataset_id: &str,

src/task/registry.rs

@@ -131,27 +131,6 @@ fn parse_test_list(raw: &str) -> Vec<String> {
         .collect()
 }
 
-fn auto_install_commands(language: &str) -> Option<Vec<String>> {
-    match language {
-        "python" => Some(vec![
-            "pip install -e '.[dev,test]' 2>/dev/null || pip install -e . 2>/dev/null || true".to_string(),
-            "pip install pytest 2>/dev/null || true".to_string(),
-        ]),
-        "javascript" | "typescript" => Some(vec![
-            "npm install --legacy-peer-deps 2>/dev/null || yarn install --frozen-lockfile 2>/dev/null || true".to_string(),
-        ]),
-        "go" => Some(vec![
-            "go mod download 2>/dev/null || true".to_string(),
-        ]),
-        "rust" => Some(vec![
-            "cargo fetch 2>/dev/null || true".to_string(),
-        ]),
-        _ => Some(vec![
-            "pip install -e . 2>/dev/null || npm install 2>/dev/null || true".to_string(),
-        ]),
-    }
-}
-
 fn convert_dataset_entry_to_task(entry: &DatasetEntry) -> Result<SweForgeTask> {
     let repo_url = build_repo_url(&entry.repo);
     let language = entry
@@ -168,13 +147,11 @@ fn convert_dataset_entry_to_task(entry: &DatasetEntry) -> Result<SweForgeTask> {
         .as_deref()
         .and_then(|s| serde_json::from_str(s).ok());
 
-    let install = auto_install_commands(&language);
-
     let workspace = WorkspaceConfig {
         repo: repo_url,
         version: entry.version.clone().unwrap_or_default(),
         base_commit: Some(entry.base_commit.clone()),
-        install,
+        install: None,
         language: Some(language),
         fail_to_pass: f2p,
         pass_to_pass: p2p,