feat: integrate HuggingFace dataset handler with task/evaluation system

- Add src/swe_forge/ module with HuggingFace dataset client and types
  - types.rs: DatasetEntry, HuggingFaceDataset, DatasetConfig for
    CortexLM/swe-forge dataset schema (repo, instance_id, base_commit,
    patch, test_patch, problem_statement, hints_text, version, etc.)
  - client.rs: HuggingFaceClient with async fetch_dataset() and
    fetch_entry() using HuggingFace Dataset Viewer REST API
- Restructure src/task.rs into src/task/ module directory
  - types.rs: SweForgeTaskFields struct for SWE-forge-specific fields
  - config.rs: TaskSource enum (Local/HuggingFace) and TaskConfig
  - registry.rs: TaskRegistry with load_from_huggingface() that converts
    DatasetEntry items to SweForgeTask format, including test script
    generation from test_patch and fail_to_pass fields
- Add swe_forge_fields: Option<SweForgeTaskFields> to SweForgeTask
- Add swe_forge module declaration to main.rs
- All existing tests pass, 27 new tests added

Author: echobt

Cargo.lock

@@ -6,6 +6,7 @@ mod executor;
 mod handlers;
 mod metrics;
 mod session;
+mod swe_forge;
 mod task;
 mod validator_whitelist;
 mod ws;

src/main.rs

@@ -0,0 +1,180 @@
+use anyhow::{Context, Result};
+use tracing::{debug, info};
+
+use super::types::{DatasetConfig, DatasetEntry, HfRowsResponse, HuggingFaceDataset};
+
+const HF_DATASET_VIEWER_BASE: &str = "https://datasets-server.huggingface.co/rows";
+const DEFAULT_TIMEOUT_SECS: u64 = 30;
+const MAX_PAGE_SIZE: usize = 100;
+
+pub struct HuggingFaceClient {
+    client: reqwest::Client,
+}
+
+impl HuggingFaceClient {
+    pub fn new() -> Result<Self> {
+        let client = reqwest::Client::builder()
+            .timeout(std::time::Duration::from_secs(DEFAULT_TIMEOUT_SECS))
+            .build()
+            .context("Failed to build HTTP client for HuggingFace")?;
+        Ok(Self { client })
+    }
+
+    pub async fn fetch_dataset(&self, config: &DatasetConfig) -> Result<HuggingFaceDataset> {
+        info!(
+            "Fetching HuggingFace dataset: {} (split={}, offset={}, limit={})",
+            config.dataset_id, config.split, config.offset, config.limit
+        );
+
+        let mut all_entries = Vec::new();
+        let mut offset = config.offset;
+        let mut total_count = 0;
+        let remaining = config.limit;
+
+        while all_entries.len() < remaining {
+            let page_size = MAX_PAGE_SIZE.min(remaining - all_entries.len());
+
+            let response = self
+                .fetch_page(&config.dataset_id, &config.split, offset, page_size)
+                .await?;
+
+            if let Some(total) = response.num_rows_total {
+                total_count = total;
+            }
+
+            let row_count = response.rows.len();
+            if row_count == 0 {
+                break;
+            }
+
+            for wrapper in response.rows {
+                all_entries.push(wrapper.row);
+            }
+
+            offset += row_count;
+
+            if row_count < page_size {
+                break;
+            }
+        }
+
+        info!(
+            "Fetched {} entries from {} (total available: {})",
+            all_entries.len(),
+            config.dataset_id,
+            total_count
+        );
+
+        Ok(HuggingFaceDataset {
+            dataset_id: config.dataset_id.clone(),
+            split: config.split.clone(),
+            entries: all_entries,
+            total_count,
+        })
+    }
+
+    pub async fn fetch_entry(
+        &self,
+        dataset_id: &str,
+        split: &str,
+        index: usize,
+    ) -> Result<DatasetEntry> {
+        debug!(
+            "Fetching single entry from {} at index {}",
+            dataset_id, index
+        );
+
+        let response = self.fetch_page(dataset_id, split, index, 1).await?;
+
+        response
+            .rows
+            .into_iter()
+            .next()
+            .map(|w| w.row)
+            .context(format!(
+                "No entry found at index {} in dataset {}",
+                index, dataset_id
+            ))
+    }
+
+    async fn fetch_page(
+        &self,
+        dataset_id: &str,
+        split: &str,
+        offset: usize,
+        length: usize,
+    ) -> Result<HfRowsResponse> {
+        let url = format!(
+            "{}?dataset={}&config=default&split={}&offset={}&length={}",
+            HF_DATASET_VIEWER_BASE, dataset_id, split, offset, length
+        );
+
+        debug!("Requesting HuggingFace API: {}", url);
+
+        let resp = self
+            .client
+            .get(&url)
+            .send()
+            .await
+            .context("Failed to send request to HuggingFace dataset viewer")?;
+
+        let status = resp.status();
+        if !status.is_success() {
+            let body = resp.text().await.unwrap_or_default();
+            anyhow::bail!(
+                "HuggingFace API returned HTTP {}: {}",
+                status.as_u16(),
+                &body[..body.len().min(500)]
+            );
+        }
+
+        let response: HfRowsResponse = resp
+            .json()
+            .await
+            .context("Failed to parse HuggingFace API response")?;
+
+        Ok(response)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_client_creation() {
+        let client = HuggingFaceClient::new();
+        assert!(client.is_ok());
+    }
+
+    #[test]
+    fn test_hf_rows_response_deserialize() {
+        let json = r#"{
+            "rows": [
+                {
+                    "row": {
+                        "repo": "psf/requests",
+                        "instance_id": "psf__requests-1234",
+                        "base_commit": "abc123",
+                        "patch": "diff --git a/f.py b/f.py",
+                        "test_patch": "diff --git a/t.py b/t.py",
+                        "problem_statement": "Fix bug"
+                    }
+                }
+            ],
+            "num_rows_total": 2294
+        }"#;
+        let resp: HfRowsResponse = serde_json::from_str(json).expect("should parse");
+        assert_eq!(resp.rows.len(), 1);
+        assert_eq!(resp.num_rows_total, Some(2294));
+        assert_eq!(resp.rows[0].row.repo, "psf/requests");
+    }
+
+    #[test]
+    fn test_hf_rows_response_empty() {
+        let json = r#"{"rows": []}"#;
+        let resp: HfRowsResponse = serde_json::from_str(json).expect("should parse");
+        assert!(resp.rows.is_empty());
+        assert!(resp.num_rows_total.is_none());
+    }
+}

src/swe_forge/client.rs

@@ -0,0 +1,4 @@
+#[allow(dead_code)]
+pub mod client;
+#[allow(dead_code)]
+pub mod types;

src/swe_forge/mod.rs

@@ -0,0 +1,138 @@
+use serde::{Deserialize, Serialize};
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DatasetEntry {
+    pub repo: String,
+    pub instance_id: String,
+    pub base_commit: String,
+    pub patch: String,
+    pub test_patch: String,
+    pub problem_statement: String,
+    #[serde(default)]
+    pub hints_text: Option<String>,
+    #[serde(default)]
+    pub created_at: Option<String>,
+    #[serde(default)]
+    pub version: Option<String>,
+    #[serde(default)]
+    pub fail_to_pass: Option<String>,
+    #[serde(default)]
+    pub pass_to_pass: Option<String>,
+    #[serde(default)]
+    pub environment_setup_commit: Option<String>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct HuggingFaceDataset {
+    pub dataset_id: String,
+    pub split: String,
+    pub entries: Vec<DatasetEntry>,
+    pub total_count: usize,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct DatasetConfig {
+    pub dataset_id: String,
+    #[serde(default = "default_split")]
+    pub split: String,
+    #[serde(default = "default_limit")]
+    pub limit: usize,
+    #[serde(default)]
+    pub offset: usize,
+}
+
+fn default_split() -> String {
+    "test".to_string()
+}
+
+fn default_limit() -> usize {
+    100
+}
+
+impl Default for DatasetConfig {
+    fn default() -> Self {
+        Self {
+            dataset_id: "CortexLM/swe-forge".to_string(),
+            split: default_split(),
+            limit: default_limit(),
+            offset: 0,
+        }
+    }
+}
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct HfRowsResponse {
+    pub rows: Vec<HfRowWrapper>,
+    #[serde(default)]
+    pub num_rows_total: Option<usize>,
+}
+
+#[derive(Debug, Deserialize)]
+pub(crate) struct HfRowWrapper {
+    pub row: DatasetEntry,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_dataset_config_default() {
+        let config = DatasetConfig::default();
+        assert_eq!(config.dataset_id, "CortexLM/swe-forge");
+        assert_eq!(config.split, "test");
+        assert_eq!(config.limit, 100);
+        assert_eq!(config.offset, 0);
+    }
+
+    #[test]
+    fn test_dataset_entry_deserialize() {
+        let json = r#"{
+            "repo": "psf/requests",
+            "instance_id": "psf__requests-1234",
+            "base_commit": "abc123def456",
+            "patch": "diff --git a/file.py b/file.py",
+            "test_patch": "diff --git a/test_file.py b/test_file.py",
+            "problem_statement": "Fix the bug in requests library"
+        }"#;
+        let entry: DatasetEntry = serde_json::from_str(json).expect("should deserialize");
+        assert_eq!(entry.repo, "psf/requests");
+        assert_eq!(entry.instance_id, "psf__requests-1234");
+        assert!(entry.hints_text.is_none());
+        assert!(entry.version.is_none());
+    }
+
+    #[test]
+    fn test_dataset_entry_deserialize_with_optional_fields() {
+        let json = r#"{
+            "repo": "psf/requests",
+            "instance_id": "psf__requests-1234",
+            "base_commit": "abc123def456",
+            "patch": "diff --git a/file.py b/file.py",
+            "test_patch": "diff --git a/test_file.py b/test_file.py",
+            "problem_statement": "Fix the bug",
+            "hints_text": "Check the encoding",
+            "version": "2.31.0",
+            "fail_to_pass": "[\"test_requests.py::test_encoding\"]",
+            "pass_to_pass": "[\"test_requests.py::test_basic\"]"
+        }"#;
+        let entry: DatasetEntry = serde_json::from_str(json).expect("should deserialize");
+        assert_eq!(entry.hints_text.as_deref(), Some("Check the encoding"));
+        assert_eq!(entry.version.as_deref(), Some("2.31.0"));
+        assert!(entry.fail_to_pass.is_some());
+    }
+
+    #[test]
+    fn test_huggingface_dataset_serialize_roundtrip() {
+        let dataset = HuggingFaceDataset {
+            dataset_id: "CortexLM/swe-forge".to_string(),
+            split: "test".to_string(),
+            entries: vec![],
+            total_count: 0,
+        };
+        let json = serde_json::to_string(&dataset).expect("should serialize");
+        let back: HuggingFaceDataset = serde_json::from_str(&json).expect("should deserialize");
+        assert_eq!(back.dataset_id, "CortexLM/swe-forge");
+        assert_eq!(back.total_count, 0);
+    }
+}