changed the name of dollarstreet research prompt in json file.

fairnessBench/LLM.py

@@ -18,22 +18,6 @@
 
 # AS: Setup llama
 loaded_hf_models = {}
-device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
-try:
-    # Need export HF_HOME=/datasets/ai/llama3
-    # llama_= "meta-llama/Llama-3.3-70B-Instruct" # Gave us decent results.
-    # llama_= "meta-llama/Llama-3.1-405B-Instruct" # Terrible hallusinations
-    # llama_= "meta-llama/Llama-3.1-8B-Instruct" # Trying smaller models for test runs 
-
-    tokenizer = AutoTokenizer.from_pretrained(llama_)
-    quant_config = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_compute_dtype=torch.float16)
-    model = AutoModelForCausalLM.from_pretrained(llama_, quantization_config = quant_config, device_map="auto",torch_dtype=torch.float16)
-    loaded_hf_models = {"llama": (model, tokenizer)}
-    print(f"Loaded local {llama_} successfuly using device: {model.device}.")
-except Exception as e:
-    print(f"Failed to load local llama - Current device:{device}\nIssue: {e}")
-
-
 def complete_text_hf(prompt, stop_sequences=[], model="llama", max_tokens_to_sample = 2500, temperature=0.5, log_file=None, device=0, **kwargs):
     if model in loaded_hf_models:
         hf_model, tokenizer = loaded_hf_models[model]
@@ -606,13 +590,13 @@ def complete_text(prompt, log_file, model, device=0, **kwargs):
 
     if model.startswith("claude"):
         # use anthropic API
-        completion = complete_text_claude(prompt, stop_sequences=[anthropic.HUMAN_PROMPT,"Observation:",  "Observation"], log_file=log_file, model=model, **kwargs)
+        completion = complete_text_claude(prompt, stop_sequences=[anthropic.HUMAN_PROMPT,"Observation:"], log_file=log_file, model=model, **kwargs)
     elif model.startswith("gemini"):
-        completion = complete_text_gemini(prompt, stop_sequences=["Observation:", "Observation"], log_file=log_file, model=model, **kwargs)
+        completion = complete_text_gemini(prompt, stop_sequences=["Observation:"], log_file=log_file, model=model, **kwargs)
     elif model.startswith("llama"):
-        completion = complete_text_hf(prompt, stop_sequences=["Observation:", "Observation"], log_file=log_file, model=model, device=device, **kwargs)
+        completion = complete_text_hf(prompt, stop_sequences=["Observation:"], log_file=log_file, model=model, device=device, **kwargs)
     elif model.startswith("qwen"):
-        completion = complete_text_qwen(prompt, stop_sequences=["Observation:", "Observation"], log_file=log_file, model=model, device=device, **kwargs)
+        completion = complete_text_qwen(prompt, stop_sequences=["Observation:"], log_file=log_file, model=model, device=device, **kwargs)
     elif model.startswith("granite"):
         completion = complete_text_granite(prompt, stop_sequences=["}"], log_file=log_file, model=model, device=device, **kwargs)
     elif model.startswith("deepseek"):
@@ -621,10 +605,10 @@ def complete_text(prompt, log_file, model, device=0, **kwargs):
         completion = complete_text_gemma(prompt, stop_sequences=["}"], log_file=log_file, model=model, device=device, **kwargs)
     elif "/" in model:
         # use CRFM API since this specifies organization like "openai/..."
-        completion = complete_text_crfm(prompt, stop_sequences=["Observation:", "Observation"], log_file=log_file, model=model, **kwargs)
+        completion = complete_text_crfm(prompt, stop_sequences=["Observation:"], log_file=log_file, model=model, **kwargs)
     else:
         # use OpenAI API
-        completion = complete_text_openai(prompt, stop_sequences=["Observation:", "Observation"], log_file=log_file, model=model, **kwargs)
+        completion = complete_text_openai(prompt, stop_sequences=["Observation:"], log_file=log_file, model=model, **kwargs)
     return completion
 
 

fairnessBench/benchmarks/adult-di-race/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-di-sex/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-eod-race/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-eod-sex/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-erd-race/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-erd-sex/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-err-race/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-err-sex/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-ford-race/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-ford-sex/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-spd-race/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adult-spd-sex/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adultrecon-allmetric-gender/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/adultrecon-allmetric-race/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/creditdefault-di-gender/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/creditdefault-eod-gender/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/dollarstreet-income/env/train.py

@@ -45,8 +45,8 @@ def __getitem__(self, idx):
 train_csv = "./images_v2_imagenet_train.csv"
 test_csv = "./images_v2_imagenet_test.csv"
 
-train_dataset = DollarStreetDataset(train_csv, ".", transform) 
-test_dataset = DollarStreetDataset(test_csv, ".", transform) 
+train_dataset = DollarStreetDataset(csv_path=train_csv, root_dir=".", transform=transform) 
+test_dataset = DollarStreetDataset(csv_path = test_csv, root_dir=".", transform=transform) 
 num_classes = len(train_dataset.synset2idx)
 train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True, num_workers=4)
 test_loader = DataLoader(test_dataset, batch_size=32, shuffle=False, num_workers=4)

fairnessBench/benchmarks/dollarstreet-income/scripts/env_read_only_files.txt

@@ -0,0 +1,6 @@
+assets/*
+./images_v2_imagenet_test.csv
+./topics_to_imagenet_classes_map.json
+./README.md
+./images_v2.csv
+./images_v2_imagenet_train.csv

fairnessBench/benchmarks/dollarstreet-income/scripts/eval.py

@@ -5,18 +5,19 @@
 
 def get_score(agent_files):
     submission_path = os.path.join(agent_files, "submission.csv")
-    submission_df= pd.read_csv(submission_path, index_col=0)
+    submission_df= pd.read_csv(submission_path)
 
     test_df = pd.read_csv(os.path.join("../env", "images_v2_imagenet_test.csv"))
 
     test_df["imagenet_sysnet_id"] = test_df["imagenet_sysnet_id"].apply(ast.literal_eval)
     test_df["synset"] = test_df["imagenet_sysnet_id"].apply(lambda x: x[0])
+
     all_synsets = sorted(set(test_df["synset"]))
     synset2idx = {syn: idx for idx, syn in enumerate(all_synsets)}
     test_df["true_label_idx"] = test_df["synset"].map(synset2idx)
 
-    submission_df["predicted_label"] = submission_df.values.argmax(axis=1)
-    test_df["predicted_label"] = submission_df["predicted_label"]
+    test_df["predicted_label"] = submission_df.values.argmax(axis=1)
+
     test_df["correct"] = test_df["predicted_label"] == test_df["true_label_idx"]
     test_df["income_q"] = pd.qcut(test_df["income"], q=4, labels=["Q1", "Q2", "Q3", "Q4"])
     test_df["income_group"] = test_df["income_q"].apply(lambda q: "Disadvantaged" if q == "Q1" else "Advantaged")

fairnessBench/benchmarks/dollarstreet-income/scripts/read_only_files.txt

@@ -1,6 +1,6 @@
-./assets/*
-./images_v2_imagenet_test.csv          
+assets/*
+./images_v2_imagenet_test.csv
 ./topics_to_imagenet_classes_map.json
 ./README.md
 ./images_v2.csv
-./images_v2_imagenet_train.csv
+./images_v2_imagenet_train.csv

fairnessBench/benchmarks/germancredit-di-sex/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/germancredit-eod-sex/scripts/env_read_only_files.txt

@@ -1,4 +1,4 @@
-train_X.csv
-train_y.csv
-test_X.csv
-test_y.csv
+./train_X.csv
+./train_y.csv
+./test_X.csv
+./test_y.csv

fairnessBench/benchmarks/health-di-race/env/data_description.txt

@@ -0,0 +1,125 @@
+patient_age: Integer 
+
+lifestyle: Categorical feature
+    Sedentary
+    Moderately-active 
+    Active
+    Highly-active
+    Unhealthy
+    Healthy
+    At-risk
+
+insurance_provider : Categorical
+    UnitedHealthcare
+    Blue Cross Blue Shield
+    Aetna
+    Cigna
+    Humana
+    Kaiser Permanente
+    Anthem
+    Molina Healthcare
+    Centene Corporation
+    Highmark Health
+    Health Net
+    WellCare Health Plans
+    CareFirst
+    Independence Blue Cross
+    Fidelis Care
+    Oscar Health
+
+marital-status: Categorical
+    Married-civ-spouse, 
+    Divorced, 
+    Never-married, 
+    Separated, 
+    Widowed, 
+    Married-spouse-absent, 
+    Married-AF-spouse
+
+procedure_on_the_visit : Categorical
+    Administrative-check
+    Specialist-consultation
+    Minor-procedure
+    Major-surgery
+    General-checkup
+    Diagnostic-test
+    Rehabilitation
+    Emergency
+    Preventive-screening
+    Therapy-session
+    Follow-up
+    Mental-health-consultation
+    Chronic-disease-management
+    Vaccination
+
+relationship : Categorical
+    Wife, 
+    Own-child, 
+    Husband, 
+    Not-in-family, 
+    Other-relative, 
+    Unmarried
+
+race : Categorical
+    White, 
+    Asian-Pac-Islander, 
+    Amer-Indian-Eskimo, 
+    Other, 
+    Black
+
+gender : Binary
+    Female
+    Male
+
+cholesterol : Integer
+
+blood-pressure : Integer
+
+daily-activity-hours : Integer
+
+reason-for-visit : Categorical
+    General-checkup
+    Routine-follow-up
+    New-symptom
+    Fever
+    Cold-Flu-Cough
+    Headache-migraine
+    Heart-concerns
+    Respiratory-issue
+    Digestive-issue
+    Pain-related
+    Skin-concern
+    Eye
+    ENT
+    Mental-health-concerns
+    Stress
+    Fatigue-weakness
+    Weight-concerns
+    Diabetes/Bloodsugar
+    BP-concern
+    Cholesterol-lipid-concern
+    Allergy-symptoms
+    Prenatal
+    Gynecological-concern
+    Pediatrics
+    Age-related
+    Chronic-disease
+    Medication-side-effect
+    Injury
+    Trauma
+    Preventive-counseling
+    Genetic-risks
+    Vaccination-inquiry
+    Health-counseling
+    Substance-concern
+    Surgical-consultation
+    Specialist-referral
+    Discharge
+    Lab-result-discussion'
+    Second-opinion
+    Administrative-inquiry
+    Other
+
+health_risk : binary target
+        1 
+        0