nanoGPT/out-eval-multi.json at master · a5c-ai/nanoGPT · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
{
  "results": [
    {
      "checkpoint": "out-grpo-multi/ckpt.pt",
      "num_domains": 4,
      "total_problems": 52,
      "overall_pass@1": {
        "mean": 0.3462,
        "ci_lower": 0.2115,
        "ci_upper": 0.462
      },
      "per_domain": {
        "logic": {
          "domain": "logic",
          "num_problems": 13,
          "k": 4,
          "pass@1": {
            "mean": 0.5385,
            "ci_lower": 0.2308,
            "ci_upper": 0.7692
          },
          "pass@4": {
            "mean": 0.6154,
            "ci_lower": 0.3077,
            "ci_upper": 0.8462
          },
          "majority@4": {
            "mean": 0.6154,
            "ci_lower": 0.3077,
            "ci_upper": 0.8462
          },
          "format_compliance": {
            "mean": 1.0,
            "ci_lower": 1.0,
            "ci_upper": 1.0
          }
        },
        "math": {
          "domain": "math",
          "num_problems": 13,
          "k": 4,
          "pass@1": {
            "mean": 0.0769,
            "ci_lower": 0.0,
            "ci_upper": 0.2308
          },
          "pass@4": {
            "mean": 0.0769,
            "ci_lower": 0.0,
            "ci_upper": 0.2308
          },
          "majority@4": {
            "mean": 0.0769,
            "ci_lower": 0.0,
            "ci_upper": 0.2308
          },
          "format_compliance": {
            "mean": 0.9231,
            "ci_lower": 0.8654,
            "ci_upper": 0.9808
          }
        },
        "physical": {
          "domain": "physical",
          "num_problems": 13,
          "k": 4,
          "pass@1": {
            "mean": 0.5385,
            "ci_lower": 0.3077,
            "ci_upper": 0.8462
          },
          "pass@4": {
            "mean": 1.0,
            "ci_lower": 1.0,
            "ci_upper": 1.0
          },
          "majority@4": {
            "mean": 0.5385,
            "ci_lower": 0.3077,
            "ci_upper": 0.7692
          },
          "format_compliance": {
            "mean": 1.0,
            "ci_lower": 1.0,
            "ci_upper": 1.0
          }
        },
        "science": {
          "domain": "science",
          "num_problems": 13,
          "k": 4,
          "pass@1": {
            "mean": 0.2308,
            "ci_lower": 0.0,
            "ci_upper": 0.4615
          },
          "pass@4": {
            "mean": 0.5385,
            "ci_lower": 0.2308,
            "ci_upper": 0.7712
          },
          "majority@4": {
            "mean": 0.2308,
            "ci_lower": 0.0,
            "ci_upper": 0.4615
          },
          "format_compliance": {
            "mean": 1.0,
            "ci_lower": 1.0,
            "ci_upper": 1.0
          }
        }
      },
      "benchmark": "multi_domain"
    }
  ]
}