{
  "snapshotUTC": "2026-10-15T06:55:03.592078+00:00",
  "source": "https://github.com/DennisKoshta/upbringing/blob/main/site/data/rlvr/posthoc.json",
  "sourceSHA256": "77d6bd99875bba6173a14cf0ca47f642a0c5536e77bb1bbea28f217ae49a9562",
  "scope": "256 puzzles excluded from training updates but monitored during development; 32 stochastic samples per puzzle plus greedy evaluation. Puzzle-bootstrap intervals. No transfer evaluation or independent confirmation set.",
  "results": {
    "checkpoints": [
      {
        "label": "base",
        "step": 0,
        "n_puzzles": 256,
        "samples_per_puzzle": 32,
        "settings": {
          "temperature": 1.0,
          "top_p": 1.0,
          "max_tokens": 1024,
          "stop": "</answer>",
          "seed": 0
        },
        "per_attempt_accuracy": 0.01318359375,
        "ci95": [
          0.01025390625,
          0.016357421875
        ],
        "greedy_accuracy": 0.0078125,
        "pass_at_k": {
          "1": 0.013183593749999998,
          "2": 0.025918283770161286,
          "4": 0.050112646864571744,
          "8": 0.09386166921579533,
          "16": 0.1659135150166852,
          "32": 0.265625
        },
        "mean_tokens": 221.2060546875,
        "by_size": {
          "3": {
            "n_puzzles": 119,
            "per_attempt_accuracy": 0.026785714285714284,
            "ci95": [
              0.02100840336134454,
              0.032563025210084036
            ],
            "greedy_accuracy": 0.01680672268907563,
            "pass_at_k": {
              "1": 0.02678571428571428,
              "2": 0.05260571970723772,
              "4": 0.10150283695235603,
              "8": 0.18931585982557653,
              "16": 0.3317131079350539,
              "32": 0.5210084033613446
            },
            "mean_tokens": 213.57904411764707
          },
          "4": {
            "n_puzzles": 137,
            "per_attempt_accuracy": 0.0013686131386861315,
            "ci95": [
              0.0004562043795620438,
              0.002509124087591241
            ],
            "greedy_accuracy": 0.0,
            "pass_at_k": {
              "1": 0.0013686131386861315,
              "2": 0.002737226277372263,
              "4": 0.005474452554744526,
              "8": 0.010948905109489052,
              "16": 0.021897810218978103,
              "32": 0.043795620437956206
            },
            "mean_tokens": 227.83097627737226
          }
        }
      },
      {
        "label": "rlvr-step-00050",
        "step": 50,
        "n_puzzles": 256,
        "samples_per_puzzle": 32,
        "settings": {
          "temperature": 1.0,
          "top_p": 1.0,
          "max_tokens": 1024,
          "stop": "</answer>",
          "seed": 0
        },
        "per_attempt_accuracy": 0.06494140625,
        "ci95": [
          0.0552978515625,
          0.0751953125
        ],
        "greedy_accuracy": 0.13671875,
        "pass_at_k": {
          "1": 0.06494140624999997,
          "2": 0.12066847278225803,
          "4": 0.2102305513070078,
          "8": 0.3297923789668958,
          "16": 0.45122969219355374,
          "32": 0.55078125
        },
        "mean_tokens": 109.930908203125,
        "by_size": {
          "3": {
            "n_puzzles": 119,
            "per_attempt_accuracy": 0.12289915966386554,
            "ci95": [
              0.10714285714285714,
              0.13918067226890757
            ],
            "greedy_accuracy": 0.17647058823529413,
            "pass_at_k": {
              "1": 0.12289915966386548,
              "2": 0.2265519110870154,
              "4": 0.38841593367046484,
              "8": 0.5901336430296712,
              "16": 0.7613436617735228,
              "32": 0.8571428571428571
            },
            "mean_tokens": 105.51759453781513
          },
          "4": {
            "n_puzzles": 137,
            "per_attempt_accuracy": 0.014598540145985401,
            "ci95": [
              0.010264598540145985,
              0.01893248175182482
            ],
            "greedy_accuracy": 0.10218978102189781,
            "pass_at_k": {
              "1": 0.0145985401459854,
              "2": 0.028696727101483396,
              "4": 0.05545638706429691,
              "8": 0.10365653645981343,
              "16": 0.18186062372628142,
              "32": 0.2846715328467153
            },
            "mean_tokens": 113.7643704379562
          }
        }
      },
      {
        "label": "rlvr-step-00100",
        "step": 100,
        "n_puzzles": 256,
        "samples_per_puzzle": 32,
        "settings": {
          "temperature": 1.0,
          "top_p": 1.0,
          "max_tokens": 1024,
          "stop": "</answer>",
          "seed": 0
        },
        "per_attempt_accuracy": 0.22412109375,
        "ci95": [
          0.1920166015625,
          0.256591796875
        ],
        "greedy_accuracy": 0.2890625,
        "pass_at_k": {
          "1": 0.22412109374999994,
          "2": 0.3303931451612903,
          "4": 0.43130529146968855,
          "8": 0.5045905086997186,
          "16": 0.5436567655778747,
          "32": 0.55859375
        },
        "mean_tokens": 145.4105224609375,
        "by_size": {
          "3": {
            "n_puzzles": 119,
            "per_attempt_accuracy": 0.3807773109243697,
            "ci95": [
              0.3269432773109244,
              0.4372373949579832
            ],
            "greedy_accuracy": 0.5042016806722689,
            "pass_at_k": {
              "1": 0.38077731092436967,
              "2": 0.5362903225806451,
              "4": 0.6601660575242333,
              "8": 0.7334886840278452,
              "16": 0.7686770323599443,
              "32": 0.7815126050420168
            },
            "mean_tokens": 113.28098739495799
          },
          "4": {
            "n_puzzles": 137,
            "per_attempt_accuracy": 0.08804744525547445,
            "ci95": [
              0.06546532846715329,
              0.11268248175182481
            ],
            "greedy_accuracy": 0.10218978102189781,
            "pass_at_k": {
              "1": 0.08804744525547442,
              "2": 0.15154815163644922,
              "4": 0.23251382314493801,
              "8": 0.30576654618842614,
              "16": 0.3482012053803105,
              "32": 0.36496350364963503
            },
            "mean_tokens": 173.31865875912408
          }
        }
      },
      {
        "label": "rlvr-step-00200",
        "step": 200,
        "n_puzzles": 256,
        "samples_per_puzzle": 32,
        "settings": {
          "temperature": 1.0,
          "top_p": 1.0,
          "max_tokens": 1024,
          "stop": "</answer>",
          "seed": 0
        },
        "per_attempt_accuracy": 0.47705078125,
        "ci95": [
          0.4229736328125,
          0.5335693359375
        ],
        "greedy_accuracy": 0.5078125,
        "pass_at_k": {
          "1": 0.47705078125,
          "2": 0.5232957409274194,
          "4": 0.5496262122844827,
          "8": 0.5602256836745244,
          "16": 0.5623876991676189,
          "32": 0.5625
        },
        "mean_tokens": 274.845947265625,
        "by_size": {
          "3": {
            "n_puzzles": 119,
            "per_attempt_accuracy": 0.7439600840336135,
            "ci95": [
              0.6680672268907563,
              0.8161764705882353
            ],
            "greedy_accuracy": 0.7647058823529411,
            "pass_at_k": {
              "1": 0.7439600840336135,
              "2": 0.7676199512062889,
              "4": 0.7721076639777157,
              "8": 0.77304872732014,
              "16": 0.7731092301504439,
              "32": 0.773109243697479
            },
            "mean_tokens": 185.7610294117647
          },
          "4": {
            "n_puzzles": 137,
            "per_attempt_accuracy": 0.24520985401459855,
            "ci95": [
              0.18613138686131386,
              0.3074817518248175
            ],
            "greedy_accuracy": 0.2846715328467153,
            "pass_at_k": {
              "1": 0.24520985401459855,
              "2": 0.31107252178008005,
              "4": 0.3563759002297768,
              "8": 0.3753647917487707,
              "16": 0.3793522087518803,
              "32": 0.3795620437956204
            },
            "mean_tokens": 352.22627737226276
          }
        }
      },
      {
        "label": "rlvr-step-00300",
        "step": 300,
        "n_puzzles": 256,
        "samples_per_puzzle": 32,
        "settings": {
          "temperature": 1.0,
          "top_p": 1.0,
          "max_tokens": 1024,
          "stop": "</answer>",
          "seed": 0
        },
        "per_attempt_accuracy": 0.5439453125,
        "ci95": [
          0.48388671875,
          0.60400390625
        ],
        "greedy_accuracy": 0.546875,
        "pass_at_k": {
          "1": 0.5439453125,
          "2": 0.5551915322580645,
          "4": 0.5604160873192436,
          "8": 0.5623017569676183,
          "16": 0.5624994816365162,
          "32": 0.5625
        },
        "mean_tokens": 295.9779052734375,
        "by_size": {
          "3": {
            "n_puzzles": 119,
            "per_attempt_accuracy": 0.7694327731092437,
            "ci95": [
              0.6940651260504201,
              0.8445378151260504
            ],
            "greedy_accuracy": 0.773109243697479,
            "pass_at_k": {
              "1": 0.7694327731092437,
              "2": 0.7726179181349959,
              "4": 0.7730928856525925,
              "8": 0.7731092428985513,
              "16": 0.773109243697479,
              "32": 0.773109243697479
            },
            "mean_tokens": 195.85582983193277
          },
          "4": {
            "n_puzzles": 137,
            "per_attempt_accuracy": 0.34808394160583944,
            "ci95": [
              0.2728102189781022,
              0.42700729927007297
            ],
            "greedy_accuracy": 0.35036496350364965,
            "pass_at_k": {
              "1": 0.34808394160583944,
              "2": 0.36633211678832117,
              "4": 0.375682225993196,
              "8": 0.37919160495461823,
              "16": 0.379561075174804,
              "32": 0.3795620437956204
            },
            "mean_tokens": 382.94525547445255
          }
        }
      },
      {
        "label": "rlvr-step-00400",
        "step": 400,
        "n_puzzles": 256,
        "samples_per_puzzle": 32,
        "settings": {
          "temperature": 1.0,
          "top_p": 1.0,
          "max_tokens": 1024,
          "stop": "</answer>",
          "seed": 0
        },
        "per_attempt_accuracy": 0.5521240234375,
        "ci95": [
          0.49365234375,
          0.611572265625
        ],
        "greedy_accuracy": 0.55078125,
        "pass_at_k": {
          "1": 0.5521240234375,
          "2": 0.559782951108871,
          "4": 0.562150762131535,
          "8": 0.5624908771349695,
          "16": 0.5624999998895218,
          "32": 0.5625
        },
        "mean_tokens": 345.510009765625,
        "by_size": {
          "3": {
            "n_puzzles": 119,
            "per_attempt_accuracy": 0.7707457983193278,
            "ci95": [
              0.6948529411764706,
              0.8463760504201681
            ],
            "greedy_accuracy": 0.773109243697479,
            "pass_at_k": {
              "1": 0.7707457983193278,
              "2": 0.7727534562211982,
              "4": 0.7731010646750357,
              "8": 0.773109243697479,
              "16": 0.773109243697479,
              "32": 0.773109243697479
            },
            "mean_tokens": 223.31617647058823
          },
          "4": {
            "n_puzzles": 137,
            "per_attempt_accuracy": 0.36222627737226276,
            "ci95": [
              0.28489963503649635,
              0.4427463503649635
            ],
            "greedy_accuracy": 0.35766423357664234,
            "pass_at_k": {
              "1": 0.36222627737226276,
              "2": 0.3747939722156816,
              "4": 0.3789165577324359,
              "8": 0.37954499669016206,
              "16": 0.3795620435891795,
              "32": 0.3795620437956204
            },
            "mean_tokens": 451.6491788321168
          }
        }
      },
      {
        "label": "rlvr-step-00600",
        "step": 600,
        "n_puzzles": 256,
        "samples_per_puzzle": 32,
        "settings": {
          "temperature": 1.0,
          "top_p": 1.0,
          "max_tokens": 1024,
          "stop": "</answer>",
          "seed": 0
        },
        "per_attempt_accuracy": 0.56005859375,
        "ci95": [
          0.5003662109375,
          0.6199951171875
        ],
        "greedy_accuracy": 0.55859375,
        "pass_at_k": {
          "1": 0.56005859375,
          "2": 0.5620983492943549,
          "4": 0.5624771881952169,
          "8": 0.5624999832880551,
          "16": 0.5625,
          "32": 0.5625
        },
        "mean_tokens": 350.00439453125,
        "by_size": {
          "3": {
            "n_puzzles": 119,
            "per_attempt_accuracy": 0.7699579831932774,
            "ci95": [
              0.6938025210084033,
              0.8455882352941176
            ],
            "greedy_accuracy": 0.7647058823529411,
            "pass_at_k": {
              "1": 0.7699579831932774,
              "2": 0.7723298997018162,
              "4": 0.7730601695628195,
              "8": 0.773109207745732,
              "16": 0.773109243697479,
              "32": 0.773109243697479
            },
            "mean_tokens": 222.65099789915968
          },
          "4": {
            "n_puzzles": 137,
            "per_attempt_accuracy": 0.3777372262773723,
            "ci95": [
              0.2979014598540146,
              0.45825729927007297
            ],
            "greedy_accuracy": 0.3795620437956204,
            "pass_at_k": {
              "1": 0.3777372262773723,
              "2": 0.3794884624440782,
              "4": 0.3795620437956204,
              "8": 0.3795620437956204,
              "16": 0.3795620437956204,
              "32": 0.3795620437956204
            },
            "mean_tokens": 460.6252281021898
          }
        }
      },
      {
        "label": "rlvr-step-00800",
        "step": 800,
        "n_puzzles": 256,
        "samples_per_puzzle": 32,
        "settings": {
          "temperature": 1.0,
          "top_p": 1.0,
          "max_tokens": 1024,
          "stop": "</answer>",
          "seed": 0
        },
        "per_attempt_accuracy": 0.5570068359375,
        "ci95": [
          0.49755859375,
          0.6168212890625
        ],
        "greedy_accuracy": 0.5546875,
        "pass_at_k": {
          "1": 0.5570068359375,
          "2": 0.5606728830645161,
          "4": 0.562163145682703,
          "8": 0.5624904938743666,
          "16": 0.5624999998895218,
          "32": 0.5625
        },
        "mean_tokens": 326.222412109375,
        "by_size": {
          "3": {
            "n_puzzles": 119,
            "per_attempt_accuracy": 0.7694327731092437,
            "ci95": [
              0.6919642857142857,
              0.845063025210084
            ],
            "greedy_accuracy": 0.7647058823529411,
            "pass_at_k": {
              "1": 0.7694327731092437,
              "2": 0.7717877473570073,
              "4": 0.7729421579532814,
              "8": 0.7731082154775147,
              "16": 0.773109243697479,
              "32": 0.773109243697479
            },
            "mean_tokens": 204.70903361344537
          },
          "4": {
            "n_puzzles": 137,
            "per_attempt_accuracy": 0.37249087591240876,
            "ci95": [
              0.29447992700729925,
              0.4532390510948905
            ],
            "greedy_accuracy": 0.3722627737226277,
            "pass_at_k": {
              "1": 0.37249087591240876,
              "2": 0.3772957381681187,
              "4": 0.37907772626519326,
              "8": 0.3795451736497343,
              "16": 0.3795620435891795,
              "32": 0.3795620437956204
            },
            "mean_tokens": 431.7705291970803
          }
        }
      },
      {
        "label": "rlvr-step-00900",
        "step": 900,
        "n_puzzles": 256,
        "samples_per_puzzle": 32,
        "settings": {
          "temperature": 1.0,
          "top_p": 1.0,
          "max_tokens": 1024,
          "stop": "</answer>",
          "seed": 0
        },
        "per_attempt_accuracy": 0.556884765625,
        "ci95": [
          0.496826171875,
          0.61669921875
        ],
        "greedy_accuracy": 0.55859375,
        "pass_at_k": {
          "1": 0.556884765625,
          "2": 0.5592710433467742,
          "4": 0.5607982393631813,
          "8": 0.5619193297336547,
          "16": 0.5624654805493758,
          "32": 0.5625
        },
        "mean_tokens": 364.1490478515625,
        "by_size": {
          "3": {
            "n_puzzles": 119,
            "per_attempt_accuracy": 0.7654936974789915,
            "ci95": [
              0.6885504201680672,
              0.8411239495798319
            ],
            "greedy_accuracy": 0.7647058823529411,
            "pass_at_k": {
              "1": 0.7654936974789915,
              "2": 0.7675860666847384,
              "4": 0.7696156326824389,
              "8": 0.7718610989074903,
              "16": 0.7730349833667245,
              "32": 0.773109243697479
            },
            "mean_tokens": 226.7636554621849
          },
          "4": {
            "n_puzzles": 137,
            "per_attempt_accuracy": 0.37568430656934304,
            "ci95": [
              0.2953923357664234,
              0.458029197080292
            ],
            "greedy_accuracy": 0.3795620437956204,
            "pass_at_k": {
              "1": 0.37568430656934304,
              "2": 0.3783258770897104,
              "4": 0.3794167079398846,
              "8": 0.37956115067025,
              "16": 0.3795620437956204,
              "32": 0.3795620437956204
            },
            "mean_tokens": 483.48380474452557
          }
        }
      }
    ]
  }
}