{ "modelVersion": "polagg-v1", "cycles": [ 2020, 2022, 2024 ], "horizonDays": 21, "trainingRows": 142, "schemes": { "by_cycle": { "folds": [ { "model": "kalman", "fold": "holdout_2020", "n_test": 31, "mae_pt": 12.534056478302814, "rmse_pt": 18.22146681464235, "brier": 0.13561428704759276, "log_loss": 0.4814686830288149, "ece": 0.13262306536448143 }, { "model": "kalman", "fold": "holdout_2022", "n_test": 68, "mae_pt": 10.560225712481648, "rmse_pt": 16.745222402760806, "brier": 0.06554084200377677, "log_loss": 0.19027844794912968, "ece": 0.06811462193944294 }, { "model": "kalman", "fold": "holdout_2024", "n_test": 43, "mae_pt": 8.376879212285063, "rmse_pt": 13.389073188867298, "brier": 0.0607383575392098, "log_loss": 0.18800209823320496, "ece": 0.041753276004787676 }, { "model": "qgbt", "fold": "holdout_2020", "n_test": 31, "mae_pt": 6.6326684318117595, "rmse_pt": 9.357259981733609, "brier": 0.07591739373084758, "log_loss": 0.2737028544144438, "ece": 0.07144469813976934 }, { "model": "qgbt", "fold": "holdout_2022", "n_test": 68, "mae_pt": 7.450331928353106, "rmse_pt": 11.16889102589608, "brier": 0.10932309255487764, "log_loss": 0.47169770632021735, "ece": 0.12172215707073823 }, { "model": "qgbt", "fold": "holdout_2024", "n_test": 43, "mae_pt": 7.757484764874477, "rmse_pt": 15.094195032222933, "brier": 0.056049008924475, "log_loss": 0.17292663308735826, "ece": 0.0702943204858904 }, { "model": "ridge", "fold": "holdout_2020", "n_test": 31, "mae_pt": 7.356983896799694, "rmse_pt": 10.444374127141998, "brier": 0.08556865817715055, "log_loss": 0.2685849366049205, "ece": 0.09259850200319839 }, { "model": "ridge", "fold": "holdout_2022", "n_test": 68, "mae_pt": 8.226826212756283, "rmse_pt": 12.110715221055386, "brier": 0.08880551104716904, "log_loss": 0.2659389256267118, "ece": 0.10001344616550335 }, { "model": "ridge", "fold": "holdout_2024", "n_test": 43, "mae_pt": 8.351399077573802, "rmse_pt": 13.646994884612718, "brier": 0.0698399451971671, "log_loss": 0.30599040246838516, "ece": 0.07244593586342825 }, { "model": "rf", "fold": "holdout_2020", "n_test": 31, "mae_pt": 6.685016513056834, "rmse_pt": 9.172941309036549, "brier": 0.05915385135365427, "log_loss": 0.1952086286684403, "ece": 0.07666313796592188 }, { "model": "rf", "fold": "holdout_2022", "n_test": 68, "mae_pt": 6.8475761554621855, "rmse_pt": 10.875151742202924, "brier": 0.0751660189639123, "log_loss": 0.2241613815574301, "ece": 0.08497176770480726 }, { "model": "rf", "fold": "holdout_2024", "n_test": 43, "mae_pt": 7.53440088593577, "rmse_pt": 13.877535374628971, "brier": 0.055804557712961445, "log_loss": 0.18214757498266387, "ece": 0.05745362891416535 }, { "model": "pvi", "fold": "holdout_2020", "n_test": 31, "mae_pt": 11.452580645161289, "rmse_pt": 14.445387141832489, "brier": 0.08106819440665902, "log_loss": 0.2824754869775721, "ece": 0.18442191685684445 }, { "model": "pvi", "fold": "holdout_2022", "n_test": 68, "mae_pt": 12.926764705882354, "rmse_pt": 16.4724569580647, "brier": 0.10480484299863678, "log_loss": 0.348112593887532, "ece": 0.12604322171823726 }, { "model": "pvi", "fold": "holdout_2024", "n_test": 43, "mae_pt": 12.504651162790699, "rmse_pt": 17.006820109372537, "brier": 0.1204085413006478, "log_loss": 0.4059826260751307, "ece": 0.14279312644766176 }, { "model": "market", "fold": "holdout_2020", "n_test": 31, "mae_pt": 18.72193548387097, "rmse_pt": 22.75201869810039, "brier": 0.25, "log_loss": 0.6931471805599448, "ece": 0.08064516129032256 }, { "model": "market", "fold": "holdout_2022", "n_test": 68, "mae_pt": 19.584852941176475, "rmse_pt": 23.662633269255856, "brier": 0.25, "log_loss": 0.6931471805599454, "ece": 0.04411764705882354 }, { "model": "market", "fold": "holdout_2024", "n_test": 43, "mae_pt": 18.5, "rmse_pt": 22.483652459219716, "brier": 0.25, "log_loss": 0.6931471805599452, "ece": 0.08139534883720928 }, { "model": "ensemble", "fold": "holdout_2020", "n_test": 31, "mae_pt": 6.211418660630171, "rmse_pt": 9.129361969962618, "brier": 0.05352677686981406, "log_loss": 0.20665211336238581, "ece": 0.11442857117448009 }, { "model": "ensemble", "fold": "holdout_2022", "n_test": 68, "mae_pt": 6.8777590799612, "rmse_pt": 11.006830456596191, "brier": 0.07316137364454618, "log_loss": 0.23852616152530626, "ece": 0.08884179533762401 }, { "model": "ensemble", "fold": "holdout_2024", "n_test": 43, "mae_pt": 6.5618720668403805, "rmse_pt": 12.000632070048662, "brier": 0.06220490747805339, "log_loss": 0.24399943946203004, "ece": 0.060456159658909825 } ], "summary": { "kalman": { "mae_pt": 10.33, "rmse_pt": 16.051, "brier": 0.0794, "log_loss": 0.2532, "ece": 0.0742, "n_folds": 3, "n_test_total": 142 }, "qgbt": { "mae_pt": 7.365, "rmse_pt": 11.962, "brier": 0.0859, "log_loss": 0.338, "ece": 0.0952, "n_folds": 3, "n_test_total": 142 }, "ridge": { "mae_pt": 8.075, "rmse_pt": 12.212, "brier": 0.0824, "log_loss": 0.2786, "ece": 0.09, "n_folds": 3, "n_test_total": 142 }, "rf": { "mae_pt": 7.02, "rmse_pt": 11.413, "brier": 0.0658, "log_loss": 0.2051, "ece": 0.0748, "n_folds": 3, "n_test_total": 142 }, "pvi": { "mae_pt": 12.477, "rmse_pt": 16.192, "brier": 0.1043, "log_loss": 0.3513, "ece": 0.1439, "n_folds": 3, "n_test_total": 142 }, "market": { "mae_pt": 19.068, "rmse_pt": 23.107, "brier": 0.25, "log_loss": 0.6931, "ece": 0.0634, "n_folds": 3, "n_test_total": 142 }, "ensemble": { "mae_pt": 6.637, "rmse_pt": 10.898, "brier": 0.0656, "log_loss": 0.2332, "ece": 0.0858, "n_folds": 3, "n_test_total": 142 } } }, "leave_one_state_out": { "folds": [ { "model": "kalman", "fold": "loso_AR", "n_test": 3, "mae_pt": 19.869142183395837, "rmse_pt": 22.28892352881095, "brier": 0.0833333333337805, "log_loss": 0.23104944659481, "ece": 0.16666705274546317 }, { "model": "kalman", "fold": "loso_AZ", "n_test": 4, "mae_pt": 2.0001049484257525, "rmse_pt": 2.3157973135333427, "brier": 0.03477667190418128, "log_loss": 0.12714301510300666, "ece": 0.10375803639442088 }, { "model": "kalman", "fold": "loso_CO", "n_test": 3, "mae_pt": 4.703294192637034, "rmse_pt": 4.87060973115779, "brier": 2.7520642400425807e-05, "log_loss": 0.0032686166172631813, "ece": 0.0032547730898976335 }, { "model": "kalman", "fold": "loso_CT", "n_test": 3, "mae_pt": 0.5919293226385646, "rmse_pt": 0.8634811715494802, "brier": 3.7357935900122833e-10, "log_loss": 1.684654813963769e-05, "ece": 1.6846361346689598e-05 }, { "model": "kalman", "fold": "loso_DE", "n_test": 3, "mae_pt": 7.525559725173147, "rmse_pt": 8.279077205885148, "brier": 3.844430279350106e-11, "log_loss": 3.580457892318605e-06, "ece": 3.579812505649649e-06 }, { "model": "kalman", "fold": "loso_FL", "n_test": 3, "mae_pt": 8.71622822900382, "rmse_pt": 9.304963998321357, "brier": 2.588353513360088e-05, "log_loss": 0.0035851657116673908, "ece": 0.0035721510246567076 }, { "model": "kalman", "fold": "loso_HI", "n_test": 3, "mae_pt": 34.73333333333334, "rmse_pt": 35.60437894416921, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "kalman", "fold": "loso_IA", "n_test": 3, "mae_pt": 2.537569439510905, "rmse_pt": 3.4635916512204723, "brier": 0.049616985842099635, "log_loss": 0.16248712697583548, "ece": 0.12860637485061238 }, { "model": "kalman", "fold": "loso_IL", "n_test": 3, "mae_pt": 2.917436523649267, "rmse_pt": 3.810819732772249, "brier": 0.00012004963737041799, "log_loss": 0.006386676801074461, "ece": 0.0063258815891238385 }, { "model": "kalman", "fold": "loso_IN", "n_test": 3, "mae_pt": 7.798539877724493, "rmse_pt": 8.402036260799367, "brier": 6.814585277868029e-06, "log_loss": 0.0020850539032884283, "ece": 0.002081638833682379 }, { "model": "kalman", "fold": "loso_KS", "n_test": 3, "mae_pt": 4.889386487913485, "rmse_pt": 5.521598453467786, "brier": 0.05164718308382873, "log_loss": 0.1871318260886927, "ece": 0.15205191610698213 }, { "model": "kalman", "fold": "loso_MA", "n_test": 3, "mae_pt": 17.520811365801645, "rmse_pt": 24.79755305794315, "brier": 0.33146989145906025, "log_loss": 1.9594881349102637, "ece": 0.33240030658931613 }, { "model": "kalman", "fold": "loso_MD", "n_test": 3, "mae_pt": 13.21380373989894, "rmse_pt": 15.723646832645166, "brier": 0.010828871529514915, "log_loss": 0.07256077474891669, "ece": 0.06640669276613032 }, { "model": "kalman", "fold": "loso_ME", "n_test": 3, "mae_pt": 5.200367373613692, "rmse_pt": 6.401084274395864, "brier": 0.16441409065179388, "log_loss": 0.40429381260308506, "ece": 0.23449517693079594 }, { "model": "kalman", "fold": "loso_MI", "n_test": 3, "mae_pt": 2.019924768093889, "rmse_pt": 2.3526580058708455, "brier": 0.0040332149648590505, "log_loss": 0.04928598623389895, "ece": 0.047130755737454955 }, { "model": "kalman", "fold": "loso_MN", "n_test": 3, "mae_pt": 9.533333333333333, "rmse_pt": 10.529691353501297, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "kalman", "fold": "loso_MO", "n_test": 3, "mae_pt": 5.303709486280525, "rmse_pt": 5.763105394737279, "brier": 1.012925107582778e-07, "log_loss": 0.0001994848175889294, "ece": 0.00019943415286779084 }, { "model": "kalman", "fold": "loso_MT", "n_test": 3, "mae_pt": 3.6817669847301953, "rmse_pt": 4.787288778271895, "brier": 0.013690928418156183, "log_loss": 0.09354862216703806, "ece": 0.08575297285043298 }, { "model": "kalman", "fold": "loso_ND", "n_test": 3, "mae_pt": 16.587984905497233, "rmse_pt": 19.65686267394628, "brier": 0.08333333333333333, "log_loss": 0.23104906085331509, "ece": 0.16666666666667143 }, { "model": "kalman", "fold": "loso_NE", "n_test": 4, "mae_pt": 17.059751801968282, "rmse_pt": 21.360881665934198, "brier": 0.1251916874322188, "log_loss": 0.35359380303681315, "ece": 0.2569225621727344 }, { "model": "kalman", "fold": "loso_NH", "n_test": 4, "mae_pt": 2.9497462594381636, "rmse_pt": 3.654142742130264, "brier": 0.004986287210692492, "log_loss": 0.039579600057557784, "ece": 0.03682444588722383 }, { "model": "kalman", "fold": "loso_NM", "n_test": 3, "mae_pt": 2.6685210642973054, "rmse_pt": 3.261460987875674, "brier": 0.00014370326205416482, "log_loss": 0.009559176700476714, "ece": 0.009486520670941467 }, { "model": "kalman", "fold": "loso_NV", "n_test": 3, "mae_pt": 3.135511078684768, "rmse_pt": 3.633010542049964, "brier": 0.23824957665634552, "log_loss": 0.6423528620269195, "ece": 0.1690293138053526 }, { "model": "kalman", "fold": "loso_NY", "n_test": 3, "mae_pt": 1.9500235065436822, "rmse_pt": 2.4583495585861614, "brier": 1.5610203889876802e-11, "log_loss": 2.287204338528096e-06, "ece": 2.2869056023333556e-06 }, { "model": "kalman", "fold": "loso_OH", "n_test": 3, "mae_pt": 5.88220914654476, "rmse_pt": 5.984656751886815, "brier": 0.21100022240494964, "log_loss": 0.5528553559238191, "ece": 0.37252561115241367 }, { "model": "kalman", "fold": "loso_OK", "n_test": 4, "mae_pt": 6.073615944777645, "rmse_pt": 6.694128417535803, "brier": 6.514941135756919e-17, "log_loss": 4.7857590720106805e-09, "ece": 4.037046558158508e-09 }, { "model": "kalman", "fold": "loso_OR", "n_test": 3, "mae_pt": 2.2663872723793603, "rmse_pt": 2.690005100853029, "brier": 0.13100450684071374, "log_loss": 0.32864952513589624, "ece": 0.20897549847323615 }, { "model": "kalman", "fold": "loso_RI", "n_test": 3, "mae_pt": 12.109988013544845, "rmse_pt": 19.174153355198808, "brier": 0.08333333333333352, "log_loss": 0.23104906838002923, "ece": 0.1666666745902181 }, { "model": "kalman", "fold": "loso_SC", "n_test": 3, "mae_pt": 7.467063874957764, "rmse_pt": 7.889161387871755, "brier": 0.00012007545427438896, "log_loss": 0.009007690679164468, "ece": 0.00894711032290274 }, { "model": "kalman", "fold": "loso_SD", "n_test": 3, "mae_pt": 25.47993398629005, "rmse_pt": 25.832151973959167, "brier": 0.09359743299341639, "log_loss": 0.2953659476488428, "ece": 0.2251591153859795 }, { "model": "kalman", "fold": "loso_TN", "n_test": 3, "mae_pt": 10.963262615704586, "rmse_pt": 11.271427105948266, "brier": 4.296739666394125e-12, "log_loss": 1.2109911299120171e-06, "ece": 1.2109889815740498e-06 }, { "model": "kalman", "fold": "loso_TX", "n_test": 3, "mae_pt": 1.3760362094956609, "rmse_pt": 1.997417500944057, "brier": 1.645293552350237e-06, "log_loss": 0.0010058090405181304, "ece": 0.0010049853970371763 }, { "model": "kalman", "fold": "loso_UT", "n_test": 3, "mae_pt": 6.572105926564595, "rmse_pt": 8.614748732086737, "brier": 5.033096921331976e-32, "log_loss": 9.999999722180686e-10, "ece": 1.295260195396016e-16 }, { "model": "kalman", "fold": "loso_VT", "n_test": 4, "mae_pt": 30.543857327542604, "rmse_pt": 32.24430562580104, "brier": 0.125, "log_loss": 0.34657359077997263, "ece": 0.25000000000127454 }, { "model": "kalman", "fold": "loso_WA", "n_test": 3, "mae_pt": 5.380624346901747, "rmse_pt": 5.657862960126032, "brier": 0.0018548938767353794, "log_loss": 0.0259976464413275, "ece": 0.025021334699658282 }, { "model": "kalman", "fold": "loso_WI", "n_test": 3, "mae_pt": 1.573490171111058, "rmse_pt": 1.8358454738646155, "brier": 0.07983330183873111, "log_loss": 0.2954606930004379, "ece": 0.24047046578032016 }, { "model": "kalman", "fold": "loso_WV", "n_test": 3, "mae_pt": 22.96604643832595, "rmse_pt": 23.463463932056932, "brier": 9.503065160806331e-09, "log_loss": 6.471633373064834e-05, "ece": 6.471158167568418e-05 }, { "model": "kalman", "fold": "loso_WY", "n_test": 3, "mae_pt": 51.77666666666667, "rmse_pt": 52.01708277095132, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "qgbt", "fold": "loso_AR", "n_test": 3, "mae_pt": 4.097653821880617, "rmse_pt": 5.317145268792009, "brier": 1.6618726270711719e-21, "log_loss": 9.999999722180686e-10, "ece": 2.3538596997478105e-11 }, { "model": "qgbt", "fold": "loso_AZ", "n_test": 4, "mae_pt": 1.671842864152787, "rmse_pt": 2.136197872033367, "brier": 0.1467691266966436, "log_loss": 0.45513538548123106, "ece": 0.33500667910728277 }, { "model": "qgbt", "fold": "loso_CO", "n_test": 3, "mae_pt": 4.420256532106696, "rmse_pt": 4.910079552337823, "brier": 0.0007481944344352648, "log_loss": 0.020082998034833, "ece": 0.019697886018565924 }, { "model": "qgbt", "fold": "loso_CT", "n_test": 3, "mae_pt": 1.4515604050826443, "rmse_pt": 1.9701371920742077, "brier": 1.940860682058087e-05, "log_loss": 0.0025576413516025464, "ece": 0.002547887108196978 }, { "model": "qgbt", "fold": "loso_DE", "n_test": 3, "mae_pt": 1.690606104313293, "rmse_pt": 2.533830139459276, "brier": 7.68759824193287e-12, "log_loss": 1.6174203988336029e-06, "ece": 1.6170920206226569e-06 }, { "model": "qgbt", "fold": "loso_FL", "n_test": 3, "mae_pt": 7.941782476182733, "rmse_pt": 8.068266449282842, "brier": 0.0026208353937562964, "log_loss": 0.043028325160443935, "ece": 0.04165979230991549 }, { "model": "qgbt", "fold": "loso_HI", "n_test": 3, "mae_pt": 51.661491415350724, "rmse_pt": 52.022737568175536, "brier": 0.5539911088348202, "log_loss": 1.365562392339536, "ece": 0.744077962242575 }, { "model": "qgbt", "fold": "loso_IA", "n_test": 3, "mae_pt": 3.5791000403258995, "rmse_pt": 4.658741647464768, "brier": 0.0040473419610202515, "log_loss": 0.0390099091672941, "ece": 0.036824100858724874 }, { "model": "qgbt", "fold": "loso_IL", "n_test": 3, "mae_pt": 4.167723022145785, "rmse_pt": 4.373066774462119, "brier": 0.02506539843920559, "log_loss": 0.10683583937172246, "ece": 0.09140648743664402 }, { "model": "qgbt", "fold": "loso_IN", "n_test": 3, "mae_pt": 4.757266360400686, "rmse_pt": 7.317453838973911, "brier": 4.102938436585533e-05, "log_loss": 0.0037370929807840486, "ece": 0.0037164249533779468 }, { "model": "qgbt", "fold": "loso_KS", "n_test": 3, "mae_pt": 3.356200183327489, "rmse_pt": 5.598591270440236, "brier": 0.3263278371727345, "log_loss": 1.5168616098721481, "ece": 0.32971541891862877 }, { "model": "qgbt", "fold": "loso_MA", "n_test": 3, "mae_pt": 9.59192559189415, "rmse_pt": 11.259556289742413, "brier": 5.54097993170698e-09, "log_loss": 5.261830604488907e-05, "ece": 5.261553533231211e-05 }, { "model": "qgbt", "fold": "loso_MD", "n_test": 3, "mae_pt": 3.433851592186638, "rmse_pt": 5.236001566708228, "brier": 0.00022076394302162047, "log_loss": 0.013004837851834187, "ece": 0.012893019810235273 }, { "model": "qgbt", "fold": "loso_ME", "n_test": 3, "mae_pt": 6.387998553321528, "rmse_pt": 8.274256557196145, "brier": 0.2637325056526944, "log_loss": 0.734885177186435, "ece": 0.29708106933047673 }, { "model": "qgbt", "fold": "loso_MI", "n_test": 3, "mae_pt": 3.907646742656672, "rmse_pt": 4.234343169112752, "brier": 0.3225685477506044, "log_loss": 0.8256282765507986, "ece": 0.47828999159186286 }, { "model": "qgbt", "fold": "loso_MN", "n_test": 3, "mae_pt": 10.714104312368773, "rmse_pt": 11.875550148290685, "brier": 0.26905683379822765, "log_loss": 0.7319323173151405, "ece": 0.5160914169611244 }, { "model": "qgbt", "fold": "loso_MO", "n_test": 3, "mae_pt": 4.1785944875512335, "rmse_pt": 4.732394581704145, "brier": 3.964965584128028e-25, "log_loss": 9.999999722180686e-10, "ece": 3.82601358002906e-13 }, { "model": "qgbt", "fold": "loso_MT", "n_test": 3, "mae_pt": 5.279052045480074, "rmse_pt": 6.5379334316176365, "brier": 7.372739647090554e-06, "log_loss": 0.0015713848740116039, "ece": 0.0015676869052653197 }, { "model": "qgbt", "fold": "loso_ND", "n_test": 3, "mae_pt": 4.514858186053643, "rmse_pt": 4.62041877328003, "brier": 5.806413024152374e-13, "log_loss": 4.406068401590735e-07, "ece": 4.399403043716295e-07 }, { "model": "qgbt", "fold": "loso_NE", "n_test": 4, "mae_pt": 10.047949260580161, "rmse_pt": 12.70428923755669, "brier": 1.1716702363495793e-11, "log_loss": 1.717882789584642e-06, "ece": 1.7175247047918063e-06 }, { "model": "qgbt", "fold": "loso_NH", "n_test": 4, "mae_pt": 4.478458309010391, "rmse_pt": 5.302217226568531, "brier": 0.06329723239451435, "log_loss": 0.19260390847613856, "ece": 0.14396249826282204 }, { "model": "qgbt", "fold": "loso_NM", "n_test": 3, "mae_pt": 4.285917517304832, "rmse_pt": 5.012790096191883, "brier": 1.3996296870273502e-07, "log_loss": 0.00023132984253679557, "ece": 0.0002312598309212044 }, { "model": "qgbt", "fold": "loso_NV", "n_test": 3, "mae_pt": 3.9033855073779677, "rmse_pt": 4.662256163311817, "brier": 0.060621410292426646, "log_loss": 0.22325827431695466, "ece": 0.18180469347086792 }, { "model": "qgbt", "fold": "loso_NY", "n_test": 3, "mae_pt": 3.530900579471686, "rmse_pt": 4.416213092187915, "brier": 4.637293099681626e-06, "log_loss": 0.001255269044372653, "ece": 0.0012529446166814617 }, { "model": "qgbt", "fold": "loso_OH", "n_test": 3, "mae_pt": 2.9015546335310645, "rmse_pt": 3.4855986373433523, "brier": 0.027310619598824876, "log_loss": 0.11416226803328566, "ece": 0.09717499751877384 }, { "model": "qgbt", "fold": "loso_OK", "n_test": 4, "mae_pt": 3.0634977384351805, "rmse_pt": 3.579068200782722, "brier": 9.576022074498584e-10, "log_loss": 1.5473803004278627e-05, "ece": 1.5472574183619803e-05 }, { "model": "qgbt", "fold": "loso_OR", "n_test": 3, "mae_pt": 2.2199291423140775, "rmse_pt": 2.3993857594965453, "brier": 0.0038276691103585705, "log_loss": 0.0377833056609152, "ece": 0.03572072537512627 }, { "model": "qgbt", "fold": "loso_RI", "n_test": 3, "mae_pt": 11.28022530873154, "rmse_pt": 15.82311205704353, "brier": 0.048573945757943195, "log_loss": 0.16028234332313177, "ece": 0.12724791184274595 }, { "model": "qgbt", "fold": "loso_SC", "n_test": 3, "mae_pt": 1.6642219036251504, "rmse_pt": 1.666100751999858, "brier": 5.893655089548935e-07, "log_loss": 0.0004440165585127575, "ece": 0.0004437212879990586 }, { "model": "qgbt", "fold": "loso_SD", "n_test": 3, "mae_pt": 6.628048598909614, "rmse_pt": 7.428214871666661, "brier": 3.4900024829315493e-16, "log_loss": 1.5113093173952775e-08, "ece": 1.4779759693883676e-08 }, { "model": "qgbt", "fold": "loso_TN", "n_test": 3, "mae_pt": 5.352925601609438, "rmse_pt": 5.998037200386916, "brier": 2.9485173297169406e-12, "log_loss": 9.946292904075793e-07, "ece": 9.942944828219187e-07 }, { "model": "qgbt", "fold": "loso_TX", "n_test": 3, "mae_pt": 1.9044293203614142, "rmse_pt": 2.2504091102854855, "brier": 0.00011637370638464497, "log_loss": 0.007239682196490017, "ece": 0.00718077386056019 }, { "model": "qgbt", "fold": "loso_UT", "n_test": 3, "mae_pt": 9.51849222692591, "rmse_pt": 9.66016649853931, "brier": 2.6655815700195095e-10, "log_loss": 9.532260042356771e-06, "ece": 9.532126760765832e-06 }, { "model": "qgbt", "fold": "loso_VT", "n_test": 4, "mae_pt": 52.548137671278695, "rmse_pt": 59.15214977859921, "brier": 0.6705065616521708, "log_loss": 2.649644747222866, "ece": 0.7077443242798904 }, { "model": "qgbt", "fold": "loso_WA", "n_test": 3, "mae_pt": 3.5721840527243156, "rmse_pt": 4.295237023698124, "brier": 0.00029618923290357656, "log_loss": 0.010087395123827031, "ece": 0.009936289763459505 }, { "model": "qgbt", "fold": "loso_WI", "n_test": 3, "mae_pt": 2.101131916358057, "rmse_pt": 2.470308896919306, "brier": 0.28643798225536216, "log_loss": 0.7664346198256955, "ece": 0.20968569281770438 }, { "model": "qgbt", "fold": "loso_WV", "n_test": 3, "mae_pt": 4.147600779768474, "rmse_pt": 4.962667943598541, "brier": 5.412368199478832e-08, "log_loss": 0.00015574675902280135, "ece": 0.00015571935686816327 }, { "model": "qgbt", "fold": "loso_WY", "n_test": 3, "mae_pt": 19.834675435599127, "rmse_pt": 20.51776668320902, "brier": 1.938966695854996e-09, "log_loss": 2.5498451622738544e-05, "ece": 2.549748209009402e-05 }, { "model": "ridge", "fold": "loso_AR", "n_test": 3, "mae_pt": 3.991968022356238, "rmse_pt": 6.776742802170498, "brier": 1.013780971685397e-06, "log_loss": 0.0006046053266392946, "ece": 0.0006040978473873156 }, { "model": "ridge", "fold": "loso_AZ", "n_test": 4, "mae_pt": 4.037396352792806, "rmse_pt": 4.224535322565525, "brier": 0.2127117430747678, "log_loss": 0.6119656722587059, "ece": 0.4115567280357221 }, { "model": "ridge", "fold": "loso_CO", "n_test": 3, "mae_pt": 6.068123007883886, "rmse_pt": 6.673996680007039, "brier": 0.02479998342807065, "log_loss": 0.1609137389843549, "ece": 0.14663851836032696 }, { "model": "ridge", "fold": "loso_CT", "n_test": 3, "mae_pt": 1.1909557301646998, "rmse_pt": 1.1932574362973085, "brier": 0.0024493858870413437, "log_loss": 0.048371737719418995, "ece": 0.04709985579752807 }, { "model": "ridge", "fold": "loso_DE", "n_test": 3, "mae_pt": 1.846574931943291, "rmse_pt": 2.418437453485124, "brier": 0.0001825607410846775, "log_loss": 0.011665611206867996, "ece": 0.011573317160583296 }, { "model": "ridge", "fold": "loso_FL", "n_test": 3, "mae_pt": 7.861766195463424, "rmse_pt": 7.905624150030438, "brier": 0.025773500927572985, "log_loss": 0.1655626105849856, "ece": 0.15079055273205294 }, { "model": "ridge", "fold": "loso_HI", "n_test": 3, "mae_pt": 26.21772196164054, "rmse_pt": 27.161977265645906, "brier": 0.017637685528590227, "log_loss": 0.14063081129259267, "ece": 0.13089820416370568 }, { "model": "ridge", "fold": "loso_IA", "n_test": 3, "mae_pt": 1.818227869527523, "rmse_pt": 2.1687198303102266, "brier": 0.018577317858985625, "log_loss": 0.1127514716059566, "ece": 0.10194990401362226 }, { "model": "ridge", "fold": "loso_IL", "n_test": 3, "mae_pt": 11.40033580414324, "rmse_pt": 17.242324993757276, "brier": 0.3126875128683864, "log_loss": 1.160133507380191, "ece": 0.33357830965576196 }, { "model": "ridge", "fold": "loso_IN", "n_test": 3, "mae_pt": 4.961181524594314, "rmse_pt": 5.486813680826909, "brier": 0.000268679186983387, "log_loss": 0.011643639212872288, "ece": 0.01150680092093076 }, { "model": "ridge", "fold": "loso_KS", "n_test": 3, "mae_pt": 6.420206783548752, "rmse_pt": 9.552839744415214, "brier": 0.31459702700253817, "log_loss": 1.1903999342298552, "ece": 0.3091735216128119 }, { "model": "ridge", "fold": "loso_MA", "n_test": 3, "mae_pt": 7.047939338475836, "rmse_pt": 7.501353708920159, "brier": 1.9169136089066534e-07, "log_loss": 0.0003446186083499632, "ece": 0.00034452271995688566 }, { "model": "ridge", "fold": "loso_MD", "n_test": 3, "mae_pt": 8.302909242601544, "rmse_pt": 9.317684262219226, "brier": 7.139803221976769e-06, "log_loss": 0.0015664157474734527, "ece": 0.0015628347901848194 }, { "model": "ridge", "fold": "loso_ME", "n_test": 3, "mae_pt": 9.799762276495244, "rmse_pt": 9.939375043336591, "brier": 0.18614978796864515, "log_loss": 0.5327684063298933, "ece": 0.16394862315599923 }, { "model": "ridge", "fold": "loso_MI", "n_test": 3, "mae_pt": 4.995709429379704, "rmse_pt": 5.6344491069316485, "brier": 0.11445789482905938, "log_loss": 0.409187888914525, "ece": 0.3332318570832542 }, { "model": "ridge", "fold": "loso_MN", "n_test": 3, "mae_pt": 13.034105749349417, "rmse_pt": 13.763291948059695, "brier": 0.462401858158395, "log_loss": 1.1420690382163274, "ece": 0.679309542851752 }, { "model": "ridge", "fold": "loso_MO", "n_test": 3, "mae_pt": 6.165065091822111, "rmse_pt": 6.308224826063696, "brier": 6.900462289767768e-06, "log_loss": 0.002193481871913263, "ece": 0.002190023576620348 }, { "model": "ridge", "fold": "loso_MT", "n_test": 3, "mae_pt": 10.8989580537051, "rmse_pt": 11.68949087483012, "brier": 0.00015169666110643136, "log_loss": 0.007643924593096016, "ece": 0.007566986876968755 }, { "model": "ridge", "fold": "loso_ND", "n_test": 3, "mae_pt": 3.357043515735223, "rmse_pt": 4.24254930696347, "brier": 1.275632072546903e-07, "log_loss": 0.00020990707117565509, "ece": 0.0002098432632678208 }, { "model": "ridge", "fold": "loso_NE", "n_test": 4, "mae_pt": 9.309339150189317, "rmse_pt": 12.180174285739266, "brier": 1.6361744350706324e-05, "log_loss": 0.002948217699214616, "ece": 0.002940003074423353 }, { "model": "ridge", "fold": "loso_NH", "n_test": 4, "mae_pt": 5.287127590294578, "rmse_pt": 5.742706124504079, "brier": 0.04163921608035248, "log_loss": 0.21573230057787782, "ece": 0.19099771902302398 }, { "model": "ridge", "fold": "loso_NM", "n_test": 3, "mae_pt": 2.3992994284334035, "rmse_pt": 3.5369448294238817, "brier": 0.06241293019833022, "log_loss": 0.2846767097316895, "ece": 0.24669790293084226 }, { "model": "ridge", "fold": "loso_NV", "n_test": 3, "mae_pt": 2.8176794202860407, "rmse_pt": 3.1890903151673466, "brier": 0.12166844991949645, "log_loss": 0.41456221764572326, "ece": 0.327605785191037 }, { "model": "ridge", "fold": "loso_NY", "n_test": 3, "mae_pt": 3.068949102667966, "rmse_pt": 3.239657167811202, "brier": 0.0024658167435771647, "log_loss": 0.03468779626992328, "ece": 0.03338096090985643 }, { "model": "ridge", "fold": "loso_OH", "n_test": 3, "mae_pt": 4.0683923554192125, "rmse_pt": 4.11510224668071, "brier": 0.010556900115828943, "log_loss": 0.09035819868859844, "ece": 0.08458153744992848 }, { "model": "ridge", "fold": "loso_OK", "n_test": 4, "mae_pt": 7.862541592767771, "rmse_pt": 8.681134784654532, "brier": 1.1223950470970324e-08, "log_loss": 5.4458123994422754e-05, "ece": 5.4452511226821265e-05 }, { "model": "ridge", "fold": "loso_OR", "n_test": 3, "mae_pt": 1.8951793920674291, "rmse_pt": 2.3748077922147215, "brier": 0.010145237140827127, "log_loss": 0.07812100030963652, "ece": 0.07240449248549918 }, { "model": "ridge", "fold": "loso_RI", "n_test": 3, "mae_pt": 12.104200388448584, "rmse_pt": 17.346364827967893, "brier": 0.03409553140554829, "log_loss": 0.14179630496773674, "ece": 0.1200319395114486 }, { "model": "ridge", "fold": "loso_SC", "n_test": 3, "mae_pt": 2.853869642158594, "rmse_pt": 3.4519183612849687, "brier": 0.0015662774309395716, "log_loss": 0.024498587314266018, "ece": 0.02367775388409658 }, { "model": "ridge", "fold": "loso_SD", "n_test": 3, "mae_pt": 8.923506955166113, "rmse_pt": 10.25256699120131, "brier": 1.0721503669200252e-06, "log_loss": 0.000724209198699345, "ece": 0.0007236725195434909 }, { "model": "ridge", "fold": "loso_TN", "n_test": 3, "mae_pt": 8.133999002273082, "rmse_pt": 9.03226050210184, "brier": 0.00010261105037498334, "log_loss": 0.005907238222467043, "ece": 0.005855324579711514 }, { "model": "ridge", "fold": "loso_TX", "n_test": 3, "mae_pt": 1.260111712095437, "rmse_pt": 1.5423496053733803, "brier": 0.008023918913070525, "log_loss": 0.08797551565190032, "ece": 0.08366211137615293 }, { "model": "ridge", "fold": "loso_UT", "n_test": 3, "mae_pt": 8.58615925535412, "rmse_pt": 12.114592394449245, "brier": 9.81197656962912e-10, "log_loss": 2.9795517034349113e-05, "ece": 2.9795026423714194e-05 }, { "model": "ridge", "fold": "loso_VT", "n_test": 4, "mae_pt": 46.67205413561246, "rmse_pt": 53.35098229765505, "brier": 0.7101673467956813, "log_loss": 3.8587962703808616, "ece": 0.7292929929405868 }, { "model": "ridge", "fold": "loso_WA", "n_test": 3, "mae_pt": 0.4677683531826311, "rmse_pt": 0.5663399560638946, "brier": 0.001473343102091534, "log_loss": 0.032642326392946924, "ece": 0.0318786133868848 }, { "model": "ridge", "fold": "loso_WI", "n_test": 3, "mae_pt": 3.5711122917240536, "rmse_pt": 4.766970005096896, "brier": 0.295157792983799, "log_loss": 0.7953081396449244, "ece": 0.2874184661997007 }, { "model": "ridge", "fold": "loso_WV", "n_test": 3, "mae_pt": 14.427176395848297, "rmse_pt": 16.201932488749623, "brier": 7.430188653474764e-15, "log_loss": 5.043342019549831e-08, "ece": 4.9779561620925485e-08 }, { "model": "ridge", "fold": "loso_WY", "n_test": 3, "mae_pt": 23.338666183148717, "rmse_pt": 23.777901203696736, "brier": 2.8914184219414888e-09, "log_loss": 5.12308570686793e-05, "ece": 5.1229411301293805e-05 }, { "model": "rf", "fold": "loso_AR", "n_test": 3, "mae_pt": 2.3830555555555555, "rmse_pt": 2.931867570969689, "brier": 9.866610116893429e-08, "log_loss": 0.0001845623712445955, "ece": 0.00018451268697041354 }, { "model": "rf", "fold": "loso_AZ", "n_test": 4, "mae_pt": 2.1013125, "rmse_pt": 2.6848450286981183, "brier": 0.15468568620955747, "log_loss": 0.4856375800213108, "ece": 0.3741145509262789 }, { "model": "rf", "fold": "loso_CO", "n_test": 3, "mae_pt": 4.662361111111111, "rmse_pt": 5.251265196976961, "brier": 0.004104586832296544, "log_loss": 0.05352860463707238, "ece": 0.05135513824702509 }, { "model": "rf", "fold": "loso_CT", "n_test": 3, "mae_pt": 1.097555555555557, "rmse_pt": 1.2912372978113904, "brier": 1.5783252347853928e-05, "log_loss": 0.0023283916056237226, "ece": 0.002320463596742761 }, { "model": "rf", "fold": "loso_DE", "n_test": 3, "mae_pt": 3.1724999999999994, "rmse_pt": 3.902172620144758, "brier": 5.571284159663212e-05, "log_loss": 0.005157084278323167, "ece": 0.005128999472738038 }, { "model": "rf", "fold": "loso_FL", "n_test": 3, "mae_pt": 10.595, "rmse_pt": 10.808103891165288, "brier": 0.02567564835429043, "log_loss": 0.17262450889468584, "ece": 0.1581544363077141 }, { "model": "rf", "fold": "loso_HI", "n_test": 3, "mae_pt": 73.76333333333334, "rmse_pt": 74.17746760303969, "brier": 0.8913086996906477, "log_loss": 2.893858690238485, "ece": 0.9440563386693268 }, { "model": "rf", "fold": "loso_IA", "n_test": 3, "mae_pt": 2.8418055555555544, "rmse_pt": 3.365369154879017, "brier": 0.04963334966020505, "log_loss": 0.16606794404889083, "ece": 0.13218601487431567 }, { "model": "rf", "fold": "loso_IL", "n_test": 3, "mae_pt": 1.9255992063492062, "rmse_pt": 2.1582628977663734, "brier": 0.003911427300965268, "log_loss": 0.04103214883852486, "ece": 0.038924112103334796 }, { "model": "rf", "fold": "loso_IN", "n_test": 3, "mae_pt": 4.4574404761904765, "rmse_pt": 6.331328991321138, "brier": 9.185909819366509e-05, "log_loss": 0.007609872540445439, "ece": 0.007563545560364149 }, { "model": "rf", "fold": "loso_KS", "n_test": 3, "mae_pt": 5.015666666666667, "rmse_pt": 7.1447889868537535, "brier": 0.24654060821006984, "log_loss": 0.6556555981792659, "ece": 0.2864156356406982 }, { "model": "rf", "fold": "loso_MA", "n_test": 3, "mae_pt": 8.959944444444446, "rmse_pt": 10.422333037050064, "brier": 0.0009249630588588322, "log_loss": 0.02891597868416942, "ece": 0.028441927761015506 }, { "model": "rf", "fold": "loso_MD", "n_test": 3, "mae_pt": 4.237555555555555, "rmse_pt": 4.684868921839201, "brier": 0.006009892031452258, "log_loss": 0.05013835332247484, "ece": 0.04683500191908374 }, { "model": "rf", "fold": "loso_ME", "n_test": 3, "mae_pt": 7.156416666666668, "rmse_pt": 8.390343369264853, "brier": 0.23145181814837282, "log_loss": 0.6248058262006495, "ece": 0.3102113199610461 }, { "model": "rf", "fold": "loso_MI", "n_test": 3, "mae_pt": 2.325805555555555, "rmse_pt": 3.7982663628852995, "brier": 0.0997278829645043, "log_loss": 0.3601377624109099, "ece": 0.2926983913205577 }, { "model": "rf", "fold": "loso_MN", "n_test": 3, "mae_pt": 9.090416666666668, "rmse_pt": 11.230128666791186, "brier": 0.24375662609588256, "log_loss": 0.6799670129091663, "ece": 0.48622502579419985 }, { "model": "rf", "fold": "loso_MO", "n_test": 3, "mae_pt": 5.879361111111112, "rmse_pt": 6.176633857358746, "brier": 1.8362434434464882e-09, "log_loss": 2.4791701084843687e-05, "ece": 2.4790539954540797e-05 }, { "model": "rf", "fold": "loso_MT", "n_test": 3, "mae_pt": 4.690611111111113, "rmse_pt": 5.88241676917713, "brier": 0.0008062242594779702, "log_loss": 0.017442271232117165, "ece": 0.017025464915015425 }, { "model": "rf", "fold": "loso_ND", "n_test": 3, "mae_pt": 4.51188888888889, "rmse_pt": 4.824062745277095, "brier": 3.647918166505455e-10, "log_loss": 1.1115482657588863e-05, "ece": 1.1114966924352654e-05 }, { "model": "rf", "fold": "loso_NE", "n_test": 4, "mae_pt": 10.97215277777778, "rmse_pt": 14.44208119222464, "brier": 7.171488402857915e-15, "log_loss": 4.30923237705209e-08, "ece": 4.234695646920805e-08 }, { "model": "rf", "fold": "loso_NH", "n_test": 4, "mae_pt": 4.948958333333334, "rmse_pt": 5.634326143406838, "brier": 0.07803575560129723, "log_loss": 0.2726863956971117, "ece": 0.21507100844495344 }, { "model": "rf", "fold": "loso_NM", "n_test": 3, "mae_pt": 5.624166666666667, "rmse_pt": 6.187119554334604, "brier": 6.500508565888374e-07, "log_loss": 0.0006962385939160714, "ece": 0.0006959133444008891 }, { "model": "rf", "fold": "loso_NV", "n_test": 3, "mae_pt": 3.132638888888889, "rmse_pt": 4.21609315516813, "brier": 0.11679685966330718, "log_loss": 0.40252180700503165, "ece": 0.3194660736380456 }, { "model": "rf", "fold": "loso_NY", "n_test": 3, "mae_pt": 0.8676388888888885, "rmse_pt": 0.8781700841860066, "brier": 0.007210451235102774, "log_loss": 0.05305232743950019, "ece": 0.04904938256650393 }, { "model": "rf", "fold": "loso_OH", "n_test": 3, "mae_pt": 3.208194444444444, "rmse_pt": 3.5214702990554416, "brier": 0.04778367251668885, "log_loss": 0.17689372923476523, "ece": 0.1448348030410822 }, { "model": "rf", "fold": "loso_OK", "n_test": 4, "mae_pt": 6.922625000000001, "rmse_pt": 10.681052294509916, "brier": 1.6467236329201588e-09, "log_loss": 2.8376374734275864e-05, "ece": 2.837555133637648e-05 }, { "model": "rf", "fold": "loso_OR", "n_test": 3, "mae_pt": 2.3035833333333335, "rmse_pt": 3.1029830573444426, "brier": 0.005809336634234701, "log_loss": 0.047203809805957765, "ece": 0.044015185097863174 }, { "model": "rf", "fold": "loso_RI", "n_test": 3, "mae_pt": 11.608000000000002, "rmse_pt": 14.366200899900305, "brier": 0.020463377149494526, "log_loss": 0.10709846679024683, "ece": 0.09484993806006418 }, { "model": "rf", "fold": "loso_SC", "n_test": 3, "mae_pt": 0.9722222222222244, "rmse_pt": 1.2014102746972635, "brier": 7.5070832943972706e-06, "log_loss": 0.002014146382414907, "ece": 0.0020103821408710143 }, { "model": "rf", "fold": "loso_SD", "n_test": 3, "mae_pt": 6.619896825396826, "rmse_pt": 7.607421420353655, "brier": 1.3529003478097084e-11, "log_loss": 2.2043572569675937e-06, "ece": 2.204017159113114e-06 }, { "model": "rf", "fold": "loso_TN", "n_test": 3, "mae_pt": 7.135138888888888, "rmse_pt": 8.232013856961352, "brier": 3.578119947982204e-06, "log_loss": 0.001093933273590876, "ece": 0.0010921399683709125 }, { "model": "rf", "fold": "loso_TX", "n_test": 3, "mae_pt": 0.8859722222222217, "rmse_pt": 1.1324917473740668, "brier": 0.000809085211560309, "log_loss": 0.023531530320553537, "ece": 0.023116015168416843 }, { "model": "rf", "fold": "loso_UT", "n_test": 3, "mae_pt": 10.417222222222222, "rmse_pt": 11.424501002150954, "brier": 6.2359763267107004e-06, "log_loss": 0.0015496095160686474, "ece": 0.0015464825773002973 }, { "model": "rf", "fold": "loso_VT", "n_test": 4, "mae_pt": 58.538537698412696, "rmse_pt": 65.73556796350451, "brier": 0.7499996917243046, "log_loss": 12.26643145249632, "ece": 0.7499998373275467 }, { "model": "rf", "fold": "loso_WA", "n_test": 3, "mae_pt": 1.817999999999999, "rmse_pt": 2.1098440700677377, "brier": 0.0016048739360767165, "log_loss": 0.024158803268130496, "ece": 0.023317205120153006 }, { "model": "rf", "fold": "loso_WI", "n_test": 3, "mae_pt": 1.559236111111111, "rmse_pt": 1.8031043871984391, "brier": 0.233746539312797, "log_loss": 0.6605244807124565, "ece": 0.1235634885939676 }, { "model": "rf", "fold": "loso_WV", "n_test": 3, "mae_pt": 7.038325396825396, "rmse_pt": 8.298063720418487, "brier": 3.596373511155088e-08, "log_loss": 0.00010986561557295401, "ece": 0.00010984762976682753 }, { "model": "rf", "fold": "loso_WY", "n_test": 3, "mae_pt": 18.26827777777778, "rmse_pt": 18.630469716598967, "brier": 7.66154586701315e-14, "log_loss": 1.6047435880666858e-07, "ece": 1.598076538322779e-07 }, { "model": "pvi", "fold": "loso_AR", "n_test": 3, "mae_pt": 18.220000000000002, "rmse_pt": 18.456063141056564, "brier": 0.0011322989291008114, "log_loss": 0.03422882911675678, "ece": 0.0336496497619338 }, { "model": "pvi", "fold": "loso_AZ", "n_test": 4, "mae_pt": 4.2775, "rmse_pt": 4.533428614194779, "brier": 0.34863398142869056, "log_loss": 0.8927020289161415, "ece": 0.5904523532247886 }, { "model": "pvi", "fold": "loso_CO", "n_test": 3, "mae_pt": 11.030000000000001, "rmse_pt": 11.766297916790425, "brier": 0.1047722991294981, "log_loss": 0.3910970658065389, "ece": 0.3236854941598374 }, { "model": "pvi", "fold": "loso_CT", "n_test": 3, "mae_pt": 7.603333333333333, "rmse_pt": 7.848968085041498, "brier": 0.044823746121023945, "log_loss": 0.2378970920280866, "ece": 0.21171619239213602 }, { "model": "pvi", "fold": "loso_DE", "n_test": 3, "mae_pt": 11.823333333333332, "rmse_pt": 12.431080671714211, "brier": 0.06067194111958924, "log_loss": 0.2827830915078763, "ece": 0.246316749571744 }, { "model": "pvi", "fold": "loso_FL", "n_test": 3, "mae_pt": 13.646666666666667, "rmse_pt": 13.912524333611543, "brier": 0.13379170600374563, "log_loss": 0.45535226386965705, "ece": 0.3657754857884077 }, { "model": "pvi", "fold": "loso_HI", "n_test": 3, "mae_pt": 22.833333333333332, "rmse_pt": 24.1376980399264, "brier": 0.002991297017024184, "log_loss": 0.05624527316425471, "ece": 0.05469275104640625 }, { "model": "pvi", "fold": "loso_IA", "n_test": 3, "mae_pt": 7.323333333333334, "rmse_pt": 8.794564609272404, "brier": 0.060671941119589214, "log_loss": 0.2827830915078763, "ece": 0.2463167495717439 }, { "model": "pvi", "fold": "loso_IL", "n_test": 3, "mae_pt": 8.69, "rmse_pt": 8.821131824582755, "brier": 0.044823746121023945, "log_loss": 0.2378970920280866, "ece": 0.21171619239213602 }, { "model": "pvi", "fold": "loso_IN", "n_test": 3, "mae_pt": 10.313333333333334, "rmse_pt": 10.840092250529976, "brier": 0.023011478334305807, "log_loss": 0.16451544722332767, "ece": 0.15169534710829402 }, { "model": "pvi", "fold": "loso_KS", "n_test": 3, "mae_pt": 9.366666666666667, "rmse_pt": 10.525850717796322, "brier": 0.26504095705794695, "log_loss": 0.7795109605847937, "ece": 0.20692688282512928 }, { "model": "pvi", "fold": "loso_MA", "n_test": 3, "mae_pt": 16.310000000000002, "rmse_pt": 17.23634241943459, "brier": 0.004700382486940788, "log_loss": 0.07102278986358658, "ece": 0.06855933551997706 }, { "model": "pvi", "fold": "loso_MD", "n_test": 3, "mae_pt": 13.469999999999999, "rmse_pt": 16.459142342985754, "brier": 0.002991297017024184, "log_loss": 0.05624527316425471, "ece": 0.05469275104640625 }, { "model": "pvi", "fold": "loso_ME", "n_test": 3, "mae_pt": 13.65, "rmse_pt": 14.196597010081913, "brier": 0.28833241261216486, "log_loss": 0.7707567977660837, "ece": 0.2571190198914553 }, { "model": "pvi", "fold": "loso_MI", "n_test": 3, "mae_pt": 5.033333333333332, "rmse_pt": 6.765633747107509, "brier": 0.29759377234023227, "log_loss": 0.7886048030628189, "ece": 0.5455215599224583 }, { "model": "pvi", "fold": "loso_MN", "n_test": 3, "mae_pt": 8.683333333333334, "rmse_pt": 9.766792376892903, "brier": 0.20655065249531565, "log_loss": 0.6060129512591295, "ece": 0.4544784400775417 }, { "model": "pvi", "fold": "loso_MO", "n_test": 3, "mae_pt": 8.150000000000002, "rmse_pt": 8.782545189180642, "brier": 0.023011478334305807, "log_loss": 0.16451544722332767, "ece": 0.15169534710829402 }, { "model": "pvi", "fold": "loso_MT", "n_test": 3, "mae_pt": 4.59, "rmse_pt": 6.427158521980507, "brier": 0.01086062052985007, "log_loss": 0.11005406982094113, "ece": 0.10421430098527779 }, { "model": "pvi", "fold": "loso_ND", "n_test": 3, "mae_pt": 18.546666666666667, "rmse_pt": 19.15783738665023, "brier": 0.00012310697325520218, "log_loss": 0.011157370799804249, "ece": 0.011095358185079118 }, { "model": "pvi", "fold": "loso_NE", "n_test": 4, "mae_pt": 14.485, "rmse_pt": 16.658182673989383, "brier": 0.004700382486940788, "log_loss": 0.07102278986358658, "ece": 0.06855933551997706 }, { "model": "pvi", "fold": "loso_NH", "n_test": 4, "mae_pt": 12.41, "rmse_pt": 12.836664286332333, "brier": 0.25207221241777394, "log_loss": 0.6973088771609743, "ece": 0.04552155992245832 }, { "model": "pvi", "fold": "loso_NM", "n_test": 3, "mae_pt": 4.986666666666667, "rmse_pt": 5.31185466668658, "brier": 0.13379170600374563, "log_loss": 0.45535226386965705, "ece": 0.3657754857884077 }, { "model": "pvi", "fold": "loso_NV", "n_test": 3, "mae_pt": 1.31, "rmse_pt": 1.3657110480136954, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.16666666666666663 }, { "model": "pvi", "fold": "loso_NY", "n_test": 3, "mae_pt": 5.536666666666668, "rmse_pt": 6.7843275274709445, "brier": 0.023011478334305807, "log_loss": 0.16451544722332767, "ece": 0.1516953471082939 }, { "model": "pvi", "fold": "loso_OH", "n_test": 3, "mae_pt": 7.473333333333334, "rmse_pt": 11.552999610490776, "brier": 0.060671941119589214, "log_loss": 0.2827830915078763, "ece": 0.2463167495717439 }, { "model": "pvi", "fold": "loso_OK", "n_test": 4, "mae_pt": 10.307500000000001, "rmse_pt": 11.249467765187827, "brier": 0.00012310697325520218, "log_loss": 0.011157370799804249, "ece": 0.011095358185079118 }, { "model": "pvi", "fold": "loso_OR", "n_test": 3, "mae_pt": 7.993333333333333, "rmse_pt": 9.220603740175225, "brier": 0.06067194111958924, "log_loss": 0.2827830915078763, "ece": 0.246316749571744 }, { "model": "pvi", "fold": "loso_RI", "n_test": 3, "mae_pt": 17.28, "rmse_pt": 18.431304891406903, "brier": 0.032450105741978506, "log_loss": 0.19862062244566944, "ece": 0.1801391288476174 }, { "model": "pvi", "fold": "loso_SC", "n_test": 3, "mae_pt": 11.036666666666667, "rmse_pt": 12.74664269523548, "brier": 0.032450105741978506, "log_loss": 0.19862062244566944, "ece": 0.1801391288476174 }, { "model": "pvi", "fold": "loso_SD", "n_test": 3, "mae_pt": 20.323333333333334, "rmse_pt": 21.502037887915026, "brier": 0.0011322989291008114, "log_loss": 0.03422882911675678, "ece": 0.0336496497619338 }, { "model": "pvi", "fold": "loso_TN", "n_test": 3, "mae_pt": 17.656666666666666, "rmse_pt": 17.772031397676518, "brier": 0.002991297017024178, "log_loss": 0.05624527316425471, "ece": 0.05469275104640631 }, { "model": "pvi", "fold": "loso_TX", "n_test": 3, "mae_pt": 5.426666666666667, "rmse_pt": 5.515197185958087, "brier": 0.08050740406871067, "log_loss": 0.3337096297105279, "ece": 0.28373826683884334 }, { "model": "pvi", "fold": "loso_UT", "n_test": 3, "mae_pt": 11.610000000000001, "rmse_pt": 14.381021057861878, "brier": 0.0072237120598574525, "log_loss": 0.08882293346095334, "ece": 0.08499242354385156 }, { "model": "pvi", "fold": "loso_VT", "n_test": 4, "mae_pt": 48.475, "rmse_pt": 50.399897321323984, "brier": 0.6709521704474146, "log_loss": 2.193579393545732, "ece": 0.6953072489535936 }, { "model": "pvi", "fold": "loso_WA", "n_test": 3, "mae_pt": 7.933333333333333, "rmse_pt": 8.464549604084082, "brier": 0.032450105741978506, "log_loss": 0.19862062244566944, "ece": 0.1801391288476174 }, { "model": "pvi", "fold": "loso_WI", "n_test": 3, "mae_pt": 2.783333333333333, "rmse_pt": 3.3167504177030462, "brier": 0.28833241261216486, "log_loss": 0.7707567977660837, "ece": 0.2571190198914553 }, { "model": "pvi", "fold": "loso_WV", "n_test": 3, "mae_pt": 19.52666666666667, "rmse_pt": 20.3153849746114, "brier": 3.526211168452984e-05, "log_loss": 0.005955892109655916, "ece": 0.005938190943758026 }, { "model": "pvi", "fold": "loso_WY", "n_test": 3, "mae_pt": 30.52666666666667, "rmse_pt": 30.93268605644629, "brier": 4.519179064644729e-06, "log_loss": 0.00212809888301609, "ece": 0.0021258360860246794 }, { "model": "market", "fold": "loso_AR", "n_test": 3, "mae_pt": 31.820000000000004, "rmse_pt": 31.95575482861681, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_AZ", "n_test": 4, "mae_pt": 2.5775, "rmse_pt": 2.9830311765048654, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_CO", "n_test": 3, "mae_pt": 14.43, "rmse_pt": 15.000325552022753, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_CT", "n_test": 3, "mae_pt": 13.553333333333333, "rmse_pt": 13.692642793364131, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_DE", "n_test": 3, "mae_pt": 16.923333333333332, "rmse_pt": 17.353379113782616, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_FL", "n_test": 3, "mae_pt": 16.19666666666667, "rmse_pt": 16.421292072590795, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_HI", "n_test": 3, "mae_pt": 34.73333333333334, "rmse_pt": 35.60437894416921, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_IA", "n_test": 3, "mae_pt": 12.423333333333334, "rmse_pt": 13.343626443612198, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_IL", "n_test": 3, "mae_pt": 14.64, "rmse_pt": 14.718215471539565, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_IN", "n_test": 3, "mae_pt": 17.963333333333335, "rmse_pt": 18.270853838833037, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_KS", "n_test": 3, "mae_pt": 12.200000000000001, "rmse_pt": 14.86259959316225, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.16666666666666669 }, { "model": "market", "fold": "loso_MA", "n_test": 3, "mae_pt": 27.36, "rmse_pt": 27.92212384472213, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_MD", "n_test": 3, "mae_pt": 25.30333333333333, "rmse_pt": 27.046491454530656, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_ME", "n_test": 3, "mae_pt": 13.083333333333334, "rmse_pt": 13.571319022114247, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.16666666666666669 }, { "model": "market", "fold": "loso_MI", "n_test": 3, "mae_pt": 4.183333333333333, "rmse_pt": 6.159515673600752, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_MN", "n_test": 3, "mae_pt": 9.533333333333333, "rmse_pt": 10.529691353501297, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_MO", "n_test": 3, "mae_pt": 15.800000000000002, "rmse_pt": 16.135383478554203, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_MT", "n_test": 3, "mae_pt": 12.466666666666667, "rmse_pt": 13.675252587551475, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_ND", "n_test": 3, "mae_pt": 35.546666666666674, "rmse_pt": 35.86933788070251, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_NE", "n_test": 4, "mae_pt": 23.345, "rmse_pt": 25.909786760990528, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_NH", "n_test": 4, "mae_pt": 12.41, "rmse_pt": 12.807495852039148, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.0 }, { "model": "market", "fold": "loso_NM", "n_test": 3, "mae_pt": 7.536666666666666, "rmse_pt": 7.755662447528257, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_NV", "n_test": 3, "mae_pt": 1.31, "rmse_pt": 1.3657110480136954, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.16666666666666663 }, { "model": "market", "fold": "loso_NY", "n_test": 3, "mae_pt": 12.933333333333332, "rmse_pt": 13.615601345515373, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_OH", "n_test": 3, "mae_pt": 11.586666666666666, "rmse_pt": 15.02151124221528, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_OK", "n_test": 4, "mae_pt": 25.6375, "rmse_pt": 26.63128845925409, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_OR", "n_test": 3, "mae_pt": 11.973333333333334, "rmse_pt": 13.45873446254637, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_RI", "n_test": 3, "mae_pt": 24.08, "rmse_pt": 24.91908906842303, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_SC", "n_test": 3, "mae_pt": 17.83666666666667, "rmse_pt": 18.942427686721327, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_SD", "n_test": 3, "mae_pt": 33.92333333333333, "rmse_pt": 34.642348361506905, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_TN", "n_test": 3, "mae_pt": 29.55666666666667, "rmse_pt": 29.62572812044738, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_TX", "n_test": 3, "mae_pt": 9.676666666666668, "rmse_pt": 9.726590701097003, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_UT", "n_test": 3, "mae_pt": 21.810000000000002, "rmse_pt": 23.402943547055504, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_VT", "n_test": 4, "mae_pt": 42.525, "rmse_pt": 43.21667097313258, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.25 }, { "model": "market", "fold": "loso_WA", "n_test": 3, "mae_pt": 14.733333333333334, "rmse_pt": 15.02604183853264, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_WI", "n_test": 3, "mae_pt": 1.75, "rmse_pt": 2.1041625412500813, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.16666666666666663 }, { "model": "market", "fold": "loso_WV", "n_test": 3, "mae_pt": 38.22666666666667, "rmse_pt": 38.635504396862736, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "market", "fold": "loso_WY", "n_test": 3, "mae_pt": 51.77666666666667, "rmse_pt": 52.01708277095132, "brier": 0.25, "log_loss": 0.6931471805599453, "ece": 0.5 }, { "model": "ensemble", "fold": "loso_AR", "n_test": 3, "mae_pt": 7.268033401355331, "rmse_pt": 8.428436313170089, "brier": 0.0009764987792719571, "log_loss": 0.022254733252630676, "ece": 0.021748749600974077 }, { "model": "ensemble", "fold": "loso_AZ", "n_test": 4, "mae_pt": 3.325556472798694, "rmse_pt": 3.401178642682185, "brier": 0.27671283846049166, "log_loss": 0.7473319971021763, "ece": 0.5240857029462818 }, { "model": "ensemble", "fold": "loso_CO", "n_test": 3, "mae_pt": 5.041360012892331, "rmse_pt": 5.465374392241152, "brier": 0.053442691599647076, "log_loss": 0.2576580277101294, "ece": 0.2254945741454716 }, { "model": "ensemble", "fold": "loso_CT", "n_test": 3, "mae_pt": 1.3838048672761822, "rmse_pt": 1.6301646665992544, "brier": 0.013086539105621154, "log_loss": 0.11990006242425022, "ece": 0.11278689884678472 }, { "model": "ensemble", "fold": "loso_DE", "n_test": 3, "mae_pt": 3.952550157238548, "rmse_pt": 4.954052236558246, "brier": 0.00263911571600429, "log_loss": 0.04911835373552259, "ece": 0.04774304742430713 }, { "model": "ensemble", "fold": "loso_FL", "n_test": 3, "mae_pt": 6.776723500269273, "rmse_pt": 7.4215678182684615, "brier": 0.04922676599165902, "log_loss": 0.25042116116535124, "ece": 0.22138965593040116 }, { "model": "ensemble", "fold": "loso_HI", "n_test": 3, "mae_pt": 19.864470257998168, "rmse_pt": 21.350984384483294, "brier": 0.012674270702202143, "log_loss": 0.11943697411902714, "ece": 0.11258006352015515 }, { "model": "ensemble", "fold": "loso_IA", "n_test": 3, "mae_pt": 1.959456496704, "rmse_pt": 2.299258342839834, "brier": 0.023828332862422242, "log_loss": 0.1482872678865564, "ece": 0.1343584573400602 }, { "model": "ensemble", "fold": "loso_IL", "n_test": 3, "mae_pt": 2.568146607649479, "rmse_pt": 2.8184258642318474, "brier": 0.012597692890809087, "log_loss": 0.11641586578462632, "ece": 0.10955642753101669 }, { "model": "ensemble", "fold": "loso_IN", "n_test": 3, "mae_pt": 2.5488531214178445, "rmse_pt": 2.93623374040964, "brier": 0.005563425047833526, "log_loss": 0.0755100217888524, "ece": 0.07257080817801344 }, { "model": "ensemble", "fold": "loso_KS", "n_test": 3, "mae_pt": 6.329305896630996, "rmse_pt": 8.692635936930813, "brier": 0.24072289075943262, "log_loss": 0.6631970665696639, "ece": 0.2364352428597119 }, { "model": "ensemble", "fold": "loso_MA", "n_test": 3, "mae_pt": 11.569216965498734, "rmse_pt": 14.852510193394343, "brier": 0.020427761626269424, "log_loss": 0.10667376363477883, "ece": 0.09441932468311494 }, { "model": "ensemble", "fold": "loso_MD", "n_test": 3, "mae_pt": 5.958743917506889, "rmse_pt": 7.790480084417428, "brier": 0.00395544960495579, "log_loss": 0.053974199799888566, "ece": 0.05188228238472847 }, { "model": "ensemble", "fold": "loso_ME", "n_test": 3, "mae_pt": 8.140712533139792, "rmse_pt": 8.306819052168999, "brier": 0.1486902234549707, "log_loss": 0.46855807868230565, "ece": 0.3541609609251941 }, { "model": "ensemble", "fold": "loso_MI", "n_test": 3, "mae_pt": 2.8940978741603396, "rmse_pt": 3.9142986818148584, "brier": 0.21412144289584165, "log_loss": 0.6199106417033348, "ece": 0.4587578733298997 }, { "model": "ensemble", "fold": "loso_MN", "n_test": 3, "mae_pt": 9.671979167533049, "rmse_pt": 10.655380639505509, "brier": 0.25453228685689416, "log_loss": 0.7022120008811862, "ece": 0.504511929350431 }, { "model": "ensemble", "fold": "loso_MO", "n_test": 3, "mae_pt": 3.238073357123467, "rmse_pt": 3.5547806151840224, "brier": 0.004854625082909467, "log_loss": 0.07157400669541976, "ece": 0.06902400451414363 }, { "model": "ensemble", "fold": "loso_MT", "n_test": 3, "mae_pt": 7.001919522546605, "rmse_pt": 7.206421910050015, "brier": 0.006314580707228662, "log_loss": 0.07281868647744563, "ece": 0.06944204573777339 }, { "model": "ensemble", "fold": "loso_ND", "n_test": 3, "mae_pt": 5.353443118953404, "rmse_pt": 5.542133644319551, "brier": 0.0001820778991721291, "log_loss": 0.008197458730645688, "ece": 0.008104978913442232 }, { "model": "ensemble", "fold": "loso_NE", "n_test": 4, "mae_pt": 9.16643429641366, "rmse_pt": 12.142894015067817, "brier": 0.004778431295640005, "log_loss": 0.06295888411520306, "ece": 0.06042260621024284 }, { "model": "ensemble", "fold": "loso_NH", "n_test": 4, "mae_pt": 5.612319839249473, "rmse_pt": 5.73083663650123, "brier": 0.1010710534655955, "log_loss": 0.3673799697108051, "ece": 0.2989613876558753 }, { "model": "ensemble", "fold": "loso_NM", "n_test": 3, "mae_pt": 2.2444152486317726, "rmse_pt": 2.796920467474372, "brier": 0.086670250425176, "log_loss": 0.3474228058832145, "ece": 0.29286998841130807 }, { "model": "ensemble", "fold": "loso_NV", "n_test": 3, "mae_pt": 1.323903499540984, "rmse_pt": 1.334762166752633, "brier": 0.2266364290438034, "log_loss": 0.645785406167609, "ece": 0.1392668169217437 }, { "model": "ensemble", "fold": "loso_NY", "n_test": 3, "mae_pt": 5.442286429346024, "rmse_pt": 5.992343028248257, "brier": 0.005128908032613742, "log_loss": 0.0717123461521081, "ece": 0.06900348466472306 }, { "model": "ensemble", "fold": "loso_OH", "n_test": 3, "mae_pt": 3.956553647926178, "rmse_pt": 4.209710351026161, "brier": 0.045399631564053654, "log_loss": 0.21827616014513518, "ece": 0.19080760753999387 }, { "model": "ensemble", "fold": "loso_OK", "n_test": 4, "mae_pt": 11.374865855233228, "rmse_pt": 12.504269069611702, "brier": 2.917700952454723e-06, "log_loss": 0.0014652557597877368, "ece": 0.0014637945077530107 }, { "model": "ensemble", "fold": "loso_OR", "n_test": 3, "mae_pt": 1.4048681198029689, "rmse_pt": 1.5284372365200412, "brier": 0.04657528523087662, "log_loss": 0.21652679860857404, "ece": 0.1870594390497774 }, { "model": "ensemble", "fold": "loso_RI", "n_test": 3, "mae_pt": 8.675098836202423, "rmse_pt": 14.546146652646758, "brier": 0.024649617569542592, "log_loss": 0.14055695275988553, "ece": 0.1257917514032666 }, { "model": "ensemble", "fold": "loso_SC", "n_test": 3, "mae_pt": 3.408888305085977, "rmse_pt": 3.4463003707849316, "brier": 0.010059112600960178, "log_loss": 0.0973135444161143, "ece": 0.0918524809432847 }, { "model": "ensemble", "fold": "loso_SD", "n_test": 3, "mae_pt": 8.895533198533075, "rmse_pt": 9.425645428377395, "brier": 0.0013582377702674104, "log_loss": 0.0312569820810329, "ece": 0.030555403111852175 }, { "model": "ensemble", "fold": "loso_TN", "n_test": 3, "mae_pt": 1.2626730653867202, "rmse_pt": 1.6066516789775467, "brier": 0.00011623963533472527, "log_loss": 0.009984243849043296, "ece": 0.009925631906181712 }, { "model": "ensemble", "fold": "loso_TX", "n_test": 3, "mae_pt": 2.663234877058919, "rmse_pt": 2.962285935463799, "brier": 0.024924314731381417, "log_loss": 0.171356528911817, "ece": 0.15738843690914034 }, { "model": "ensemble", "fold": "loso_UT", "n_test": 3, "mae_pt": 10.254833054793124, "rmse_pt": 12.655495386438915, "brier": 2.251829600382151e-05, "log_loss": 0.0045984786800030595, "ece": 0.004587180094141589 }, { "model": "ensemble", "fold": "loso_VT", "n_test": 4, "mae_pt": 42.48991156405001, "rmse_pt": 46.25173672537638, "brier": 0.44139127921472876, "log_loss": 1.2382427317571083, "ece": 0.5499514186727494 }, { "model": "ensemble", "fold": "loso_WA", "n_test": 3, "mae_pt": 0.9601627623637145, "rmse_pt": 0.9642183282665889, "brier": 0.021349772793321748, "log_loss": 0.14937747711713192, "ece": 0.13730484747608396 }, { "model": "ensemble", "fold": "loso_WI", "n_test": 3, "mae_pt": 3.5706221773215496, "rmse_pt": 3.8847618393416976, "brier": 0.2550933132551256, "log_loss": 0.7023900781799847, "ece": 0.49755259569187066 }, { "model": "ensemble", "fold": "loso_WV", "n_test": 3, "mae_pt": 5.14527703774414, "rmse_pt": 5.25639118075264, "brier": 3.770653463236871e-06, "log_loss": 0.0017793998243596629, "ece": 0.0017775116214953275 }, { "model": "ensemble", "fold": "loso_WY", "n_test": 3, "mae_pt": 21.62300301339719, "rmse_pt": 22.192520095885296, "brier": 4.832025472617076e-05, "log_loss": 0.006975549877908823, "ece": 0.006951277201073969 } ], "summary": { "kalman": { "mae_pt": 10.263, "rmse_pt": 11.51, "brier": 0.0698, "log_loss": 0.2277, "ece": 0.1307, "n_folds": 38, "n_test_total": 119 }, "qgbt": { "mae_pt": 7.909, "rmse_pt": 9.053, "brier": 0.085, "log_loss": 0.2804, "ece": 0.1217, "n_folds": 38, "n_test_total": 119 }, "ridge": { "mae_pt": 8.341, "rmse_pt": 9.649, "brier": 0.0835, "log_loss": 0.3364, "ece": 0.1394, "n_folds": 38, "n_test_total": 119 }, "rf": { "mae_pt": 8.812, "rmse_pt": 10.027, "brier": 0.0895, "log_loss": 0.6208, "ece": 0.1367, "n_folds": 38, "n_test_total": 119 }, "pvi": { "mae_pt": 12.621, "rmse_pt": 13.652, "brier": 0.108, "log_loss": 0.3636, "ece": 0.2022, "n_folds": 38, "n_test_total": 119 }, "market": { "mae_pt": 19.131, "rmse_pt": 19.944, "brier": 0.25, "log_loss": 0.6931, "ece": 0.4412, "n_folds": 38, "n_test_total": 119 }, "ensemble": { "mae_pt": 7.268, "rmse_pt": 8.252, "brier": 0.0735, "log_loss": 0.2512, "ece": 0.1704, "n_folds": 38, "n_test_total": 119 } } }, "random_holdout": { "folds": [ { "model": "kalman", "fold": "random_holdout", "n_test": 28, "mae_pt": 9.234574222117775, "rmse_pt": 13.945461417744673, "brier": 0.1008806784515461, "log_loss": 0.28142821778092475, "ece": 0.09341855416450034 }, { "model": "qgbt", "fold": "random_holdout", "n_test": 28, "mae_pt": 6.895128400537834, "rmse_pt": 11.909889989511047, "brier": 0.03930351163140996, "log_loss": 0.12562522508101154, "ece": 0.04039894248950468 }, { "model": "ridge", "fold": "random_holdout", "n_test": 28, "mae_pt": 6.819203927899319, "rmse_pt": 11.417299809151427, "brier": 0.05112028133875991, "log_loss": 0.16577244124197013, "ece": 0.05170494683582781 }, { "model": "rf", "fold": "random_holdout", "n_test": 28, "mae_pt": 7.537154549319728, "rmse_pt": 12.52885832253739, "brier": 0.044244766919580604, "log_loss": 0.14452980430949178, "ece": 0.06200205444700552 }, { "model": "pvi", "fold": "random_holdout", "n_test": 28, "mae_pt": 13.010000000000003, "rmse_pt": 18.002786292285823, "brier": 0.10053976746042981, "log_loss": 0.3522190839357372, "ece": 0.17304475368840394 }, { "model": "market", "fold": "random_holdout", "n_test": 28, "mae_pt": 19.402499999999996, "rmse_pt": 23.402234554479133, "brier": 0.25, "log_loss": 0.6931471805599451, "ece": 0.10714285714285715 }, { "model": "ensemble", "fold": "random_holdout", "n_test": 28, "mae_pt": 5.5883607243310776, "rmse_pt": 9.241615175556863, "brier": 0.04257600017400011, "log_loss": 0.1545369015321833, "ece": 0.08736424053891292 } ], "summary": { "kalman": { "mae_pt": 9.235, "rmse_pt": 13.945, "brier": 0.1009, "log_loss": 0.2814, "ece": 0.0934, "n_folds": 1, "n_test_total": 28 }, "qgbt": { "mae_pt": 6.895, "rmse_pt": 11.91, "brier": 0.0393, "log_loss": 0.1256, "ece": 0.0404, "n_folds": 1, "n_test_total": 28 }, "ridge": { "mae_pt": 6.819, "rmse_pt": 11.417, "brier": 0.0511, "log_loss": 0.1658, "ece": 0.0517, "n_folds": 1, "n_test_total": 28 }, "rf": { "mae_pt": 7.537, "rmse_pt": 12.529, "brier": 0.0442, "log_loss": 0.1445, "ece": 0.062, "n_folds": 1, "n_test_total": 28 }, "pvi": { "mae_pt": 13.01, "rmse_pt": 18.003, "brier": 0.1005, "log_loss": 0.3522, "ece": 0.173, "n_folds": 1, "n_test_total": 28 }, "market": { "mae_pt": 19.402, "rmse_pt": 23.402, "brier": 0.25, "log_loss": 0.6931, "ece": 0.1071, "n_folds": 1, "n_test_total": 28 }, "ensemble": { "mae_pt": 5.588, "rmse_pt": 9.242, "brier": 0.0426, "log_loss": 0.1545, "ece": 0.0874, "n_folds": 1, "n_test_total": 28 } } } }, "generatedAt": "2026-05-04T18:55:26.794537+00:00" }