TO_EVALUATE = [
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TA1_raw_v54/pred_BAS_smt_it_stm_p8_TA1_raw_v54_epoch=13-step=242605/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TA1_raw_v54/pred_BAS_smt_it_stm_p8_TA1_raw_v54_epoch=26-step=467882/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TA1_raw_v54/pred_BAS_smt_it_stm_p8_TA1_raw_v54_epoch=5-step=103973/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_scratch_v64/pred_BOTH_smt_it_stm_p8_TA1_RAW_scratch_v64_epoch=44-step=92159/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_scratch_v64/pred_BOTH_smt_it_stm_p8_TA1_RAW_scratch_v64_epoch=65-step=135167/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_scratch_v64/pred_BOTH_smt_it_stm_p8_TA1_RAW_scratch_v64_epoch=7-step=16383/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_scratch_v64/pred_BOTH_smt_it_stm_p8_TA1_RAW_scratch_v64_epoch=8-step=18431/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v61/pred_BOTH_smt_it_stm_p8_TA1_RAW_v61_epoch=33-step=69631/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v61/pred_BOTH_smt_it_stm_p8_TA1_RAW_v61_epoch=71-step=147455/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v62/pred_BOTH_smt_it_stm_p8_TA1_RAW_v62_epoch=15-step=32767/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v62/pred_BOTH_smt_it_stm_p8_TA1_RAW_v62_epoch=16-step=34815/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v62/pred_BOTH_smt_it_stm_p8_TA1_RAW_v62_epoch=18-step=38911/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v62/pred_BOTH_smt_it_stm_p8_TA1_RAW_v62_epoch=23-step=49151/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v62/pred_BOTH_smt_it_stm_p8_TA1_RAW_v62_epoch=30-step=63487/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v62/pred_BOTH_smt_it_stm_p8_TA1_RAW_v62_epoch=32-step=67583/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v62/pred_BOTH_smt_it_stm_p8_TA1_RAW_v62_epoch=35-step=73727/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v62/pred_BOTH_smt_it_stm_p8_TA1_RAW_v62_epoch=42-step=88063/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_TA1_RAW_v62/pred_BOTH_smt_it_stm_p8_TA1_RAW_v62_epoch=5-step=12287/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json'
]


TO_EVALUATE = [
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_L1_raw_v48/pred_BAS_smt_it_stm_p8_L1_raw_v48_epoch=20-step=815282/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_L1_raw_v53/pred_BAS_smt_it_stm_p8_L1_raw_v53_epoch=15-step=340047/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_L1_raw_v53/pred_BAS_smt_it_stm_p8_L1_raw_v53_epoch=3-step=85011/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TA1_raw_v54/pred_BAS_smt_it_stm_p8_TA1_raw_v54_epoch=5-step=103973/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TUNE_L1_DIL_v57/pred_BAS_smt_it_stm_p8_TUNE_L1_DIL_v57_epoch=3-step=81135/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TUNE_L1_DIL_v57/pred_BAS_smt_it_stm_p8_TUNE_L1_DIL_v57_epoch=4-step=101419/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TUNE_L1_I2L8_v59/pred_BAS_smt_it_stm_p8_TUNE_L1_I2L8_v59_epoch=1-step=31531/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TUNE_L1_I2L8_v59/pred_BAS_smt_it_stm_p8_TUNE_L1_I2L8_v59_epoch=2-step=47297/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TUNE_L1_I8L8_v60/pred_BAS_smt_it_stm_p8_TUNE_L1_I8L8_v60_epoch=1-step=31531/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TUNE_L1_RAW_v58/pred_BAS_smt_it_stm_p8_TUNE_L1_RAW_v58_epoch=3-step=81135/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TUNE_L1_RAW_v58/pred_BAS_smt_it_stm_p8_TUNE_L1_RAW_v58_epoch=8-step=182555/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_L1_DIL_v55/pred_BOTH_smt_it_stm_p8_L1_DIL_v55_epoch=5-step=53819/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
]

NAMEK_TO_EVALUATE_SC = [
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_st_s12_newanns_weighted_pfnorm_rgb_v25/pred_SC_smt_it_st_s12_newanns_weighted_pfnorm_rgb_v25_epoch=13-step=599381/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_centerannot_raw_v42/pred_SC_smt_it_stm_p8_centerannot_raw_v42_epoch=13-step=208753/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_centerannot_raw_v42/pred_SC_smt_it_stm_p8_centerannot_raw_v42_epoch=3-step=59643/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_centerannot_raw_v42/pred_SC_smt_it_stm_p8_centerannot_raw_v42_epoch=5-step=89465/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_centerannot_raw_v42/pred_SC_smt_it_stm_p8_centerannot_raw_v42_epoch=55-step=835015/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_centerannot_raw_v42/pred_SC_smt_it_stm_p8_centerannot_raw_v42_epoch=7-step=119287/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_cs64_t5_perframe_rgb_v30/pred_SC_smt_it_stm_p8_newanns_cs64_t5_perframe_rgb_v30_epoch=26-step=1155950/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_cs96_t3_perframe_rgb_v28/pred_SC_smt_it_stm_p8_newanns_cs96_t3_perframe_rgb_v28_epoch=51-step=1274623/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_cs96_t3_perframe_rgb_v28/pred_SC_smt_it_stm_p8_newanns_cs96_t3_perframe_rgb_v28_epoch=57-step=1421695/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_mat6raw6_v41/pred_SC_smt_it_stm_p8_newanns_weighted_mat6raw6_v41_epoch=27-step=1198763/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_mat6raw6_v41/pred_SC_smt_it_stm_p8_newanns_weighted_mat6raw6_v41_epoch=32-step=1412828/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_mat6raw6_v41/pred_SC_smt_it_stm_p8_newanns_weighted_mat6raw6_v41_epoch=42-step=1840958/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_raw_v38/pred_SC_smt_it_stm_p8_newanns_weighted_raw_v38_epoch=85-step=3681917/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_raw_v39/pred_SC_smt_it_stm_p8_newanns_weighted_raw_v39_epoch=13-step=599381/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_raw_v39/pred_SC_smt_it_stm_p8_newanns_weighted_raw_v39_epoch=23-step=1027511/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_raw_v39/pred_SC_smt_it_stm_p8_newanns_weighted_raw_v39_epoch=46-step=2012210/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_raw_v39/pred_SC_smt_it_stm_p8_newanns_weighted_raw_v39_epoch=52-step=2269088/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_raw_v39/pred_SC_smt_it_stm_p8_newanns_weighted_raw_v39_epoch=66-step=2868470/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_raw_v39/pred_SC_smt_it_stm_p8_newanns_weighted_raw_v39_epoch=67-step=2911283/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_rgb_v26/pred_SC_smt_it_stm_p8_newanns_weighted_rgb_v26_epoch=66-step=2868470/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_smt_it_stm_p8_newanns_weighted_rgb_v26/pred_SC_smt_it_stm_p8_newanns_weighted_rgb_v26_epoch=92-step=3981608/Drop1-Aligned-L1-2022-01_vali_data_nowv.kwcoco/pred.kwcoco.json',
]


BEST_MODELS = [

    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_TUNE_L1_RAW_v58/pred_BAS_smt_it_stm_p8_TUNE_L1_RAW_v58_epoch=3-step=81135/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',

    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_L1_DIL_v55/pred_BOTH_smt_it_stm_p8_L1_DIL_v55_epoch=5-step=53819/Drop1-Aligned-L1-2022-01_combo_DILM_nowv_vali.kwcoco/pred.kwcoco.json',
]


TO_EVALUATE = [
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/SC_TA1_ALL_REGIONS_c002_v078/pred_SC_TA1_ALL_REGIONS_c002_v078_epoch=14-step=30719/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v082/pred_BAS_TA1_c001_v082_epoch=59-step=122879/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v073/pred_BAS_TA1_c001_v073_epoch=13-step=28671/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v080/pred_BAS_TA1_c001_v080_epoch=14-step=30719/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v076/pred_BAS_TA1_c001_v076_epoch=90-step=186367/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v075/pred_BAS_TA1_c001_v075_epoch=31-step=65535/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v074/pred_BAS_TA1_c001_v074_epoch=27-step=57343/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v074/pred_BAS_TA1_c001_v074_epoch=10-step=22527/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v074/pred_BAS_TA1_c001_v074_epoch=64-step=133119/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v076/pred_BAS_TA1_c001_v076_epoch=75-step=155647/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v076/pred_BAS_TA1_c001_v076_epoch=12-step=26623/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_c001_v076/pred_BAS_TA1_c001_v076_epoch=4-step=10239/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_ALL_REGIONS_v084/pred_BAS_TA1_ALL_REGIONS_v084_epoch=3-step=34611/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_ALL_REGIONS_v084/pred_BAS_TA1_ALL_REGIONS_v084_epoch=1-step=17305/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=5-step=11189/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=4-step=9324/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
    '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=3-step=7459/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json',
]

# TO_EVALUATE = [
# '/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_smt_it_stm_p8_L1_raw_v53/pred_BAS_smt_it_stm_p8_L1_raw_v53_epoch=3-step=85011/Drop2-Aligned-TA1-2022-01_data_nowv_vali.kwcoco/pred.kwcoco.json'
# ]

"""
/home/joncrall/remote/ooo/

rsync -avprR ooo:data/dvc-repos/smart_watch_dvc/models/fusion/./SC-20201117/ ~/data/dvc-repos/smart_watch_dvc/models/fusion/
"""


def main():
    """
    python -m geowatch.cli.run_tracker \
        /home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=5-step=11189/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=4-step=9324/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json \
        --track_fn watch.tasks.tracking.from_heatmap.TimeAggregatedBAS --track_kwargs '{"thresh": 0.1}' \
        --out_dir /home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=5-step=11189/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=4-step=9324/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/eval/tracked/iarpa-task-thresh=0.1/tracked_sites \
            score -- --merge --gt_dpath /home/joncrall/data/dvc-repos/smart_watch_dvc/annotations --metrics_dpath /home/joncrall/code/metrics-and-test-framework --tmp_dir /home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=5-step=11189/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=4-step=9324/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/eval/tracked/iarpa-task-thresh=0.1/tmp --out_dir /home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=5-step=11189/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/pred.kwcoco.json/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BAS_TA1_KOREA_v083/pred_BAS_TA1_KOREA_v083_epoch=4-step=9324/Drop2-Aligned-TA1-2022-01_combo_L_nowv_vali.kwcoco/eval/tracked/iarpa-task-thresh=0.1/scores
    """
    run = 1

    import watch
    import ubelt as ub
    from watch.utils import tmux_queue
    dvc_dpath = watch.find_smart_dvc_dpath()
    tmux_schedule_dpath = dvc_dpath / '_tmp_tmux_schedule'
    tmux_schedule_dpath.ensuredir()

    serial = False

    environ = {
        'DVC_DPATH': dvc_dpath,
    }

    IARPA_METRICS_DPATH = ub.cmd(
        'python -c "import iarpa_smart_metrics, ubelt; '
        'print(ubelt.Path(iarpa_smart_metrics.__file__).parent.parent)"')['out'].strip()

    task = 'bas'
    # task = 'sc'
    recompute_eval = 0
    skip_existing = 1

    # if task == 'sc':
    #     TO_EVALUATE = [if 'SC' in TO_EVALUATE]
    expected_outputs = []
    stamp = ub.timestamp() + '_' + ub.hash_data([])[0:8]
    tq = tmux_queue.TMUXMultiQueue(
        name=stamp, size=5, environ=environ,
        dpath=tmux_schedule_dpath)

    import numpy as np
    track_kwargs_basis = {
        'thresh': np.linspace(0.1, 0.4, 10).round(3).tolist()[::3],
        # 'thresh': np.linspace(0.17, 0.22, 5).round(3).tolist(),
    }
    for track_kwargs in ub.named_product(track_kwargs_basis):
        for pred_fpath in TO_EVALUATE:
            pred_fpath = ub.Path(pred_fpath)
            pred_bundle_dpath = pred_fpath.parent

            if task == 'bas':
                import shlex
                import json
                track_kwargs_str = shlex.quote(json.dumps(track_kwargs))
                bas_args = f'--track_fn watch.tasks.tracking.from_heatmap.TimeAggregatedBAS --track_kwargs {track_kwargs_str}'  # NOQA
                task_args = bas_args
            elif task == 'sc':
                sc_args = r'--track_fn watch.tasks.tracking.from_heatmap.TimeAggregatedHybrid --track_kwargs "{\"coco_dset_sc\": \"' + str(pred_fpath) + r'\"}"'  # NOQA
                task_args = bas_args
            else:
                raise KeyError

            suffix = 'task-' + ub.urepr(track_kwargs, compact=1)
            IARPA_EVAL_DPATH = pred_bundle_dpath / f'eval/tracked/iarpa-{suffix}'
            summary_fpath = IARPA_EVAL_DPATH / 'scores/merged/summary2.json'
            expected_outputs.append(summary_fpath)

            command = ub.codeblock(
                fr'''
                python -m geowatch.cli.run_tracker \
                    "{pred_fpath}" \
                     {task_args} \
                    --out_dir "{IARPA_EVAL_DPATH}/tracked_sites" \
                    score  -- \
                        --merge \
                        --gt_dpath "{dvc_dpath}/annotations" \
                        --metrics_dpath "{IARPA_METRICS_DPATH}" \
                        --tmp_dir "{IARPA_EVAL_DPATH}/tmp" \
                        --out_dir "{IARPA_EVAL_DPATH}/scores"
                ''')

            if recompute_eval or not (skip_existing and summary_fpath.exists()):
                tq.submit(command)

    print('tq = {!r}'.format(tq))
    tq.rprint()

    if run:
        # ub.cmd('bash ' + str(driver_fpath), verbose=3, check=True)
        if serial:
            tq.serial_run()
        else:
            tq.run()
        agg_state = tq.monitor()
        if not agg_state['errored']:
            tq.kill()
    else:
        driver_fpath = tq.write()
        print('Wrote script: to run execute:\n{}'.format(driver_fpath))

    # if IARPA_EVAL_DPATH.exists():
    #     print(IARPA_EVAL_DPATH)
    #     break

    # expected_outputs = []
    # for pred_fpath in TO_EVALUATE:
    #     pred_fpath = ub.Path(pred_fpath)
    #     pred_bundle_dpath = pred_fpath.parent
    #     IARPA_EVAL_DPATH = pred_bundle_dpath / 'eval/iarpa'
    #     summary_fpath = IARPA_EVAL_DPATH / 'scores/merged/summary2.json'
    #     expected_outputs.append(summary_fpath)

    have_outputs = [s for s in expected_outputs if s.exists()]
    have_outputs[0].exists()
    import json
    import pandas as pd

    @ub.memoize
    def to_df(bas_dpath, region_id):
        import parse
        scoreboard_fpaths = sorted(bas_dpath.glob('scoreboard_rho=*.csv'))
        rho_parser = parse.Parser('scoreboard_rho={rho:f}.csv')
        dfs = []
        for pth in scoreboard_fpaths:
            rho = rho_parser.parse(pth.name).named['rho']
            df = pd.read_csv(pth)
            df['rho'] = rho
            df['region_id'] = region_id
            # MultiIndex with rho, tau and region_id
            df = df.set_index(['region_id', 'rho', 'tau'])
            dfs.append(df)
        return pd.concat(dfs)

    rows = []
    for summary_fpath in have_outputs:
        summary_fpath.parent
        summary_text = summary_fpath.read_text()
        if not summary_text:
            print('bad summary_fpath = {!r}'.format(summary_fpath))
        else:
            # hack to get thresh:
            import parse
            thresh = float(parse.parse('{}thresh={thresh}/{}', str(summary_fpath)).named['thresh'])

            summary = json.loads(summary_text)

            pred_bundle_dpath = summary_fpath.parent.parent.parent.parent.parent
            # dataset_key = pred_bundle_dpath.name
            model_key = pred_bundle_dpath.parent.parent.name

            # best_bas_rows = pd.read_json(json.dumps(summary['best_bas_rows']), orient='table')
            # merged_row = best_bas_rows.loc['merged']
            # merged_subrow = merged_row[['F1']]

            if 1:
                # More complete measures
                sites_dpath = summary_fpath.parent.parent
                unmerged = [p for p in sites_dpath.glob('*') if p.name != 'merged']
                dfs = []
                for p in unmerged:
                    bas_dpath = p / 'bas'
                    unmerged_df = to_df(bas_dpath, p.name)
                    dfs.append(unmerged_df)
                concat_df = pd.concat(dfs)
                result_df = pd.DataFrame(index=dfs[0].droplevel('region_id').index)
                sum_cols = [
                    'tp sites', 'fp sites', 'fn sites', 'truth sites', 'proposed sites',
                    'total sites', 'truth slices', 'proposed slices'
                ]
                result_df[sum_cols] = concat_df.groupby(['rho', 'tau'])[sum_cols].sum()
                # ref: metrics-and-test-framework.evaluation.Metric
                (_, tp), (_, fp), (_, fn) = result_df[['tp sites', 'fp sites',
                                                       'fn sites']].iteritems()
                result_df['precision'] = np.where(tp > 0, tp / (tp + fp), 0)
                result_df['recall (PD)'] = np.where(tp > 0, tp / (tp + fn), 0)


                result_df['F1'] = np.where(tp > 0, tp / (tp + 0.5 * (fp + fn)), 0)
                def fbeta(tp, fp, fn, beta):
                    b2 = (beta * beta)
                    opb2 = (1 + b2)
                    numer = opb2 * tp
                    denom = opb2 * tp + b2 * fn + fp
                    bad = (denom == 0)
                    denom[bad] = 1
                    numer[bad] = 0
                    score = numer / denom
                    return score
                result_df['F2'] = fbeta(tp, fp, fn, beta=2)

                flags1 = result_df.index.get_level_values('rho') >= 0.1
                flags2 = result_df.index.get_level_values('tau') >= 0.1
                sub_df = result_df[flags1 & flags2]

                loc1 = result_df['F1'].idxmax()
                loc2 = result_df['F2'].idxmax()
                loc3 = sub_df['F1'].idxmax()
                loc4 = sub_df['F2'].idxmax()

                merged_subrow = result_df.loc[[
                    # loc1,
                    # loc2,
                    loc3,
                    loc4
                ]]
                merged_subrow = merged_subrow.rename(
                    {'tp sites': 'tp',
                     'fp sites': 'fp',
                     'fn sites': 'fn',
                     'truth sites': 'truth',
                     'proposed sites': 'proposed',
                     'total sites': 'total'}, axis=1)
                merged_subrow = merged_subrow.drop([
                    'truth slices',
                    'proposed slices', 'precision', 'recall (PD)'], axis=1)
                mF1 = result_df['F1'].mean()

                merged_subrow = merged_subrow.assign(mF1=mF1)

            # merged_subrow = merged_subrow.assign(dataset=[dataset_key])
            merged_subrow = merged_subrow.assign(thresh=thresh)
            merged_subrow = merged_subrow.assign(model=model_key)
            merged_subrow = merged_subrow.reset_index()
            rows.append(merged_subrow)

    summary_df = pd.concat(rows)
    summary_df = summary_df.reset_index()
    summary_df = summary_df.sort_values('F1')
    print(summary_df.to_string())
    print(summary_df.drop(['proposed', 'total', 'truth', 'tp',  'fp',  'fn', 'F2'], axis=1).iloc[-10:].set_index(['rho', 'tau'], drop=1).to_string())

    print(summary_df.loc[summary_df.groupby('model')['mF1'].idxmax()].sort_values('mF1').to_string())
    # print(summary_df.loc[summary_df.groupby('model')['sub_f1'].idxmax()].sort_values('sub_f1').to_string())

    # summary_df = summary_df.sort_values('thresh')
    print(summary_df.to_string())
    # summary_df = summary_df.iloc[3:]
    # print(summary_df.to_string())

    if 1:
        import dataframe_image as dfi
        dfi.export(
            summary_df,
            "./tmp.png",
            table_conversion="chrome",
            fontsize=28,
            max_rows=-1,
        )
        import kwplot
        fig, _ = kwplot.imshow('./tmp.png', fnum=10)
        fig.tight_layout()
        kwplot.autompl()
        kwplot.imshow('./tmp.png', fnum=101)


"""

HACK

PRED_BUNDLE_DPATH="/home/joncrall/data/dvc-repos/smart_watch_dvc/models/fusion/SC-20201117/BOTH_smt_it_stm_p8_L1_DIL_v55/pred_BOTH_smt_it_stm_p8_L1_DIL_v55_epoch=6-step=62789/combo_DILM.kwcoco_vali"
PRED_DATASET="$PRED_BUNDLE_DPATH/pred.kwcoco.json"
EVAL_DPATH="$PRED_BUNDLE_DPATH/eval"
IARPA_EVAL_DPATH="$EVAL_DPATH/iarpa-sc-thresh-0.1"
IARPA_METRICS_DPATH=$(python -c "import iarpa_smart_metrics, os; print(os.path.dirname(os.path.dirname(iarpa_smart_metrics.__file__)))")

REGION_FILE=/home/joncrall/data/dvc-repos/smart_watch_dvc/annotations/region_models/KR_R002.geojson

echo $PRED_DATASET
echo $REGION_FILE

python -m geowatch.cli.run_tracker \
    "$PRED_DATASET" \
    --track_fn watch.tasks.tracking.from_heatmap.TimeAggregatedSC --track_kwargs "{\"thresh\": 0.01}" \
    --site_summary $REGION_FILE \
    score  -- \
        --merge \
        --gt_dpath "$DVC_DPATH/annotations" \
        --metrics_dpath "$IARPA_METRICS_DPATH" \
        --tmp_dir "$IARPA_EVAL_DPATH/tmp" \
        --out_dir "$IARPA_EVAL_DPATH/scores"

nautilus $IARPA_EVAL_DPATH


python -m geowatch.cli.run_tracker \
    "$PRED_DATASET" \
    --default_track_fn watch.tasks.tracking.from_heatmap.TimeAggregatedSC --track_kwargs "{\"thresh\": 0.01}" \
    score  -- \
        --merge \
        --gt_dpath "$DVC_DPATH/annotations" \
        --metrics_dpath "$IARPA_METRICS_DPATH" \
        --tmp_dir "$IARPA_EVAL_DPATH/tmp" \
        --out_dir "$IARPA_EVAL_DPATH/scores"
"""