""" python -m geowatch mlops "status" --dataset_codes Drop4-SC python -m geowatch mlops "push packages" --dataset_codes Drop4-SC python -m geowatch mlops "pull packages" --dataset_codes Drop4-SC models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/Drop4_tune_V30_V2_epoch=6-step=83790.pt.pt models/fusion/Drop4-SC/packages/Drop4_tune_V30_8GSD_V3/package_epoch3_step22551.pt.pt models/fusion/Drop4-SC/packages/Drop4_tune_V30_2GSD_V3/package_epoch0_step57.pt.pt # Quick validation sites KR_R001_0000_box KR_R001_0015_box KR_R002_0025_box KR_R002_0030_box # negative US_R007_0045_box US_R007_0015_box python -m geowatch find_dvc --hardware=auto --tags=phase2_expt DATASET_CODE=Drop4-SC DATA_DVC_DPATH=$(geowatch_dvc --tags="phase2_data" --hardware="auto") EXPT_DVC_DPATH=$(geowatch_dvc --tags="phase2_expt" --hardware="auto") echo " DATA_DVC_DPATH = $DATA_DVC_DPATH EXPT_DVC_DPATH = $EXPT_DVC_DPATH " # Setup the small dataset kwcoco subset \ --src $DATA_DVC_DPATH/Drop4-SC/data_vali.kwcoco.json \ --dst $DATA_DVC_DPATH/Drop4-SC/data_vali_small.kwcoco.json \ --select_videos '((.name == "KR_R001_0000_box") or (.name == "KR_R001_0015_box") or (.name == "KR_R002_0025_box") or (.name == "KR_R002_0030_box") or (.name == "US_R007_0045_box") or (.name == "US_R007_0015_box"))' {'Drop4_tune_V30_8GSD_V3_epoch=0-step=5778-v1.pt', 'Drop4_tune_V30_8GSD_V3_epoch=1-step=11556.pt', 'Drop4_tune_V30_8GSD_V3_epoch=2-step=17334.pt', 'Drop4_tune_V30_V2_epoch=1-step=23940.pt', 'Drop4_tune_V30_V2_epoch=2-step=35910-v1.pt', 'Drop4_tune_V30_V2_epoch=3-step=47880-v1.pt', 'package_epoch0_step171.pt', 'package_epoch0_step57.pt', 'package_epoch0_step6587.pt', 'package_epoch3_step22551.pt'} DATASET_CODE=Drop4-SC DATA_DVC_DPATH=$(geowatch_dvc --tags="phase2_data" --hardware="auto") EXPT_DVC_DPATH=$(geowatch_dvc --tags="phase2_expt" --hardware="auto") python -m geowatch.mlops.schedule_evaluation \ --params=" matrix: trk.pxl.package_fpath: unused trk.pxl.data.test_dataset: unused trk.pxl.data.window_space_scale: unused trk.pxl.data.time_sampling: unused trk.pxl.data.input_space_scale: unused trk.poly.thresh: unused crop.src: unused crop.regions: truth act.pxl.data.test_dataset: - $DATA_DVC_DPATH/$DATASET_CODE/data_vali_small.kwcoco.json act.pxl.data.input_space_scale: # - auto - 8GSD act.pxl.data.time_steps: - auto act.pxl.data.chip_overlap: - 0.3 act.poly.thresh: - 0.07 - 0.1 - 0.13 act.poly.use_viterbi: - 0 act.pxl.package_fpath: - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/Drop4_tune_V30_V2_epoch=1-step=23940.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/Drop4_tune_V30_V2_epoch=2-step=35910-v1.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/Drop4_tune_V30_V2_epoch=3-step=47880-v1.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_8GSD_V3/package_epoch3_step22551.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_2GSD_V3/package_epoch0_step57.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/package_epoch0_step171.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/package_epoch0_step6587.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_8GSD_V3/Drop4_tune_V30_8GSD_V3_epoch=0-step=5778-v1.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_8GSD_V3/Drop4_tune_V30_8GSD_V3_epoch=1-step=11556.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_8GSD_V3/Drop4_tune_V30_8GSD_V3_epoch=2-step=17334.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/Drop4_tune_V30_V2_epoch=4-step=59850.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/Drop4_tune_V30_V2_epoch=5-step=71820.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/Drop4_tune_V30_V2_epoch=6-step=83790.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/Drop4_tune_V30_V2_epoch=0-step=11970.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/package_epoch0_step0.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/package_epoch0_step11970.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/package_epoch0_step1661.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-SC/packages/Drop4_tune_V30_V2/package_epoch7_step95760.pt.pt include: - act.pxl.data.chip_dims: 256,256 act.pxl.data.window_space_scale: 8GSD act.pxl.data.input_space_scale: 8GSD act.pxl.data.output_space_scale: 8GSD # - act.pxl.data.chip_dims: 256,256 # act.pxl.data.window_space_scale: 4GSD # act.pxl.data.input_space_scale: 4GSD # act.pxl.data.output_space_scale: 4GSD " \ --enable_pred_trk_pxl=0 \ --enable_pred_trk_poly=0 \ --enable_eval_trk_pxl=0 \ --enable_eval_trk_poly=0 \ --enable_crop=0 \ --enable_pred_act_pxl=1 \ --enable_pred_act_poly=1 \ --enable_eval_act_pxl=1 \ --enable_eval_act_poly=1 \ --enable_viz_pred_trk_poly=0 \ --enable_viz_pred_act_poly=0 \ --enable_links=1 \ --devices="0,1" --queue_size=2 \ --queue_name='nov-sc-eval2' \ --backend=tmux --skip_existing=1 \ --run=1 python -m geowatch.cli.run_tracker /home/joncrall/remote/Ooo/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-SC/pred/act/package_epoch3_step22551.pt/Drop4-SC_data_vali_small.kwcoco/act_pxl_e836a34c/pred.kwcoco.json --default_track_fn class_heatmaps --track_kwargs '{"boundaries_as": "polys", "thresh": 0.1, "use_viterbi": 0}' --site_summary /home/joncrall/remote/Ooo/data/dvc-repos/smart_expt_dvc/models/fusion/dset_code_unknown/pred/trk/foo/Drop4-SC_combo_US_R007_I.kwcoco/trk_pxl_ca8e6033/trk_poly_9f08fb8c/site_summary_tracks_manifest.json --out_sites_fpath /home/joncrall/remote/Ooo/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-SC/pred/act/package_epoch3_step22551.pt/Drop4-SC_data_vali_small.kwcoco/act_pxl_e836a34c/act_poly_e50c1c4f/site_activity_manifest.json --out_sites_dir /home/joncrall/remote/Ooo/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-SC/pred/act/package_epoch3_step22551.pt/Drop4-SC_data_vali_small.kwcoco/act_pxl_e836a34c/act_poly_e50c1c4f/sites --out_kwcoco /home/joncrall/remote/Ooo/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-SC/pred/act/package_epoch3_step22551.pt/Drop4-SC_data_vali_small.kwcoco/act_pxl_e836a34c/act_poly_e50c1c4f/activity_tracks.kwcoco.json TEST_DATASET=$DATA_DVC_DPATH/$DATASET_CODE/BR_R001.kwcoco.json """ def bas_report(): import rich # NOQA import ubelt as ub # NOQA import pandas as pd # NOQA from watch import heuristics # from watch.mlops import expt_state from watch.mlops import expt_manager as expt_state from watch.mlops import expt_report expt_dvc_dpath = heuristics.auto_expt_dvc() data_dvc_dpath = heuristics.auto_expt_dvc() # I messed up the name of the dataset I was working on. # it is marked as train, but it should have been vali. # dataset_code = ['Drop4-BAS', 'dset_code_unknown', 'eval3_candidates', 'Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC'] dataset_code = ['Drop4-BAS'] # 'Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC'] # dataset_code = '*' state = expt_state.ExperimentState( expt_dvc_dpath, dataset_code=dataset_code, data_dvc_dpath=data_dvc_dpath, # model_pattern='*' model_pattern='package_epoch0_step41.pt.pt', # model_pattern='*' ) self = state # NOQA state._build_path_patterns() state.summarize() reporter = expt_report.EvaluationReporter(state) reporter.load1() reporter.load2() df = reporter.orig_merged_df # df = DotDictDataFrame(df) groupid_to_shortlist = reporter.report_best(show_configs=True, verbose=1, top_k=10) if 1: df2 = df[df['type'] == 'eval_trk_poly_fpath'] df2 = df2.sort_values('trk.poly.metrics.bas_f1') rich.print(df2[[ 'trk_model', 'type', # 'trk.pxl.metrics.salient_AP', 'trk.poly.metrics.bas_f1', 'trk.pxl.input_space_scale', 'trk.pxl.window_space_scale', 'trk.pxl.output_space_scale', # 'trk.pxl.tta_time', # 'trk.pxl.tta_fliprot', 'trk.pxl.chip_dims', 'trk.pxl.time_steps', # 'trk.pxl.time_span', # 'trk.pxl.time_sampling', 'trk.pxl.set_cover_algo', # 'trk.pxl.resample_invalid_frames', # 'trk.pxl.use_cloudmask', 'trk.pxl.resource.total_hours', 'trk.pxl.resource.gpu_name', 'trk.poly.thresh', 'trk.poly.moving_window_size', # 'trk.poly.polygon_fn', ]].to_string()) if 0: g = groupid_to_shortlist[('Drop4-BAS_KR_R001.kwcoco', 'eval_trk_poly_fpath')] g = DotDictDataFrame(g) import kwplot sns = kwplot.autosns() df['trk.pxl.model'] = df['trk.pxl.package_fpath'].apply(lambda x: ub.Path(x).name) df['trk.pxl.model'] = df['trk.pxl.package_fpath'].apply(lambda x: ub.Path(x).name) df['trk.pxl.model'] = df['trk.pxl.package_fpath'].apply(lambda x: ub.Path(x).name) df['trk.gsd'] = df['trk.pxl.input_space_scale'].apply(lambda x: int(x[:-3])) df['trk.efficiency'] = df['trk.poly.metrics.bas_faa_f1'] / df['trk.both.total_hours'] fig = kwplot.figure(fnum=1, doclf=True) sns.scatterplot(data=df, x='trk.gsd', y='trk.both.total_hours', hue='trk.pxl.package_fpath', ax=fig.gca()) # sns.scatterplot(data=df, x='trk.pxl.properties.step', y='act.poly.metrics.sc_macro_f1', hue='act.pxl.model_name') fig = kwplot.figure(fnum=2, doclf=True) sns.scatterplot(data=df, x='trk.gsd', y='trk.poly.metrics.bas_faa_f1', hue='trk.pxl.package_fpath', legend=False) fig = kwplot.figure(fnum=3, doclf=True) sns.scatterplot(data=df, x='trk.gsd', y='trk.efficiency', hue='trk.pxl.package_fpath', legend=False) def main(): """ import sys, ubelt sys.path.append(ubelt.expandpath('~/code/watch/dev/reports')) from report_2022_09_xx import * # NOQA """ import rich import ubelt as ub import pandas as pd # NOQA from watch import heuristics from watch.mlops import expt_state from watch.mlops import expt_report expt_dvc_dpath = heuristics.auto_expt_dvc() data_dvc_dpath = heuristics.auto_expt_dvc() # I messed up the name of the dataset I was working on. # it is marked as train, but it should have been vali. dataset_code = 'Drop4-SC' state = expt_state.ExperimentState( expt_dvc_dpath, dataset_code=dataset_code, data_dvc_dpath=data_dvc_dpath, model_pattern='*' # model_pattern='*Drop4_tune_V30_8GSD_V3_epoch=2-step=17334*' ) self = state # NOQA state._build_path_patterns() state.summarize() reporter = expt_report.EvaluationReporter(state) reporter.load1() reporter.load2() reporter.state.summarize() df = reporter.orig_merged_df from watch.utils.util_pandas import DotDictDataFrame df = DotDictDataFrame(df) groupid_to_shortlist = reporter.report_best(show_configs=True, verbose=1, top_k=4) if 1: df2 = df[df['type'] == 'eval_act_poly_fpath'] df2 = df2.sort_values('act.poly.metrics.sc_macro_f1') rich.print(df2[[ 'act_model', # 'type', 'act.poly.metrics.sc_macro_f1', 'act.pxl.input_space_scale', 'act.pxl.window_space_scale', 'act.pxl.output_space_scale', # 'act.pxl.tta_time', # 'act.pxl.tta_fliprot', 'act.pxl.chip_dims', 'act.pxl.time_steps', # 'act.pxl.time_span', # 'act.pxl.time_sampling', # 'act.pxl.set_cover_algo', # 'act.pxl.resample_invalid_frames', 'act.pxl.use_cloudmask', 'act.pxl.resource.total_hours', 'act.poly.resource.total_hours', 'act.pxl.resource.gpu_name', 'act.poly.thresh', # 'act.poly.moving_window_size', 'act.poly.boundaries_as', ]].iloc[-100:].to_string()) if 1: import kwplot sns = kwplot.autosns() df['act.pxl.package_fpath'] = df['act.pxl.package_fpath'].apply(lambda x: ub.Path(x).name) df['act.gsd'] = df['act.pxl.input_space_scale'].apply(lambda x: int(x[:-3])) df['act.both.total_hours'] = df[['act.pxl.resource.total_hours', 'act.poly.resource.total_hours']].sum(axis=1) df['act.efficiency'] = df['act.poly.metrics.sc_macro_f1'] / df['act.both.total_hours'] fig = kwplot.figure(fnum=1, doclf=True) sns.scatterplot(data=df, x='act.gsd', y='act.both.total_hours', hue='act.pxl.package_fpath', ax=fig.gca(), legend=False) fig.set_size_inches([6, 3]) fig.subplots_adjust(bottom=0.2) # sns.scatterplot(data=df, x='act.pxl.properties.step', y='act.poly.metrics.sc_macro_f1', hue='act.pxl.model_name') fig = kwplot.figure(fnum=2, doclf=True) sns.scatterplot(data=df, x='act.gsd', y='act.poly.metrics.sc_macro_f1', hue='act.pxl.package_fpath', legend=False) fig.set_size_inches([6, 3]) fig.subplots_adjust(bottom=0.2) fig = kwplot.figure(fnum=3, doclf=True) sns.scatterplot(data=df, x='act.gsd', y='act.efficiency', hue='act.pxl.package_fpath', legend=False) fig.set_size_inches([6, 3]) fig.subplots_adjust(bottom=0.2) from watch.utils.util_pandas import DotDictDataFrame sdf = DotDictDataFrame(groupid_to_shortlist[('Drop4-SC_data_vali_small.kwcoco', 'eval_act_poly_fpath')]) keepers = [] for x in groupid_to_shortlist.values(): keepers.extend(x['act_model']) from watch.utils.util_pandas import DotDictDataFrame df = DotDictDataFrame(df) print('df.nested_columns = {}'.format(ub.urepr(df.nested_columns, nl=True))) space = DotDictDataFrame(df['input_space_scale']) (space['act.fit.input_space_scale'] != space['act.pxl.input_space_scale']).sum() act_poly_df = DotDictDataFrame(df[df['type'] == 'eval_act_poly_fpath']) rich.print(act_poly_df.sort_values('act.poly.metrics.sc_macro_f1').iloc[-3:].drop(['raw'], axis=1).T.to_string()) dotted = df.find_columns('*.*') metric_cols = df.find_columns('*metrics.*') meta_cols = df.find_columns('*meta.*') resource_cols = df.find_columns('*resource.*') fit_cols = df.find_columns('*fit.*') param_cols = dotted - (metric_cols | fit_cols | resource_cols | meta_cols) # df['trk.pxl.properties.test_dataset_fname'] = df['trk.pxl.test_dataset'].apply(lambda x: ub.Path(x).name) # flags = df['trk.pxl.properties.test_dataset_fname'] == 'data.kwcoco.json' # df = df[flags] # TODO: shrink rows where data isn't all nan # subdf = df[~df['trk.poly.thresh'].isnull()] # subdf = subdf[['trk.poly.thresh', 'trk.poly.metrics.bas_f1']] # ~subdf.isnull(axis=1) # trk.poly.metrics.]] # df[~df['trk.poly.thresh'].isnull()] cols_of_interest = [ 'trk.poly.thresh', 'trk.poly.metrics.bas_f1', 'trk.poly.metrics.bas_tp', 'trk.poly.metrics.bas_fp', 'trk.poly.metrics.bas_fn', 'trk.poly.metrics.bas_ppv', 'trk.poly.metrics.bas_tpr', 'act.poly.metrics.sc_macro_f1', 'act.poly.resource.total_hours', 'act.pxl.resource.total_hours', 'act.fit.input_space_scale', 'act.pxl.package_fpath', 'act.fit.input_space_scale', 'act.poly.pxl.input_space_scale', ] col_order = (ub.oset(cols_of_interest) | metric_cols) & ub.oset(df.columns) df2 = df[col_order] df['total_hours'] df['act.total_hours'] = df['act.resource.total_hours'].sum(axis=1) df._clear_column_caches() df['act.pxl.model_name'] = df['act.pxl.package_fpath'].apply(lambda x: ub.Path(x).name) df['act.pxl.properties.step'] rich.print(df2) import kwplot sns = kwplot.autosns() sns.scatterplot(data=df, x='act.pxl.properties.step', y='act.poly.metrics.sc_macro_f1', hue='act.pxl.model_name') sns.scatterplot(data=df, x='act.pxl.input_space_scale', y='act.poly.metrics.sc_macro_f1', hue='act.pxl.model_name') sns.scatterplot(data=df, x='act.total_hours', y='act.poly.metrics.sc_macro_f1', hue='act.pxl.model_name') (df['act.fit.input_space_scale'] != df['act.pxl.input_space_scale']) (df['act.fit.input_space_scale'] != df['act.pxl.input_space_scale']) sns.lineplot(data=df2, x='trk.poly.metrics.bas_fp', y='trk.poly.metrics.bas_fp') sns.lineplot(data=df2, x='trk.poly.metrics.bas_tpr', y='trk.poly.metrics.bas_ppv') sns.lineplot(data=df2, x='trk.poly.thresh', y='trk.poly.metrics.bas_f1') sns.lineplot(data=df2, x='trk.poly.metrics.bas_tpr', y='trk.poly.metrics.bas_ppv') sns.lineplot(data=df2, x='trk.poly.metrics.bas_tpr', y='trk.poly.metrics.bas_ppv') sns.lineplot(data=df2, x='trk.poly.metrics.bas_fp', y='trk.poly.metrics.bas_tpr') # hue='trk.poly.thresh') df.loc[:, df.columns & (list(metric_cols) + ['trk.poly.thresh'])] # dpath = reporter.dpath dpath = ub.Path.appdir('watch/expt-report/2022-10-xx').ensuredir() # Dump details out about the best models cohort = ub.timestamp() best_models_dpath = (dpath / 'best_models' / cohort).ensuredir() def shrink_na_cols(df): import pandas as pd isnull = pd.isnull(df) df = df.loc[:, ~isnull.all(axis=0)] return df """ Test connors models: DATASET_CODE=Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC DATA_DVC_DPATH=$(geowatch_dvc --tags="phase2_data" --hardware="ssd") EXPT_DVC_DPATH=$(geowatch_dvc --tags="phase2_expt") # TEST_DATASET=$DATA_DVC_DPATH/$DATASET_CODE/data.kwcoco.json TEST_DATASET=$DATA_DVC_DPATH/$DATASET_CODE/BR_R001.kwcoco.json python -m geowatch.mlops.schedule_evaluation \ --params=" matrix: trk.pxl.package_fpath: # - $EXPT_DVC_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/packages/Drop4_BAS_Multi_Native/package_epoch10_step200000.pt - /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/training/toothbrush/joncrall/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/runs/Drop4_BAS_invariants_30GSD_V016/lightning_logs/version_4/checkpoints/Drop4_BAS_invariants_30GSD_V016_epoch=10-step=5632.pt # - $EXPT_DVC_DPATH/models/fusion/eval3_candidates/packages/Drop3_SpotCheck_V323/Drop3_SpotCheck_V323_epoch=18-step=12976.pt trk.pxl.data.test_dataset: - $DATA_DVC_DPATH/$DATASET_CODE/combo_BR_R001_I.kwcoco.json - $DATA_DVC_DPATH/$DATASET_CODE/combo_KR_R001_I.kwcoco.json - $DATA_DVC_DPATH/$DATASET_CODE/combo_KR_R002_I.kwcoco.json - $DATA_DVC_DPATH/$DATASET_CODE/combo_US_R007_I.kwcoco.json trk.pxl.data.window_space_scale: - "auto" trk.pxl.data.time_sampling: - "auto" trk.pxl.data.input_space_scale: - "auto" trk.poly.thresh: - 0.1 crop.src: - /home/joncrall/remote/toothbrush/data/dvc-repos/smart_data_dvc/online_v1/kwcoco_for_sc_fielded.json crop.regions: - trk.poly.output act.pxl.data.test_dataset: - /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/crop/online_v1_kwcoco_for_sc_fielded/trk_poly_id_0408400f/crop_f64d5b9a/crop_id_59ed6e1b/crop.kwcoco.json act.pxl.data.input_space_scale: - 3GSD act.pxl.data.time_steps: - 3 act.pxl.data.chip_overlap: - 0.35 act.poly.thresh: - 0.01 act.poly.use_viterbi: - 0 act.pxl.package_fpath: - $EXPT_DVC_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-WV-PD-ACC/packages/Drop4_SC_RGB_scratch_V002/Drop4_SC_RGB_scratch_V002_epoch=99-step=50300-v1.pt.pt include: - act.pxl.data.chip_dims: 256,256 act.pxl.data.window_space_scale: 3GSD act.pxl.data.input_space_scale: 3GSD act.pxl.data.output_space_scale: 3GSD " \ --enable_pred_trk_pxl=1 \ --enable_pred_trk_poly=1 \ --enable_eval_trk_pxl=1 \ --enable_eval_trk_poly=1 \ --enable_crop=0 \ --enable_pred_act_pxl=0 \ --enable_pred_act_poly=0 \ --enable_eval_act_pxl=0 \ --enable_eval_act_poly=0 \ --enable_viz_pred_trk_poly=0 \ --enable_viz_pred_act_poly=0 \ --enable_links=1 \ --devices="1," --queue_size=2 \ --queue_name='nov-eval' \ --backend=tmux --skip_existing=1 \ --run=0 # BAS TUNING DATASET_CODE=Drop4-BAS DATA_DVC_DPATH=$(geowatch_dvc --tags="phase2_data" --hardware="auto") EXPT_DVC_DPATH=$(geowatch_dvc --tags="phase2_expt" --hardware="auto") python -m geowatch.mlops.schedule_evaluation \ --params=" matrix: trk.pxl.package_fpath: # - $EXPT_DVC_DPATH/models/fusion/eval3_candidates/packages/Drop3_SpotCheck_V323/Drop3_SpotCheck_V323_epoch=18-step=12976.pt # - $EXPT_DVC_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/packages/Drop4_BAS_Continue_15GSD_BGR_V004/Drop4_BAS_Continue_15GSD_BGR_V004_epoch=78-step=323584.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-BAS/packages/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2_epoch=0-step=7501.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-BAS/packages/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2_epoch=0-step=23012.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-BAS/packages/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2_epoch=0-step=7501-v1.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-BAS/packages/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2_epoch=0-step=23012-v1.pt.pt - $EXPT_DVC_DPATH/models/fusion/Drop4-BAS/packages/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2/package_epoch0_step41.pt.pt # - $EXPT_DVC_DPATH/models/fusion/Drop4-BAS/packages/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2/package_epoch0_step24.pt.pt trk.pxl.data.test_dataset: - $DATA_DVC_DPATH/$DATASET_CODE/KR_R001.kwcoco.json # - $DATA_DVC_DPATH/$DATASET_CODE/BR_R002.kwcoco.json # - $DATA_DVC_DPATH/$DATASET_CODE/KR_R002.kwcoco.json # - $DATA_DVC_DPATH/$DATASET_CODE/US_R007.kwcoco.json trk.pxl.data.window_space_scale: # - "auto" - "10GSD" - "15GSD" # - "30GSD" # - "40GSD" # - "60GSD" trk.pxl.data.set_cover_algo: - null - approx trk.pxl.data.time_steps: - 11 - 5 # - auto # - 2 # - 3 # - 4 # - 5 # - 6 trk.pxl.data.time_sampling: - "auto" trk.poly.thresh: # - 0.07 # - 0.09 - 0.1 # - 0.11 # - 0.13 crop.src: - /home/joncrall/remote/toothbrush/data/dvc-repos/smart_data_dvc/online_v1/kwcoco_for_sc_fielded.json crop.regions: - trk.poly.output act.pxl.data.test_dataset: - /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-L8-ACC/crop/online_v1_kwcoco_for_sc_fielded/trk_poly_id_0408400f/crop_f64d5b9a/crop_id_59ed6e1b/crop.kwcoco.json act.pxl.data.input_space_scale: - 3GSD act.pxl.data.time_steps: - 3 act.pxl.data.chip_overlap: - 0.35 act.poly.thresh: - 0.01 act.poly.use_viterbi: - 0 act.pxl.package_fpath: - $EXPT_DVC_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-WV-PD-ACC/packages/Drop4_SC_RGB_scratch_V002/Drop4_SC_RGB_scratch_V002_epoch=99-step=50300-v1.pt.pt include: # - trk.pxl.data.window_space_scale: 60GSD # trk.pxl.data.input_space_scale: 60GSD # trk.pxl.data.output_space_scale: 60GSD # - trk.pxl.data.window_space_scale: 40GSD # trk.pxl.data.input_space_scale: 40GSD # trk.pxl.data.output_space_scale: 40GSD # - trk.pxl.data.window_space_scale: 30GSD # trk.pxl.data.input_space_scale: 30GSD # trk.pxl.data.output_space_scale: 30GSD - trk.pxl.data.window_space_scale: 15GSD trk.pxl.data.input_space_scale: 15GSD trk.pxl.data.output_space_scale: 15GSD - trk.pxl.data.window_space_scale: 10GSD trk.pxl.data.input_space_scale: 10GSD trk.pxl.data.output_space_scale: 10GSD # - trk.pxl.data.window_space_scale: auto # trk.pxl.data.input_space_scale: auto # trk.pxl.data.output_space_scale: auto " \ --enable_pred_trk_pxl=1 \ --enable_pred_trk_poly=1 \ --enable_eval_trk_pxl=1 \ --enable_eval_trk_poly=1 \ --enable_crop=0 \ --enable_pred_act_pxl=0 \ --enable_pred_act_poly=0 \ --enable_eval_act_pxl=0 \ --enable_eval_act_poly=0 \ --enable_viz_pred_trk_poly=0 \ --enable_viz_pred_act_poly=0 \ --enable_links=1 \ --devices="0,1" --queue_size=2 \ --queue_name='bas-eval' \ --backend=tmux --skip_existing=0 \ --run=0 """ # ------ """ #### BAS CHECKS python -m geowatch mlops "status" --dataset_codes Drop4-BAS python -m geowatch mlops "push packages" --dataset_codes Drop4-BAS python -m geowatch mlops "pull packages" --dataset_codes Drop4-BAS python -m geowatch.tasks.fusion.predict \ --package_fpath=/home/local/KHQ/jon.crall/remote/horologic/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-BAS/packages/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2/package_epoch0_step41.pt.pt \ --test_dataset=/home/local/KHQ/jon.crall/remote/horologic/data/dvc-repos/smart_data_dvc/Drop4-BAS/KR_R001.kwcoco.json \ --pred_dataset=/home/local/KHQ/jon.crall/remote/horologic/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-BAS/pred/trk/package_epoch0_step41.pt.pt/Drop4-BAS_KR_R001.kwcoco/trk_pxl_16f221bd/pred.kwcoco.json \ --time_steps=auto \ --time_sampling=auto \ --window_space_scale=15GSD \ --input_space_scale=15GSD \ --output_space_scale=15GSD \ --num_workers=4 \ --devices=0, \ --accelerator=gpu \ --batch_size=1 python -m geowatch.cli.run_tracker \ "/home/local/KHQ/jon.crall/remote/horologic/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-BAS/pred/trk/package_epoch0_step41.pt.pt/Drop4-BAS_KR_R001.kwcoco/trk_pxl_16f221bd/pred.kwcoco.json" \ --default_track_fn saliency_heatmaps \ --track_kwargs '{"thresh": 0.1, "moving_window_size": null, "polygon_fn": "heatmaps_to_polys"}' \ --clear_annots \ --out_sites_dir "/home/local/KHQ/jon.crall/remote/horologic/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-BAS/pred/trk/package_epoch0_step41.pt.pt/Drop4-BAS_KR_R001.kwcoco/trk_pxl_16f221bd/trk_poly_9f08fb8c/sites" \ --out_site_summaries_dir "/home/local/KHQ/jon.crall/remote/horologic/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-BAS/pred/trk/package_epoch0_step41.pt.pt/Drop4-BAS_KR_R001.kwcoco/trk_pxl_16f221bd/trk_poly_9f08fb8c/site-summaries" \ --out_sites_fpath "/home/local/KHQ/jon.crall/remote/horologic/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-BAS/pred/trk/package_epoch0_step41.pt.pt/Drop4-BAS_KR_R001.kwcoco/trk_pxl_16f221bd/trk_poly_9f08fb8c/site_tracks_manifest.json" \ --out_site_summaries_fpath "/home/local/KHQ/jon.crall/remote/horologic/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-BAS/pred/trk/package_epoch0_step41.pt.pt/Drop4-BAS_KR_R001.kwcoco/trk_pxl_16f221bd/trk_poly_9f08fb8c/site_summary_tracks_manifest.json" \ --out_kwcoco "/home/local/KHQ/jon.crall/remote/horologic/data/dvc-repos/smart_expt_dvc/models/fusion/Drop4-BAS/pred/trk/package_epoch0_step41.pt.pt/Drop4-BAS_KR_R001.kwcoco/trk_pxl_16f221bd/trk_poly_9f08fb8c/tracks.kwcoco.json" # END_TO_END TUNING DATASET_CODE=Drop4-BAS DATA_DVC_DPATH=$(geowatch_dvc --tags="phase2_data" --hardware="auto") EXPT_DVC_DPATH=$(geowatch_dvc --tags="phase2_expt" --hardware="auto") python -m geowatch.mlops.schedule_evaluation \ --params=" matrix: trk.pxl.package_fpath: - $EXPT_DVC_DPATH/models/fusion/Drop4-BAS/packages/Drop4_TuneV323_BAS_30GSD_BGRNSH_V2/package_epoch0_step41.pt.pt trk.pxl.data.test_dataset: # - $DATA_DVC_DPATH/$DATASET_CODE/KR_R001.kwcoco.json # - $DATA_DVC_DPATH/$DATASET_CODE/BR_R002.kwcoco.json - $DATA_DVC_DPATH/$DATASET_CODE/KR_R002.kwcoco.json # - $DATA_DVC_DPATH/$DATASET_CODE/US_R007.kwcoco.json trk.pxl.data.window_space_scale: # - "auto" # - "10GSD" - "15GSD" # - "30GSD" # - "40GSD" # - "60GSD" trk.pxl.data.set_cover_algo: - null - approx trk.pxl.data.time_steps: - 11 # - 5 # - auto # - 2 trk.pxl.data.time_sampling: - "auto" trk.poly.thresh: # - 0.07 # - 0.09 - 0.1 # - 0.11 # - 0.13 crop.src: - /home/joncrall/remote/toothbrush/data/dvc-repos/smart_data_dvc/online_v1/kwcoco_for_sc_fielded.json crop.regions: - trk.poly.output act.pxl.data.test_dataset: - crop.dst act.pxl.data.input_space_scale: - 8GSD act.pxl.data.time_steps: - 3 act.pxl.data.chip_overlap: - 0.3 act.poly.thresh: - 0.1 act.poly.use_viterbi: - 0 act.pxl.package_fpath: - $EXPT_DVC_DPATH/models/fusion/Aligned-Drop4-2022-08-08-TA1-S2-WV-PD-ACC/packages/Drop4_SC_RGB_scratch_V002/Drop4_SC_RGB_scratch_V002_epoch=99-step=50300-v1.pt.pt " \ --out_dpath=$EXPT_DVC_DPATH/tmp \ --enable_pred_trk_pxl=1 \ --enable_pred_trk_poly=1 \ --enable_eval_trk_pxl=1 \ --enable_eval_trk_poly=1 \ --enable_crop=1 \ --enable_pred_act_pxl=1 \ --enable_pred_act_poly=1 \ --enable_eval_act_pxl=1 \ --enable_eval_act_poly=1 \ --enable_viz_pred_trk_poly=1 \ --enable_viz_pred_act_poly=1 \ --enable_links=1 \ --devices="0,1" --queue_size=2 \ --queue_name='bas-eval' \ --backend=serial --skip_existing=0 \ --run=0 python -m geowatch.cli.run_tracker /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/models/fusion/fixme/pred/trk/package_epoch0_step41.pt/Drop4-BAS_KR_R002.kwcoco/trk_pxl_77402b79/pred.kwcoco.json --default_track_fn saliency_heatmaps --track_kwargs '{"thresh": 0.1, "moving_window_size": null}' --clear_annots --out_sites_dir /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/models/fusion/fixme/pred/trk/package_epoch0_step41.pt/Drop4-BAS_KR_R002.kwcoco/trk_pxl_77402b79/trk_poly_9f08fb8c/sites --out_site_summaries_dir /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/models/fusion/fixme/pred/trk/package_epoch0_step41.pt/Drop4-BAS_KR_R002.kwcoco/trk_pxl_77402b79/trk_poly_9f08fb8c/site-summaries --out_sites_fpath /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/models/fusion/fixme/pred/trk/package_epoch0_step41.pt/Drop4-BAS_KR_R002.kwcoco/trk_pxl_77402b79/trk_poly_9f08fb8c/site_tracks_manifest.json --out_site_summaries_fpath /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/models/fusion/fixme/pred/trk/package_epoch0_step41.pt/Drop4-BAS_KR_R002.kwcoco/trk_pxl_77402b79/trk_poly_9f08fb8c/site_summary_tracks_manifest.json --out_kwcoco /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/models/fusion/fixme/pred/trk/package_epoch0_step41.pt/Drop4-BAS_KR_R002.kwcoco/trk_pxl_77402b79/trk_poly_9f08fb8c/tracks.kwcoco.json """