#!/bin/bash export DVC_DATA_DPATH=$(geowatch_dvc --tags="phase3_data") export DVC_EXPT_DPATH=$(geowatch_dvc --tags="phase3_expt") cd "$DVC_EXPT_DPATH" python -m geowatch.mlops.manager "status" --dataset_codes "Drop8-ARA-Median10GSD-V1" python -m geowatch.mlops.manager "list checkpoints" --dataset_codes "Drop8-ARA-Median10GSD-V1" python -m geowatch.mlops.manager "repackage checkpoints" --dataset_codes "Drop8-ARA-Median10GSD-V1" python -m geowatch.mlops.manager "gather packages" --dataset_codes "Drop8-ARA-Median10GSD-V1" python -m geowatch.mlops.manager "push packages" --dataset_codes "Drop8-ARA-Median10GSD-V1" export DVC_DATA_DPATH=$(geowatch_dvc --tags="phase3_data") export DVC_EXPT_DPATH=$(geowatch_dvc --tags="phase3_expt") cd "$DVC_EXPT_DPATH" python -m geowatch.mlops.manager "status" --dataset_codes "Drop8-ARA-Median10GSD-V1" python -m geowatch.mlops.manager "list checkpoints" --dataset_codes "Drop8-ARA-Median10GSD-V1" python -m geowatch.mlops.manager "repackage checkpoints" --dataset_codes "Drop8-ARA-Median10GSD-V1" python -m geowatch.mlops.manager "gather packages" --dataset_codes "Drop8-ARA-Median10GSD-V1" python -m geowatch.mlops.manager "push packages" --dataset_codes "Drop8-ARA-Median10GSD-V1" # Point based mlops evaluation export DVC_DATA_DPATH=$(geowatch_dvc --tags='phase3_data' --hardware=ssd) export DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto) TRUTH_DPATH=$DVC_DATA_DPATH/annotations/drop8-v1 MLOPS_NAME=_preeval23_point_bas_grid MLOPS_DPATH=$DVC_EXPT_DPATH/$MLOPS_NAME #TODO: perform query showing only v4 models AND last model in list (aka baseline) # Avoid jank way using agregate hundreds of times MODEL_SHORTLIST=" - /home/local/KHQ/vincenzo.dimatteo/Desktop/dvc_repos/smart_phase3_expt/models/fusion/Drop8-ARA-Median10GSD-V1/packages/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4_epoch128_step10836.pt - $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Median10GSD-V1/packages/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4_epoch117_step9912.pt - $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Median10GSD-V1/packages/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4_epoch129_step10920.pt - $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Median10GSD-V1/packages/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4_epoch165_step13944.pt - $DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Median10GSD-V1/packages/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4_epoch131_step11088.pt - /home/local/KHQ/vincenzo.dimatteo/Desktop/dvc_repos/smart_phase3_expt//models/fusion/Drop8-ARA-Median10GSD-V1/packages/Drop8_ARA_Median10GSD_allsensors_pointannsv1_v1/Drop8_ARA_Median10GSD_allsensors_pointannsv1_v1_epoch56_step4788.pt " mkdir -p "$MLOPS_DPATH" echo "$MODEL_SHORTLIST" > "$MLOPS_DPATH/shortlist.yaml" cat "$MLOPS_DPATH/shortlist.yaml" # FIXME: make sdvc request works with YAML lists of models # sdvc request --verbose=3 "$MLOPS_DPATH/shortlist.yaml" geowatch schedule --params=" pipeline: bas matrix: bas_pxl.package_fpath: $MLOPS_DPATH/shortlist.yaml bas_pxl.test_dataset: - $DVC_DATA_DPATH/Drop8-ARA-Median10GSD-V1/KR_R002/imganns-KR_R002-rawbands.kwcoco.zip - $DVC_DATA_DPATH/Drop8-ARA-Median10GSD-V1/CN_C000/imganns-CN_C000-rawbands.kwcoco.zip - $DVC_DATA_DPATH/Drop8-ARA-Median10GSD-V1/KW_C001/imganns-KW_C001-rawbands.kwcoco.zip #- $DVC_DATA_DPATH/Drop8-ARA-Median10GSD-V1/CO_C001/imganns-CO_C001-rawbands.kwcoco.zip bas_pxl.chip_overlap: 0.3 bas_pxl.chip_dims: auto bas_pxl.time_span: auto bas_pxl.time_sampling: soft4 bas_poly.thresh: - 0.36 #- 0.37 - 0.375 #- 0.38 - 0.39 - 0.40 bas_poly.inner_window_size: 1y bas_poly.inner_agg_fn: mean bas_poly.norm_ord: inf bas_poly.polygon_simplify_tolerance: 1 bas_poly.agg_fn: probs bas_poly.time_thresh: - 0.85 #- 0.8 - 0.75 bas_poly.time_pad_after: #- 0 months - 3 months #- 12 months bas_poly.resolution: 10GSD bas_poly.moving_window_size: null bas_poly.poly_merge_method: 'v2' bas_poly.min_area_square_meters: 7200 bas_poly.max_area_square_meters: 8000000 bas_poly.boundary_region: $TRUTH_DPATH/region_models bas_poly_eval.true_site_dpath: $TRUTH_DPATH/site_models bas_poly_eval.true_region_dpath: $TRUTH_DPATH/region_models bas_pxl.enabled: 1 bas_pxl_eval.enabled: 0 bas_poly_viz.enabled: 0 bas_poly.enabled: 1 bas_poly_eval.enabled: 1 include: - bas_poly.time_pad_after: 0 months bas_poly.time_pad_before: 0 months - bas_poly.time_pad_after: 3 months bas_poly.time_pad_before: 3 months - bas_poly.time_pad_after: 12 months bas_poly.time_pad_before: 12 months " \ --root_dpath="$MLOPS_DPATH" \ --devices="1,2,3" --tmux_workers=8 \ --backend=tmux --queue_name "$MLOPS_NAME" \ --skip_existing=1 \ --run=1 # Point based mlops evaluation export DVC_DATA_DPATH=$(geowatch_dvc --tags='phase3_data' --hardware=ssd) export DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto) TRUTH_DPATH=$DVC_DATA_DPATH/annotations/drop8-v1 MLOPS_NAME=_preeval23_point_bas_grid MLOPS_DPATH=$DVC_EXPT_DPATH/$MLOPS_NAME DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto) echo "DVC_EXPT_DPATH = $DVC_EXPT_DPATH" python -m geowatch.mlops.aggregate \ --embed \ --pipeline=bas \ --target " - $MLOPS_DPATH - $DVC_EXPT_DPATH/_preeval22_point_bas_grid " \ --output_dpath="$MLOPS_DPATH/aggregate" \ --resource_report=0 \ --eval_nodes=" - bas_poly_eval #- bas_pxl_eval " \ --plot_params=" enabled: 0 stats_ranking: 0 min_variations: 1 params_of_interest: - params.bas_poly.thresh - resolved_params.bas_pxl.package_fpath - resolved_params.bas_pxl.channels " \ --stdout_report=" top_k: 10 per_group: 1 macro_analysis: 0 analyze: 0 print_models: True reference_region: final concise: 0 show_csv: 0 " export DVC_DATA_DPATH=$(geowatch_dvc --tags='phase3_data' --hardware=ssd) export DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto) TRUTH_DPATH=$DVC_DATA_DPATH/annotations/drop8-v1 MLOPS_NAME=_preeval23_point_bas_grid MLOPS_DPATH=$DVC_EXPT_DPATH/$MLOPS_NAME DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase3_expt' --hardware=auto) echo "DVC_EXPT_DPATH = $DVC_EXPT_DPATH" python -m geowatch.mlops.aggregate \ --embed \ --pipeline=bas \ --target " - $MLOPS_DPATH - $DVC_EXPT_DPATH/_preeval22_point_bas_grid " \ --output_dpath="$MLOPS_DPATH/aggregate" \ --resource_report=0 \ --eval_nodes=" - bas_poly_eval #- bas_pxl_eval " \ --plot_params=" enabled: 0 stats_ranking: 0 min_variations: 1 params_of_interest: - params.bas_poly.thresh - resolved_params.bas_pxl.package_fpath - resolved_params.bas_pxl.channels " \ --stdout_report=" top_k: 100000000000 per_group: 1 macro_analysis: 0 analyze: 0 print_models: True reference_region: final concise: 0 show_csv: 0 " \ --rois="KR_R002,CN_C000,KW_C001"\ --query="df.table['params.bas_pxl.package_fpath'].str.contains('$DVC_EXPT_DPATH/models/fusion/Drop8-ARA-Median10GSD-V1/packages/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4/Drop8_ARA_Median10GSD_allsensors_pointannsv1_fixeddataloader_v4_epoch131_step11088.pt') | df.table['params.bas_pxl.package_fpath'].str.contains('/home/local/KHQ/jon.crall/data/dvc-repos/smart_phase3_expt/models/fusion/Drop8-ARA-Median10GSD-V1/packages/Drop8_ARA_Median10GSD_allsensors_pointannsv1_v1/Drop8_ARA_Median10GSD_allsensors_pointannsv1_v1_epoch56_step4788.pt')" #--rois="KR_R002,CN_C000,KW_C001" #--rois="KR_R002" #--rois="KR_R002,CN_C000,KW_C001,CO_C001" #" --rois="KR_R002,CN_C000" #--rois="CN_C000"