source "$HOME"/code/watch/secrets/secrets DVC_DATA_DPATH=$(geowatch_dvc --tags=phase2_data --hardware="hdd") SENSORS=TA1-S2-L8-WV-PD-ACC-3 DATASET_SUFFIX=Drop7 test -e "$DVC_DATA_DPATH/annotations/drop7/region_models" echo $? REGION_GLOBSTR="$DVC_DATA_DPATH/annotations/drop7/region_models/*_C*.geojson" SITE_GLOBSTR="$DVC_DATA_DPATH/annotations/drop7/site_models/*_C*.geojson" export GDAL_DISABLE_READDIR_ON_OPEN=EMPTY_DIR simple_dvc request "$DVC_DATA_DPATH/annotations/drop7" --verbose # Train Regions # US_C001, NG_C001, QA_C001, RU_C001, CN_C001, PE_C004, IN_C000, IN_C000, # SN_C000, CO_C009, US_C016 # Valiation Regions # CN_C000, KW_C001, SA_C001, CO_C001, VN_C002 # # # WOrldview-only # MY_C000, BR_C010, BO_C001, PH_C001 #simple_dvc validate_sidecar "$DVC_DATA_DPATH/annotations/drop7" #--regions=" # - $DVC_DATA_DPATH/annotations/drop7/region_models/CN_C000.geojson # - $DVC_DATA_DPATH/annotations/drop7/region_models/KW_C001.geojson # - $DVC_DATA_DPATH/annotations/drop7/region_models/SA_C001.geojson # - $DVC_DATA_DPATH/annotations/drop7/region_models/CO_C001.geojson # - $DVC_DATA_DPATH/annotations/drop7/region_models/VN_C002.geojson #" \ #--sites=" # - $DVC_DATA_DPATH/annotations/drop7/site_models/CN_C000_*.geojson # - $DVC_DATA_DPATH/annotations/drop7/site_models/KW_C001_*.geojson # - $DVC_DATA_DPATH/annotations/drop7/site_models/SA_C001_*.geojson # - $DVC_DATA_DPATH/annotations/drop7/site_models/CO_C001_*.geojson # - $DVC_DATA_DPATH/annotations/drop7/site_models/VN_C002_*.geojson #" \ # # # #rm -rf Aligned-Drop6-2022-12-01-c30-TA1-S2-L8-WV-PD-ACC-2/ Drop6-MeanYear10GSD/ Drop6_Mean3Month10GSD/ Drop6_MeanYear/ Drop7-Cropped2GSD-V2/ # Construct the TA2-ready dataset python -m geowatch.cli.queue_cli.prepare_ta2_dataset \ --dataset_suffix=$DATASET_SUFFIX \ --stac_query_mode=auto \ --cloud_cover=20 \ --sensors="$SENSORS" \ --api_key=env:SMART_STAC_API_KEY \ --collated True \ --dvc_dpath="$DVC_DATA_DPATH" \ --regions="$REGION_GLOBSTR" \ --sites="$SITE_GLOBSTR" \ --aws_profile=iarpa \ --requester_pays=False \ --fields_workers=8 \ --convert_workers=0 \ --align_workers=8 \ --align_aux_workers=0 \ --ignore_duplicates=1 \ --visualize=0 \ --target_gsd=10 \ --cache=1 \ --verbose=100 \ --skip_existing=1 \ --force_min_gsd=2.0 \ --force_nodata=-9999 \ --hack_lazy=False \ --backend=tmux \ --tmux_workers=8 \ --run=1 dvc add CN_C000/L8 KW_C001/L8 SA_C001/L8 CO_C001/L8 VN_C002/L8 CN_C000/S2 KW_C001/S2 SA_C001/S2 CO_C001/S2 VN_C002/S2 KW_C001/WV dvc push -vv -r aws CN_C000/*.dvc KW_C001/*.dvc SA_C001/*.dvc CO_C001/*.dvc VN_C002/*.dvc dvc pull -vv -r namek_hdd CN_C000/*.dvc KW_C001/*.dvc SA_C001/*.dvc CO_C001/*.dvc VN_C002/*.dvc python -c "if 1: import ubelt as ub root = ub.Path('.') region_dpaths = [p for p in root.ls() if p.is_dir()] region_dpaths = [p for p in region_dpaths if '_C' in p.name] rois = 'CN_C000,KW_C001,SA_C001,CO_C001,VN_C002'.split(',') region_dpaths = [root / n for n in rois] for dpath in region_dpaths: import xdev xdev.tree_repr(dpath, max_depth=1) print(dpath.ls()) " # Reorganize kwcoco files to be inside of their region folder python -c "if 1: import os os.chdir('/home/joncrall/remote/namek/data/dvc-repos/smart_data_dvc/Aligned-Drop7') import ubelt as ub bundle_dpath = ub.Path('.').absolute() all_paths = list(bundle_dpath.glob('*_[C]*')) region_dpaths = [] for p in all_paths: if p.is_dir() and str(p.name)[2] == '_': region_dpaths.append(p) region_dpaths import cmd_queue queue = cmd_queue.Queue.create('tmux', size=16) for dpath in region_dpaths: print(ub.urepr(dpath.ls())) existing = [] for dpath in region_dpaths: region_id = dpath.name fnames = [ f'imgonly-{region_id}.kwcoco.zip', f'imganns-{region_id}.kwcoco.zip', ] for fname in fnames: old_fpath = bundle_dpath / fname new_fpath = dpath / fname if old_fpath.exists(): if new_fpath.exists(): existing.append(new_fpath) if not new_fpath.exists(): queue.submit(f'kwcoco move {old_fpath} {new_fpath}') queue.write_network_text() queue.run() " dvc add CN_C000/*.kwcoco.zip KW_C001/*.kwcoco.zip SA_C001/*.kwcoco.zip CO_C001/*.kwcoco.zip VN_C002/*.kwcoco.zip dvc push -r aws CN_C000/*.kwcoco.zip KW_C001/*.kwcoco.zip SA_C001/*.kwcoco.zip CO_C001/*.kwcoco.zip VN_C002/*.kwcoco.zip dvc pull -r namek_hdd -- */*.kwcoco.zip.dvc dvc pull -R namek_hdd -- CN_C000 KW_C001 SA_C001 CO_C001 VN_C002 #python ~/code/watch-smartflow-dags/reproduce_mlops.py imgonly-US_R006.kwcoco.zip # ~/code/watch/dev/poc/prepare_time_combined_dataset.py DVC_DATA_DPATH=$(geowatch_dvc --tags='phase2_data' --hardware=hdd) python ~/code/watch/watch/cli/queue_cli/prepare_time_combined_dataset.py \ --regions="['US_C010', 'NG_C000', 'KW_C001', 'SA_C005', 'US_C001', 'CO_C009', 'US_C014', 'CN_C000', 'PE_C004', 'IN_C000', 'PE_C001', 'SA_C001', 'US_C016', 'AE_C001', 'US_C011', 'AE_C002', 'PE_C003', 'RU_C000', 'CO_C001', 'US_C000', 'US_C012', 'AE_C003', 'CN_C001', 'QA_C001', 'SN_C000'] # 'VN_C002', " \ --input_bundle_dpath="$DVC_DATA_DPATH"/Aligned-Drop7 \ --output_bundle_dpath="$DVC_DATA_DPATH"/Drop7-MedianNoWinter10GSD-iMERIT \ --true_site_dpath="$DVC_DATA_DPATH"/annotations/drop7/site_models \ --true_region_dpath="$DVC_DATA_DPATH"/annotations/drop7/region_models \ --spatial_tile_size=512 \ --merge_method=median \ --remove_seasons=winter \ --tmux_workers=2 \ --time_window=1y \ --combine_workers=4 \ --resolution=10GSD \ --backend=tmux \ --skip_existing=0 \ --cache=1 \ --run=1 DVC_DATA_DPATH=$(geowatch_dvc --tags=phase2_data --hardware="hdd") python -m geowatch.cli.queue_cli.prepare_splits \ --base_fpath="$DVC_DATA_DPATH"/Drop7-MedianNoWinter10GSD-iMERIT/*/imganns*-*_[RC]*.kwcoco.zip \ --suffix=rawbands \ --backend=tmux --tmux_workers=6 \ --run=0 DVC_DATA_DPATH=$(geowatch_dvc --tags=phase2_data --hardware="hdd") kwcoco move \ "$DVC_DATA_DPATH"/Drop7-MedianNoWinter10GSD-iMERIT/US_C010/data_train_rawbands_split6.kwcoco.zip \ "$DVC_DATA_DPATH"/Drop7-MedianNoWinter10GSD-iMERIT/data_train_rawbands_split6.kwcoco.zip kwcoco move \ "$DVC_DATA_DPATH"/Drop7-MedianNoWinter10GSD-iMERIT/US_C010/data_vali_rawbands_split6.kwcoco.zip \ "$DVC_DATA_DPATH"/Drop7-MedianNoWinter10GSD-iMERIT/data_vali_rawbands_split6.kwcoco.zip python -m geowatch reproject \ --src "$DVC_DATA_DPATH"/Drop7-MedianNoWinter10GSD-iMERIT/.kwcoco.zip \ --dst "$DVC_DATA_DPATH"/Drop7-MedianNoWinter10GSD-iMERIT/.kwcoco.zip \ --status_to_catname="positive_excluded: positive" \ --regions="$DVC_DATA_DPATH"/drop7/region_models/*.geojson \ --sites="$DVC_DATA_DPATH"/annotations/drop7/site_models/*.geojson dvc add -vv -- */raw_bands dvc add -vv -- */imganns-*.kwcoco.zip ###--- # geowatch site_stats --regions /home/joncrall/remote/toothbrush/data/dvc-repos/smart_data_dvc/annotations/drop6_hard_v1/region_models/CN_C000.geojson python -m geowatch reproject \ --src /home/joncrall/remote/toothbrush/data/dvc-repos/smart_data_dvc/Drop7-MedianNoWinter10GSD-iMERIT/CN_C000/imgonly-CN_C000-fielded.kwcoco.zip \ --dst /home/joncrall/remote/toothbrush/data/dvc-repos/smart_data_dvc/Drop7-MedianNoWinter10GSD-iMERIT/CN_C000/imganns-CN_C000.kwcoco.zip \ --status_to_catname="positive_excluded: positive" \ --regions="/home/joncrall/remote/toothbrush/data/dvc-repos/smart_data_dvc/annotations/drop6_hard_v1/region_models/CN_C000.geojson" \ --sites="/home/joncrall/remote/toothbrush/data/dvc-repos/smart_data_dvc/annotations/drop6_hard_v1/site_models/CN_C000_*.geojson" # Run BAS on iMERIT # PREEVAL 14 BAS+SV Grid DVC_DATA_DPATH=$(geowatch_dvc --tags='phase2_data' --hardware=hdd) DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase2_expt' --hardware=auto) python -m geowatch.utils.simple_dvc request \ "$DVC_EXPT_DPATH"/models/fusion/Drop6-MeanYear10GSD-V2/packages/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47_epoch47_step3026.pt \ "$DVC_EXPT_DPATH"/models/fusion/Drop7-MedianNoWinter10GSD/packages/Drop7-MedianNoWinter10GSD_bgrn_split6_V68/Drop7-MedianNoWinter10GSD_bgrn_split6_V68_epoch34_stepNone.pt \ "$DVC_EXPT_DPATH"/models/fusion/Drop6-MeanYear10GSD-V2/packages/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47_epoch47_step3026.pt \ "$DVC_EXPT_DPATH"/models/fusion/uconn/D7-MNW10_coldL8S2-cv-a0-a1-b1-c1-rmse-split6_eval11_Norm_lr1e4_bs48_focal/epoch=16-step=374.pt \ "$DVC_EXPT_DPATH"/models/fusion/Drop7-MedianNoWinter10GSD/packages/Drop7-MedianNoWinter10GSD_bgr_cold_split6_V62/Drop7-MedianNoWinter10GSD_bgr_cold_split6_V62_epoch359_step15480.pt DVC_DATA_DPATH=$(geowatch_dvc --tags='phase2_data' --hardware=hdd) DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase2_expt' --hardware=auto) TEST_DPATH=$DVC_EXPT_DPATH/_test/_imeritbas geowatch schedule --params=" matrix: bas_pxl.package_fpath: - $DVC_EXPT_DPATH/models/fusion/Drop6-MeanYear10GSD-V2/packages/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47/Drop6_TCombo1Year_BAS_10GSD_V2_landcover_split6_V47_epoch47_step3026.pt - $DVC_EXPT_DPATH/models/fusion/Drop7-MedianNoWinter10GSD/packages/Drop7-MedianNoWinter10GSD_bgrn_split6_V68/Drop7-MedianNoWinter10GSD_bgrn_split6_V68_epoch34_stepNone.pt bas_pxl.test_dataset: - $DVC_DATA_DPATH/Drop7-MedianNoWinter10GSD-iMERIT/CN_C000/imganns-CN_C000.kwcoco.zip bas_pxl.chip_overlap: 0.3 bas_pxl.chip_dims: auto bas_pxl.time_span: auto bas_pxl.time_sampling: soft4 bas_poly.thresh: - 0.2 - 0.25 - 0.3 - 0.325 - 0.35 - 0.375 - 0.39 - 0.4 - 0.4125 - 0.425 - 0.435 - 0.45 - 0.5 - 0.6 bas_poly.inner_window_size: 1y bas_poly.inner_agg_fn: mean bas_poly.norm_ord: inf bas_poly.polygon_simplify_tolerance: 1 bas_poly.agg_fn: probs bas_poly.time_thresh: - 0.95 - 0.925 - 0.9 - 0.85 - 0.825 - 0.8 - 0.775 - 0.75 - 0.7 - 0.6 bas_poly.resolution: 10GSD bas_poly.moving_window_size: null bas_poly.poly_merge_method: 'v2' bas_poly.min_area_square_meters: 7200 bas_poly.max_area_square_meters: 8000000 bas_poly.boundary_region: $DVC_DATA_DPATH/annotations/drop7/region_models bas_poly_eval.true_site_dpath: $DVC_DATA_DPATH/annotations/drop7/site_models bas_poly_eval.true_region_dpath: $DVC_DATA_DPATH/annotations/drop7/region_models bas_pxl.enabled: 1 bas_pxl_eval.enabled: 0 bas_poly.enabled: 1 bas_poly_eval.enabled: 1 bas_poly_viz.enabled: 0 " \ --root_dpath="$TEST_DPATH" \ --devices="0,1" --tmux_workers=2 \ --backend=tmux \ --pipeline=bas \ --skip_existing=1 \ --run=1 --help DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase2_expt' --hardware=auto) TEST_DPATH=$DVC_EXPT_DPATH/_test/_imeritbas DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase2_expt' --hardware=auto) python -m geowatch.mlops.aggregate \ --pipeline=bas \ --target " - $TEST_DPATH " \ --output_dpath="$TEST_DPATH/aggregate" \ --resource_report=0 \ --eval_nodes=" - bas_poly_eval - bas_pxl_eval " \ --plot_params=" enabled: 1 stats_ranking: 0 min_variations: 1 params_of_interest: - resolved_params.bas_poly.time_thresh - resolved_params.bas_poly.thresh - resolved_params.bas_pxl.package_fpath " \ --stdout_report=" top_k: 13 per_group: 1 macro_analysis: 0 analyze: 0 print_models: True reference_region: null concise: 1 show_csv: 0 " DVC_DATA_DPATH=$(geowatch_dvc --tags='phase2_data' --hardware=hdd) python -m geowatch.mlops.confusor_analysis \ --metrics_node_dpath /home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/_test/_imeritbas/eval/flat/bas_poly_eval/bas_poly_eval_id_fd88699a/ \ --true_region_dpath="$DVC_DATA_DPATH"/annotations/drop7/region_models \ --true_site_dpath="$DVC_DATA_DPATH"/annotations/drop7/site_models \ --viz_site_case=True # Compute all feature except COLD export CUDA_VISIBLE_DEVICES="0" DVC_DATA_DPATH=$(geowatch_dvc --tags=phase2_data --hardware="ssd") DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase2_expt' --hardware='auto') BUNDLE_DPATH=$DVC_DATA_DPATH/Drop7-MedianNoWinter10GSD-iMERIT python -m geowatch.cli.queue_cli.prepare_teamfeats \ --src_kwcocos "$BUNDLE_DPATH"/*/imganns-*[0-9].kwcoco.zip \ --expt_dvc_dpath="$DVC_EXPT_DPATH" \ --with_landcover=1 \ --with_invariants2=1 \ --with_sam=0 \ --with_materials=0 \ --with_mae=1 \ --with_depth=0 \ --with_cold=0 \ --skip_existing=1 \ --assets_dname=teamfeats \ --gres=0, --tmux_workers=8 --backend=tmux --run=1 dvc add -- */teamfeats