# flake8: noqa import math import pandas as pd import ubelt as ub from watch.mlops.aggregate import hash_param from watch.mlops.aggregate import fix_duplicate_param_hashids from watch.utils import util_pandas def _debug_roi_issue(): import numpy as np import watch from watch.mlops.aggregate import AggregateLoader expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') loader = AggregateLoader( pipeline='bas_depth_vali', # pipeline='bas_building_vali', target=[ expt_dvc_dpath / '_mlops_test_depth_pcd', # expt_dvc_dpath / '_toothbrush_split6_landcover_MeanYear10GSD-V2', ]) eval_type_to_aggregator = loader.coerce_aggregators() pxl_agg = eval_type_to_aggregator['bas_pxl_eval'] poly_agg = eval_type_to_aggregator['bas_poly_eval'] sv_poly_agg = eval_type_to_aggregator['sv_poly_eval'] # poly_agg.build_macro_tables(rois) # _ = poly_agg.report_best() rois = ['CH_R001', 'KR_R001', 'KR_R002', 'NZ_R001'] print(ub.urepr(ub.udict(sv_poly_agg.macro_compatible).map_values(len))) print(ub.urepr(ub.udict(poly_agg.macro_compatible).map_values(len))) agg = sv_poly_agg # rois = 'KR_R001,KR_R002,CH_R001,NZ_R001,BR_R002'.split(',') # rois = 'KR_R001,KR_R002,CH_R001,NZ_R001,BR_R002,AE_R001'.split(',') rois = 'KR_R002,CH_R001,NZ_R001'.split(',') rois = ['KR_R002'] # agg.build_macro_tables(rois) flags = agg.table['params.sv_crop.crop_src_fpath'].isnull() subagg = agg.compress(~flags) subagg.build_macro_tables(rois) _ = subagg.report_best() macro_key = list(subagg.region_to_tables.keys())[-1] macro = subagg.region_to_tables[macro_key] macro = macro.sort_values('metrics.sv_poly_eval.bas_f1') points = macro[['region_id', 'param_hashid', 'metrics.bas_poly_eval.bas_f1', 'metrics.sv_poly_eval.bas_f1', 'metrics.bas_poly_eval.bas_tpr', 'metrics.sv_poly_eval.bas_tpr']] import kwplot import matplotlib as mpl kwplot.autosns() kwplot.plt.ion() kwplot.figure() ax = kwplot.plt.gca() ax.cla() segments = [] # min_x = float('inf') # max_x = -float('inf') for row in macro.to_dict('records'): x1 = row['metrics.bas_poly_eval.bas_tpr'] x2 = row['metrics.sv_poly_eval.bas_tpr'] # y1 = row['metrics.bas_poly_eval.bas_fp'] # y2 = row['metrics.sv_poly_eval.bas_fp'] y1 = row['metrics.bas_poly_eval.bas_f1'] y2 = row['metrics.sv_poly_eval.bas_f1'] segments.append([(x1, y1), (x2, y2)]) macro['metrics.bas_poly_eval.bas_f1'] pts1 = [s[0] for s in segments] pts2 = [s[1] for s in segments] data_lines = mpl.collections.LineCollection(segments, color='blue', alpha=0.5, linewidths=1) ax.add_collection(data_lines) ax.plot(*zip(*pts1), 'rx', label='before SV') ax.plot(*zip(*pts2), 'bo', label='after SV') ax.legend() ax.set_xlabel('bas_tpr') ax.set_ylabel('bas_f1') # ax.set_ylabel('bas_fp') ax.set_title(f'Effect of SV: {rois}') # kwplot.sns.scatterplot(data=macro, y='metrics.bas_poly_eval.bas_f1', x='metrics.bas_poly_eval.bas_tpr', markers='x', ax=ax, hue='resolved_params.bas_poly.thresh') # kwplot.sns.scatterplot(data=macro, y='metrics.sv_poly_eval.bas_f1', x='metrics.sv_poly_eval.bas_tpr', markers='o', hue='resolved_params.sv_dino_filter.end_min_score', ax=ax) # kwplot.sns.scatterplot(data=macro, y='metrics.sv_poly_eval.bas_f1', x='metrics.sv_poly_eval.bas_tpr', markers='o', hue='resolved_params.bas_poly.thresh', ax=ax) if 1: # Point A target_pt = (0.6227, 0.57730) delta = points[['metrics.bas_poly_eval.bas_tpr', 'metrics.bas_poly_eval.bas_f1']].values - target_pt dist = np.linalg.norm(delta, axis=1) point_A = points.iloc[np.argmin(dist)] ax.plot(point_A['metrics.bas_poly_eval.bas_tpr'], point_A['metrics.bas_poly_eval.bas_f1'], '*', markersize=20, color='orange') ax.plot(point_A['metrics.sv_poly_eval.bas_tpr'], point_A['metrics.sv_poly_eval.bas_f1'], '*', markersize=20, color='orange') ax.text(*target_pt, 'A', fontdict={'weight': 'bold'}) # Pt C target_pt = (0.656, 0.60) delta = points[['metrics.sv_poly_eval.bas_tpr', 'metrics.sv_poly_eval.bas_f1']].values - target_pt dist = np.linalg.norm(delta, axis=1) point_C = points.iloc[np.argmin(dist)] ax.plot(point_C['metrics.bas_poly_eval.bas_tpr'], point_C['metrics.bas_poly_eval.bas_f1'], '*', markersize=20, color='orange') ax.plot(point_C['metrics.sv_poly_eval.bas_tpr'], point_C['metrics.sv_poly_eval.bas_f1'], '*', markersize=20, color='orange') ax.text(*target_pt, 'C', fontdict={'weight': 'bold'}) # Point B target_pt = (0.7952, 0.5581) delta = points[['metrics.sv_poly_eval.bas_tpr', 'metrics.sv_poly_eval.bas_f1']].values - target_pt dist = np.linalg.norm(delta, axis=1) point_B = points.iloc[np.argmin(dist)] ax.plot(point_B['metrics.bas_poly_eval.bas_tpr'], point_B['metrics.bas_poly_eval.bas_f1'], '*', markersize=20, color='orange') ax.plot(point_B['metrics.sv_poly_eval.bas_tpr'], point_B['metrics.sv_poly_eval.bas_f1'], '*', markersize=20, color='orange') ax.text(*target_pt, 'B', fontdict={'weight': 'bold'}) special_params = {} special_params['point_A'] = agg.hashid_to_params[point_A['param_hashid']] special_params['point_B'] = agg.hashid_to_params[point_B['param_hashid']] special_params['point_C'] = agg.hashid_to_params[point_C['param_hashid']] print('special_params = {}'.format(ub.urepr(special_params, nl=2))) if 0: chosen_param_hashid = point_C['param_hashid'] chosen_agg = agg.filterto(param_hashids=[chosen_param_hashid]) _ = chosen_agg.report_best() print(ub.repr2(chosen_agg.table['fpath'].tolist())) out_dpath = ub.Path('./chosen') eval_links = (out_dpath / 'eval_links').ensuredir() for _, row in chosen_agg.table.iterrows(): node_dpath = ub.Path(row['fpath']).parent link_fpath = eval_links / (row['region_id'] + '_' + node_dpath.name) ub.symlink(node_dpath, link_fpath) if 0: # Find high TPR example: idx = subagg.table['metrics.bas_poly_eval.bas_tpr'].idxmax() high_tpr_param_hashid = subagg.table.loc[idx]['param_hashid'] high_tpr_agg = agg.filterto(param_hashids=[high_tpr_param_hashid]) _ = high_tpr_agg.report_best() print(ub.repr2(high_tpr_agg.table['fpath'].tolist())) out_dpath = ub.Path('./high-tpr') eval_links = (out_dpath / 'eval_links').ensuredir() for _, row in high_tpr_agg.table.iterrows(): node_dpath = ub.Path(row['fpath']).parent link_fpath = eval_links / (row['region_id'] + '_' + node_dpath.name) ub.symlink(node_dpath, link_fpath) ... if 0: # Inspect one of the best ones. ### out_dpath = ub.Path('./chosen') root_dpath = expt_dvc_dpath / '_toothbrush_split6_landcover_MeanYear10GSD-V2/' out_dpath = root_dpath / '_custom' max_id = macro['metrics.sv_poly_eval.bas_f1'].idxmax() chosen_param_hashid = macro.loc[max_id]['param_hashid'] chosen_agg = agg.filterto(param_hashids=[chosen_param_hashid]) _ = chosen_agg.report_best() print(ub.repr2(chosen_agg.table['fpath'].tolist())) eval_links = (out_dpath / 'eval_links').ensuredir() for _, row in chosen_agg.table.iterrows(): node_dpath = ub.Path(row['fpath']).parent link_fpath = eval_links / (row['region_id'] + '_' + node_dpath.name) ub.symlink(node_dpath, link_fpath) agg.table['resolved_params.sv_crop.src'].unique() bad_values['resolved_params.sv_crop.src'].unique() good_values['resolved_params.sv_crop.src'].unique() bad_values = agg.table[flags] good_values = agg.table[~flags] node_dpath = '/home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/_toothbrush_split6_landcover_MeanYear10GSD-V2/eval/flat/sv_poly_eval/sv_poly_eval_id_cb328495/.pred/sv_dino_filter/sv_dino_filter_id_321a821e/.pred/sv_dino_boxes/sv_dino_boxes_id_c6cc58bf/.pred/sv_crop/sv_crop_id_55fc5fce/' from watch.mlops.aggregate_loader import load_result_resolved r = load_result_resolved(node_dpath) fpath = '/home/joncrall/remote/toothbrush/data/dvc-repos/smart_expt_dvc/_toothbrush_split6_landcover_MeanYear10GSD-V2/eval/flat/sv_poly_eval/sv_poly_eval_id_cb328495/poly_eval.json' node_name = agg.type out_node_key = 'sv_poly_eval.eval_fpath' def _sitevisit_2023_april_report(): import watch from watch.mlops.aggregate import AggregateLoader expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') loader = AggregateLoader( target=[ expt_dvc_dpath / 'aggregate_results/toothbrush/*.csv.zip', expt_dvc_dpath / 'aggregate_results/ooo/*.csv.zip', expt_dvc_dpath / 'aggregate_results/namek/*.csv.zip', ]) eval_type_to_aggregator = loader.coerce_aggregators() pxl_agg = eval_type_to_aggregator['bas_pxl_eval'] poly_agg = eval_type_to_aggregator['bas_poly_eval'] sv_poly_agg = eval_type_to_aggregator['sv_poly_eval'] # sv_poly_agg.resource_summary_table() import kwplot from watch.utils import util_kwplot kwplot.plt.ion() resource_summary_df = poly_agg.resource_summary_table() util_kwplot.dataframe_table(resource_summary_df, 'resource_summary_bas.png') resource_summary_df2 = sv_poly_agg.resource_summary_table() util_kwplot.dataframe_table(resource_summary_df2, 'resource_summary_sv.png') _ = poly_agg.report_best(top_k=10, print_models=True) agg = poly_agg agg.output_dpath = expt_dvc_dpath / 'site_visit_plots' print(ub.urepr(ub.udict(agg.macro_compatible).map_values(len).sorted_values())) rois = ['KR_R001', 'KR_R002'] # rois = ['AE_R001', 'BR_R002', 'CH_R001', 'KR_R001', 'KR_R002', 'NZ_R001'] from watch.utils.util_pandas import DotDictDataFrame table = DotDictDataFrame(agg.table) import rich rich.print('resolved.nested_columns = {}'.format(ub.urepr(table.nested_columns, nl=True))) from watch.mlops.smart_global_helper import SmartGlobalHelper def experiment_timeline(): import pandas as pd from kwutil import util_time resources = agg.resources duration_cols = [k for k in resources.keys() if k.endswith('.duration')] start_time_cols = [k for k in agg.table.keys() if k.endswith('.start_timestamp')] end_time_cols = [k for k in agg.table.keys() if k.endswith('.stop_timestamp')] for k in duration_cols: table.loc[:, k] = table.loc[:, k].apply(lambda x: util_time.coerce_timedelta(x) if not pd.isnull(x) else x) for k in start_time_cols: table.loc[:, k] = table.loc[:, k].apply(lambda x: util_time.coerce_datetime(x) if not pd.isnull(x) else x) for k in end_time_cols: table.loc[:, k] = table.loc[:, k].apply(lambda x: util_time.coerce_datetime(x) if not pd.isnull(x) else x) run_rows = [] for k in duration_cols: a, b, c = k.split('.') uuid_key = f'context.{b}.uuid' start_time_col = f'context.{b}.start_timestamp' end_time_col = f'context.{b}.stop_timestamp' co2_key = f'{a}.{b}.co2_kg' kwh_key = f'{a}.{b}.kwh' chosen = [] for _, group in table.groupby(uuid_key): idx = group[k].idxmax() chosen.append(idx) unique_rows = table.loc[chosen] for _row in unique_rows.to_dict('records'): ... row = { 'node': b, 'resource': c, 'duration': _row[k], 'start_time': _row[start_time_col], 'end_time': _row[end_time_col], 'co2_kg': _row.get(co2_key, None), 'kwh': _row.get(kwh_key, None), } run_rows.append(row) start_times = [] end_times = [] for idx, row in enumerate(run_rows): start_times.append(row['start_time']) end_times.append(row['end_time']) end_times = np.array(end_times) start_times = np.array(start_times) earliest = start_times.min() latest = end_times.max() num_buckets = 5000 bucketsize = (latest - earliest) / num_buckets buckets = earliest + np.arange(num_buckets) * bucketsize interval_assign = np.searchsorted(buckets, start_times) import kwarray bkt_to_idxs = ub.dzip(*kwarray.group_indices(interval_assign)) y_values = np.empty(len(interval_assign)) for bkt, groupxs in bkt_to_idxs.items(): y_values[groupxs] = np.arange(len(groupxs)) # ub.group_items(start_times, interval_assign) import kwplot kwplot.autompl() import matplotlib as mpl segments = [] min_x = float('inf') max_x = -float('inf') for y, row in zip(y_values, run_rows): x1 = mpl.dates.date2num(row['start_time']) x2 = mpl.dates.date2num(row['end_time']) max_x = max(max_x, x2) min_x = min(min_x, x1) segments.append([(x1, y), (x2, y)]) data_lines = mpl.collections.LineCollection(segments, color='blue', alpha=0.5, linewidths=10) ax = kwplot.plt.gca() ax.set_xlim(min_x, max_x) ax.set_ylim(0, max(y_values) + 1) ax.add_collection(data_lines) ax.autoscale_view() ax.xaxis_date() # from collections import Counter # c = Counter() # for # c[1] max(ub.dict_hist().values()) np.sea # starts_after1 = start_times[None, :] >= start_times[:, None] # ends_after1 = start_times[None, :] <= end_times[:, None] # interval_overlaps = (is_after_start) & (is_before_end) interval_overlaps[0] start_times >= start_times if 0: fig = px.timeline(df, x_start="Start", x_end="Finish", y="Task", color="Completion_pct") resource_summary_df = pd.DataFrame(resource_summary) import rich rich.print(resource_summary_df.to_string()) table['resolved_params.bas_pxl.package_fpath'] rois = None def _gather_namek_shortlist_results(): """ geowatch model_stats models/fusion/Drop4-BAS/packages/Drop4_BAS_2022_12_15GSD_BGRNSH_BGR_V4/Drop4_BAS_2022_12_15GSD_BGRNSH_BGR_V4_v0_epoch44_step46014.pt """ from watch.mlops.aggregate import AggregateEvluationConfig from watch.mlops.aggregate import coerce_aggregators import watch expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') cmdline = 0 kwargs = { 'target': expt_dvc_dpath / '_namek_split1_eval_small', # 'target': expt_dvc_dpath / '_namek_split2_eval_small', 'pipeline': 'bas', 'io_workers': 20, } config = AggregateEvluationConfig.cli(cmdline=cmdline, data=kwargs) eval_type_to_aggregator = coerce_aggregators(config) agg = eval_type_to_aggregator['bas_pxl_eval'] region_id_to_summary, top_param_lut = agg.report_best(1000, verbose=0) tocombine_indexes = [] for region, summary in region_id_to_summary.items(): tocombine_indexes.append(list(summary.index)) import itertools as it top_indexes = list(ub.oset([x for x in ub.flatten( it.zip_longest(*tocombine_indexes)) if x is not None])) table = agg.table.copy() table.loc[top_indexes, 'rank'] = list(range(len(top_indexes))) table = table.sort_values('rank') chosen_indexes = [] for expt, group in table.groupby('resolved_params.bas_pxl_fit.name'): group['params.bas_pxl.package_fpath'].tolist() group = group.sort_values('rank') chosen_indexes.extend(group.index[0:2]) chosen_indexes = table.loc[chosen_indexes, 'rank'].sort_values().index all_models_fpath = ub.Path('$HOME/code/watch/dev/reports/split1_all_models.yaml').expand() from kwutil.util_yaml import Yaml known_models = Yaml.coerce(all_models_fpath) top_k = 40 chosen_indexes = chosen_indexes[:top_k] chosen_table = table.loc[chosen_indexes] # Need to remove invariants for now flags = ~np.array(['invariants' in chan for chan in chosen_table['resolved_params.bas_pxl_fit.channels']]) chosen_table = chosen_table[flags] chosen_models = chosen_table['params.bas_pxl.package_fpath'].tolist() set(known_models).issuperset(set(chosen_models)) new_models_fpath = ub.Path('$HOME/code/watch/dev/reports/split1_models_filter1.yaml').expand() new_models_fpath.write_text(Yaml.dumps(chosen_models)) subagg = agg.filterto(index=chosen_indexes) subagg.table['resolved_params.bas_pxl_fit.name'] # top_table = agg.table.loc[top_indexes] # top_models = top_table['resolved_params.bas_pxl_fit.name'].unique() # from watch.utils.util_pandas import DotDictDataFrame # resolved = DotDictDataFrame(agg.resolved_params) # fit_params = resolved.subframe('resolved_params.bas_pxl_fit') # top_fit_params = fit_params.loc[top_indexes] num_top_models = len(top_models) num_total_models = len(agg.params['params.bas_pxl.package_fpath'].unique()) print(f'num_top_models={num_top_models} / {num_total_models}') is_highres = [float(a.split('G')[0]) < 8 for a in agg.table['resolved_params.bas_pxl_fit.window_space_scale']] has_wv = ['WV' in a for a in agg.table['resolved_params.bas_pxl_fit.channels']] flags = (np.array(has_wv) & np.array(is_highres)) subagg = agg.compress(flags) def _namek_check_pipeline_status(): from watch.mlops import aggregate_loader import watch expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') root_dpath = expt_dvc_dpath / '_namek_eval' pipeline = 'bas' io_workers = 16 # eval_type_to_results = aggregate_loader.build_tables(root_dpath, pipeline, io_workers) # eval_type_to_results['bas_pxl_eval'] from watch.mlops import smart_pipeline dag = smart_pipeline.make_smart_pipeline(pipeline) dag.configure(config=None, root_dpath=root_dpath) # node_to_fpaths = {} # for node_name, node in ub.ProgIter(dag.nodes.items()): # node_fpaths = {} # for out_node_key, out_node in node.outputs.items(): # node_fpaths[out_node_key] = aggregate_loader.out_node_matching_fpaths(out_node) # node_to_fpaths[node_name] = node_fpaths # node_to_fpaths = ub.udict(node_to_fpaths).map_values(ub.udict) # num_existing_outs = node_to_fpaths.map_values(lambda x: x.map_values(len)) # stages_of_interest = ['bas_pxl', 'bas_pxl_eval', 'bas_poly', 'bas_poly_eval'] stages_of_interest = ['bas_pxl', 'bas_pxl_eval'] existing = {} for stage in stages_of_interest: rows = dag.nodes[stage].find_template_outputs(workers=8) if 1: rows = [row for row in rows if row.get('request.bas_pxl.test_dataset', None) is not None] existing[stage] = rows bad_rows = [] for stage in stages_of_interest: from watch.utils import util_dotdict # NOQA for row in ub.ProgIter(existing[stage]): node = dag.nodes[stage] row = util_dotdict.DotDict(row) config = row.prefix_get('request') dag.configure(config, cache=False) row_dpath = ub.Path(row['dpath']) row['process_id'] = node.process_id # print(node.final_command()) if node.final_node_dpath != row['dpath']: # print('config = {}'.format(ub.urepr(config, nl=1))) # print((row_dpath / 'job_config.json').read_text()) # print((row_dpath / 'invoke.sh').read_text()) # print(node.final_command()) bad_rows.append(row) dfs = {} from watch.utils.util_pandas import DotDictDataFrame for stage in stages_of_interest: dfs[stage] = DotDictDataFrame(existing[stage]) stage1 = 'bas_pxl' stage2 = 'bas_pxl_eval' df1 = dfs[stage1] df2 = dfs[stage2] from watch.utils import util_dotdict # NOQA bad_rows = [] for row in existing[stage1]: wanted_succ = ub.Path(row['dpath']) / '.succ' / stage2 if not wanted_succ.exists(): node = dag.nodes[stage] row = util_dotdict.DotDict(row) config = row.prefix_get('request') dag.configure(config, cache=False) if node.does_exist: raise Exception node1 = dag.nodes[stage1] node2 = dag.nodes[stage2] ... df1 = dfs['bas_pxl'].subframe('request') df2 = dfs['bas_pxl_eval'].subframe('request') df1 = df1[~df1['bas_pxl.package_fpath'].isnull()] df2 = df2[~df2['bas_pxl.package_fpath'].isnull()] from kwcoco._helpers import _delitems hashids1 = [ub.hash_data(row) for row in df1.to_dict('records')] hashids2 = [ub.hash_data(row) for row in df2.to_dict('records')] # There shouldn't be duplicates here, except in pathological circumstances to_drop1 = [] for _, idxs in ub.find_duplicates(hashids1).items(): print('Dropping') print(df1.iloc[idxs]) to_drop1.extend(idxs) to_drop2 = [] for _, idxs in ub.find_duplicates(hashids2).items(): print('Dropping') print(df2.iloc[idxs]) to_drop2.extend(idxs) df1 = df1.drop(df1.index[to_drop1], axis=0) df2 = df2.drop(df2.index[to_drop2], axis=0) _delitems(hashids1, to_drop1) _delitems(hashids2, to_drop2) assert not ub.find_duplicates(hashids1) assert not ub.find_duplicates(hashids2) hashids1 = ub.oset(hashids1) hashids2 = ub.oset(hashids2) df1['hashid'] = hashids1 df2['hashid'] = hashids2 common = hashids1 & hashids2 df1 = df1.set_index('hashid') df2 = df2.set_index('hashid') missing1 = hashids2 - hashids1 assert len(missing1) == 0, 'should not be possible without error' missing2 = hashids1 - hashids2 # These have not had a computation done for them. missing_df = df1.loc[missing2] # WANTED: Algorithm that takes a list of grid points and determines if any # of the rows can be expressed conciesly as a matrix. """ given = [ {'x': 1, 'y': 1, 'z': 1}, {'x': 1, 'y': 2, 'z': 1}, {'x': 1, 'y': 3, 'z': 1}, {'x': 2, 'y': 1, 'z': 2}, {'x': 2, 'y': 2, 'z': 2}, {'x': 2, 'y': 3, 'z': 2}, {'x': 3, 'y': 1, 'z': 2}, {'x': 4, 'y': 3, 'z': 2}, {'x': 4, 'y': 4, 'z': 2}, {'x': 4, 'y': 3, 'z': 3}, {'x': 4, 'y': 4, 'z': 3}, ] should output: - x: [1, 2] y: [1, 2, 3] z: [1, 2] - x: [3] y: [1] z: [2] - x: [4] y: [3, 4] z: [2, 3] It seems like a solution here is non-unique, or at least the greedy way of finding the solution will result in different answers depending on which variable you try to collapse first. Is this a well-studied algorithm? Seems like a decomposition problem. """ import sys, ubelt from watch.utils.result_analysis import varied_value_counts submatrices = [] for _, group in missing_df.groupby('bas_pxl.test_dataset'): group_records = group.to_dict('records') group_varied = varied_value_counts(group_records) for row in group_records: row = {k: v for k, v in row.items() if not isinstance(v, float) or not math.isnan(v)} row['bas_pxl.enabled'] = False row['bas_pxl_eval.enabled'] = True row['bas_poly.enabled'] = True row['bas_poly_eval.enabled'] = True submatrices.append(row) from kwutil import util_yaml submat_text = util_yaml.Yaml.dumps({'submatrices': submatrices}) fpath = ub.Path('foo.yaml').absolute() fpath.write_text(submat_text) # Generate code to run the missing experiments invocation = ub.codeblock( fr''' python -m geowatch.mlops.schedule_evaluation \ --params={fpath} \ --root_dpath="{root_dpath}" \ --devices="0,1" --queue_size=4 \ --backend=tmux --queue_name "_explicit_recompute" \ --pipeline=bas --skip_existing=0 \ --print_varied=1 \ --run=0 ''') print(invocation) # For two levels in the node figure out: # What paths on the parent are are in common. # What paths on the child have yet to be computed. node_fpaths = node_to_fpaths['bas_pxl'] def _gather_all_results(): r""" # On Namek DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase2_expt' --hardware=auto) python -m geowatch.mlops.aggregate \ --pipeline=bas \ --target " - $DVC_EXPT_DPATH/_timekernel_test_drop4 - $DVC_EXPT_DPATH/_namek_eval " \ --export_tables=True \ --output_dpath="$DVC_EXPT_DPATH/namek_agg" # On Toothbrush DVC_EXPT_DPATH=$(geowatch_dvc --tags='phase2_expt' --hardware=auto) rsync -avprPR namek:data/dvc-repos/smart_expt_dvc/./namek_agg "$DVC_EXPT_DPATH" python -m geowatch.mlops.aggregate \ --pipeline=bas \ --target " - namek_agg/*.csv.zip - $DVC_EXPT_DPATH/_timekernel_test_drop4 - $DVC_EXPT_DPATH/_testpipe - $DVC_EXPT_DPATH/_evaluations - $DVC_EXPT_DPATH/_testpipe2 " \ --export_tables=True \ --output_dpath="$DVC_EXPT_DPATH/all_agg_2022-02-24" python -m geowatch.mlops.aggregate \ --pipeline=bas \ --target " - all_agg_2022-02-24/*.csv.zip " \ --stdout_report=True \ --output_dpath="$DVC_EXPT_DPATH/all_agg_2022-02-24/reports" """ from watch.mlops.aggregate import AggregateEvluationConfig from watch.mlops.aggregate import coerce_aggregators import watch expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') cmdline = 0 kwargs = { # 'target': expt_dvc_dpath / 'all_agg_2022-02-24/*.csv.zip', 'target': expt_dvc_dpath / '_split6_toothbrush_meanyear', 'pipeline': 'bas', 'io_workers': 20, } config = AggregateEvluationConfig.cli(cmdline=cmdline, data=kwargs) eval_type_to_aggregator = coerce_aggregators(config) agg = eval_type_to_aggregator.get('bas_pxl_eval', None) agg = eval_type_to_aggregator.get('bas_poly_eval', None) from watch.mlops.aggregate import coerce_aggregators build_all_param_plots(agg, rois, config) def resource_usage_report(agg): import pandas as pd from kwutil import util_time table = agg.table.copy() resources = agg.resources duration_cols = [ k for k in agg.resources.keys() if k.endswith('.duration') ] for k in duration_cols: table.loc[:, k] = table.loc[:, k].apply(lambda x: util_time.coerce_timedelta(x) if not pd.isnull(x) else x) resource_summary = [] for k in duration_cols: a, b, c = k.split('.') uuid_key = f'context.{b}.uuid' chosen = [] for _, group in table.groupby(uuid_key): idx = group[k].idxmax() chosen.append(idx) unique_rows = table.loc[chosen] row = { 'node': b, 'resource': c, 'total': unique_rows[k].sum(), 'mean': unique_rows[k].mean(), 'num': len(chosen), } resource_summary.append(row) co2_key = f'{a}.{b}.co2_kg' if co2_key in table: unique_rows[co2_key] row = { 'node': b, 'resource': 'co2_kg', 'total': unique_rows[co2_key].sum(), 'mean': unique_rows[co2_key].mean(), 'num': len(chosen), } resource_summary.append(row) co2_key = f'{a}.{b}.kwh' if co2_key in table: unique_rows[co2_key] row = { 'node': b, 'resource': 'kwh', 'total': unique_rows[co2_key].sum(), 'mean': unique_rows[co2_key].mean(), 'num': len(chosen), } resource_summary.append(row) pd.DataFrame(resource_summary) agg.resources['resources.bas_poly_eval.duration'] agg.resources['resources.bas_poly.duration'] agg.resources['resources.bas_poly.co2_kg'] unique_resources = {} unique_resources['bas_pxl'] = poly_agg.table.groupby('context.bas_pxl.uuid') unique_resources['bas_poly'] = poly_agg.table.groupby('context.bas_poly.uuid') unique_resources['bas_poly_eval'] = poly_agg.table.groupby('context.bas_poly_eval.uuid') for k, v in unique_resources.items(): v[f'resources.{k}.duration'].max().apply(util_time.coerce_timedelta).sum() co2_key = f'resources.{k}.co2_kg' if co2_key in v: ... # v[].max().sum() pxl_time = poly_agg.table.groupby('context.bas_pxl.uuid')['resources.bas_pxl.duration'].max().apply(util_time.coerce_timedelta).sum() poly_time = poly_agg.table.groupby('context.bas_poly.uuid')['resources.bas_poly.duration'].max().apply(util_time.coerce_timedelta).sum() poly_eval_time = poly_agg.table.groupby('context.bas_poly_eval.uuid')['resources.bas_poly.duration'].max().apply(util_time.coerce_timedelta).sum() total_time = pxl_time + poly_time + poly_eval_time poly_agg.table['context.bas_poly.uuid'] poly_agg.table['context.bas_poly_eval.uuid'] def _check_high_tpr_case(agg, config): macro_results = agg.region_to_tables[agg.primary_macro_region].copy() from watch.utils.util_pandas import DotDictDataFrame macro_metrics = DotDictDataFrame(macro_results['metrics']) tpr_col = macro_metrics.find_column('bas_tpr') macro_metrics = macro_metrics.sort_values(tpr_col, ascending=False) inspect_idxs = macro_metrics.index[0:1] agg_dpath = ub.Path(config['root_dpath'] / 'aggregate') agg_group_dpath = (agg_dpath / ('top_tpr_cases' + ub.timestamp())).ensuredir() for rank, idx in enumerate(inspect_idxs): param_hashid = macro_results['index'].loc[idx]['param_hashid'] subagg = agg.filterto(param_hashids=[param_hashid]) subagg.build_macro_tables(rois) subagg.report_best() dpath = (agg_group_dpath / f'top_{rank:03d}_tpr_case').ensuredir() for loc in subagg.fpaths.index: index_row = subagg.index.loc[loc] eval_fpath = subagg.fpaths.loc[loc] eval_dpath = eval_fpath.parent link_dpath = dpath / f'eval_link_{index_row.region_id}_{index_row.param_hashid}' ub.symlink(real_path=eval_dpath, link_path=link_dpath) ... agg.index['param_hashid'] == param_hashid subagg.fpaths.tolist() from watch.mlops.aggregate import make_summary_analysis agg1 = subagg make_summary_analysis(agg1, config, dpath) ... def _namek_eval(): from watch.mlops.aggregate import AggregateEvluationConfig from watch.mlops.aggregate import coerce_aggregators import watch data_dvc_dpath = watch.find_dvc_dpath(tags='phase2_data', hardware='auto') expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') cmdline = 0 kwargs = { 'target': expt_dvc_dpath / '_namek_eval', 'pipeline': 'bas', 'io_workers': 0, } config = AggregateEvluationConfig.cli(cmdline=cmdline, data=kwargs) eval_type_to_aggregator = coerce_aggregators(config) poly_agg = eval_type_to_aggregator.get('bas_poly_eval', None) pxl_agg = eval_type_to_aggregator.get('bas_pxl_eval', None) if 1: # from watch.utils import util_dotdict # NOQA # Join the pixel and polygon results from watch.utils import util_pandas a = util_pandas.DotDictDataFrame(poly_agg.params).subframe('params.bas_pxl') b = util_pandas.DotDictDataFrame(pxl_agg.params).subframe('params.bas_pxl') a_hashids = [ub.hash_data(row) for row in a.to_dict('records')] b_hashids = [ub.hash_data(row) for row in b.to_dict('records')] a['hashid.bas_pxl'] = a_hashids b['hashid.bas_pxl'] = b_hashids hashid_to_idxs1 = ub.find_duplicates(a_hashids, k=0) hashid_to_idxs2 = ub.find_duplicates(b_hashids, k=0) missing1 = set(hashid_to_idxs1) - set(hashid_to_idxs2) missing2 = set(hashid_to_idxs2) - set(hashid_to_idxs1) common = set(hashid_to_idxs2) & set(hashid_to_idxs1) pxl_eval_cols = [c for c in pxl_agg.table.columns if 'bas_pxl_eval' in c.split('.')] sub_table = pxl_agg.table[pxl_eval_cols].copy() sub_table.loc[:, 'hashid.bas_pxl'] = b_hashids main_table = poly_agg.table.copy() main_table.loc[:, 'hashid.bas_pxl'] = a_hashids joined_table = pd.merge(main_table, sub_table, on='hashid.bas_pxl') joined_table[['metrics.bas_poly_eval.bas_f1', 'metrics.bas_pxl_eval.salient_AP']] joined_table[['resolved_params.bas_poly.agg_fn']] import kwplot if 0: sns = kwplot.autosns() plt = kwplot.autoplt() # sns.scatterplot( # data=joined_table, # x='metrics.bas_poly_eval.bas_f1', # y='metrics.bas_pxl_eval.salient_AP', # hue='params.bas_pxl.package_fpath', # legend=False, # ) sns.scatterplot( data=joined_table, # x='metrics.bas_poly_eval.bas_f1', x='metrics.bas_poly_eval.bas_f1', y='metrics.bas_pxl_eval.salient_AP', hue='resolved_params.bas_poly.agg_fn', legend=False, ) # ['params']).find_column('saliency_loss') # row = util_dotdict.DotDict(row) ... poly_agg.resources['resources.bas_poly_eval.duration'] poly_agg.resources['resources.bas_poly.duration'] from kwutil import util_time unique_resources = {} unique_resources['bas_pxl'] = poly_agg.table.groupby('context.bas_pxl.uuid') unique_resources['bas_poly'] = poly_agg.table.groupby('context.bas_poly.uuid') unique_resources['bas_poly_eval'] = poly_agg.table.groupby('context.bas_poly_eval.uuid') for k, v in unique_resources.items(): v[f'resources.{k}.duration'].max().apply(util_time.coerce_timedelta).sum() co2_key = f'resources.{k}.co2_kg' if co2_key in v: ... # v[].max().sum() pxl_time = poly_agg.table.groupby('context.bas_pxl.uuid')['resources.bas_pxl.duration'].max().apply(util_time.coerce_timedelta).sum() poly_time = poly_agg.table.groupby('context.bas_poly.uuid')['resources.bas_poly.duration'].max().apply(util_time.coerce_timedelta).sum() poly_eval_time = poly_agg.table.groupby('context.bas_poly_eval.uuid')['resources.bas_poly.duration'].max().apply(util_time.coerce_timedelta).sum() total_time = pxl_time + poly_time + poly_eval_time poly_agg.table['context.bas_poly.uuid'] poly_agg.table['context.bas_poly_eval.uuid'] agg = ub.peek(eval_type_to_aggregator.values()) agg.build_macro_tables() agg.primary_display_cols = ['bas_poly_eval.metrics.bas_faa_f1', 'bas_poly_eval.metrics.bas_f1', 'bas_poly_eval.metrics.bas_tpr', 'bas_poly_eval.metrics.bas_ppv'] agg.primary_metric_cols = ['bas_poly_eval.metrics.bas_tpr', 'bas_poly_eval.metrics.bas_ppv'] _ = agg.report_best() agg.primary_metric_cols = ['bas_poly_eval.metrics.bas_f1'] _ = agg.report_best() # agg.primary_metric_cols = ['bas_poly_eval.metrics.bas_ppv', 'bas_poly_eval.metrics.bas_tpr'] # _ = agg.report_best() from watch.mlops.aggregate import build_all_param_plots # rois = {'KR_R001', 'KR_R002'} rois = {'KR_R001'} build_all_param_plots(agg, rois, config) # Find best dicefocal model from watch.utils import util_pandas col = util_pandas.DotDictDataFrame(agg.fit_params).find_column('saliency_loss') flags = agg.fit_params[col] == 'dicefocal' subagg1 = agg.compress(flags) agg.primary_metric_cols = ['bas_poly_eval.metrics.bas_ppv', 'bas_poly_eval.metrics.bas_tpr'] _ = subagg1.report_best() def _timekernel_analysis(): from watch.mlops.aggregate import AggregateEvluationConfig from watch.mlops.aggregate import build_tables from watch.mlops.aggregate import build_aggregators import watch data_dvc_dpath = watch.find_dvc_dpath(tags='phase2_data', hardware='auto') expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') cmdline = 0 kwargs = { 'root_dpath': expt_dvc_dpath / '_timekernel_test_drop4', 'pipeline': 'bas', 'io_workers': 10, 'freeze_cache': 0, } config = AggregateEvluationConfig.cli(cmdline=cmdline, data=kwargs) eval_type_to_results = build_tables(config) agg_dpath = ub.Path(config['root_dpath']) / 'aggregate' eval_type_to_aggregator = build_aggregators(eval_type_to_results, agg_dpath) agg = ub.peek(eval_type_to_aggregator.values()) agg = eval_type_to_aggregator.get('bas_poly_eval', None) from watch.mlops.aggregate import build_all_param_plots rois = {'KR_R001', 'KR_R002'} rois = {'KR_R001'} _ = agg.report_best() build_all_param_plots(agg, rois, config) build_all_param_plots(agg, rois, config) from watch.mlops.aggregate import build_all_param_plots rois = {'KR_R001', 'KR_R002'} # rois = {'KR_R001'} build_all_param_plots(agg, rois, config) agg = eval_type_to_aggregator.get('bas_pxl_eval', None) def _setup_sc_analysis(): from watch.mlops.aggregate import AggregateEvluationConfig from watch.mlops.aggregate import build_tables from watch.mlops.aggregate import build_aggregators import watch expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') cmdline = 0 kwargs = { 'root_dpath': expt_dvc_dpath / '_testpipe', 'pipeline': 'joint_bas_sc', 'io_workers': 20, 'freeze_cache': 0, # 'pipeline': 'joint_bas_sc_nocrop', # 'root_dpath': expt_dvc_dpath / '_testsc', #'pipeline': 'sc', } config = AggregateEvluationConfig.cli(cmdline=cmdline, data=kwargs) eval_type_to_results = build_tables(config) agg_dpath = ub.Path(config['root_dpath']) / 'aggregate' eval_type_to_aggregator = build_aggregators(eval_type_to_results, agg_dpath) agg = ub.peek(eval_type_to_aggregator.values()) agg = eval_type_to_aggregator.get('sc_poly_eval', None) print(f'agg={agg}') rois = {'KR_R001', 'KR_R002', 'BR_R002'} print(f'rois={rois}') def _setup_bas(): from watch.mlops.aggregate import AggregateEvluationConfig from watch.mlops.aggregate import build_tables from watch.mlops.aggregate import build_aggregators import watch expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') cmdline = 0 kwargs = { 'root_dpath': expt_dvc_dpath / '_testpipe', 'pipeline': 'bas', 'io_workers': 20, 'freeze_cache': 0, # 'pipeline': 'joint_bas_sc_nocrop', # 'root_dpath': expt_dvc_dpath / '_testsc', #'pipeline': 'sc', } config = AggregateEvluationConfig.cli(cmdline=cmdline, data=kwargs) eval_type_to_results = build_tables(config) agg_dpath = ub.Path(config['root_dpath']) / 'aggregate' eval_type_to_aggregator = build_aggregators(eval_type_to_results, agg_dpath) agg = ub.peek(eval_type_to_aggregator.values()) agg = eval_type_to_aggregator.get('bas_poly_eval', None) fname = f'{agg.type}_{agg.agg_dpath.parent.name}.csv' agg.table().to_csv(fname, index_label=False) agg = eval_type_to_aggregator.get('bas_pxl_eval', None) fname = f'{agg.type}_{agg.agg_dpath.parent.name}.csv' agg.table().to_csv(fname, index_label=False) print(f'agg={agg}') rois = {'KR_R001', 'KR_R002'} # rois = {'KR_R001', 'KR_R002', 'BR_R002'} print(f'rois={rois}') def _resource_table(eval_type_to_aggregator): agg1 = eval_type_to_aggregator.get('bas_poly_eval', None) agg2 = eval_type_to_aggregator.get('bas_pxl_eval', None) agg2.resolved_info['resources'].sum() mapping = { 'bas_poly_eval.params.bas_pxl.package_fpath': 'models', 'bas_poly_eval.params.bas_poly.moving_window_size': 'tracking_window_size', 'bas_poly_eval.params.bas_poly.thresh': 'tracking_threshold', 'bas_poly_eval.params.bas_pxl.input_space_scale': 'heatmap_gsd', 'bas_poly_eval.params.bas_poly.max_area_sqkm': 'max_area_km_threshold', 'bas_poly_eval.params.bas_poly.min_area_sqkm': 'min_area_km_threshold', 'bas_poly_eval.params.bas_poly.min_area_square_meters': 'min_area_meters', 'bas_poly_eval.params.bas_poly.max_area_square_meters': 'max_area_meters', 'bas_poly_eval.params.bas_poly.polygon_simplify_tolerance': 'polygon_simplify_tolerence', 'bas_poly_eval.params.bas_pxl.chip_dims': 'input_window_dimensions', 'bas_poly_eval.params.bas_pxl.test_dataset': 'regions', } summary_rows = [] for colname, col in agg1.effective_params.fillna('None').T.iterrows(): histo = col.value_counts() if len(histo) > 1: if colname in mapping: summary_rows.append({ 'param': mapping[colname], 'num_unique': len(histo), }) print(f'colname={colname}') print(histo) df = pd.DataFrame(summary_rows) from watch.utils import util_kwplot dfh = humanize_dataframe(df) dataframe_table(dfh, 'foo.png') from watch.utils import util_pandas resource_df = util_pandas.pandas_shorten_columns(agg2.resolved_info['resources']) resource_df = resource_df.drop(['vram_gb'], axis=1) resouce_table = resource_df.sum(numeric_only=True, axis=0).to_frame('bas_pxl') resouce_table.loc['num_params', 'bas_pxl'] = len(agg2) resouce_table.loc[:, 'bas_poly'] = '' resouce_table.loc['num_params', 'bas_poly'] = len(agg1) dfh = humanize_dataframe(resouce_table, title='Pipeline Summary') dataframe_table(dfh, 'pipeline_summary.png') def report_top_results(agg): rois = {'KR_R001', 'KR_R002', 'BR_R002', 'AE_R001'} agg.build_macro_tables(rois) macro_results = agg.region_to_tables[agg.primary_macro_region].copy() z = agg.report_best() best_macro_param_hashid = list(z[0].values())[-1]['param_hashid'].iloc[0] agg_best = agg.filterto(param_hashids=[best_macro_param_hashid]) agg_best.build_macro_tables() agg_best.report_best() def generate_kr2_heatmaps(agg): agg1 = agg.filterto(models=['package_epoch0_step41']) rois = {'KR_R001', 'KR_R002', 'BR_R002'} agg1 = agg1.compress(agg1.params['bas_poly_eval.params.bas_poly.thresh'] == 0.12) agg1 = agg1.compress(agg1.params['bas_poly_eval.params.bas_poly.moving_window_size'].isna()) agg1 = agg1.filterto(param_hashids=['txfrewydmfeb']) agg1.build_single_macro_table(rois) agg1 = agg1.compress(agg1.index['region_id'] == 'KR_R002') eval_fpath = agg1.fpaths.iloc[0] from watch.mlops import confusion_visualization confusion_visualization.bas_poly_eval_confusion_analysis(eval_fpath) rois = {'KR_R001', 'KR_R002', 'BR_R002'} agg1 = agg.filterto(models=['Drop4_BAS_15GSD_BGRNSH_invar_V8_epoch=16-step=8704']) # agg1 = agg.filterto(param_hashids=['cvvlwbictocz']) agg1.build_single_macro_table(rois) best = agg1.report_best() param_hashid = list(best[0].values())[-1].iloc[0]['param_hashid'] agg1 = agg1.filterto(param_hashids=[param_hashid]) agg1 = agg1.compress(agg1.index['region_id'] == 'KR_R002') eval_fpath = agg1.fpaths.iloc[0] print(f'eval_fpath={eval_fpath}') from watch.mlops import confusion_visualization confusion_visualization.bas_poly_eval_confusion_analysis(eval_fpath) rois = {'KR_R001', 'KR_R002', 'BR_R002'} agg1 = agg.filterto(models=['Drop4_BAS_2022_12_15GSD_BGRN_V10_epoch=0-step=4305']) # agg1 = agg.filterto(param_hashids=['cvvlwbictocz']) agg1.build_single_macro_table(rois) best = agg1.report_best() param_hashid = list(best[0].values())[-1].iloc[0]['param_hashid'] agg1 = agg1.filterto(param_hashids=[param_hashid]) agg1 = agg1.compress(agg1.index['region_id'] == 'KR_R002') eval_fpath = agg1.fpaths.iloc[0] print(f'eval_fpath={eval_fpath}') from watch.mlops import confusion_visualization confusion_visualization.bas_poly_eval_confusion_analysis(eval_fpath) def check_baseline(eval_type_to_aggregator): from watch.utils.util_pandas import DotDictDataFrame models_of_interest = [ 'package_epoch0_step41', ] agg0 = eval_type_to_aggregator.get('bas_poly_eval', None) agg1 = agg0.filterto(models=models_of_interest) _ = agg1.report_best() DotDictDataFrame(agg1.effective_params) flags = DotDictDataFrame(agg1.effective_params)['thresh'] == 0.1 agg2 = agg1.compress(flags.values) agg3 = agg2.filterto(param_hashids=['peercprsgafm']) agg3.build_macro_tables() agg3.report_best() def compare_simplify_tolerence(agg0, config): # Filter to only parameters that are A / B comparable parameter_of_interest = 'bas_poly_eval.params.bas_poly.polygon_simplify_tolerance' # groups = agg0.effective_params.groupby(parameter_of_interest) if parameter_of_interest not in agg0.effective_params.columns: raise ValueError(f'Missing parameter of interest: {parameter_of_interest}') other_params = agg0.effective_params.columns.difference({parameter_of_interest}).tolist() candidate_groups = agg0.effective_params.groupby(other_params, dropna=False) comparable_groups = [] for value, group in candidate_groups: # if len(group) > 1: if len(group) == 4: if len(group[parameter_of_interest].unique()) == 4: comparable_groups.append(group) comparable_idxs = sorted(ub.flatten([group.index for group in comparable_groups])) comparable_agg1 = agg0.filterto(index=comparable_idxs) agg = comparable_agg1 # _ = comparable_agg1.report_best() selector = {'KR_R001', 'KR_R002', 'BR_R002'} # Find the best result for the macro region rois = selector # NOQA macro_results = comparable_agg1.build_single_macro_table(selector) top_macro_id = macro_results['metrics'].sort_values(comparable_agg1.primary_metric_cols, ascending=False).index[0] print(macro_results['index'].loc[top_macro_id]) # Find the corresponding A / B for each region. # Get the param hash each region should have top_params = macro_results['effective_params'].loc[top_macro_id] # top_value = top_params[parameter_of_interest] top_flags = macro_results['specified_params'].loc[top_macro_id] > 0 top_flags[comparable_agg1.test_dset_cols] = False comparable_agg1.hashid_to_params[macro_results['index'].loc[top_macro_id]['param_hashid']] params_of_interest = [] for value in comparable_agg1.effective_params[parameter_of_interest].unique(): params_row = top_params.copy() flags = top_flags.copy() if isinstance(value, float) and math.isnan(value): flags[parameter_of_interest] = False | flags[parameter_of_interest] else: flags[parameter_of_interest] = True | flags[parameter_of_interest] params_row[parameter_of_interest] = value params = params_row[flags].to_dict() param_hashid = hash_param(params) if not (param_hashid in comparable_agg1.index['param_hashid'].values): flags[parameter_of_interest] = not flags[parameter_of_interest] params_row[parameter_of_interest] = value params = params_row[flags].to_dict() param_hashid = hash_param(params) if not (param_hashid in comparable_agg1.index['param_hashid'].values): raise AssertionError params_of_interest.append(param_hashid) comparable_agg2 = comparable_agg1.filterto(param_hashids=params_of_interest) _ = comparable_agg2.report_best(top_k=10) agg_dpath = ub.Path(config['root_dpath'] / 'aggregate') agg_group_dpath = (agg_dpath / ('inspect_simplify_' + ub.timestamp())).ensuredir() # Given these set of A/B values, visualize each region for region_id, group in comparable_agg2.index.groupby('region_id'): group_agg = comparable_agg2.filterto(index=group.index) for id, row in group_agg.index.iterrows(): eval_fpath = group_agg.fpaths[id] param_hashid = row['param_hashid'] region_id = row['region_id'] dname = f'{region_id}_{param_hashid}' link_dpath = agg_group_dpath / dname real_dpath = eval_fpath.parent ub.symlink(real_path=real_dpath, link_path=link_dpath) import kwimage from kwcoco.metrics.drawing import concice_si_display region_viz_fpaths = list((eval_fpath.parent / 'region_viz_overall').glob('*_detailed.png')) assert len(region_viz_fpaths) == 1 region_viz_fpath = region_viz_fpaths[0] viz_img = kwimage.imread(region_viz_fpath) scores_of_interest = util_pandas.pandas_shorten_columns(comparable_agg2.metrics).loc[id, ['bas_tp', 'bas_fp', 'bas_fn', 'bas_f1']] scores_of_interest = ub.udict(scores_of_interest.to_dict()) text = ub.urepr(scores_of_interest.map_values(concice_si_display), nobr=1, si=1, compact=1) new_img = kwimage.draw_header_text(viz_img, param_hashid + '\n' + text) kwimage.imwrite(agg_group_dpath / f'summary_{region_id}_{param_hashid}.jpg', new_img) # Hacked result analysis from watch.utils import result_analysis metrics_of_interest = comparable_agg1.primary_metric_cols params = comparable_agg1.results['params'] specified = comparable_agg1.results['specified_params'] # NOQA params.loc[:, parameter_of_interest] = params.loc[:, parameter_of_interest].fillna('None') analysis = result_analysis.ResultAnalysis( comparable_agg1.results, metrics=metrics_of_interest) analysis.results analysis.analysis() ### comparable_agg1.build_macro_tables() # Check to see if simplify shrinks the size of our files rows = [] for idx, fpath in comparable_agg1.fpaths.iteritems(): poly_coco_fpath = list((fpath.parent / '.pred').glob('bas_poly/*/poly.kwcoco.json'))[0] sites_dpath = list((fpath.parent / '.pred').glob('bas_poly/*/site_summaries'))[0] size1 = poly_coco_fpath.stat().st_size size2 = sum([p.stat().st_size for p in sites_dpath.ls()]) value = comparable_agg1.effective_params.loc[idx, parameter_of_interest] rows.append({ 'coco_size': size1, 'site_size': size2, 'param_value': value, }) for val, group in pd.DataFrame(rows).groupby('param_value', dropna=False): print(f'val={val}') import xdev print(len(group)) size1 = xdev.byte_str(group['coco_size'].sum()) size2 = xdev.byte_str(group['site_size'].sum()) print(f'size1={size1}') print(f'size2={size2}') macro_region_id = list(comparable_agg1.macro_key_to_regions.keys())[-1] regions_of_interest = comparable_agg1.macro_key_to_regions[macro_region_id] # NOQA tables = comparable_agg1.region_to_tables[macro_region_id] effective_params = tables['effective_params'] metrics = tables['metrics'] index = tables['index'] table = pd.concat([index, effective_params, metrics], axis=1) table = table.fillna('None') main_metric = agg.primary_metric_cols[0] results = [] for idx, row in enumerate(table.to_dict('records')): row = ub.udict(row) row_metrics = row & set(metrics.keys()) row_params = row & set(effective_params.keys()) result = result_analysis.Result(str(idx), row_params, row_metrics) results.append(result) analysis = result_analysis.ResultAnalysis( results, metrics=[main_metric], metric_objectives={main_metric: 'max'} ) # self = analysis analysis.analysis() analysis.report() pd.Index.union(*[group.index for group in comparable_groups]) def comparse_single_param(agg0, config): agg0 = fix_duplicate_param_hashids(agg0) agg0.resolved_params = agg0.resolved_params.applymap(lambda x: str(x) if isinstance(x, list) else x) resolved_params = agg0.resolved_params.fillna('None') parameters_of_interest = [ 'bas_poly_eval.params.bas_pxl.output_space_scale', 'bas_poly_eval.params.bas_pxl.input_space_scale', 'bas_poly_eval.params.bas_pxl.window_space_scale', ] other_params = resolved_params.columns.difference(parameters_of_interest).tolist() candidate_groups = resolved_params.groupby(other_params, dropna=False) comparable_groups = [] for value, group in candidate_groups: if len(group) > 1: has_variation = ~group[parameters_of_interest].apply(ub.allsame, axis=0) if has_variation.any(): comparable_groups.append(group) else: print('unsure why this can happen') comparable_idxs = sorted(ub.flatten([group.index for group in comparable_groups])) comparable_agg1 = agg0.filterto(index=comparable_idxs) agg = comparable_agg1 # NOQA selector = {'KR_R001', 'KR_R002', 'BR_R002'} # Find the best result for the macro region rois = selector # NOQA macro_results = comparable_agg1.build_single_macro_table(selector) top_macro_id = macro_results['metrics'].sort_values(comparable_agg1.primary_metric_cols, ascending=False).index[0] print(macro_results['index'].loc[top_macro_id]) # Find the corresponding A / B for each region. # Get the param hash each region should have top_params = macro_results['resolved_params'].loc[top_macro_id] # NOQA # top_value = top_params[parameter_of_interest] top_flags = macro_results['specified_params'].loc[top_macro_id] > 0 top_flags[comparable_agg1.test_dset_cols] = False comparable_agg1.hashid_to_params[macro_results['index'].loc[top_macro_id]['param_hashid']] # comparable_agg1 # params_of_interest = [] # for value in comparable_agg1.resolved_params[parameter_of_interest].unique(): # params_row = top_params.copy() # flags = top_flags.copy() # if isinstance(value, float) and math.isnan(value): # flags[parameter_of_interest] = False | flags[parameter_of_interest] # else: # flags[parameter_of_interest] = True | flags[parameter_of_interest] # params_row[parameter_of_interest] = value # params = params_row[flags].to_dict() # param_hashid = hash_param(params) # if not (param_hashid in comparable_agg1.index['param_hashid'].values): # flags[parameter_of_interest] = not flags[parameter_of_interest] # params_row[parameter_of_interest] = value # params = params_row[flags].to_dict() # param_hashid = hash_param(params) # if not (param_hashid in comparable_agg1.index['param_hashid'].values): # raise AssertionError # params_of_interest.append(param_hashid) def visualize_cases(agg): region_id = 'macro_02_19bfe3' regions_of_interest = agg.macro_key_to_regions.get(region_id, [region_id]) param_hashid = 'ab43161b' to_inspect_idxs = [] for region_id in regions_of_interest: tables = agg.region_to_tables[region_id] grouped_indexes = tables['index'].groupby('param_hashid') rows = grouped_indexes.get_group(param_hashid) assert len(rows) == 1 to_inspect_idxs.append(rows.index[0]) for index in to_inspect_idxs: pass index = to_inspect_idxs[0] agg.index.iloc[index] agg.results['param_types'][index] eval_fpath = agg.results['fpaths'][index] # NOQA def plot_stats_tables(agg, config): """ agg = eval_type_to_aggregator.get('bas_poly_eval', None) """ # from watch.mlops import smart_result_parser # for fpath in fpaths: # ... # result = smart_result_parser.load_sc_poly_eval(fpath, None) # print(result['metrics']['sc_macro_f1']) import numpy as np from watch.utils import util_kwplot import kwplot sns = kwplot.autosns() plt = kwplot.autoplt() # metric_cols = [c for c in df.columns if 'metrics.' in c] kwplot.close_figures() metric = 'sc_poly_eval.metrics.sc_macro_f1' agg_dpath = ub.Path(config['root_dpath'] / 'aggregate') rois = {'BR_R002', 'KR_R001', 'KR_R002'} agg.build_single_macro_table(rois) macro_key = agg.primary_macro_region agg_group_dpath = (agg_dpath / (f'stats_tables_{macro_key}' + ub.timestamp())).ensuredir() # df['sc_poly_eval.metrics.macro_f1_active'] for metric in agg.primary_metric_cols: node_id = metric.split('.')[0] metric_name = metric.split('.')[-1] df = pd.concat([agg.metrics, agg.index, agg.resolved_params], axis=1) plt.figure() ax = sns.boxplot(data=df, x='region_id', y=metric) fig = ax.figure ax.set_ylabel(metric_name) ax.set_title(f'{node_id} {metric_name} n={len(agg)}') xtick_mapping = { region_id: f'{region_id}\n(n={num})' for region_id, num in df.groupby('region_id').size().to_dict().items() } util_kwplot.relabel_xticks(xtick_mapping, ax=ax) fname = f'boxplot_{metric}.png' fpath = agg_group_dpath / fname fig.set_size_inches(np.array([6.4, 4.8]) * 1.4) fig.tight_layout() fig.savefig(fpath) util_kwplot.cropwhite_ondisk(fpath) ### Macro stats analysis regions_of_interest = agg.macro_key_to_regions[macro_key] print(f'regions_of_interest={regions_of_interest}') tables = agg.region_to_tables[agg.primary_macro_region] table = pd.concat(list((ub.udict(tables) & { 'index', 'resolved_params', 'metrics'}).values()), axis=1) table = table.fillna('None') # Fix table['bas_poly_eval.params.bas_pxl.chip_dims'] = table['bas_poly_eval.params.bas_pxl.chip_dims'].apply(lambda x: str(x) if isinstance(x, list) else x) from watch.utils import result_analysis results = [] for idx, row in enumerate(table.to_dict('records')): row = ub.udict(row) row_metrics = row & set(tables['metrics'].keys()) row_params = row & set(tables['resolved_params'].keys()) result = result_analysis.Result(str(idx), row_params, row_metrics) results.append(result) analysis = result_analysis.ResultAnalysis( results, metrics=[metric], metric_objectives={metric: 'max'} ) # self = analysis analysis.analysis() analysis.report() kwplot.close_figures() for param in analysis.varied: fig = plt.figure() fig.clf() ax = sns.boxplot(data=table, x=param, y=metric) fig = ax.figure ax.set_ylabel(metric_name) ax.set_title(f'{node_id} {macro_key} {metric_name} {param} n={len(table)}') xtick_mapping = { str(value): f'{value}\nn={num}' for value, num in table.groupby(param, dropna=False).size().to_dict().items() } util_kwplot.relabel_xticks(xtick_mapping, ax=ax) # util_kwplot.relabel_xticks(xtick_mapping, ax=ax) fname = f'boxplot_{macro_key}_{metric}_{param}.png' fpath = agg_group_dpath / fname # fig.set_size_inches(np.array([6.4, 4.8]) * 1.4) fig.set_size_inches(np.array([16, 9]) * 1.0) fig.tight_layout() fig.savefig(fpath) util_kwplot.cropwhite_ondisk(fpath) def custom_analysis(eval_type_to_aggregator, config): macro_groups = [ {'KR_R001', 'KR_R002'}, {'KR_R001', 'KR_R002', 'US_R007'}, {'BR_R002', 'KR_R001', 'KR_R002', 'AE_R001'}, {'BR_R002', 'KR_R001', 'KR_R002', 'AE_R001', 'US_R007'}, ] rois = macro_groups # NOQA agg0 = eval_type_to_aggregator.get('bas_poly_eval', None) agg_dpath = ub.Path(config['root_dpath'] / 'aggregate') if agg0 is not None: agg0_ = fix_duplicate_param_hashids(agg0) # params_of_interest = ['414d0b37'] # v0_params_of_interest = [ # '414d0b37', 'ab43161b', '34bed2b3', '8ac5594b'] params_of_interest = list({ '414d0b37': 'ffmpktiwwpbx', 'ab43161b': 'nriolrrxfbco', '34bed2b3': 'eflhsmpfhgsj', '8ac5594b': 'lbxwewpkfwcd', }.values()) param_of_interest = params_of_interest[0] subagg1 = agg0_.filterto(param_hashids=param_of_interest) subagg1.build_macro_tables(macro_groups) _ = subagg1.report_best() print(ub.urepr(subagg1.results['fpaths']['fpath'].to_list())) agg_group_dpath = agg_dpath / (f'agg_params_{param_of_interest}') agg_group_dpath = agg_group_dpath.ensuredir() # Make a directory with a summary over all the regions summary_dpath = (agg_group_dpath / 'summary').ensuredir() # make a analysis link to the final product for idx, fpath in ub.ProgIter(list(subagg1.fpaths.iteritems())): region_id = subagg1.index.loc[idx]['region_id'] param_hashid = subagg1.index.loc[idx]['param_hashid'] link_dpath = agg_group_dpath / region_id ub.symlink(real_path=fpath.parent, link_path=link_dpath) # ub.symlink(real_path=region_viz_fpath, link_path=summary_dpath / region_viz_fpath.name) import kwimage from kwcoco.metrics.drawing import concice_si_display region_viz_fpaths = list((fpath.parent / 'region_viz_overall').glob('*_detailed.png')) assert len(region_viz_fpaths) == 1 region_viz_fpath = region_viz_fpaths[0] viz_img = kwimage.imread(region_viz_fpath) scores_of_interest = util_pandas.pandas_shorten_columns(subagg1.metrics).loc[idx, ['bas_tp', 'bas_fp', 'bas_fn', 'bas_f1']] scores_of_interest = ub.udict(scores_of_interest.to_dict()) text = ub.urepr(scores_of_interest.map_values(concice_si_display), nobr=1, si=1, compact=1) new_img = kwimage.draw_header_text(viz_img, param_hashid + '\n' + text) kwimage.imwrite(summary_dpath / f'summary_{region_id}.jpg', new_img) # eval_dpath = bas_poly_eval_confusion_analysis(eval_fpath) # TODO: use the region_id. # ub.symlink(real_path=eval_dpath, link_path=agg_group_dpath / eval_dpath.name) #### FIND BEST FOR EACH MODEL agg0_.build_macro_tables(macro_groups) params_of_interest = [] macro_metrics = agg0_.region_to_tables[agg0_.primary_macro_region]['metrics'] macro_params = agg0_.region_to_tables[agg0_.primary_macro_region]['effective_params'] macro_index = agg0_.region_to_tables[agg0_.primary_macro_region]['index'] for model_name, group in macro_params.groupby(agg0_.model_cols): group_metrics = macro_metrics.loc[group.index] group_metrics = group_metrics.sort_values(agg0_.primary_metric_cols, ascending=False) group_metrics.iloc[0] param_hashid = macro_index.loc[group_metrics.index[0]]['param_hashid'] params_of_interest.append(param_hashid) agg1 = agg0_.filterto(param_hashids=params_of_interest) agg1.build_macro_tables(macro_groups) # Get a shortlist of the top models hash_part = agg1.region_to_tables[agg1.primary_macro_region]['index']['param_hashid'] model_part = agg1.region_to_tables[agg1.primary_macro_region]['effective_params'][agg0_.model_cols] metric_part = agg1.region_to_tables[agg1.primary_macro_region]['metrics'][agg0_.primary_metric_cols] table = pd.concat([hash_part, model_part, metric_part], axis=1) table = table.sort_values(agg0_.primary_metric_cols, ascending=False) blocklist = { 'Drop4_BAS_2022_12_15GSD_BGRN_V10_epoch=0-step=512-v1', 'Drop4_BAS_2022_12_15GSD_BGRN_V10_epoch=6-step=3584', 'Drop4_BAS_2022_12_15GSD_BGRN_V10_epoch=4-step=2560', 'Drop4_BAS_2022_12_15GSD_BGRN_V5_epoch=1-step=77702', 'Drop4_BAS_2022_12_15GSD_BGRN_V5_epoch=5-step=233106', } flags = ~kwarray.isect_flags(table['bas_poly_eval.params.bas_pxl.package_fpath'].values, blocklist) params_of_interest = table['param_hashid'].loc[flags] agg1 = agg0_.filterto(param_hashids=params_of_interest) agg1.build_macro_tables(macro_groups) # rois = {'BR_R002', 'KR_R001', 'KR_R002', 'AE_R001', 'US_R007'}/ # rois = {'KR_R001', 'KR_R002'} # rois = {'KR_R001', 'KR_R002', 'US_R007'} # rois = {'BR_R002', 'KR_R001', 'KR_R002', 'AE_R001'} # _ = agg.build_macro_tables(rois) # params_of_interest = ['414d0b37'] # params_of_interest = ['ab43161b'] # params_of_interest = ['34bed2b3'] # params_of_interest = ['8ac5594b'] # model_of_interest = # models_of_interest = [ # 'package_epoch0_step41', # 'Drop4_BAS_15GSD_BGRNSH_invar_V8_epoch=16-step=8704', # 'Drop4_BAS_2022_12_15GSD_BGRN_V10_epoch=0-step=4305', # 'Drop4_BAS_2022_12_15GSD_BGRN_V5_epoch=1-step=77702-v1', # ] # subagg2 = agg.filterto(models=params_of_interest1) # _ = subagg2.build_macro_tables({'KR_R001', 'KR_R002'}) # _ = subagg2.build_macro_tables(rois) # subagg2.macro_analysis() # _ = subagg2.build_macro_tables('max') # _ = subagg2.build_macro_tables({'KR_R001', 'KR_R002', 'US_R007'}) # _ = subagg2.build_macro_tables({'KR_R001', 'KR_R002'}) # _ = subagg2.build_macro_tables({'AE_R001'}) # _ = subagg2.build_macro_tables({'BR_R002'}) # _ = subagg2.build_macro_tables({'US_R007'}) # _ = subagg2.build_macro_tables({'KR_R002'}) # subagg2.macro_analysis() # _ = subagg2.report_best() # rois = {'BR_R002', 'KR_R001', 'KR_R002', 'AE_R001'} # macro_results = subagg.build_macro_tables(rois) # top_idxs = macro_results['metrics'].sort_values(subagg.primary_metric_cols).index[0:3] # top_param_hashids = macro_results['index']['param_hashid'].iloc[top_idxs] # flags = kwarray.isect_flags(subagg.index['param_hashid'].values, top_param_hashids) # final_agg = subagg.compress(flags) # macro_results = final_agg.build_macro_tables(rois) # # top_idx = macro_results['metrics'].sort_values(subagg.primary_metric_cols).index[0] # final_scores = final_agg.report_best() # region_id_to_summary = subagg.report_best() # region_id_to_summary['macro_02_19bfe3'] def quick_heatmap_viz(): import ubelt as ub import kwcoco import kwarray dmj_dpath = ub.Path('/data/david.joy/DataFor2023Jan31Delivery/KW_R001_eval_8/2021-08-31/split/mono/products/bas-fusion') kwcoco_fpath = dmj_dpath / 'bas_fusion_kwcoco.json' dset = kwcoco.CocoDataset(kwcoco_fpath) video_name = 'KW_R001' video = dset.index.name_to_video[video_name] video_id = video['id'] images = dset.images(video_id=video_id) # Average all heatmaps together running = kwarray.RunningStats() for coco_img in ub.ProgIter(images.coco_images, desc='loading images'): delayed = coco_img.imdelay('salient', resolution='10 GSD', nodata_method='float') heatmap = delayed.finalize() running.update(heatmap) import kwplot import kwimage from watch.utils import util_kwimage stats = running.current() average_heatmap = stats['mean'] average_heatmap = util_kwimage.exactly_1channel(average_heatmap) canvas = kwplot.make_heatmask(average_heatmap)[:, :, 0:3] canvas = kwimage.ensure_uint255(canvas) kwimage.imwrite('average_heatmap.png', canvas) import kwplot kwplot.autompl() kwplot.imshow(stats['min'], cmap='plasma', data_colorbar=True, title='min response', fnum=1) kwplot.imshow(stats['max'], cmap='plasma', data_colorbar=True, title='max response', fnum=2) kwplot.imshow(stats['mean'], cmap='plasma', data_colorbar=True, title='mean response', fnum=3) kwplot.imshow(stats['std'], cmap='plasma', data_colorbar=True, title='std response', fnum=4) def remove_specific_runs_by_region(): """ Find specific regions and remove runs for them """ from watch.mlops import aggregate_loader import watch expt_dvc_dpath = watch.find_dvc_dpath(tags='phase2_expt', hardware='auto') root_dpath = expt_dvc_dpath / '_split6_toothbrush_meanyear' pipeline = 'bas' io_workers = 16 # eval_type_to_results = aggregate_loader.build_tables(root_dpath, pipeline, io_workers) # eval_type_to_results['bas_pxl_eval'] from watch.mlops import smart_pipeline dag = smart_pipeline.make_smart_pipeline(pipeline) dag.configure(config=None, root_dpath=root_dpath) remove_regions = [ # 'NZ_R001', # 'BR_R001', # 'BR_R002', 'CH_R001', ] stages_of_interest = ['bas_pxl_eval', 'bas_poly_eval'] existing = {} bad_dpaths = [] for stage in stages_of_interest: rows = dag.nodes[stage].find_template_outputs(workers=8) for row in rows: test_dset_fpath = ub.Path(row['request.bas_pxl.test_dataset']) if any([r in test_dset_fpath.name for r in remove_regions]): bad_dpath = row['dpath'] bad_dpaths.append(bad_dpath) def expand_pred(dpath): pred_link_dpath = dpath / '.pred' for pred_dpath in list(pred_link_dpath.glob('*/*')): yield pred_dpath yield from expand_pred(pred_dpath) def expand_succ(dpath): succ_link_dpath = dpath / '.succ' for succ_dpath in list(succ_link_dpath.glob('*/*')): yield succ_dpath yield from expand_succ(succ_dpath) complement_bad_dpaths = set() # for dpath in list(bad_dpaths): # if dpath not in complement_bad_dpaths: # complement_bad_dpaths.update(list(expand_pred(dpath))) # complement_bad_dpaths.update(list(expand_succ(dpath))) closed_bad_dpaths = sorted([p.resolve() for p in list(complement_bad_dpaths) + bad_dpaths]) print('closed_bad_dpaths = {}'.format(ub.urepr(closed_bad_dpaths, nl=1))) # Keep bas pxl jobs closed_bad_dpaths = [p for p in closed_bad_dpaths if 'bas_pxl' not in str(p)] for dpath in closed_bad_dpaths: dpath.delete()