
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "004"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder


# Use environment variables to figure out path to the csvs and auto_metrics
antenna = str(int(os.environ["ANTENNA"]))
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "86"
csv_folder = "/home/obs/src/H5C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H5C_Notebooks/auto_metrics_inspect"


display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))


import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna


# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 255 csvs in /home/obs/src/H5C_Notebooks/_rtp_summary_
Found 248 auto_metrics notebooks in /home/obs/src/H5C_Notebooks/auto_metrics_inspect


# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'


this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
            antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
        else:
            antnum = int(row['Ant'])
        if antnum != int(antenna):
            continue
        
        if np.issubdtype(type(row['Node']), np.integer):
            row['Node'] = str(row['Node'])
        if type(row['Node']) == str and row['Node'].isnumeric():
            row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break


# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])


display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))


htmls_to_display = []
for am_html in auto_metric_htmls:
    html_to_display = ''
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line:
            htmls_to_display.append(html_to_display)
            break


for i, html_to_display in enumerate(htmls_to_display):
    if i == 100:
        break
    display(HTML(html_to_display))

JDs	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	Flag Fraction Before Redcal	Flagged By Redcal chi^2 Fraction	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric	Median chi^2 Per Antenna (Jee)	Median chi^2 Per Antenna (Jnn)
2459810	digital_ok	0.00%	-	-	-	-	-	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan
2459809	digital_ok	100.00%	0.00%	4.08%	0.00%	100.00%	0.00%	1.514287	4.685490	-0.663485	0.022207	2.925508	2.391792	0.128351	-0.274851	0.7410	0.4985	0.5017	4.108931	4.483211
2459808	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	86.84%	1.954865	2.434191	0.858104	-0.636123	1.066846	0.191372	-0.185896	0.218391	0.7561	0.7101	0.4343	2.761773	2.531530
2459807	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	86.84%	1.265490	2.138180	-0.640045	-0.725206	0.184120	-0.025155	-0.401169	0.648360	0.7405	0.6935	0.4338	2.895862	2.622125
2459806	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.390311	5.893417	-0.139845	-0.301882	5.909761	3.669121	1.211174	1.995557	0.6854	0.5861	0.3683	5.283256	6.037171
2459805	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	100.00%	0.341667	2.221335	-0.443228	-0.543543	1.662606	0.419402	1.232358	1.376066	0.7519	0.7159	0.4301	3.193949	2.648379
2459804	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	94.74%	1.543608	2.573262	-0.103159	-0.536627	-0.098973	0.032252	-0.412962	0.044863	0.7503	0.7141	0.4215	3.192318	2.817471
2459803	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	13.16%	2.293579	3.247566	1.893287	-0.485709	0.380835	-0.175370	0.491236	0.502022	0.7396	0.7129	0.4090	2.902762	2.439822
2459802	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.474007	4.393267	0.839588	-0.529146	1.906955	1.308885	0.194880	0.794855	0.7228	0.5952	0.4493	4.631010	4.358225
2459801	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.831842	2.776190	-0.222118	-0.703428	0.968535	0.497921	-0.006053	-0.493228	0.0694	0.0722	0.0086	1.176945	1.178971
2459800	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.446711	2.766734	-0.757509	-0.891170	-0.499278	0.123489	-0.672718	-0.598401	0.0707	0.0764	0.0067	1.220849	1.220187
2459799	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.932723	5.450581	-0.778807	-0.217438	0.822768	5.608038	0.453275	0.826946	0.0801	0.0785	0.0113	1.234206	1.235572
2459798	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459797	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.954356	2.215808	-0.257056	-0.661461	-0.578517	0.443749	-0.672298	0.002047	0.0703	0.0645	0.0062	1.203313	1.198729
2459796	RF_maintenance	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.671716	2.623239	2.028796	-0.391769	1.307197	0.428135	-0.697972	-0.345949	0.0809	0.0710	0.0065	0.000000	0.000000
2459795	RF_maintenance	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.730453	2.893421	0.440125	-0.666799	2.685891	0.224346	-0.821902	-1.009480	0.0810	0.0716	0.0063	1.213842	1.217448
2459794	RF_maintenance	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.729166	2.648740	1.072651	-0.723788	0.430938	1.143072	1.068593	3.639759	0.0748	0.0701	0.0070	0.000000	0.000000
2459793	RF_maintenance	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.757227	1.685133	-0.093095	-0.961496	0.939104	0.067084	-0.646135	-0.555539	0.0722	0.0669	0.0053	1.206622	1.200014
2459792	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.382371	5.070740	-0.398665	0.235941	1.192589	4.369205	1.123171	1.710604	0.0724	0.0903	0.0110	1.183368	1.182149
2459791	RF_maintenance	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.154803	2.944948	-0.632167	-0.131150	-0.136742	0.581161	-0.417513	-0.299950	0.0654	0.0678	0.0066	1.201285	1.204851
2459790	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459789	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.070365	5.023714	-0.684187	-0.388684	4.690350	2.892674	1.711351	1.154361	0.7206	0.7103	0.3786	7.504424	6.313824
2459788	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459787	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.243245	4.773409	-0.599545	-0.268439	2.103607	1.277045	0.662733	-0.365094	0.7140	0.7075	0.3604	7.017137	6.004329
2459786	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.939863	5.236214	-0.535885	-0.273626	-0.884262	1.591079	1.956710	4.160747	0.6909	0.6855	0.3482	0.000000	0.000000
2459785	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.383205	6.480248	0.900748	17.785835	1.987281	13.789366	0.387997	-0.069572	0.7660	0.6756	0.3786	3.865280	3.124338
2459784	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.430975	9.460415	-0.673379	14.838404	3.128358	16.503209	0.114569	0.099534	0.6589	0.6528	0.3403	21.193891	11.596774
2459783	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	5.349688	14.865455	3.519807	7.652871	6.492882	15.312994	-1.329733	-1.595301	0.6396	0.6141	0.3382	2.905015	2.890974
2459782	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.269092	13.578130	1.445612	3.770058	2.423617	8.561653	-0.357420	-1.056175	0.5306	0.5057	0.2906	2.395041	2.111446
2459781	RF_maintenance	100.00%	49.15%	49.15%	0.00%	100.00%	0.00%	-0.675579	1.203384	-0.976425	-0.584279	15.168059	15.525540	-1.370183	-1.391713	0.6923	0.6618	0.3967	2.749552	2.546218
2459778	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	14.495049	17.641330	10.551598	11.041345	23.762385	22.058431	1.659673	4.536869	0.0346	0.0329	0.0019	1.147054	1.145596
2459776	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	20.918290	23.909655	8.185351	8.022807	36.296163	37.489931	1.175534	5.151760	0.0319	0.0292	0.0009	1.150448	1.149496
2459774	RF_maintenance	-	100.00%	100.00%	0.00%	-	-	nan	nan	nan	nan	nan	nan	nan	nan	0.0342	0.0323	0.0018	nan	nan
2459764	digital_ok	-	-	-	-	-	-	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan
2459763	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	17.211877	22.560031	35.564682	36.224281	8.643462	4.963058	5.623594	3.953099	0.0301	0.0311	0.0023	1.126363	1.114810
2459761	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459760	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459759	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459758	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459757	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000

Antenna Season Report Notebook¶

Antenna 86 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 86, Node N08:

Load antenna metric spectra and waterfalls from auto_metrics notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 86: 2459810

Antenna 86: 2459809

Antenna 86: 2459808

Antenna 86: 2459807

Antenna 86: 2459806

Antenna 86: 2459805

Antenna 86: 2459804

Antenna 86: 2459803

Antenna 86: 2459802

Antenna 86: 2459801

Antenna 86: 2459800

Antenna 86: 2459799

Antenna 86: 2459798

Antenna 86: 2459797

Antenna 86: 2459796

Antenna 86: 2459795

Antenna 86: 2459794

Antenna 86: 2459793

Antenna 86: 2459792

Antenna 86: 2459791

Antenna 86: 2459790

Antenna 86: 2459789

Antenna 86: 2459788

Antenna 86: 2459787

Antenna 86: 2459786

Antenna 86: 2459785

Antenna 86: 2459784

Antenna 86: 2459783

Antenna 86: 2459782

Antenna 86: 2459781

Antenna 86: 2459778

Antenna 86: 2459776

Antenna 86: 2459763

Antenna 86: 2459761

Antenna 86: 2459760

Antenna 86: 2459759

Antenna 86: 2459758

Antenna 86: 2459757

Antenna 86: 2459755

Antenna 86: 2459754

Antenna 86: 2459753

Antenna 86: 2459752

Antenna 86: 2459750

Antenna 86: 2459749

Antenna 86: 2459748

Antenna 86: 2459747

Antenna 86: 2459746

Antenna 86: 2459745

Antenna 86: 2459744

Antenna 86: 2459743

Antenna 86: 2459742

Antenna 86: 2459741

Antenna 86: 2459740

Antenna 86: 2459738

Antenna 86: 2459736

Antenna 86: 2459734

Antenna 86: 2459733

Antenna 86: 2459732

Antenna 86: 2459731

Antenna 86: 2459730

Antenna 86: 2459728

Antenna 86: 2459726

Antenna 86: 2459724

Antenna 86: 2459723

Antenna 86: 2459722

Antenna 86: 2459720

Antenna 86: 2459719

Antenna 86: 2459718

Antenna 86: 2459717

Antenna 86: 2459716

Antenna 86: 2459715

Antenna 86: 2459713

Antenna 86: 2459712

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶