
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "004"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder


# Use environment variables to figure out path to the csvs and auto_metrics
antenna = str(int(os.environ["ANTENNA"]))
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "102"
csv_folder = "/home/obs/src/H5C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H5C_Notebooks/auto_metrics_inspect"


display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))


import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna


# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 255 csvs in /home/obs/src/H5C_Notebooks/_rtp_summary_
Found 248 auto_metrics notebooks in /home/obs/src/H5C_Notebooks/auto_metrics_inspect


# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'


this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
            antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
        else:
            antnum = int(row['Ant'])
        if antnum != int(antenna):
            continue
        
        if np.issubdtype(type(row['Node']), np.integer):
            row['Node'] = str(row['Node'])
        if type(row['Node']) == str and row['Node'].isnumeric():
            row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break


# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])


display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))


htmls_to_display = []
for am_html in auto_metric_htmls:
    html_to_display = ''
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line:
            htmls_to_display.append(html_to_display)
            break


for i, html_to_display in enumerate(htmls_to_display):
    if i == 100:
        break
    display(HTML(html_to_display))

JDs	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	Flag Fraction Before Redcal	Flagged By Redcal chi^2 Fraction	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric	Median chi^2 Per Antenna (Jee)	Median chi^2 Per Antenna (Jnn)
2459810	RF_maintenance	0.00%	-	-	-	-	-	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan
2459809	RF_maintenance	100.00%	0.00%	40.82%	0.00%	100.00%	0.00%	5.774495	10.993382	15.503783	22.044440	269.310362	22.825560	22.922312	0.891637	0.6745	0.4054	0.4887	3.817472	3.368624
2459808	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	18.458144	17.406108	8.476428	8.057658	282.554595	275.930511	6229.910590	6192.332050	0.6404	0.6612	0.4231	0.000000	0.000000
2459807	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	18.969633	17.889758	6.825966	6.755838	271.921241	265.486435	20006.590542	19847.643677	0.6155	0.6451	0.4247	0.000000	0.000000
2459806	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	18.278740	17.029085	0.314228	0.210264	401.724222	391.472568	14968.643119	14837.381758	0.5308	0.5127	0.3236	0.000000	0.000000
2459805	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	17.288239	15.091319	9.518175	8.003192	258.651313	238.769586	4749.732996	4659.523857	0.6121	0.6819	0.4530	0.000000	0.000000
2459804	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	20.406806	17.240076	7.848726	6.588821	297.561212	267.561934	3378.126479	3321.819847	0.6433	0.6793	0.4271	0.000000	0.000000
2459803	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	25.420412	21.779368	9.899680	8.305044	335.674348	308.817882	10603.462199	10477.513912	0.5998	0.6733	0.4306	0.000000	0.000000
2459802	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	21.279482	18.073295	4.006022	2.869621	342.931241	309.332058	7962.618098	7917.307024	0.5489	0.5694	0.3963	0.000000	0.000000
2459801	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	20.168049	16.084637	7.534439	4.698869	427.849260	357.069550	14093.991715	13929.973251	0.0888	0.0901	0.0173	0.000000	0.000000
2459800	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	2.870778	2.114081	24.096118	2.460000	50.897380	3.515942	659.890665	60.258015	0.0513	0.0723	0.0075	1.228020	1.229009
2459799	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	6.749199	2.553912	13.914333	2.924648	123.148519	29.392186	255.614495	72.327165	0.0618	0.0535	0.0044	1.222884	1.225917
2459798	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	17.475209	15.030065	80.854394	77.536473	1779.670127	1612.378871	20198.939465	17104.620860	nan	nan	nan	0.000000	0.000000
2459797	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	9.745011	2.100855	37.524936	9.692096	114.423713	16.906786	1390.501607	203.861030	0.0884	0.0564	0.0132	1.157234	1.177040
2459796	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.725634	0.988470	15.800737	3.915945	16.500532	12.700199	261.244525	119.536283	0.0552	0.0638	0.0055	962.255722	230.628460
2459795	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	2.402307	1.363848	16.311488	2.432654	20.175121	7.227227	298.346325	100.018102	0.0618	0.0650	0.0082	1.185030	1.184565
2459794	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.548348	2.515994	20.189866	11.926451	32.864175	19.306670	654.520876	379.422120	0.0580	0.0667	0.0092	0.000000	0.000000
2459793	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	4.522040	1.302690	30.273088	20.548638	65.974961	36.382296	630.653756	339.726038	0.0612	0.0683	0.0104	1.169170	1.176454
2459792	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	12.955318	3.881090	26.410416	17.392320	158.873248	94.684957	808.896158	493.936058	0.1014	0.0767	0.0247	1.170910	1.179050
2459791	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	3.754050	2.122290	32.471905	22.791945	43.055577	28.136064	132.351186	82.409612	0.0613	0.0650	0.0081	1.166639	1.161073
2459790	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	24.373806	22.570403	97.027864	93.025494	2960.379662	2677.471808	18114.718507	15328.771731	nan	nan	nan	0.000000	0.000000
2459789	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.509684	0.818848	16.097027	4.463228	29.903168	21.229391	414.053776	222.630992	0.6973	0.7088	0.3877	50.715492	27.681501
2459788	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	18.684365	17.650987	108.395152	103.444962	3467.881145	3122.694343	18990.835255	16306.213376	nan	nan	nan	0.000000	0.000000
2459787	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.907287	1.893057	4.182910	1.937612	30.314987	8.207790	107.784362	63.482910	0.6998	0.7064	0.3665	11.795695	11.988098
2459786	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.392459	0.810769	5.252610	-0.748069	21.922458	14.921653	168.246635	121.877519	0.6755	0.6842	0.3608	3.529960	3.909504
2459785	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.027592	0.433981	30.741684	6.661897	97.642810	32.735048	251.957696	94.755636	0.7095	0.6848	0.4010	3.350757	3.534006
2459784	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.121828	1.002066	3.028915	-0.194616	6.048558	-0.310572	-0.246145	-0.909274	0.6530	0.6576	0.3432	10.259499	8.341960
2459783	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	10.201966	0.845779	6.264939	-1.442169	14.547051	8.536441	18.685400	84.812912	0.6191	0.6266	0.3268	3.078243	3.131866
2459782	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.187412	0.400455	2.172244	-1.010879	5.034639	4.964299	19.909032	60.296462	0.5188	0.5196	0.2882	2.291897	2.308768
2459781	RF_maintenance	100.00%	49.15%	49.15%	0.00%	100.00%	0.00%	1.211969	0.574604	-0.605342	0.491539	15.356693	15.814396	16.611723	73.419182	0.6654	0.6690	0.3945	2.416957	2.691735
2459778	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	12.226439	12.472507	9.792376	10.734563	145.082527	173.994240	1873.025784	2235.964511	0.0343	0.0334	0.0014	1.160933	1.155722
2459776	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	20.322284	23.215774	7.883813	8.439257	187.326279	82.985682	560.960759	235.920094	0.0304	0.0295	0.0015	1.155920	1.152637
2459774	RF_maintenance	-	100.00%	100.00%	0.00%	-	-	nan	nan	nan	nan	nan	nan	nan	nan	0.0333	0.0323	0.0016	nan	nan
2459764	RF_maintenance	-	-	-	-	-	-	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan
2459763	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	17.225975	20.928769	35.170294	37.323258	8.318758	4.565628	5.491628	3.501257	0.0345	0.0329	0.0006	1.140352	1.131145
2459761	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459760	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459759	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459758	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459757	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000

Antenna Season Report Notebook¶

Antenna 102 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 102, Node N08:

Load antenna metric spectra and waterfalls from auto_metrics notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 102: 2459810

Antenna 102: 2459809

Antenna 102: 2459808

Antenna 102: 2459807

Antenna 102: 2459806

Antenna 102: 2459805

Antenna 102: 2459804

Antenna 102: 2459803

Antenna 102: 2459802

Antenna 102: 2459801

Antenna 102: 2459800

Antenna 102: 2459799

Antenna 102: 2459798

Antenna 102: 2459797

Antenna 102: 2459796

Antenna 102: 2459795

Antenna 102: 2459794

Antenna 102: 2459793

Antenna 102: 2459792

Antenna 102: 2459791

Antenna 102: 2459790

Antenna 102: 2459789

Antenna 102: 2459788

Antenna 102: 2459787

Antenna 102: 2459786

Antenna 102: 2459785

Antenna 102: 2459784

Antenna 102: 2459783

Antenna 102: 2459782

Antenna 102: 2459781

Antenna 102: 2459778

Antenna 102: 2459776

Antenna 102: 2459763

Antenna 102: 2459761

Antenna 102: 2459760

Antenna 102: 2459759

Antenna 102: 2459758

Antenna 102: 2459757

Antenna 102: 2459755

Antenna 102: 2459754

Antenna 102: 2459753

Antenna 102: 2459752

Antenna 102: 2459750

Antenna 102: 2459749

Antenna 102: 2459748

Antenna 102: 2459747

Antenna 102: 2459746

Antenna 102: 2459745

Antenna 102: 2459744

Antenna 102: 2459743

Antenna 102: 2459742

Antenna 102: 2459741

Antenna 102: 2459740

Antenna 102: 2459738

Antenna 102: 2459736

Antenna 102: 2459734

Antenna 102: 2459733

Antenna 102: 2459732

Antenna 102: 2459731

Antenna 102: 2459730

Antenna 102: 2459728

Antenna 102: 2459726

Antenna 102: 2459724

Antenna 102: 2459723

Antenna 102: 2459722

Antenna 102: 2459720

Antenna 102: 2459719

Antenna 102: 2459718

Antenna 102: 2459717

Antenna 102: 2459716

Antenna 102: 2459715

Antenna 102: 2459713

Antenna 102: 2459712

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶