
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "004"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder


# Use environment variables to figure out path to the csvs and auto_metrics
antenna = str(int(os.environ["ANTENNA"]))
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "222"
csv_folder = "/home/obs/src/H5C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H5C_Notebooks/auto_metrics_inspect"


display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))


import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna


# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 255 csvs in /home/obs/src/H5C_Notebooks/_rtp_summary_
Found 248 auto_metrics notebooks in /home/obs/src/H5C_Notebooks/auto_metrics_inspect


# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'


this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
            antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
        else:
            antnum = int(row['Ant'])
        if antnum != int(antenna):
            continue
        
        if np.issubdtype(type(row['Node']), np.integer):
            row['Node'] = str(row['Node'])
        if type(row['Node']) == str and row['Node'].isnumeric():
            row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break


# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])


display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))


htmls_to_display = []
for am_html in auto_metric_htmls:
    html_to_display = ''
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line:
            htmls_to_display.append(html_to_display)
            break


for i, html_to_display in enumerate(htmls_to_display):
    if i == 100:
        break
    display(HTML(html_to_display))

JDs	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	Flag Fraction Before Redcal	Flagged By Redcal chi^2 Fraction	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric	Median chi^2 Per Antenna (Jee)	Median chi^2 Per Antenna (Jnn)
2459810	RF_ok	0.00%	-	-	-	-	-	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan
2459809	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459808	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	12.063649	11.523535	102.024651	105.438376	2138.006818	2496.993893	7839.846835	10653.663619	nan	nan	nan	0.000000	0.000000
2459807	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	11.864448	10.652663	inf	inf	2051.877417	2379.200290	24277.775717	33022.971405	nan	nan	nan	0.000000	0.000000
2459806	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	10.865156	9.332722	74.467652	78.087234	2514.876841	2957.730346	18448.556183	25264.569841	nan	nan	nan	0.000000	0.000000
2459805	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	8.704266	7.076377	95.815562	97.795559	1983.487086	2278.614683	5951.271962	7907.179285	nan	nan	nan	0.000000	0.000000
2459804	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	10.260710	7.441124	85.747159	88.018333	2278.049877	2626.844617	4182.527751	5591.765836	nan	nan	nan	0.000000	0.000000
2459803	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	12.163953	8.670072	113.369327	116.232411	2591.109768	2984.219713	13322.286837	17739.786976	nan	nan	nan	0.000000	0.000000
2459802	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	15.492523	14.884028	inf	inf	3168.017685	3740.857418	9973.658446	13679.114954	nan	nan	nan	0.000000	0.000000
2459801	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	12.424364	12.061374	86.087972	88.645275	3248.944521	3770.093258	17689.446201	23837.871183	nan	nan	nan	0.000000	0.000000
2459800	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	10.108531	7.089456	83.179467	84.663105	2624.672669	2987.728671	20101.238170	26399.746884	nan	nan	nan	0.000000	0.000000
2459799	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	9.967034	8.362035	68.756482	70.794268	3179.627724	3558.545472	7207.702488	9162.672748	nan	nan	nan	0.000000	0.000000
2459798	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	9.858014	10.141330	82.823094	80.608678	1956.730930	1962.460110	13963.766807	14116.914082	nan	nan	nan	0.000000	0.000000
2459797	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	11.272641	8.774962	inf	inf	3385.134989	3385.085555	30020.382752	30021.718025	nan	nan	nan	0.000000	0.000000
2459796	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	9.356577	6.655105	102.426270	106.302851	1942.608355	2306.566713	18672.376365	26090.314668	nan	nan	nan	0.000000	0.000000
2459795	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	13.232542	9.924868	97.830705	100.281231	2780.289220	2780.178020	30057.534159	30060.030181	nan	nan	nan	0.000000	0.000000
2459794	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	9.432693	6.090893	85.971646	88.682725	2435.928709	2858.216662	29204.766182	40398.015699	nan	nan	nan	0.000000	0.000000
2459793	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	7.634694	5.808577	85.494363	87.309389	2489.924396	2952.679694	14213.195309	20060.362669	nan	nan	nan	0.000000	0.000000
2459792	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	8.889134	7.623793	70.015149	72.820464	2054.208338	2447.096810	7184.455185	10090.023845	nan	nan	nan	0.000000	0.000000
2459791	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	12.891421	11.401766	110.031552	113.124071	2209.376567	2563.194961	4136.421850	5634.697336	nan	nan	nan	0.000000	0.000000
2459790	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	15.670918	12.728626	100.849362	103.513390	3210.224279	3779.154252	12034.025529	16896.763033	nan	nan	nan	0.000000	0.000000
2459789	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	14.018573	15.634838	109.166260	107.835395	3288.835930	3917.610006	28003.730165	39326.919488	nan	nan	nan	0.000000	0.000000
2459788	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	12.306399	12.025020	112.120653	112.193818	3723.318692	4384.456815	12806.302130	17705.824183	nan	nan	nan	0.000000	0.000000
2459787	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	11.769744	13.305803	91.836531	90.040400	2487.993837	2994.784777	12873.423198	18308.811187	nan	nan	nan	0.000000	0.000000
2459786	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	13.494971	14.337432	110.943295	109.476886	3879.975170	4654.333339	18752.358565	26759.885078	nan	nan	nan	0.000000	0.000000
2459785	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	11.423079	9.045693	107.175951	114.230437	2280.186163	2771.360355	4606.722389	7169.770732	nan	nan	nan	0.000000	0.000000
2459784	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	16.786179	11.973338	108.157393	112.620426	3754.425110	4490.367576	3250.455107	4888.014188	nan	nan	nan	0.000000	0.000000
2459783	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	14.637624	11.403713	32.447520	32.410517	2131.178096	2457.517170	10146.472264	14603.744662	nan	nan	nan	0.000000	0.000000
2459782	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	13.651875	10.490831	16.406581	16.451464	1322.443692	1520.513287	7412.907248	10684.634744	nan	nan	nan	0.000000	0.000000
2459781	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459778	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	8.754730	6.577538	19.865483	20.327689	1321.092415	1537.102781	10414.115151	15218.196718	nan	nan	nan	0.000000	0.000000
2459776	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	9.964058	9.716645	22.254404	21.028988	6476.364765	5616.875968	24447.165815	21054.357242	nan	nan	nan	0.000000	0.000000
2459773	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459772	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459771	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459770	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459769	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459768	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459767	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459766	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000

Antenna Season Report Notebook¶

Antenna 222 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 222, Node N18:

Load antenna metric spectra and waterfalls from auto_metrics notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 222: 2459810

Antenna 222: 2459809

Antenna 222: 2459808

Antenna 222: 2459807

Antenna 222: 2459806

Antenna 222: 2459805

Antenna 222: 2459804

Antenna 222: 2459803

Antenna 222: 2459802

Antenna 222: 2459801

Antenna 222: 2459800

Antenna 222: 2459799

Antenna 222: 2459798

Antenna 222: 2459797

Antenna 222: 2459796

Antenna 222: 2459795

Antenna 222: 2459794

Antenna 222: 2459793

Antenna 222: 2459792

Antenna 222: 2459791

Antenna 222: 2459790

Antenna 222: 2459789

Antenna 222: 2459788

Antenna 222: 2459787

Antenna 222: 2459786

Antenna 222: 2459785

Antenna 222: 2459784

Antenna 222: 2459783

Antenna 222: 2459782

Antenna 222: 2459781

Antenna 222: 2459778

Antenna 222: 2459776

Antenna 222: 2459773

Antenna 222: 2459772

Antenna 222: 2459771

Antenna 222: 2459770

Antenna 222: 2459769

Antenna 222: 2459768

Antenna 222: 2459767

Antenna 222: 2459766

Antenna 222: 2459765

Antenna 222: 2459763

Antenna 222: 2459761

Antenna 222: 2459760

Antenna 222: 2459759

Antenna 222: 2459758

Antenna 222: 2459757

Antenna 222: 2459755

Antenna 222: 2459754

Antenna 222: 2459753

Antenna 222: 2459752

Antenna 222: 2459750

Antenna 222: 2459749

Antenna 222: 2459748

Antenna 222: 2459747

Antenna 222: 2459746

Antenna 222: 2459745

Antenna 222: 2459744

Antenna 222: 2459743

Antenna 222: 2459742

Antenna 222: 2459741

Antenna 222: 2459740

Antenna 222: 2459738

Antenna 222: 2459736

Antenna 222: 2459734

Antenna 222: 2459733

Antenna 222: 2459732

Antenna 222: 2459731

Antenna 222: 2459730

Antenna 222: 2459728

Antenna 222: 2459726

Antenna 222: 2459724

Antenna 222: 2459723

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶