
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "004"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder


# Use environment variables to figure out path to the csvs and auto_metrics
antenna = str(int(os.environ["ANTENNA"]))
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "85"
csv_folder = "/home/obs/src/H5C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H5C_Notebooks/auto_metrics_inspect"


display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))


import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna


# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 255 csvs in /home/obs/src/H5C_Notebooks/_rtp_summary_
Found 248 auto_metrics notebooks in /home/obs/src/H5C_Notebooks/auto_metrics_inspect


# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'


this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
            antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
        else:
            antnum = int(row['Ant'])
        if antnum != int(antenna):
            continue
        
        if np.issubdtype(type(row['Node']), np.integer):
            row['Node'] = str(row['Node'])
        if type(row['Node']) == str and row['Node'].isnumeric():
            row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break


# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])


display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))


htmls_to_display = []
for am_html in auto_metric_htmls:
    html_to_display = ''
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line:
            htmls_to_display.append(html_to_display)
            break


for i, html_to_display in enumerate(htmls_to_display):
    if i == 100:
        break
    display(HTML(html_to_display))

JDs	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	Flag Fraction Before Redcal	Flagged By Redcal chi^2 Fraction	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric	Median chi^2 Per Antenna (Jee)	Median chi^2 Per Antenna (Jnn)
2459810	digital_ok	0.00%	-	-	-	-	-	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan
2459809	digital_ok	0.00%	0.00%	4.08%	0.00%	4.00%	8.00%	0.666007	0.458280	-0.720988	-0.556465	-1.448733	-1.218910	-0.928263	-1.420782	0.7463	0.5369	0.4992	1.875856	1.687148
2459808	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	76.32%	0.379793	-0.116672	-0.994838	-0.842426	-1.268745	-1.433406	-0.896016	-1.223991	0.7594	0.7234	0.4407	2.395546	2.254779
2459807	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	71.05%	-0.112678	1.160892	0.549770	1.498206	-0.419343	0.996036	-0.126558	1.104516	0.7422	0.7030	0.4374	2.317243	2.318788
2459806	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	2.14%	-0.175328	1.521040	1.013108	2.056631	-0.865051	-0.403064	-0.475094	-0.520126	0.6839	0.6084	0.3640	1.981458	1.877950
2459805	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	86.84%	0.040573	0.237705	-0.881213	-0.375210	-1.291665	-0.526266	-0.794679	-1.140935	0.7545	0.7257	0.4368	2.548050	2.408365
2459804	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	81.58%	-0.005876	1.642377	1.040063	2.592977	-0.790724	-0.220734	-0.733014	-0.669888	0.7507	0.7214	0.4283	2.609911	2.409587
2459803	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	52.63%	-0.024851	2.258253	1.145344	2.975675	-0.563599	0.195949	-0.484845	2.310838	0.7407	0.7195	0.4140	5.956845	4.894022
2459802	digital_ok	0.00%	0.00%	0.00%	0.00%	0.00%	0.00%	0.598022	1.141767	-0.905740	-0.326083	-1.270517	-1.323085	-0.711614	-0.795709	0.7182	0.6082	0.4522	1.890242	1.664852
2459801	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.215112	1.010152	-0.937414	-0.234762	-1.852372	-1.618381	-0.447875	-0.446432	0.0665	0.0677	0.0073	1.211281	1.209301
2459800	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.105136	0.780515	-1.028910	-0.460595	-1.592997	-1.164965	-0.416153	-0.748677	0.0755	0.0758	0.0080	1.213309	1.213503
2459799	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.134317	1.754999	1.042752	2.026819	-0.831215	0.289439	-0.916222	-1.019340	0.0696	0.0676	0.0072	1.224003	1.220997
2459798	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459797	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	-0.205809	0.954748	1.007934	1.936770	-0.206326	-0.033087	-0.189899	-0.018296	0.0751	0.0688	0.0086	1.192845	1.199269
2459796	RF_maintenance	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.345996	1.188852	-0.317702	0.051265	-1.094416	-1.159205	-0.881045	-1.034757	0.0717	0.0750	0.0077	0.868875	0.832626
2459795	RF_maintenance	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.472748	0.371374	-0.850528	-0.233675	-1.251910	-1.212921	-1.062932	-1.103578	0.0864	0.0778	0.0104	1.231415	1.232516
2459794	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.796342	4.093023	-0.244109	6.448451	-0.891612	5.053049	-0.605561	-2.825494	0.0832	0.0963	0.0145	0.000000	0.000000
2459793	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.393545	3.213710	-0.680677	5.306435	-1.705743	4.236568	-0.744562	-1.704187	0.0793	0.0945	0.0142	1.250025	1.253244
2459792	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.934817	34.533310	0.658805	2.261173	-1.067178	12.284781	-0.116386	5.034205	0.0773	0.0993	0.0122	1.177762	1.178819
2459791	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.630567	25.166788	2.099279	2.774071	0.166533	9.095250	-0.481629	6.274996	0.0730	0.0832	0.0084	1.196997	1.195479
2459790	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459789	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.085587	39.073098	-0.652395	3.726208	-0.883495	12.908050	0.471322	26.705379	0.7209	0.6603	0.3772	23.354555	17.596663
2459788	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459787	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.608302	33.823993	1.757436	2.310978	-0.127823	10.070134	-0.257275	10.294368	0.7185	0.6628	0.3605	14.945656	10.823316
2459786	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.019869	36.216796	0.023014	3.212243	-1.010947	13.752777	-0.082538	6.829253	0.6963	0.6370	0.3532	9.964239	14.086806
2459785	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.861338	26.929145	5.156780	5.063081	0.614077	13.835489	0.035621	4.775327	0.7776	0.6106	0.3748	4.564610	4.950237
2459784	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.045870	30.899026	4.110854	3.573297	2.078337	14.104039	-0.874525	5.219383	0.6650	0.6009	0.3422	10.474589	6.893461
2459783	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.894503	28.095958	4.333659	0.409241	6.998871	5.653575	-1.566957	3.022265	0.6404	0.5749	0.3268	3.017014	3.533738
2459782	RF_maintenance	100.00%	0.00%	24.19%	0.00%	100.00%	0.00%	7.676268	23.481836	2.576902	0.466334	4.789940	3.387783	-1.586221	-0.234995	0.5331	0.4750	0.2849	2.252119	2.788923
2459781	RF_maintenance	100.00%	49.15%	50.46%	0.00%	100.00%	0.00%	-0.404792	5.374477	-1.256422	-0.084229	15.161522	15.278786	-1.881330	2.641406	0.6940	0.6374	0.3919	2.543823	3.279252
2459778	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	13.631497	14.721437	10.774805	11.296353	23.799106	22.121836	2.600901	3.631480	0.0322	0.0329	0.0010	1.151032	1.150155
2459776	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	20.002572	23.242258	8.370385	8.228371	36.321645	37.466184	2.513419	3.665644	0.0314	0.0292	0.0016	1.163871	1.163891
2459774	RF_maintenance	-	100.00%	100.00%	0.00%	-	-	nan	nan	nan	nan	nan	nan	nan	nan	0.0324	0.0318	0.0012	nan	nan
2459764	digital_ok	-	-	-	-	-	-	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan	nan
2459763	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	17.733189	21.161128	34.576276	35.283138	8.273629	4.478406	5.645966	3.686678	0.0330	0.0314	0.0015	1.130338	1.127618
2459761	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459760	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459759	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459758	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
2459757	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000

Antenna Season Report Notebook¶

Antenna 85 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 85, Node N08:

Load antenna metric spectra and waterfalls from auto_metrics notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 85: 2459810

Antenna 85: 2459809

Antenna 85: 2459808

Antenna 85: 2459807

Antenna 85: 2459806

Antenna 85: 2459805

Antenna 85: 2459804

Antenna 85: 2459803

Antenna 85: 2459802

Antenna 85: 2459801

Antenna 85: 2459800

Antenna 85: 2459799

Antenna 85: 2459798

Antenna 85: 2459797

Antenna 85: 2459796

Antenna 85: 2459795

Antenna 85: 2459794

Antenna 85: 2459793

Antenna 85: 2459792

Antenna 85: 2459791

Antenna 85: 2459790

Antenna 85: 2459789

Antenna 85: 2459788

Antenna 85: 2459787

Antenna 85: 2459786

Antenna 85: 2459785

Antenna 85: 2459784

Antenna 85: 2459783

Antenna 85: 2459782

Antenna 85: 2459781

Antenna 85: 2459778

Antenna 85: 2459776

Antenna 85: 2459763

Antenna 85: 2459761

Antenna 85: 2459760

Antenna 85: 2459759

Antenna 85: 2459758

Antenna 85: 2459757

Antenna 85: 2459755

Antenna 85: 2459754

Antenna 85: 2459753

Antenna 85: 2459752

Antenna 85: 2459750

Antenna 85: 2459749

Antenna 85: 2459748

Antenna 85: 2459747

Antenna 85: 2459746

Antenna 85: 2459745

Antenna 85: 2459744

Antenna 85: 2459743

Antenna 85: 2459742

Antenna 85: 2459741

Antenna 85: 2459740

Antenna 85: 2459738

Antenna 85: 2459736

Antenna 85: 2459734

Antenna 85: 2459733

Antenna 85: 2459732

Antenna 85: 2459731

Antenna 85: 2459730

Antenna 85: 2459728

Antenna 85: 2459726

Antenna 85: 2459724

Antenna 85: 2459723

Antenna 85: 2459722

Antenna 85: 2459720

Antenna 85: 2459719

Antenna 85: 2459718

Antenna 85: 2459717

Antenna 85: 2459716

Antenna 85: 2459715

Antenna 85: 2459713

Antenna 85: 2459712

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶