
import os
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# antenna = "004"
# csv_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/_rtp_summary_'
# auto_metrics_folder = '/lustre/aoc/projects/hera/H5C/H5C_Notebooks/auto_metrics_inspect'
# os.environ["ANTENNA"] = antenna
# os.environ["CSV_FOLDER"] = csv_folder
# os.environ["AUTO_METRICS_FOLDER"] = auto_metrics_folder


# Use environment variables to figure out path to the csvs and auto_metrics
antenna = str(int(os.environ["ANTENNA"]))
csv_folder = os.environ["CSV_FOLDER"]
auto_metrics_folder = os.environ["AUTO_METRICS_FOLDER"]
print(f'antenna = "{antenna}"')
print(f'csv_folder = "{csv_folder}"')
print(f'auto_metrics_folder = "{auto_metrics_folder}"')

antenna = "38"
csv_folder = "/home/obs/src/H6C_Notebooks/_rtp_summary_"
auto_metrics_folder = "/home/obs/src/H6C_Notebooks/auto_metrics_inspect"


display(HTML(f'<h1 style=font-size:50px><u>Antenna {antenna} Report</u><p></p></h1>'))


import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import glob
import re
from hera_notebook_templates.utils import status_colors, Antenna


# load csvs and auto_metrics htmls in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')
auto_metric_htmls = sorted(glob.glob(auto_metrics_folder + '/auto_metrics_inspect_*.html'))[::-1]
print(f'Found {len(auto_metric_htmls)} auto_metrics notebooks in {auto_metrics_folder}')

Found 143 csvs in /home/obs/src/H6C_Notebooks/_rtp_summary_
Found 143 auto_metrics notebooks in /home/obs/src/H6C_Notebooks/auto_metrics_inspect


# Per-season options
mean_round_modz_cut = 4
dead_cut = 0.4
crossed_cut = 0.0

def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def jd_to_auto_metrics_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/auto_metrics_inspect/auto_metrics_inspect_{jd}.html'


this_antenna = None
jds = []

# parse information about antennas and nodes
for csv in csvs:
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
            antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
        else:
            antnum = int(row['Ant'])
        if antnum != int(antenna):
            continue
        
        if np.issubdtype(type(row['Node']), np.integer):
            row['Node'] = str(row['Node'])
        if type(row['Node']) == str and row['Node'].isnumeric():
            row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
            
        if this_antenna is None:
            this_antenna = Antenna(row['Ant'], row['Node'])
        jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
        jds.append(jd)
        this_antenna.add_day(jd, row)
        break

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/core/indexes/base.py:3802, in Index.get_loc(self, key, method, tolerance)
   3801 try:
-> 3802     return self._engine.get_loc(casted_key)
   3803 except KeyError as err:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/_libs/index.pyx:138, in pandas._libs.index.IndexEngine.get_loc()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/_libs/index.pyx:165, in pandas._libs.index.IndexEngine.get_loc()

File pandas/_libs/hashtable_class_helper.pxi:5745, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas/_libs/hashtable_class_helper.pxi:5753, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'ee Shape Modified Z-Score'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
Cell In [8], line 26
     24 jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
     25 jds.append(jd)
---> 26 this_antenna.add_day(jd, row)
     27 break

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:2879, in Antenna.add_day(self, jd, csv_row)
   2877 if 'Auto Metrics Flags' in csv_row:
   2878     self.auto_flags[jd] = csv_row['Auto Metrics Flags']
-> 2879     self.ee_shape_zs[jd] = csv_row['ee Shape Modified Z-Score']
   2880     self.nn_shape_zs[jd] = csv_row['nn Shape Modified Z-Score']
   2881     self.ee_power_zs[jd] = csv_row['ee Power Modified Z-Score']

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/core/series.py:981, in Series.__getitem__(self, key)
    978     return self._values[key]
    980 elif key_is_scalar:
--> 981     return self._get_value(key)
    983 if is_hashable(key):
    984     # Otherwise index.get_value will raise InvalidIndexError
    985     try:
    986         # For labels that don't resolve as scalars like tuples and frozensets

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/core/series.py:1089, in Series._get_value(self, label, takeable)
   1086     return self._values[label]
   1088 # Similar to Index.get_value, but we do not fall back to positional
-> 1089 loc = self.index.get_loc(label)
   1090 return self.index._get_values_for_loc(self, loc, label)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/core/indexes/base.py:3804, in Index.get_loc(self, key, method, tolerance)
   3802     return self._engine.get_loc(casted_key)
   3803 except KeyError as err:
-> 3804     raise KeyError(key) from err
   3805 except TypeError:
   3806     # If we have a listlike key, _check_indexing_error will raise
   3807     #  InvalidIndexError. Otherwise we fall through and re-raise
   3808     #  the TypeError.
   3809     self._check_indexing_error(key)

KeyError: 'ee Shape Modified Z-Score'


# build dataframe
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['A Priori Status'] = [this_antenna.statuses[jd] for jd in jds]

df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]
bar_cols['Flag Fraction Before Redcal'] = [this_antenna.flags_before_redcal[jd] for jd in jds]
bar_cols['Flagged By Redcal chi^2 Fraction'] = [this_antenna.redcal_flags[jd] for jd in jds]
for col in bar_cols:
    df[col] = bar_cols[col]

z_score_cols = {}
z_score_cols['ee Shape Modified Z-Score'] = [this_antenna.ee_shape_zs[jd] for jd in jds]
z_score_cols['nn Shape Modified Z-Score'] = [this_antenna.nn_shape_zs[jd] for jd in jds]
z_score_cols['ee Power Modified Z-Score'] = [this_antenna.ee_power_zs[jd] for jd in jds]
z_score_cols['nn Power Modified Z-Score'] = [this_antenna.nn_power_zs[jd] for jd in jds]
z_score_cols['ee Temporal Variability Modified Z-Score'] = [this_antenna.ee_temp_var_zs[jd] for jd in jds]
z_score_cols['nn Temporal Variability Modified Z-Score'] = [this_antenna.nn_temp_var_zs[jd] for jd in jds]
z_score_cols['ee Temporal Discontinuties Modified Z-Score'] = [this_antenna.ee_temp_discon_zs[jd] for jd in jds]
z_score_cols['nn Temporal Discontinuties Modified Z-Score'] = [this_antenna.nn_temp_discon_zs[jd] for jd in jds]
for col in z_score_cols:
    df[col] = z_score_cols[col]

ant_metrics_cols = {}
ant_metrics_cols['Average Dead Ant Metric (Jee)'] = [this_antenna.Jee_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Dead Ant Metric (Jnn)'] = [this_antenna.Jnn_dead_metrics[jd] for jd in jds]
ant_metrics_cols['Average Crossed Ant Metric'] = [this_antenna.crossed_metrics[jd] for jd in jds]
for col in ant_metrics_cols:
    df[col] = ant_metrics_cols[col]

redcal_cols = {}
redcal_cols['Median chi^2 Per Antenna (Jee)'] = [this_antenna.Jee_chisqs[jd] for jd in jds]
redcal_cols['Median chi^2 Per Antenna (Jnn)'] = [this_antenna.Jnn_chisqs[jd] for jd in jds]   
for col in redcal_cols:
    df[col] = redcal_cols[col]

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=list(z_score_cols.keys())) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=list(redcal_cols.keys())) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=list(z_score_cols.keys())) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format('{:,.2%}', na_rep='-', subset=list(bar_cols.keys())) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
Cell In [9], line 10
      8 bar_cols = {}
      9 bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
---> 10 bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
     11 bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
     12 bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]

Cell In [9], line 10, in <listcomp>(.0)
      8 bar_cols = {}
      9 bar_cols['Auto Metrics Flags'] = [this_antenna.auto_flags[jd] for jd in jds]
---> 10 bar_cols[f'Dead Fraction in Ant Metrics (Jee)'] = [this_antenna.dead_flags_Jee[jd] for jd in jds]
     11 bar_cols[f'Dead Fraction in Ant Metrics (Jnn)'] = [this_antenna.dead_flags_Jnn[jd] for jd in jds]
     12 bar_cols['Crossed Fraction in Ant Metrics'] = [this_antenna.crossed_flags[jd] for jd in jds]

KeyError: 2460100


display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
HTML(table.render(render_links=True, escape=False))

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In [10], line 2
      1 display(HTML(f'<h2>Antenna {antenna}, Node {this_antenna.node}:</h2>'))
----> 2 HTML(table.render(render_links=True, escape=False))

NameError: name 'table' is not defined


htmls_to_display = []
for am_html in auto_metric_htmls:
    html_to_display = ''
    # read html into a list of lines
    with open(am_html) as f:
        lines = f.readlines()
    
    # find section with this antenna's metric plots and add to html_to_display
    jd = [int(s) for s in re.split('_|\.', am_html) if s.isdigit()][-1]
    try:
        section_start_line = lines.index(f'<h2>Antenna {antenna}: {jd}</h2>\n')
    except ValueError:
        continue
    html_to_display += lines[section_start_line].replace(str(jd), f'<a href="{jd_to_auto_metrics_url(jd)}" target="_blank">{jd}</a>')
    for line in lines[section_start_line + 1:]:
        html_to_display += line
        if '<hr' in line:
            htmls_to_display.append(html_to_display)
            break


for i, html_to_display in enumerate(htmls_to_display):
    if i == 100:
        break
    display(HTML(html_to_display))

Antenna Season Report Notebook¶

Antenna 38 Report

Load relevant info from summary CSVs¶

Table 1: Per-Night RTP Summary Info For This Atenna¶

Antenna 38, Node N03:

Load antenna metric spectra and waterfalls from auto_metrics notebooks.¶

Figure 1: Antenna autocorrelation metric spectra and waterfalls.¶

Antenna 38: 2460136

Antenna 38: 2460135

Antenna 38: 2460134

Antenna 38: 2460133

Antenna 38: 2460132

Antenna 38: 2460131

Antenna 38: 2460130

Antenna 38: 2460129

Antenna 38: 2460128

Antenna 38: 2460127

Antenna 38: 2460126

Antenna 38: 2460125

Antenna 38: 2460124

Antenna 38: 2460123

Antenna 38: 2460122

Antenna 38: 2460121

Antenna 38: 2460120

Antenna 38: 2460119

Antenna 38: 2460118

Antenna 38: 2460117

Antenna 38: 2460116

Antenna 38: 2460113

Antenna 38: 2460112

Antenna 38: 2460111

Antenna 38: 2460110

Antenna 38: 2460109

Antenna 38: 2460108

Antenna 38: 2460106

Antenna 38: 2460105

Antenna 38: 2460104

Antenna 38: 2460102

Antenna 38: 2460101

Antenna 38: 2460099

Antenna 38: 2460098

Antenna 38: 2460097

Antenna 38: 2460096

Antenna 38: 2460095

Antenna 38: 2460094

Antenna 38: 2460093

Antenna 38: 2460092

Antenna 38: 2460091

Antenna 38: 2460089

Antenna 38: 2460088

Antenna 38: 2460087

Antenna 38: 2460086

Antenna 38: 2460085

Antenna 38: 2460084

Antenna 38: 2460083

Antenna 38: 2460082

Antenna 38: 2460081

Antenna 38: 2460080

Antenna 38: 2460079

Antenna 38: 2460078

Antenna 38: 2460077

Antenna 38: 2460076

Antenna 38: 2460075

Antenna 38: 2460073

Antenna 38: 2460072

Antenna 38: 2460071

Antenna 38: 2460070

Antenna 38: 2460069

Antenna 38: 2460068

Antenna 38: 2460067

Antenna 38: 2460066

Antenna 38: 2460065

Antenna 38: 2460064

Antenna 38: 2460062

Antenna 38: 2460061

Antenna 38: 2460060

Antenna 38: 2460059

Antenna 38: 2460058

Antenna 38: 2460056

Antenna 38: 2460055

Load antenna metric spectra and waterfalls from `auto_metrics` notebooks.¶