
import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import matplotlib.pyplot as plt
import matplotlib
import glob
import re
import os
from astropy.time import Time
from IPython.display import display, HTML
from hera_notebook_templates.utils import status_colors, Antenna
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the next line of this cell.
# csv_folder = '/mnt/sn1/H5C_Notebooks/_rtp_summary_'
# os.environ["CSV_FOLDER"] = csv_folder


# Use environment variables to figure out path to the csvs
csv_folder = os.environ["CSV_FOLDER"]
print(f'csv_folder = "{csv_folder}"')

csv_folder = "/home/obs/src/H6C_Notebooks/_rtp_summary_"


# load csvs in reverse chronological order
csvs = sorted(glob.glob(os.path.join(csv_folder, 'rtp_summary_table*.csv')))[::-1]
print(f'Found {len(csvs)} csvs in {csv_folder}')

Found 143 csvs in /home/obs/src/H6C_Notebooks/_rtp_summary_


# Per-season options
def jd_to_summary_url(jd):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/_rtp_summary_/rtp_summary_{jd}.html'

def ant_to_report_url(ant):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/antenna_report/antenna_{ant}_report.html'


antennas = {}
nodes = {}
jds = []

# parse information about antennas and nodes
for csv in csvs:
    jd = [int(s) for s in re.split('_|\.', csv) if s.isdigit()][-1]
    jds.append(jd)
    df = pd.read_csv(csv)
    for n in range(len(df)):
        # Add this day to the antenna
        row = df.loc[n]
        if isinstance(row['Ant'], str) and '<a href' in row['Ant']:
            antnum = int(row['Ant'].split('</a>')[0].split('>')[-1]) # it's a link, extract antnum
        else:
            antnum = int(row['Ant'])
        if np.issubdtype(type(row['Node']), np.integer):
            row['Node'] = str(row['Node'])
        if type(row['Node']) == str and row['Node'].isnumeric():
            row['Node'] = 'N' + ('0' if len(row['Node']) == 1 else '') + row['Node']
            
        if antnum not in antennas:
            antennas[antnum] = Antenna(row['Ant'], row['Node'])
        antennas[antnum].add_day(jd, row)
        if antennas[antnum].node not in nodes:
            nodes[antennas[antnum].node] = [antennas[antnum]]
        elif antennas[antnum] not in nodes[antennas[antnum].node]:
            nodes[antennas[antnum].node].append(antennas[antnum])
            nodes[antennas[antnum].node] = sorted(nodes[antennas[antnum].node], key=lambda ant: ant.node)

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/core/indexes/base.py:3802, in Index.get_loc(self, key, method, tolerance)
   3801 try:
-> 3802     return self._engine.get_loc(casted_key)
   3803 except KeyError as err:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/_libs/index.pyx:138, in pandas._libs.index.IndexEngine.get_loc()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/_libs/index.pyx:165, in pandas._libs.index.IndexEngine.get_loc()

File pandas/_libs/hashtable_class_helper.pxi:5745, in pandas._libs.hashtable.PyObjectHashTable.get_item()

File pandas/_libs/hashtable_class_helper.pxi:5753, in pandas._libs.hashtable.PyObjectHashTable.get_item()

KeyError: 'ee Shape Modified Z-Score'

The above exception was the direct cause of the following exception:

KeyError                                  Traceback (most recent call last)
Cell In [6], line 24
     22 if antnum not in antennas:
     23     antennas[antnum] = Antenna(row['Ant'], row['Node'])
---> 24 antennas[antnum].add_day(jd, row)
     25 if antennas[antnum].node not in nodes:
     26     nodes[antennas[antnum].node] = [antennas[antnum]]

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:2879, in Antenna.add_day(self, jd, csv_row)
   2877 if 'Auto Metrics Flags' in csv_row:
   2878     self.auto_flags[jd] = csv_row['Auto Metrics Flags']
-> 2879     self.ee_shape_zs[jd] = csv_row['ee Shape Modified Z-Score']
   2880     self.nn_shape_zs[jd] = csv_row['nn Shape Modified Z-Score']
   2881     self.ee_power_zs[jd] = csv_row['ee Power Modified Z-Score']

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/core/series.py:981, in Series.__getitem__(self, key)
    978     return self._values[key]
    980 elif key_is_scalar:
--> 981     return self._get_value(key)
    983 if is_hashable(key):
    984     # Otherwise index.get_value will raise InvalidIndexError
    985     try:
    986         # For labels that don't resolve as scalars like tuples and frozensets

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/core/series.py:1089, in Series._get_value(self, label, takeable)
   1086     return self._values[label]
   1088 # Similar to Index.get_value, but we do not fall back to positional
-> 1089 loc = self.index.get_loc(label)
   1090 return self.index._get_values_for_loc(self, loc, label)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/pandas/core/indexes/base.py:3804, in Index.get_loc(self, key, method, tolerance)
   3802     return self._engine.get_loc(casted_key)
   3803 except KeyError as err:
-> 3804     raise KeyError(key) from err
   3805 except TypeError:
   3806     # If we have a listlike key, _check_indexing_error will raise
   3807     #  InvalidIndexError. Otherwise we fall through and re-raise
   3808     #  the TypeError.
   3809     self._check_indexing_error(key)

KeyError: 'ee Shape Modified Z-Score'


# build per-night flagging table
antnums = sorted(antennas.keys())
to_show = {'JDs': [f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>' for jd in jds]}
to_show['Ants in Data'] = [np.sum([jd in antennas[ant].statuses for ant in antnums])  for jd in jds]
to_show['Unflagged Antenna-Days'] = [np.sum([antennas[ant].unflagged_days([jd]) for ant in antnums])  for jd in jds]
to_show['auto_metrics Flags'] = [np.nansum([antennas[ant].auto_flags[jd] for ant in antnums if jd in antennas[ant].auto_flags]) / 
                                 np.sum([np.isfinite(antennas[ant].auto_flags[jd]) for ant in antnums if jd in antennas[ant].auto_flags]) for jd in jds]
to_show['ant_metrics Flags'] = [x if np.isfinite(x) else np.nan for x in 
                                [np.nansum([antennas[ant].ant_metrics_flag_frac(jd) for ant in antnums]) / 
                                np.sum([np.isfinite(antennas[ant].dead_flags_Jee[jd]) for ant in antnums if jd in antennas[ant].dead_flags_Jee]) for jd in jds]]
to_show['redcal Flags'] = [np.nansum([antennas[ant].redcal_flags[jd] for ant in antnums if jd in antennas[ant].redcal_flags]) / 
                           np.sum([np.isfinite(antennas[ant].redcal_flags[jd]) for ant in antnums if jd in antennas[ant].redcal_flags]) for jd in jds]

df = pd.DataFrame(to_show)
table = df.style.hide_index() \
          .format('{:,.1f}', na_rep='-', subset=['Unflagged Antenna-Days']) \
          .background_gradient(cmap='gray', axis=None, subset=['Unflagged Antenna-Days']) \
          .format('{:,.0%}', na_rep='-', subset=['ant_metrics Flags', 'auto_metrics Flags', 'redcal Flags']) \
          .background_gradient(cmap='plasma', axis=None, subset=['ant_metrics Flags', 'auto_metrics Flags', 'redcal Flags']) \


def antenna_flag_plot():
    fig, ax = plt.subplots(1, 1, figsize=(14,7), dpi=100)
    utc = Time(jds, format='jd').datetime
    floor_jd = int(np.floor(np.min(jds) / 1000) * 1000)
    ax.plot_date(utc, to_show['Ants in Data'], '.-', ms=10, label='Antennas in Data')
    ax.plot(utc, to_show['Unflagged Antenna-Days'], 'r.-', ms=10, label='Unflagged Antenna-Days')
    ax2 = ax.twiny()
    ax2.plot(np.array(jds) - floor_jd, to_show['Unflagged Antenna-Days'], 'r.', ms=10)
    ax2.set_xlabel(f'JD - {floor_jd}')
    ax.set_ylabel('Number of Antennas')
    ax.set_xlabel('Date')
    ax.legend()
    ax2.grid()
    ax.grid(axis='y')


antenna_flag_plot()


HTML(table.render(render_links=True, escape=False))


# Build table for antenna flagging
antnums = sorted(antennas.keys())
to_show = {'Ant': [f'<a href="{ant_to_report_url(antnum)}" target="_blank">{antnum}</a>' for antnum in antnums]}
to_show['Node'] = [antennas[ant].node for ant in antnums]

to_show['Most Recent Status'] = [antennas[ant].statuses[np.max(list(antennas[ant].statuses.keys()))] for ant in antnums]

to_show['Total Days This Season'] = [len(antennas[ant].total_flags) for ant in antnums]
to_show['Unflagged Days This Season'] = [antennas[ant].unflagged_days() for ant in antnums]
to_show['Flag % This Season'] = 1 - np.array(to_show['Unflagged Days This Season']) / np.array(to_show['Total Days This Season'])
to_show['Top Flag Rationale (Season)'] = [antennas[ant].most_common_flag_rationale() for ant in antnums]

to_show['Total Days This Week'] = [len([antennas[ant].total_flags[jd] for jd in jds[0:min(len(jds), 7)] if jd in antennas[ant].total_flags]) for ant in antnums]
to_show['Unflagged Days This Week'] = [antennas[ant].unflagged_days(jds[0:min(len(jds), 7)]) for ant in antnums]
to_show['Flag % This Week'] = 1 - np.array(to_show['Unflagged Days This Week']) / np.array(to_show['Total Days This Week'])
to_show['Top Flag Rationale (Week)'] = [antennas[ant].most_common_flag_rationale(jds[0:min(len(jds), 7)]) for ant in antnums]

jd_links = []
for jd in jds:
    jd_links.append(f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>')
    to_show[jd_links[-1]] = [antennas[ant].total_flags[jd] if jd in antennas[ant].total_flags else np.nan for ant in antnums ]
df = pd.DataFrame(to_show)

table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['Most Recent Status']) \
          .format('{:,.0%}', na_rep='-', subset=jd_links + ['Flag % This Season', 'Flag % This Week']) \
          .format('{:}', na_rep='-', subset=['Top Flag Rationale (Season)', 'Top Flag Rationale (Week)']) \
          .format('{:,.1f}', na_rep='-', subset=['Unflagged Days This Season', 'Unflagged Days This Week'] ) \
          .background_gradient(cmap='plasma', vmax=1, vmin=0, axis=None, subset=jd_links) \
          .bar(subset=['Flag % This Season', 'Flag % This Week'], vmin=0, vmax=1)


HTML(table.render(render_links=True, escape=False))


# Build table for node flagging
nodenums = sorted([k for k in nodes.keys() if type(k) == str])

to_show = {'Node': nodenums, 'Number of Ants': [len(nodes[n]) for n in nodenums]}

to_show['Total Antenna-Days This Season'] = [np.sum([jd in antenna.total_flags for antenna in nodes[n] for jd in jds]) for n in nodenums]
to_show['Unflagged Antenna-Days This Season'] = [np.sum([antenna.unflagged_days() for antenna in nodes[n]]) for n in nodenums]
to_show['Flag % This Season'] = 1 - np.array(to_show['Unflagged Antenna-Days This Season']) / np.array(to_show['Total Antenna-Days This Season'])

to_show['Total Antenna-Days This Week'] = [np.sum([jd in antenna.total_flags for antenna in nodes[n] for jd in jds[0:min(len(jds), 7)]]) for n in nodenums]
to_show['Unflagged Antenna-Days This Week'] = [np.sum([antenna.unflagged_days(jds[0:min(len(jds), 7)]) for antenna in nodes[n]]) for n in nodenums]
to_show['Flag % This Week'] = 1 - np.array(to_show['Unflagged Antenna-Days This Week']) / np.array(to_show['Total Antenna-Days This Week'])

jd_links = []
for jd in jds:
    jd_links.append(f'<a href="{jd_to_summary_url(jd)}" target="_blank">{jd}</a>')
    to_show[jd_links[-1]] = [np.nansum([antenna.total_flags[jd] if jd in antenna.total_flags else np.nan for antenna in nodes[n]]) / 
                             np.sum([jd in antenna.total_flags for antenna in nodes[n]]) for n in nodenums]
df = pd.DataFrame(to_show)
table = df.style.hide_index() \
          .format('{:,.0%}', na_rep='-', subset=jd_links + ['Flag % This Season', 'Flag % This Week'] ) \
          .format('{:,.1f}', na_rep='-', subset=['Unflagged Antenna-Days This Season', 'Unflagged Antenna-Days This Week'] ) \
          .background_gradient(cmap='plasma', vmax=1, vmin=0, axis=None, subset=jd_links) \
          .bar(subset=['Flag % This Season', 'Flag % This Week'], vmin=0, vmax=1)


HTML(table.render(render_links=True, escape=False))

JDs	Ants in Data	auto_metrics Flags	ant_metrics Flags	redcal Flags
2460136	205	67%	90%	-
2460135	205	58%	41%	-
2460134	205	57%	39%	-
2460133	205	60%	44%	-
2460132	205	57%	38%	-
2460131	205	58%	40%	-
2460130	205	57%	40%	-
2460129	205	58%	39%	-
2460128	205	49%	29%	-
2460127	205	47%	28%	-
2460126	205	44%	27%	-
2460125	205	46%	27%	-
2460124	205	44%	27%	-
2460123	205	41%	30%	-
2460122	205	34%	-	-
2460121	205	55%	39%	-
2460120	205	49%	33%	-
2460119	205	53%	37%	-
2460118	202	48%	33%	-
2460117	202	41%	34%	-
2460116	202	65%	44%	-
2460113	202	47%	33%	-
2460112	202	45%	30%	-
2460111	202	39%	100%	-
2460110	202	42%	28%	-
2460109	202	46%	27%	-
2460108	202	0%	97%	-
2460106	202	45%	16%	-
2460105	202	42%	21%	-
2460104	202	46%	27%	-
2460102	202	55%	25%	-
2460101	202	41%	25%	-
2460100	1	100%	-	-

Season Data Quality Summary¶

Load Data¶

Figure 1: (Unflagged) Antennas in the Array¶

Table 1: Per Night Flagging¶

Table 2: Per-Antenna, Per-Night Flagging¶

Table 3: Per-Node, Per-Night Flagging¶