
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import pandas as pd
pd.set_option('display.max_rows', 1000)
from hera_qm.metrics_io import load_metric_file
from hera_cal import utils, io, redcal
import glob
import h5py
from copy import deepcopy
from IPython.display import display, HTML
from hera_notebook_templates.utils import status_colors
from hera_mc import mc
from pyuvdata import UVData

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the first few lines of this cell.

# JD = "2459122"
# data_path = '/lustre/aoc/projects/hera/H4C/2459122'
# ant_metrics_ext = ".ant_metrics.hdf5"
# redcal_ext = ".maybe_good.omni.calfits"
# nb_outdir = '/lustre/aoc/projects/hera/H4C/h4c_software/H4C_Notebooks/_rtp_summary_'
# good_statuses = "digital_ok,calibration_maintenance,calibration_triage,calibration_ok"
# os.environ["JULIANDATE"] = JD
# os.environ["DATA_PATH"] = data_path
# os.environ["ANT_METRICS_EXT"] = ant_metrics_ext
# os.environ["REDCAL_EXT"] = redcal_ext
# os.environ["NB_OUTDIR"] = nb_outdir
# os.environ["GOOD_STATUSES"] = good_statuses


# Use environment variables to figure out path to data
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
ant_metrics_ext = os.environ['ANT_METRICS_EXT']
redcal_ext = os.environ['REDCAL_EXT']
nb_outdir = os.environ['NB_OUTDIR']
good_statuses = os.environ['GOOD_STATUSES']
print(f'JD = "{JD}"')
print(f'data_path = "{data_path}"')
print(f'ant_metrics_ext = "{ant_metrics_ext}"')
print(f'redcal_ext = "{redcal_ext}"')
print(f'nb_outdir = "{nb_outdir}"')
print(f'good_statuses = "{good_statuses}"')

JD = "2459897"
data_path = "/mnt/sn1/2459897"
ant_metrics_ext = ".ant_metrics.hdf5"
redcal_ext = ".known_good.omni.calfits"
nb_outdir = "/home/obs/src/H6C_Notebooks/_rtp_summary_"
good_statuses = "digital_ok,calibration_maintenance,calibration_triage,calibration_ok"


from astropy.time import Time, TimeDelta
utc = Time(JD, format='jd').datetime
print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 11-13-2022


# Per-season options
def ant_to_report_url(ant):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/antenna_report/antenna_{ant}_report.html'


use_auto_metrics = False

# find the auto_metrics file
glob_str = os.path.join(data_path, f'zen.{JD}*.auto_metrics.h5')
auto_metrics_file = sorted(glob.glob(glob_str))

# if it exists, load and extract relevant information
if len(auto_metrics_file) > 0:
    auto_metrics_file = auto_metrics_file[0]
    print(f'Found auto_metrics results file at {auto_metrics_file}.')
    
    auto_metrics = load_metric_file(auto_metrics_file)
    mean_round_modz_cut = auto_metrics['parameters']['mean_round_modz_cut']
    auto_ex_ants = auto_metrics['ex_ants']['r2_ex_ants']
    
    use_auto_metrics = True
else:
    print(f'No files found matching glob {glob_str}. Skipping auto_metrics.')

Found auto_metrics results file at /mnt/sn1/2459897/zen.2459897.25283.sum.auto_metrics.h5.


use_ant_metrics = False

# get a list of all ant_metrics files
glob_str = os.path.join(data_path, f'zen.{JD}.?????.sum{ant_metrics_ext}')
ant_metrics_files = sorted(glob.glob(glob_str))

# if they exist, load as many of them as possible
if len(ant_metrics_files) > 0:
    print(f'Found {len(ant_metrics_files)} ant_metrics files matching glob {glob_str}')
    ant_metrics_apriori_exants = {}
    ant_metrics_xants_dict = {}
    ant_metrics_dead_ants_dict = {}
    ant_metrics_crossed_ants_dict = {}
    ant_metrics_dead_metrics = {}
    ant_metrics_crossed_metrics = {}
    dead_cuts = {}
    crossed_cuts = {}
    for amf in ant_metrics_files:
        with h5py.File(amf, "r") as infile: # use h5py directly since it's much faster than load_metric_file
            # get out results for this file
            dead_cuts[amf] = infile['Metrics']['dead_ant_cut'][()]
            crossed_cuts[amf] = infile['Metrics']['cross_pol_cut'][()]
            xants = infile['Metrics']['xants'][:]
            dead_ants = infile['Metrics']['dead_ants'][:]
            crossed_ants = infile['Metrics']['crossed_ants'][:]        
            try:
                # look for ex_ants in history
                ex_ants_string = infile['Header']['history'][()].decode()
                ex_ants_string = ex_ants_string.split('--apriori_xants')[1]
                ex_ants_string = ex_ants_string.split('--')[0].strip()
            except:
                ex_ants_string = ''
                    
            # This only works for the new correlation-matrix-based ant_metrics
            if 'corr' in infile['Metrics']['final_metrics'] and 'corrXPol' in infile['Metrics']['final_metrics']:
                ant_metrics_dead_metrics[amf] = {eval(ant): infile['Metrics']['final_metrics']['corr'][ant][()]
                                                 for ant in infile['Metrics']['final_metrics']['corr']}
                ant_metrics_crossed_metrics[amf] = {eval(ant): infile['Metrics']['final_metrics']['corrXPol'][ant][()]
                                                    for ant in infile['Metrics']['final_metrics']['corrXPol']}                       
            else:
                raise(KeywordError)
        
        # organize results by file
        ant_metrics_xants_dict[amf] = [(int(ant[0]), ant[1].decode()) for ant in xants]
        ant_metrics_dead_ants_dict[amf] = [(int(ant[0]), ant[1].decode()) for ant in dead_ants]
        ant_metrics_crossed_ants_dict[amf] = [(int(ant[0]), ant[1].decode()) for ant in crossed_ants]
        ant_metrics_apriori_exants[amf] = [int(ant) for ant in ex_ants_string.split()]
    
    dead_cut = np.median(list(dead_cuts.values()))
    crossed_cut = np.median(list(crossed_cuts.values()))
        
    use_ant_metrics = True
else:
    print(f'No files found matching glob {glob_str}. Skipping ant_metrics.')

Found 1849 ant_metrics files matching glob /mnt/sn1/2459897/zen.2459897.?????.sum.ant_metrics.hdf5


use_redcal = False
glob_str = os.path.join(data_path, f'zen.{JD}.?????.sum{redcal_ext}')

redcal_files = sorted(glob.glob(glob_str))
if len(redcal_files) > 0:
    print(f'Found {len(redcal_files)} ant_metrics files matching glob {glob_str}')
    post_redcal_ant_flags_dict = {}
    flagged_by_redcal_dict = {}
    cspa_med_dict = {}
    for cal in redcal_files:
        hc = io.HERACal(cal)
        _, flags, cspa, chisq = hc.read()
        cspa_med_dict[cal] = {ant: np.nanmedian(cspa[ant], axis=1) for ant in cspa}

        post_redcal_ant_flags_dict[cal] = {ant: np.all(flags[ant]) for ant in flags}
        # check history to distinguish antennas flagged going into redcal from ones flagged during redcal
        tossed_antenna_lines =  hc.history.replace('\n','').split('Throwing out antenna ')[1:]
        flagged_by_redcal_dict[cal] = sorted([int(line.split(' ')[0]) for line in tossed_antenna_lines])
        
    use_redcal = True
else:
    print(f'No files found matching glob {glob_str}. Skipping redcal chisq.')

No files found matching glob /mnt/sn1/2459897/zen.2459897.?????.sum.known_good.omni.calfits. Skipping redcal chisq.


# Parse some general array properties, taking into account the fact that we might be missing some of the metrics
ants = []
pols = []
antpol_pairs = []

if use_auto_metrics:
    ants = sorted(set(bl[0] for bl in auto_metrics['modzs']['r2_shape_modzs']))
    pols = sorted(set(bl[2] for bl in auto_metrics['modzs']['r2_shape_modzs']))
if use_ant_metrics:
    antpol_pairs = sorted(set([antpol for dms in ant_metrics_dead_metrics.values() for antpol in dms.keys()]))
    antpols = sorted(set(antpol[1] for antpol in antpol_pairs))
    ants = sorted(set(antpol[0] for antpol in antpol_pairs) | set(ants))
    pols = sorted(set(utils.join_pol(ap, ap) for ap in antpols) | set(pols))
if use_redcal:
    antpol_pairs = sorted(set([ant for cspa in cspa_med_dict.values() for ant in cspa.keys()]) | set(antpol_pairs))
    antpols = sorted(set(antpol[1] for antpol in antpol_pairs))
    ants = sorted(set(antpol[0] for antpol in antpol_pairs) | set(ants))
    pols = sorted(set(utils.join_pol(ap, ap) for ap in antpols) | set(pols))

# Figure out remaining antennas not in data and also LST range
data_files = sorted(glob.glob(os.path.join(data_path, 'zen.*.sum.uvh5')))
hd = io.HERAData(data_files[0])
unused_ants = [ant for ant in hd.antpos if ant not in ants]    
hd_last = io.HERAData(data_files[-1])


# try to load a priori antenna statusesm but fail gracefully if this doesn't work.
a_priori_statuses = {ant: 'Not Found' for ant in ants}
nodes = {ant: np.nan for ant in ants + unused_ants}
try:
    from hera_mc import cm_hookup

    # get node numbers
    hookup = cm_hookup.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in nodes:
            if hookup[ant_name].get_part_from_type('node')['E<ground'] is not None:
                nodes[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])
    
    # get apriori antenna status
    for ant_name, data in hookup.items():
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in a_priori_statuses:
            a_priori_statuses[ant] = data.apriori

except Exception as err:
    print(f'Could not load node numbers and a priori antenna statuses.\nEncountered {type(err)} with message: {err}')


if use_auto_metrics:
    # Parse modzs
    modzs_to_check = {'Shape': 'r2_shape_modzs', 'Power': 'r2_power_modzs', 
                      'Temporal Variability': 'r2_temp_var_modzs', 'Temporal Discontinuties': 'r2_temp_diff_modzs'}
    worst_metrics = []
    worst_zs = []
    all_modzs = {}
    binary_flags = {rationale: [] for rationale in modzs_to_check}

    for ant in ants:
        # parse modzs and figure out flag counts
        modzs = {f'{pol} {rationale}': auto_metrics['modzs'][dict_name][(ant, ant, pol)] 
                 for rationale, dict_name in modzs_to_check.items() for pol in pols}
        for pol in pols:
            for rationale, dict_name in modzs_to_check.items():
                binary_flags[rationale].append(auto_metrics['modzs'][dict_name][(ant, ant, pol)] > mean_round_modz_cut)

        # parse out all metrics for dataframe
        for k in modzs:
            col_label = k + ' Modified Z-Score'
            if col_label in all_modzs:
                all_modzs[col_label].append(modzs[k])
            else:
                all_modzs[col_label] = [modzs[k]]
                
    mean_round_modz_cut = auto_metrics['parameters']['mean_round_modz_cut']
else:
    mean_round_modz_cut = 0


if use_ant_metrics:
    a_priori_flag_frac = {ant: np.mean([ant in apxa for apxa in ant_metrics_apriori_exants.values()]) for ant in ants}
    dead_ant_frac = {ap: {ant: np.mean([(ant, ap) in das for das in ant_metrics_dead_ants_dict.values()])
                                 for ant in ants} for ap in antpols}
    crossed_ant_frac = {ant: np.mean([np.any([(ant, ap) in cas for ap in antpols])
                                      for cas in ant_metrics_crossed_ants_dict.values()]) for ant in ants}
    ant_metrics_xants_frac_by_antpol = {antpol: np.mean([antpol in amx for amx in ant_metrics_xants_dict.values()]) for antpol in antpol_pairs}
    ant_metrics_xants_frac_by_ant = {ant: np.mean([np.any([(ant, ap) in amx for ap in antpols])
                                     for amx in ant_metrics_xants_dict.values()]) for ant in ants}
    average_dead_metrics = {ap: {ant: np.nanmean([dm.get((ant, ap), np.nan) for dm in ant_metrics_dead_metrics.values()]) 
                                 for ant in ants} for ap in antpols}
    average_crossed_metrics = {ant: np.nanmean([cm.get((ant, ap), np.nan) for ap in antpols 
                                                for cm in ant_metrics_crossed_metrics.values()]) for ant in ants}
else:
    dead_cut = 0.4
    crossed_cut = 0.0


if use_redcal:
    cspa = {ant: np.nanmedian(np.hstack([cspa_med_dict[cal][ant] for cal in redcal_files])) for ant in antpol_pairs}
    redcal_prior_flag_frac = {ant: np.mean([np.any([afd[ant, ap] and not ant in flagged_by_redcal_dict[cal] for ap in antpols])
                                            for cal, afd in post_redcal_ant_flags_dict.items()]) for ant in ants}
    redcal_flagged_frac = {ant: np.mean([ant in fbr for fbr in flagged_by_redcal_dict.values()]) for ant in ants}


HHautos = sorted(glob.glob(f"{data_path}/zen.{JD}.*.sum.autos.uvh5"))
diffautos = sorted(glob.glob(f"{data_path}/zen.{JD}.*.diff.autos.uvh5"))

try:
    db = mc.connect_to_mc_db(None)
    session = db.sessionmaker()
    startJD = float(HHautos[0].split('zen.')[1].split('.sum')[0])
    stopJD = float(HHautos[-1].split('zen.')[1].split('.sum')[0])
    start_time = Time(startJD,format='jd')
    stop_time = Time(stopJD,format='jd')

    # get initial state by looking for commands up to 3 hours before the starttime
    # this logic can be improved after an upcoming hera_mc PR
    # which will return the most recent command before a particular time.
    search_start_time = start_time - TimeDelta(3*3600, format="sec")
    initial_command_res = session.get_array_signal_source(starttime=search_start_time, stoptime=start_time)
    if len(initial_command_res) == 0:
        initial_source = "Unknown"
    elif len(command_res) == 1:
        initial_source = initial_command_res[0].source
    else:
        # multiple commands
        times = []
        sources = []
        for obj in command_res:
            times.append(obj.time)
            sources.append(obj.source)
        initial_source = sources[np.argmax(times)]
    
    # check for any changes during observing
    command_res = session.get_array_signal_source(starttime=start_time, stoptime=stop_time)
    if len(command_res) == 0:
        # still nothing, set it to None
        obs_source = None
    else:
        obs_source_times = []
        obs_source = []
        for obj in command_res:
            obs_source_times.append(obj.time)
            obs_source.append(obj.source)

    if obs_source is not None:
        command_source = [initial_source] + obs_source
    else:
        command_source = initial_source
    
    res = session.get_antenna_status(starttime=startTime, stoptime=stopTime)
    fem_switches = {}
    right_rep_ant = []
    if len(res) > 0:
        for obj in res:
            if obj.antenna_number not in fem_switches.keys():
                fem_switches[obj.antenna_number] = {}
            fem_switches[obj.antenna_number][obj.antenna_feed_pol] = obj.fem_switch
        for ant, pol_dict in fem_switches.items():
            if pol_dict['e'] == initial_source and pol_dict['n'] == initial_source:
                right_rep_ant.append(ant)
except Exception as e:
    print(e)
    initial_source = None
    command_source = None
    right_rep_ant = []

name 'command_res' is not defined


read_inds = [1, len(HHautos)//2, -2]
x_status = [1,1,1,1,1,1,1,1]
s = UVData()
s.read(HHautos[1])

nants = len(s.get_ants())
freqs = s.freq_array[0]*1e-6
nfreqs = len(freqs)

antCon = {a: None for a in ants}
rightAnts = []
for i in read_inds:
    s = UVData()
    d = UVData()
    s.read(HHautos[i])
    d.read(diffautos[i])
    for pol in [0,1]:
        sm = np.abs(s.data_array[:,0,:,pol])
        df = np.abs(d.data_array[:,0,:,pol])
        sm = np.r_[sm, np.nan + np.zeros((-len(sm) % nants,len(freqs)))]
        sm = np.nanmean(sm.reshape(-1,nants,nfreqs),axis=1)
        df = np.r_[df, np.nan + np.zeros((-len(df) % nants,len(freqs)))]
        df = np.nanmean(df.reshape(-1,nants,nfreqs),axis=1)

        evens = (sm + df)/2
        odds = (sm - df)/2
        rat = np.divide(evens,odds)
        rat = np.nan_to_num(rat)
        for xbox in range(0,8):
            xavg = np.nanmean(rat[:,xbox*192:(xbox+1)*192],axis=1)
            if np.nanmax(xavg)>1.5 or np.nanmin(xavg)<0.5:
                x_status[xbox] = 0
    for ant in ants:
        for pol in ["xx", "yy"]:
            if antCon[ant] is False:
                continue
            spectrum = s.get_data(ant, ant, pol)
            stdev = np.std(spectrum)
            med = np.median(np.abs(spectrum))
            if (initial_source == 'digital_noise_same' or initial_source == 'digital_noise_different') and med < 10:
                antCon[ant] = True
            elif (initial_source == "load" or initial_source == 'noise') and 80000 < stdev <= 4000000 and antCon[ant] is not False:
                antCon[ant] = True
            elif initial_source == "antenna" and stdev > 500000 and med > 950000 and antCon[ant] is not False:
                antCon[ant] = True
            else:
                antCon[ant] = False
            if np.min(np.abs(spectrum)) < 100000:
                antCon[ant] = False
for ant in ants:
    if antCon[ant] is True:
        rightAnts.append(ant)
            
x_status_str = ''
for i,x in enumerate(x_status):
    if x==0:
        x_status_str += '\u274C '
    else:
        x_status_str += '\u2705 '


def comma_sep_paragraph(vals, chars_per_line=40):
    outstrs = []
    for val in vals:
        if (len(outstrs) == 0) or (len(outstrs[-1]) > chars_per_line):
            outstrs.append(str(val))
        else:
            outstrs[-1] += ', ' + str(val)
    return ',<br>'.join(outstrs)


# Time data
to_show = {'JD': [JD]}
to_show['Date'] = f'{utc.month}-{utc.day}-{utc.year}'
to_show['LST Range'] = f'{hd.lsts[0] * 12 / np.pi:.3f} -- {hd_last.lsts[-1] * 12 / np.pi:.3f} hours'

# X-engine status
to_show['X-Engine Status'] = x_status_str

# Files
to_show['Number of Files'] = len(data_files)

# Antenna Calculations
to_show['Total Number of Antennas'] = len(ants)

to_show[' '] = ''
to_show['OPERATIONAL STATUS SUMMARY'] = ''

status_count = {status: 0 for status in status_colors}
for ant, status in a_priori_statuses.items():
    if status in status_count:
        status_count[status] = status_count[status] + 1
    else:
        status_count[status] = 1
to_show['Antenna A Priori Status Count'] = '<br>'.join([f'{status}: {status_count[status]}' for status in status_colors if status in status_count and status_count[status] > 0])

to_show['Commanded Signal Source'] = ', '.join(command_source if hasattr(command_source, '__iter__') else [str(command_source)])
to_show['Antennas in Commanded State (reported)'] = f'{len(right_rep_ant)} / {len(ants)} ({len(right_rep_ant) / len(ants):.1%})'
to_show['Antennas in Commanded State (observed)'] = f'{len(rightAnts)} / {len(ants)} ({len(rightAnts) / len(ants):.1%})'

if use_ant_metrics:
    to_show['Cross-Polarized Antennas'] = ', '.join([str(ant) for ant in ants if (np.max([dead_ant_frac[ap][ant] for ap in antpols]) + crossed_ant_frac[ant] == 1) 
                                                                                 and (crossed_ant_frac[ant] > .5)])

# Node calculations
nodes_used = set([nodes[ant] for ant in ants if np.isfinite(nodes[ant])])
to_show['Total Number of Nodes'] = len(nodes_used)
if use_ant_metrics:
    node_off = {node: True for node in nodes_used}
    not_correlating = {node: True for node in nodes_used}
    for ant in ants:
        for ap in antpols:
            if np.isfinite(nodes[ant]):
                if np.isfinite(average_dead_metrics[ap][ant]):
                    node_off[nodes[ant]] = False
                if dead_ant_frac[ap][ant] < 1:
                    not_correlating[nodes[ant]] = False
    to_show['Nodes Registering 0s'] = ', '.join([f'N{n:02}' for n in sorted([node for node in node_off if node_off[node]])])
    to_show['Nodes Not Correlating'] = ', '.join([f'N{n:02}' for n in sorted([node for node in not_correlating if not_correlating[node] and not node_off[node]])])

# Pipeline calculations    
to_show['  '] = ''
to_show['NIGHTLY ANALYSIS SUMMARY'] = ''
    
all_flagged_ants = []
if use_ant_metrics:
    to_show['Ant Metrics Done?'] = '\u2705'
    ant_metrics_flagged_ants = [ant for ant in ants if ant_metrics_xants_frac_by_ant[ant] > 0]
    all_flagged_ants.extend(ant_metrics_flagged_ants)
    to_show['Ant Metrics Flagged Antennas'] = f'{len(ant_metrics_flagged_ants)} / {len(ants)} ({len(ant_metrics_flagged_ants) / len(ants):.1%})' 
else:
    to_show['Ant Metrics Done?'] = '\u274C'
if use_auto_metrics:
    to_show['Auto Metrics Done?'] = '\u2705'
    auto_metrics_flagged_ants = [ant for ant in ants if ant in auto_ex_ants]
    all_flagged_ants.extend(auto_metrics_flagged_ants)    
    to_show['Auto Metrics Flagged Antennas'] = f'{len(auto_metrics_flagged_ants)} / {len(ants)} ({len(auto_metrics_flagged_ants) / len(ants):.1%})' 
else:
    to_show['Auto Metrics Done?'] = '\u274C'
if use_redcal:
    to_show['Redcal Done?'] = '\u2705'    
    redcal_flagged_ants = [ant for ant in ants if redcal_flagged_frac[ant] > 0]
    all_flagged_ants.extend(redcal_flagged_ants)    
    to_show['Redcal Flagged Antennas'] = f'{len(redcal_flagged_ants)} / {len(ants)} ({len(redcal_flagged_ants) / len(ants):.1%})' 
else:
    to_show['Redcal Done?'] = '\u274C' 
to_show['Never Flagged Antennas'] = f'{len(ants) - len(set(all_flagged_ants))} / {len(ants)} ({(len(ants) - len(set(all_flagged_ants))) / len(ants):.1%})'

# Count bad antennas with good statuses and vice versa
n_apriori_good = len([ant for ant in ants if a_priori_statuses[ant] in good_statuses.split(',')])
apriori_good_flagged = []
aprior_bad_unflagged = []
for ant in ants:
    if ant in set(all_flagged_ants) and a_priori_statuses[ant] in good_statuses.split(','):
        apriori_good_flagged.append(ant)
    elif ant not in set(all_flagged_ants) and a_priori_statuses[ant] not in good_statuses.split(','):
        aprior_bad_unflagged.append(ant)
to_show['A Priori Good Antennas Flagged'] = f'{len(apriori_good_flagged)} / {n_apriori_good} total a priori good antennas:<br>' + \
                                            comma_sep_paragraph(apriori_good_flagged)
to_show['A Priori Bad Antennas Not Flagged'] = f'{len(aprior_bad_unflagged)} / {len(ants) - n_apriori_good} total a priori bad antennas:<br>' + \
                                            comma_sep_paragraph(aprior_bad_unflagged)

# Apply Styling
df = pd.DataFrame(to_show)
divider_cols = [df.columns.get_loc(col) for col in ['NIGHTLY ANALYSIS SUMMARY', 'OPERATIONAL STATUS SUMMARY']]
try:
    to_red_columns = [df.columns.get_loc(col) for col in ['Cross-Polarized Antennas', 'Nodes Registering 0s', 
                                                          'Nodes Not Correlating', 'A Priori Good Antennas Flagged']]
except:
    to_red_columns = []
def red_specific_cells(x):
    df1 = pd.DataFrame('', index=x.index, columns=x.columns)
    for col in to_red_columns:
        df1.iloc[col] = 'color: red'
    return df1

df = df.T
table = df.style.hide_columns().apply(red_specific_cells, axis=None)
for col in divider_cols:
    table = table.set_table_styles([{"selector":f"tr:nth-child({col+1})", "props": [("background-color", "black"), ("color", "white")]}], overwrite=False)


HTML(table.render())


# write to csv
outpath = os.path.join(nb_outdir, f'array_health_table_{JD}.csv')
print(f'Now saving Table 2 to a csv at {outpath}')
df.replace({'\u2705': 'Y'}, regex=True).replace({'\u274C': 'N'}, regex=True).replace({'<br>': ' '}, regex=True).to_csv(outpath)

Now saving Table 2 to a csv at /home/obs/src/H6C_Notebooks/_rtp_summary_/array_health_table_2459897.csv


# build dataframe
to_show = {'Ant': [f'<a href="{ant_to_report_url(ant)}" target="_blank">{ant}</a>' for ant in ants],
           'Node': [f'N{nodes[ant]:02}' for ant in ants], 
           'A Priori Status': [a_priori_statuses[ant] for ant in ants]}
           #'Worst Metric': worst_metrics, 'Worst Modified Z-Score': worst_zs}
df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
if use_auto_metrics:
    bar_cols['Auto Metrics Flags'] = [float(ant in auto_ex_ants) for ant in ants]
if use_ant_metrics:
    if np.sum(list(a_priori_flag_frac.values())) > 0:  # only include this col if there are any a priori flags
        bar_cols['A Priori Flag Fraction in Ant Metrics'] = [a_priori_flag_frac[ant] for ant in ants]
    for ap in antpols:
        bar_cols[f'Dead Fraction in Ant Metrics ({ap})'] = [dead_ant_frac[ap][ant] for ant in ants]
    bar_cols['Crossed Fraction in Ant Metrics'] = [crossed_ant_frac[ant] for ant in ants]
if use_redcal:
    bar_cols['Flag Fraction Before Redcal'] = [redcal_prior_flag_frac[ant] for ant in ants]
    bar_cols['Flagged By Redcal chi^2 Fraction'] = [redcal_flagged_frac[ant] for ant in ants]  
for col in bar_cols:
    df[col] = bar_cols[col]

# add auto_metrics
if use_auto_metrics:
    for label, modz in all_modzs.items():
        df[label] = modz
z_score_cols = [col for col in df.columns if 'Modified Z-Score' in col]        
        
# add ant_metrics
ant_metrics_cols = {}
if use_ant_metrics:
    for ap in antpols:
        ant_metrics_cols[f'Average Dead Ant Metric ({ap})'] = [average_dead_metrics[ap][ant] for ant in ants]
    ant_metrics_cols['Average Crossed Ant Metric'] = [average_crossed_metrics[ant] for ant in ants]
    for col in ant_metrics_cols:
        df[col] = ant_metrics_cols[col]   

# add redcal chisq
redcal_cols = []
if use_redcal:
    for ap in antpols:
        col_title = f'Median chi^2 Per Antenna ({ap})'
        df[col_title] = [cspa[ant, ap] for ant in ants]
        redcal_cols.append(col_title)

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=z_score_cols) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=redcal_cols) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=z_score_cols) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=z_score_cols) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format({col: '{:,.2%}'.format for col in bar_cols}) \
          .applymap(lambda val: 'font-weight: bold', subset=['Ant']) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])


HTML(table.render())


# print ex_ants for easy copy-pasting to YAML file
proposed_ex_ants = [ant for i, ant in enumerate(ants) if np.any([col[i] > 0 for col in bar_cols.values()])]
print('ex_ants: [' + ", ".join(str(ant) for ant in proposed_ex_ants) + ']')
print(f'\nunflagged_ants: [{", ".join([str(ant) for ant in ants if ant not in proposed_ex_ants])}]')
# "golden" means no flags and good a priori status
golden_ants = ", ".join([str(ant) for ant in ants if ((ant not in proposed_ex_ants) and (a_priori_statuses[ant] in good_statuses.split(',')))])
print(f'\ngolden_ants: [{golden_ants}]')

ex_ants: [3, 4, 7, 8, 9, 16, 18, 19, 21, 22, 27, 28, 29, 30, 31, 32, 34, 35, 36, 37, 38, 43, 44, 45, 46, 47, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 63, 68, 71, 72, 73, 74, 77, 78, 80, 81, 82, 83, 84, 86, 87, 90, 92, 93, 94, 96, 97, 98, 99, 100, 101, 102, 103, 104, 107, 108, 109, 110, 111, 113, 114, 116, 117, 118, 119, 121, 122, 123, 126, 130, 131, 133, 135, 136, 137, 138, 140, 142, 143, 144, 145, 146, 150, 155, 156, 158, 159, 161, 162, 163, 164, 165, 166, 167, 170, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 190, 191, 200, 201, 202, 203, 205, 206, 207, 208, 209, 210, 211, 219, 222, 224, 225, 226, 227, 228, 229, 237, 239, 240, 241, 242, 243, 244, 245, 246, 261, 262, 320]

unflagged_ants: [5, 10, 15, 17, 20, 40, 41, 42, 48, 49, 62, 64, 65, 66, 67, 69, 70, 79, 85, 88, 89, 91, 95, 105, 106, 112, 115, 120, 124, 125, 127, 128, 129, 132, 139, 141, 147, 148, 149, 157, 160, 168, 169, 220, 221, 223, 238, 324, 325, 329, 333]

golden_ants: [5, 10, 15, 17, 20, 40, 41, 42, 65, 66, 67, 69, 70, 85, 88, 91, 105, 106, 112, 124, 127, 128, 129, 141, 147, 157, 160, 169]


# write to csv
outpath = os.path.join(nb_outdir, f'rtp_summary_table_{JD}.csv')
print(f'Now saving Table 2 to a csv at {outpath}')
df.to_csv(outpath)

Now saving Table 2 to a csv at /home/obs/src/H6C_Notebooks/_rtp_summary_/rtp_summary_table_2459897.csv


# Load antenna positions
data_list = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}.?????.sum.uvh5')))
hd = io.HERAData(data_list[len(data_list) // 2])

# Figure out where to draw the nodes
node_centers = {}
for node in sorted(set(list(nodes.values()))):
    if np.isfinite(node):
        this_node_ants = [ant for ant in ants + unused_ants if nodes[ant] == node]
        if len(this_node_ants) == 1:
            # put the node label just to the west of the lone antenna 
            node_centers[node] = hd.antpos[ant][node] + np.array([-14.6 / 2, 0, 0])
        else:
            # put the node label between the two antennas closest to the node center
            node_centers[node] = np.mean([hd.antpos[ant] for ant in this_node_ants], axis=0)
            closest_two_pos = sorted([hd.antpos[ant] for ant in this_node_ants], 
                                     key=lambda pos: np.linalg.norm(pos - node_centers[node]))[0:2]
            node_centers[node] = np.mean(closest_two_pos, axis=0)


def Plot_Array(ants, unused_ants, outriggers):
    plt.figure(figsize=(16,16))
    
    plt.scatter(np.array([hd.antpos[ant][0] for ant in hd.data_ants if ant in ants]), 
                np.array([hd.antpos[ant][1] for ant in hd.data_ants if ant in ants]), c='w', s=0)

    # connect every antenna to their node
    for ant in ants:
        if nodes[ant] in node_centers:
            plt.plot([hd.antpos[ant][0], node_centers[nodes[ant]][0]], 
                     [hd.antpos[ant][1], node_centers[nodes[ant]][1]], 'k', zorder=0)

    rc_color = '#0000ff'
    antm_color = '#ffa500'
    autom_color = '#ff1493'

    # Plot 
    unflagged_ants = []
    for i, ant in enumerate(ants):
        ant_has_flag = False
        # plot large blue annuli for redcal flags
        if use_redcal:
            if redcal_flagged_frac[ant] > 0:
                ant_has_flag = True
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=7 * (2 - 1 * float(not outriggers)), fill=True, lw=0,
                                                color=rc_color, alpha=redcal_flagged_frac[ant]))
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=6 * (2 - 1 * float(not outriggers)), fill=True, color='w'))
        
        # plot medium green annuli for ant_metrics flags
        if use_ant_metrics: 
            if ant_metrics_xants_frac_by_ant[ant] > 0:
                ant_has_flag = True
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=6 * (2 - 1 * float(not outriggers)), fill=True, lw=0,
                                                color=antm_color, alpha=ant_metrics_xants_frac_by_ant[ant]))
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=5 * (2 - 1 * float(not outriggers)), fill=True, color='w'))
        
        # plot small red annuli for auto_metrics
        if use_auto_metrics:
            if ant in auto_ex_ants:
                ant_has_flag = True                
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=5 * (2 - 1 * float(not outriggers)), fill=True, lw=0, color=autom_color)) 
        
        # plot black/white circles with black outlines for antennas
        plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=4 * (2 - 1 * float(not outriggers)), fill=True, color=['w', 'k'][ant_has_flag], ec='k'))
        if not ant_has_flag:
            unflagged_ants.append(ant)

        # label antennas, using apriori statuses if available
        try:
            bgc = matplotlib.colors.to_rgb(status_colors[a_priori_statuses[ant]])
            c = 'black' if (bgc[0]*0.299 + bgc[1]*0.587 + bgc[2]*0.114) > 186 / 256 else 'white'
        except:
            c = 'k'
            bgc='white'
        plt.text(hd.antpos[ant][0], hd.antpos[ant][1], str(ant), va='center', ha='center', color=c, backgroundcolor=bgc)

    # label nodes
    for node in sorted(set(list(nodes.values()))):
        if not np.isnan(node) and not np.all(np.isnan(node_centers[node])):
            plt.text(node_centers[node][0], node_centers[node][1], str(node), va='center', ha='center', bbox={'color': 'w', 'ec': 'k'})
    
    # build legend 
    legend_objs = []
    legend_labels = []
    
    # use circles for annuli 
    legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgecolor='k', markerfacecolor='w', markersize=13))
    legend_labels.append(f'{len(unflagged_ants)} / {len(ants)} Total {["Core", "Outrigger"][outriggers]} Antennas Never Flagged')
    legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markerfacecolor='k', markersize=15))
    legend_labels.append(f'{len(ants) - len(unflagged_ants)} Antennas {["Core", "Outrigger"][outriggers]} Flagged for Any Reason')

    if use_auto_metrics:
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgewidth=2, markeredgecolor=autom_color, markersize=15))
        legend_labels.append(f'{len([ant for ant in auto_ex_ants if ant in ants])} {["Core", "Outrigger"][outriggers]} Antennas Flagged by Auto Metrics')
    if use_ant_metrics: 
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgewidth=2, markeredgecolor=antm_color, markersize=15))
        legend_labels.append(f'{np.round(np.sum([frac for ant, frac in ant_metrics_xants_frac_by_ant.items() if ant in ants]), 2)} Antenna-Nights on' 
                             f'\n{np.sum([frac > 0 for ant, frac in ant_metrics_xants_frac_by_ant.items() if ant in ants])} {["Core", "Outrigger"][outriggers]} Antennas '
                             'Flagged by Ant Metrics\n(alpha indicates fraction of time)')        
    if use_redcal:
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgewidth=2, markeredgecolor=rc_color, markersize=15))
        legend_labels.append(f'{np.round(np.sum(list(redcal_flagged_frac.values())), 2)} Antenna-Nights on' 
                             f'\n{np.sum([frac > 0 for ant, frac in redcal_flagged_frac.items() if ant in ants])} {["Core", "Outrigger"][outriggers]} Antennas '
                             'Flagged by Redcal\n(alpha indicates fraction of time)')

    # use rectangular patches for a priori statuses that appear in the array
    for aps in sorted(list(set(list(a_priori_statuses.values())))):
        if aps != 'Not Found':
            legend_objs.append(plt.Circle((0, 0), radius=7, fill=True, color=status_colors[aps]))
            legend_labels.append(f'A Priori Status:\n{aps} ({[status for ant, status in a_priori_statuses.items() if ant in ants].count(aps)} {["Core", "Outrigger"][outriggers]} Antennas)')

    # label nodes as a white box with black outline
    if len(node_centers) > 0:
        legend_objs.append(matplotlib.patches.Patch(facecolor='w', edgecolor='k'))
        legend_labels.append('Node Number')

    if len(unused_ants) > 0:
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markerfacecolor='grey', markersize=15, alpha=.2))
        legend_labels.append(f'Anntenna Not In Data')
        
    
    plt.legend(legend_objs, legend_labels, ncol=2, fontsize='large', framealpha=1)
    
    if outriggers:
        pass
    else:
        plt.xlim([-200, 150])
        plt.ylim([-150, 150])        
       
    # set axis equal and label everything
    plt.axis('equal')
    plt.tight_layout()
    plt.title(f'Summary of {["Core", "Outrigger"][outriggers]} Antenna Statuses and Metrics on {JD}', size=20)    
    plt.xlabel("Antenna East-West Position (meters)", size=12)
    plt.ylabel("Antenna North-South Position (meters)", size=12)
    plt.xticks(fontsize=12)
    plt.yticks(fontsize=12)
    xlim = plt.gca().get_xlim()
    ylim = plt.gca().get_ylim()    
        
    # plot unused antennas
    plt.autoscale(False)    
    for ant in unused_ants:
        if nodes[ant] in node_centers:
            plt.plot([hd.antpos[ant][0], node_centers[nodes[ant]][0]], 
                     [hd.antpos[ant][1], node_centers[nodes[ant]][1]], 'k', alpha=.2, zorder=0)
        
        plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=4, fill=True, color='w', ec=None, alpha=1, zorder=0))
        plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=4, fill=True, color='grey', ec=None, alpha=.2, zorder=0))
        if hd.antpos[ant][0] < xlim[1] and hd.antpos[ant][0] > xlim[0]:
            if hd.antpos[ant][1] < ylim[1] and hd.antpos[ant][1] > ylim[0]:
                plt.text(hd.antpos[ant][0], hd.antpos[ant][1], str(ant), va='center', ha='center', color='k', alpha=.2)


core_ants = [ant for ant in ants if ant < 320]
outrigger_ants = [ant for ant in ants if ant >= 320]
Plot_Array(ants=core_ants, unused_ants=unused_ants, outriggers=False)
if len(outrigger_ants) > 0:
    Plot_Array(ants=outrigger_ants, unused_ants=sorted(set(unused_ants + core_ants)), outriggers=True)


from hera_qm import __version__
print(__version__)
from hera_cal import __version__
print(__version__)

2.0.5.dev4+g1a49ae0
3.1.5.dev171+gc8e6162

Nightly Per-Antenna Quality Summary Notebook¶

Contents:¶

Load Auto Metrics¶

Load Ant Metrics¶

Load chi^2 info from redcal¶

Figure out some general properties¶

Load a priori antenna statuses and node numbers¶

Summarize auto metrics¶

Summarize ant metrics¶

Summarize redcal chi^2 metrics¶

Get FEM switch states¶

Find X-engine Failures¶

Build Overall Health `DataFrame`¶

Table 1: Overall Array Health¶

Build `DataFrame`¶

Table 2: RTP Per-Antenna Metrics Summary Table¶

Figure 1: Array Plot of Flags and A Priori Statuses¶

Metadata¶

JD	2459897
Date	11-13-2022
LST Range	23.017 -- 8.969 hours
X-Engine Status	✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅
Number of Files	1849
Total Number of Antennas	201

OPERATIONAL STATUS SUMMARY
Antenna A Priori Status Count	dish_maintenance: 7 dish_ok: 1 RF_maintenance: 54 RF_ok: 19 digital_ok: 96 not_connected: 24
Commanded Signal Source	None
Antennas in Commanded State (reported)	0 / 201 (0.0%)
Antennas in Commanded State (observed)	0 / 201 (0.0%)
Cross-Polarized Antennas
Total Number of Nodes	18
Nodes Registering 0s	N20
Nodes Not Correlating	N05, N07, N14

NIGHTLY ANALYSIS SUMMARY
Ant Metrics Done?	✅
Ant Metrics Flagged Antennas	91 / 201 (45.3%)
Auto Metrics Done?	✅
Auto Metrics Flagged Antennas	134 / 201 (66.7%)
Redcal Done?	❌
Never Flagged Antennas	51 / 201 (25.4%)
A Priori Good Antennas Flagged	68 / 96 total a priori good antennas: 3, 7, 9, 16, 19, 21, 29, 30, 31, 37, 38, 44, 45, 51, 53, 54, 55, 56, 59, 68, 71, 72, 81, 83, 84, 86, 93, 94, 98, 99, 100, 101, 103, 107, 108, 109, 111, 116, 117, 118, 121, 122, 123, 130, 136, 140, 142, 143, 144, 146, 158, 161, 162, 163, 164, 165, 167, 170, 181, 183, 184, 185, 186, 187, 189, 190, 191, 202
A Priori Bad Antennas Not Flagged	23 / 105 total a priori bad antennas: 48, 49, 62, 64, 79, 89, 95, 115, 120, 125, 132, 139, 148, 149, 168, 220, 221, 223, 238, 324, 325, 329, 333

Ant	Node	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric
3	N01	digital_ok	100.00%	100.00%	0.00%	0.00%	9.433298	-0.485478	9.175304	0.333308	9.417381	0.476345	1.285874	3.051625	0.032663	0.667088	0.456544
4	N01	RF_maintenance	100.00%	0.00%	0.00%	0.00%	0.960329	1.081744	2.208060	0.793229	4.869568	16.245073	4.938082	8.090713	0.672774	0.665186	0.406276
5	N01	digital_ok	0.00%	0.00%	0.00%	0.00%	0.534997	-0.022363	-0.592622	-0.364824	-0.174855	1.186230	-0.321437	-0.446950	0.672345	0.670350	0.401332
7	N02	digital_ok	100.00%	0.00%	0.00%	0.00%	-0.870433	-0.643533	0.067261	0.410767	0.200499	1.519096	17.297468	18.334858	0.664123	0.670110	0.399520
8	N02	RF_maintenance	100.00%	0.00%	0.00%	0.00%	0.529367	-1.154617	-0.861691	-0.268392	-0.590273	0.644040	6.655339	1.159983	0.655092	0.664832	0.393760
9	N02	digital_ok	100.00%	0.00%	0.00%	0.00%	3.202945	0.387021	7.278660	0.043888	3.586933	0.315702	0.475111	1.722022	0.510803	0.661482	0.462728
10	N02	digital_ok	0.00%	0.00%	0.00%	0.00%	-0.310655	-0.745527	-1.639531	-1.085405	-0.845360	0.328349	0.778326	-0.132616	0.646524	0.657004	0.412417
15	N01	digital_ok	0.00%	0.00%	0.00%	0.00%	-0.688945	-0.048591	-0.039906	0.103989	0.016441	1.288652	2.704581	2.010714	0.670633	0.675684	0.405022
16	N01	digital_ok	100.00%	0.00%	0.00%	0.00%	-0.880380	-0.877982	-0.814793	0.209093	0.547520	0.965093	3.043162	5.175514	0.673263	0.673524	0.397636
17	N01	digital_ok	0.00%	0.00%	0.00%	0.00%	-0.269795	0.759274	-0.379385	-0.046130	0.133348	0.649761	3.827123	3.014559	0.673710	0.679715	0.398933
18	N01	RF_maintenance	100.00%	0.00%	0.00%	0.00%	2.874013	10.214648	-0.822986	0.189949	0.603116	1.949526	18.880779	27.331018	0.652229	0.456872	0.474401
19	N02	digital_ok	100.00%	0.00%	0.00%	0.00%	0.716469	-1.083665	0.499657	-0.536680	0.257093	3.695947	10.803421	8.774968	0.658780	0.682730	0.401697
20	N02	digital_ok	0.00%	0.00%	0.00%	0.00%	-1.485410	-1.040293	3.096193	-1.618044	0.507497	0.802651	0.980649	-0.624121	0.653769	0.683290	0.411758
21	N02	digital_ok	100.00%	0.00%	0.00%	0.00%	0.296414	0.154119	-0.637823	4.097599	0.676924	-0.617407	0.414530	0.084900	0.652051	0.631568	0.408986
22	N06	not_connected	100.00%	100.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan
27	N01	RF_maintenance	100.00%	100.00%	100.00%	0.00%	9.192438	10.634915	9.204823	9.955516	9.550129	10.072616	3.423811	2.906070	0.031743	0.035609	0.004487
28	N01	RF_maintenance	100.00%	0.00%	84.37%	0.00%	11.703071	24.449042	0.885404	0.800818	4.813473	9.104036	7.012946	21.215999	0.361301	0.155700	0.261645
29	N01	digital_ok	100.00%	0.00%	100.00%	0.00%	-1.098053	11.079984	-0.504711	9.568663	-0.153083	10.046068	-0.289575	0.641196	0.679028	0.033719	0.484732
30	N01	digital_ok	100.00%	0.00%	0.00%	0.00%	-0.606225	-0.512863	-1.127039	0.231892	0.749929	-0.061475	19.316484	0.641975	0.676037	0.683740	0.391916
31	N02	digital_ok	100.00%	0.00%	0.00%	0.00%	0.008584	-1.198886	0.739489	1.145945	2.159437	0.800948	4.709266	1.121803	0.681667	0.683040	0.400358
32	N02	RF_maintenance	100.00%	0.00%	0.00%	0.00%	9.349464	21.598143	-0.273665	2.144007	14.671546	5.500422	5.245060	6.374573	0.611072	0.577014	0.318429
34	N06	not_connected	100.00%	100.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan
35	N06	not_connected	100.00%	100.00%	100.00%	0.00%	228.227832	228.348814	inf	inf	5121.960595	5139.496387	8664.070227	8767.742189	nan	nan	nan
36	N03	RF_maintenance	100.00%	0.00%	0.00%	0.00%	8.217823	7.320104	-0.324642	0.144660	0.890415	2.212280	0.512990	1.926163	0.657605	0.671490	0.406625
37	N03	digital_ok	100.00%	0.00%	0.00%	0.00%	0.401608	0.225966	-1.027981	0.435975	0.165591	0.919957	-0.856756	12.694377	0.672063	0.681125	0.413594
38	N03	digital_ok	100.00%	0.00%	0.00%	0.00%	0.008280	-0.042086	-0.317733	0.399424	0.683306	0.832028	8.734561	3.110257	0.675732	0.686634	0.412125
40	N04	digital_ok	0.00%	0.00%	0.00%	0.00%	0.113620	0.465400	-0.384780	0.272761	-0.791352	0.646717	-0.609768	-0.205173	0.677513	0.682390	0.392352
41	N04	digital_ok	0.00%	0.00%	0.00%	0.00%	0.143438	0.407826	-1.061920	-0.187631	0.959418	-0.041894	-0.358192	0.934835	0.683589	0.683422	0.380475
42	N04	digital_ok	0.00%	0.00%	0.00%	0.00%	0.117354	0.936923	-0.429477	0.677718	1.092507	0.388460	-0.054055	-0.156091	0.691356	0.687132	0.395536
43	N05	RF_maintenance	0.00%	100.00%	100.00%	0.00%	-0.432541	0.394820	0.186927	0.262850	-0.736147	0.171330	-1.666091	0.896264	0.057515	0.052793	0.007420
44	N05	digital_ok	0.00%	100.00%	100.00%	0.00%	-1.614819	-0.025833	-0.477393	-0.396991	-0.331128	0.594941	-1.273660	-0.532489	0.054307	0.049921	0.005397
45	N05	digital_ok	0.00%	100.00%	100.00%	0.00%	-0.551838	2.106691	-0.268259	0.020273	0.137042	1.333270	-0.113116	3.922013	0.060220	0.052583	0.006570
46	N05	RF_maintenance	0.00%	100.00%	100.00%	0.00%	-0.915511	1.443336	0.941123	1.777085	-0.179890	0.566956	-0.225412	-2.665624	0.070085	0.059883	0.011857
47	N06	not_connected	100.00%	100.00%	0.00%	0.00%	10.317965	1.716096	3.662683	-1.435466	9.443581	-1.209003	1.461359	6.012936	0.036865	0.648575	0.410431
48	N06	not_connected	0.00%	0.00%	0.00%	0.00%	1.011859	1.312998	1.180843	1.237743	-0.393104	0.287784	-1.844658	-2.607459	0.642581	0.664394	0.406985
49	N06	not_connected	0.00%	0.00%	0.00%	0.00%	0.109764	0.438898	-1.509918	0.621588	-0.900388	-1.558000	0.302527	-0.768741	0.611798	0.648251	0.401344
50	N03	RF_maintenance	100.00%	0.00%	0.00%	0.00%	4.625082	2.600004	-0.221371	0.607844	7.565264	3.336276	34.207482	29.345620	0.627644	0.657560	0.387101
51	N03	digital_ok	100.00%	100.00%	0.00%	0.00%	22.634367	0.703078	11.871250	-0.099513	9.686330	2.505956	12.458398	6.825454	0.036265	0.682232	0.440573
52	N03	RF_maintenance	100.00%	0.00%	0.00%	0.00%	6.594273	5.903894	-0.780417	0.221650	0.018999	0.908811	0.895639	1.415749	0.677026	0.690278	0.401007
53	N03	digital_ok	100.00%	0.00%	0.00%	0.00%	0.234069	2.480084	-0.635201	-0.193949	1.046417	1.348495	3.705301	7.384899	0.686892	0.697617	0.400133

Nightly Per-Antenna Quality Summary Notebook¶

Contents:¶

Load Auto Metrics¶

Load Ant Metrics¶

Load chi^2 info from redcal¶

Figure out some general properties¶

Load a priori antenna statuses and node numbers¶

Summarize auto metrics¶

Summarize ant metrics¶

Summarize redcal chi^2 metrics¶

Get FEM switch states¶

Find X-engine Failures¶

Build Overall Health DataFrame¶

Table 1: Overall Array Health¶

Build DataFrame¶

Table 2: RTP Per-Antenna Metrics Summary Table¶

Figure 1: Array Plot of Flags and A Priori Statuses¶

Metadata¶

Build Overall Health `DataFrame`¶

Build `DataFrame`¶