Nightly Per-Antenna Quality Summary Notebook¶

Josh Dillon, Last Revised February 2021

This notebooks brings together as much information as possible from ant_metrics, auto_metrics and redcal to help figure out which antennas are working properly and summarizes it in a single giant table. It is meant to be lightweight and re-run as often as necessary over the night, so it can be run when any of those is done and then be updated when another one completes.

Contents:¶

  • Table 1: Overall Array Health
  • Table 2: RTP Per-Antenna Metrics Summary Table
  • Figure 1: Array Plot of Flags and A Priori Statuses
In [1]:
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import pandas as pd
pd.set_option('display.max_rows', 1000)
from hera_qm.metrics_io import load_metric_file
from hera_cal import utils, io, redcal
import glob
import h5py
from copy import deepcopy
from IPython.display import display, HTML
from hera_notebook_templates.utils import status_colors
from hera_mc import mc
from pyuvdata import UVData

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))
In [2]:
# If you want to run this notebook locally, copy the output of the next cell into the first few lines of this cell.

# JD = "2459122"
# data_path = '/lustre/aoc/projects/hera/H4C/2459122'
# ant_metrics_ext = ".ant_metrics.hdf5"
# redcal_ext = ".maybe_good.omni.calfits"
# nb_outdir = '/lustre/aoc/projects/hera/H4C/h4c_software/H4C_Notebooks/_rtp_summary_'
# good_statuses = "digital_ok,calibration_maintenance,calibration_triage,calibration_ok"
# os.environ["JULIANDATE"] = JD
# os.environ["DATA_PATH"] = data_path
# os.environ["ANT_METRICS_EXT"] = ant_metrics_ext
# os.environ["REDCAL_EXT"] = redcal_ext
# os.environ["NB_OUTDIR"] = nb_outdir
# os.environ["GOOD_STATUSES"] = good_statuses
In [3]:
# Use environment variables to figure out path to data
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
ant_metrics_ext = os.environ['ANT_METRICS_EXT']
redcal_ext = os.environ['REDCAL_EXT']
nb_outdir = os.environ['NB_OUTDIR']
good_statuses = os.environ['GOOD_STATUSES']
print(f'JD = "{JD}"')
print(f'data_path = "{data_path}"')
print(f'ant_metrics_ext = "{ant_metrics_ext}"')
print(f'redcal_ext = "{redcal_ext}"')
print(f'nb_outdir = "{nb_outdir}"')
print(f'good_statuses = "{good_statuses}"')

JD = "2459964"
data_path = "/mnt/sn1/2459964"
ant_metrics_ext = ".ant_metrics.hdf5"
redcal_ext = ".known_good.omni.calfits"
nb_outdir = "/home/obs/src/H6C_Notebooks/_rtp_summary_"
good_statuses = "digital_ok,calibration_maintenance,calibration_triage,calibration_ok"
In [4]:
from astropy.time import Time, TimeDelta
utc = Time(JD, format='jd').datetime
print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 1-19-2023
In [5]:
# Per-season options
def ant_to_report_url(ant):
    return f'https://htmlpreview.github.io/?https://github.com/HERA-Team/H6C_Notebooks/blob/main/antenna_report/antenna_{ant}_report.html'

Load Auto Metrics¶

In [6]:
use_auto_metrics = False

# find the auto_metrics file
glob_str = os.path.join(data_path, f'zen.{JD}*.auto_metrics.h5')
auto_metrics_file = sorted(glob.glob(glob_str))

# if it exists, load and extract relevant information
if len(auto_metrics_file) > 0:
    auto_metrics_file = auto_metrics_file[0]
    print(f'Found auto_metrics results file at {auto_metrics_file}.')
    
    auto_metrics = load_metric_file(auto_metrics_file)
    mean_round_modz_cut = auto_metrics['parameters']['mean_round_modz_cut']
    auto_ex_ants = auto_metrics['ex_ants']['r2_ex_ants']
    
    use_auto_metrics = True
else:
    print(f'No files found matching glob {glob_str}. Skipping auto_metrics.')

Found auto_metrics results file at /mnt/sn1/2459964/zen.2459964.21320.sum.auto_metrics.h5.

Load Ant Metrics¶

In [7]:
use_ant_metrics = False

# get a list of all ant_metrics files
glob_str = os.path.join(data_path, f'zen.{JD}.?????.sum{ant_metrics_ext}')
ant_metrics_files = sorted(glob.glob(glob_str))

# if they exist, load as many of them as possible
if len(ant_metrics_files) > 0:
    print(f'Found {len(ant_metrics_files)} ant_metrics files matching glob {glob_str}')
    ant_metrics_apriori_exants = {}
    ant_metrics_xants_dict = {}
    ant_metrics_dead_ants_dict = {}
    ant_metrics_crossed_ants_dict = {}
    ant_metrics_dead_metrics = {}
    ant_metrics_crossed_metrics = {}
    dead_cuts = {}
    crossed_cuts = {}
    for amf in ant_metrics_files:
        with h5py.File(amf, "r") as infile: # use h5py directly since it's much faster than load_metric_file
            # get out results for this file
            dead_cuts[amf] = infile['Metrics']['dead_ant_cut'][()]
            crossed_cuts[amf] = infile['Metrics']['cross_pol_cut'][()]
            xants = infile['Metrics']['xants'][:]
            dead_ants = infile['Metrics']['dead_ants'][:]
            crossed_ants = infile['Metrics']['crossed_ants'][:]        
            try:
                # look for ex_ants in history
                ex_ants_string = infile['Header']['history'][()].decode()
                ex_ants_string = ex_ants_string.split('--apriori_xants')[1]
                ex_ants_string = ex_ants_string.split('--')[0].strip()
            except:
                ex_ants_string = ''
                    
            # This only works for the new correlation-matrix-based ant_metrics
            if 'corr' in infile['Metrics']['final_metrics'] and 'corrXPol' in infile['Metrics']['final_metrics']:
                ant_metrics_dead_metrics[amf] = {eval(ant): infile['Metrics']['final_metrics']['corr'][ant][()]
                                                 for ant in infile['Metrics']['final_metrics']['corr']}
                ant_metrics_crossed_metrics[amf] = {eval(ant): infile['Metrics']['final_metrics']['corrXPol'][ant][()]
                                                    for ant in infile['Metrics']['final_metrics']['corrXPol']}                       
            else:
                raise(KeywordError)
        
        # organize results by file
        ant_metrics_xants_dict[amf] = [(int(ant[0]), ant[1].decode()) for ant in xants]
        ant_metrics_dead_ants_dict[amf] = [(int(ant[0]), ant[1].decode()) for ant in dead_ants]
        ant_metrics_crossed_ants_dict[amf] = [(int(ant[0]), ant[1].decode()) for ant in crossed_ants]
        ant_metrics_apriori_exants[amf] = [int(ant) for ant in ex_ants_string.split()]
    
    dead_cut = np.median(list(dead_cuts.values()))
    crossed_cut = np.median(list(crossed_cuts.values()))
        
    use_ant_metrics = True
else:
    print(f'No files found matching glob {glob_str}. Skipping ant_metrics.')

Found 1849 ant_metrics files matching glob /mnt/sn1/2459964/zen.2459964.?????.sum.ant_metrics.hdf5

Load chi^2 info from redcal¶

In [8]:
use_redcal = False
glob_str = os.path.join(data_path, f'zen.{JD}.?????.sum{redcal_ext}')

redcal_files = sorted(glob.glob(glob_str))
if len(redcal_files) > 0:
    print(f'Found {len(redcal_files)} ant_metrics files matching glob {glob_str}')
    post_redcal_ant_flags_dict = {}
    flagged_by_redcal_dict = {}
    cspa_med_dict = {}
    for cal in redcal_files:
        hc = io.HERACal(cal)
        _, flags, cspa, chisq = hc.read()
        cspa_med_dict[cal] = {ant: np.nanmedian(cspa[ant], axis=1) for ant in cspa}

        post_redcal_ant_flags_dict[cal] = {ant: np.all(flags[ant]) for ant in flags}
        # check history to distinguish antennas flagged going into redcal from ones flagged during redcal
        tossed_antenna_lines =  hc.history.replace('\n','').split('Throwing out antenna ')[1:]
        flagged_by_redcal_dict[cal] = sorted([int(line.split(' ')[0]) for line in tossed_antenna_lines])
        
    use_redcal = True
else:
    print(f'No files found matching glob {glob_str}. Skipping redcal chisq.')

No files found matching glob /mnt/sn1/2459964/zen.2459964.?????.sum.known_good.omni.calfits. Skipping redcal chisq.

Figure out some general properties¶

In [9]:
# Parse some general array properties, taking into account the fact that we might be missing some of the metrics
ants = []
pols = []
antpol_pairs = []

if use_auto_metrics:
    ants = sorted(set(bl[0] for bl in auto_metrics['modzs']['r2_shape_modzs']))
    pols = sorted(set(bl[2] for bl in auto_metrics['modzs']['r2_shape_modzs']))
if use_ant_metrics:
    antpol_pairs = sorted(set([antpol for dms in ant_metrics_dead_metrics.values() for antpol in dms.keys()]))
    antpols = sorted(set(antpol[1] for antpol in antpol_pairs))
    ants = sorted(set(antpol[0] for antpol in antpol_pairs) | set(ants))
    pols = sorted(set(utils.join_pol(ap, ap) for ap in antpols) | set(pols))
if use_redcal:
    antpol_pairs = sorted(set([ant for cspa in cspa_med_dict.values() for ant in cspa.keys()]) | set(antpol_pairs))
    antpols = sorted(set(antpol[1] for antpol in antpol_pairs))
    ants = sorted(set(antpol[0] for antpol in antpol_pairs) | set(ants))
    pols = sorted(set(utils.join_pol(ap, ap) for ap in antpols) | set(pols))

# Figure out remaining antennas not in data and also LST range
data_files = sorted(glob.glob(os.path.join(data_path, 'zen.*.sum.uvh5')))
hd = io.HERAData(data_files[0])
unused_ants = [ant for ant in hd.antpos if ant not in ants]    
hd_last = io.HERAData(data_files[-1])

Load a priori antenna statuses and node numbers¶

In [10]:
# try to load a priori antenna statusesm but fail gracefully if this doesn't work.
a_priori_statuses = {ant: 'Not Found' for ant in ants}
nodes = {ant: np.nan for ant in ants + unused_ants}
try:
    from hera_mc import cm_hookup

    # get node numbers
    hookup = cm_hookup.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in nodes:
            if hookup[ant_name].get_part_from_type('node')['E<ground'] is not None:
                nodes[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])
    
    # get apriori antenna status
    for ant_name, data in hookup.items():
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in a_priori_statuses:
            a_priori_statuses[ant] = data.apriori

except Exception as err:
    print(f'Could not load node numbers and a priori antenna statuses.\nEncountered {type(err)} with message: {err}')

Summarize auto metrics¶

In [11]:
if use_auto_metrics:
    # Parse modzs
    modzs_to_check = {'Shape': 'r2_shape_modzs', 'Power': 'r2_power_modzs', 
                      'Temporal Variability': 'r2_temp_var_modzs', 'Temporal Discontinuties': 'r2_temp_diff_modzs'}
    worst_metrics = []
    worst_zs = []
    all_modzs = {}
    binary_flags = {rationale: [] for rationale in modzs_to_check}

    for ant in ants:
        # parse modzs and figure out flag counts
        modzs = {f'{pol} {rationale}': auto_metrics['modzs'][dict_name][(ant, ant, pol)] 
                 for rationale, dict_name in modzs_to_check.items() for pol in pols}
        for pol in pols:
            for rationale, dict_name in modzs_to_check.items():
                binary_flags[rationale].append(auto_metrics['modzs'][dict_name][(ant, ant, pol)] > mean_round_modz_cut)

        # parse out all metrics for dataframe
        for k in modzs:
            col_label = k + ' Modified Z-Score'
            if col_label in all_modzs:
                all_modzs[col_label].append(modzs[k])
            else:
                all_modzs[col_label] = [modzs[k]]
                
    mean_round_modz_cut = auto_metrics['parameters']['mean_round_modz_cut']
else:
    mean_round_modz_cut = 0

Summarize ant metrics¶

In [12]:
if use_ant_metrics:
    a_priori_flag_frac = {ant: np.mean([ant in apxa for apxa in ant_metrics_apriori_exants.values()]) for ant in ants}
    dead_ant_frac = {ap: {ant: np.mean([(ant, ap) in das for das in ant_metrics_dead_ants_dict.values()])
                                 for ant in ants} for ap in antpols}
    crossed_ant_frac = {ant: np.mean([np.any([(ant, ap) in cas for ap in antpols])
                                      for cas in ant_metrics_crossed_ants_dict.values()]) for ant in ants}
    ant_metrics_xants_frac_by_antpol = {antpol: np.mean([antpol in amx for amx in ant_metrics_xants_dict.values()]) for antpol in antpol_pairs}
    ant_metrics_xants_frac_by_ant = {ant: np.mean([np.any([(ant, ap) in amx for ap in antpols])
                                     for amx in ant_metrics_xants_dict.values()]) for ant in ants}
    average_dead_metrics = {ap: {ant: np.nanmean([dm.get((ant, ap), np.nan) for dm in ant_metrics_dead_metrics.values()]) 
                                 for ant in ants} for ap in antpols}
    average_crossed_metrics = {ant: np.nanmean([cm.get((ant, ap), np.nan) for ap in antpols 
                                                for cm in ant_metrics_crossed_metrics.values()]) for ant in ants}
else:
    dead_cut = 0.4
    crossed_cut = 0.0

Summarize redcal chi^2 metrics¶

In [13]:
if use_redcal:
    cspa = {ant: np.nanmedian(np.hstack([cspa_med_dict[cal][ant] for cal in redcal_files])) for ant in antpol_pairs}
    redcal_prior_flag_frac = {ant: np.mean([np.any([afd[ant, ap] and not ant in flagged_by_redcal_dict[cal] for ap in antpols])
                                            for cal, afd in post_redcal_ant_flags_dict.items()]) for ant in ants}
    redcal_flagged_frac = {ant: np.mean([ant in fbr for fbr in flagged_by_redcal_dict.values()]) for ant in ants}

Get FEM switch states¶

In [14]:
HHautos = sorted(glob.glob(f"{data_path}/zen.{JD}.*.sum.autos.uvh5"))
diffautos = sorted(glob.glob(f"{data_path}/zen.{JD}.*.diff.autos.uvh5"))

try:
    db = mc.connect_to_mc_db(None)
    session = db.sessionmaker()
    startJD = float(HHautos[0].split('zen.')[1].split('.sum')[0])
    stopJD = float(HHautos[-1].split('zen.')[1].split('.sum')[0])
    start_time = Time(startJD,format='jd')
    stop_time = Time(stopJD,format='jd')

    # get initial state by looking for commands up to 3 hours before the starttime
    # this logic can be improved after an upcoming hera_mc PR
    # which will return the most recent command before a particular time.
    search_start_time = start_time - TimeDelta(3*3600, format="sec")
    initial_command_res = session.get_array_signal_source(starttime=search_start_time, stoptime=start_time)
    if len(initial_command_res) == 0:
        initial_source = "Unknown"
    elif len(command_res) == 1:
        initial_source = initial_command_res[0].source
    else:
        # multiple commands
        times = []
        sources = []
        for obj in command_res:
            times.append(obj.time)
            sources.append(obj.source)
        initial_source = sources[np.argmax(times)]
    
    # check for any changes during observing
    command_res = session.get_array_signal_source(starttime=start_time, stoptime=stop_time)
    if len(command_res) == 0:
        # still nothing, set it to None
        obs_source = None
    else:
        obs_source_times = []
        obs_source = []
        for obj in command_res:
            obs_source_times.append(obj.time)
            obs_source.append(obj.source)

    if obs_source is not None:
        command_source = [initial_source] + obs_source
    else:
        command_source = initial_source
    
    res = session.get_antenna_status(starttime=startTime, stoptime=stopTime)
    fem_switches = {}
    right_rep_ant = []
    if len(res) > 0:
        for obj in res:
            if obj.antenna_number not in fem_switches.keys():
                fem_switches[obj.antenna_number] = {}
            fem_switches[obj.antenna_number][obj.antenna_feed_pol] = obj.fem_switch
        for ant, pol_dict in fem_switches.items():
            if pol_dict['e'] == initial_source and pol_dict['n'] == initial_source:
                right_rep_ant.append(ant)
except Exception as e:
    print(e)
    initial_source = None
    command_source = None
    right_rep_ant = []

name 'command_res' is not defined

Find X-engine Failures¶

In [15]:
read_inds = [1, len(HHautos)//2, -2]
x_status = [1,1,1,1,1,1,1,1]
s = UVData()
s.read(HHautos[1])

nants = len(s.get_ants())
freqs = s.freq_array[0]*1e-6
nfreqs = len(freqs)

antCon = {a: None for a in ants}
rightAnts = []
for i in read_inds:
    s = UVData()
    d = UVData()
    s.read(HHautos[i])
    d.read(diffautos[i])
    for pol in [0,1]:
        sm = np.abs(s.data_array[:,0,:,pol])
        df = np.abs(d.data_array[:,0,:,pol])
        sm = np.r_[sm, np.nan + np.zeros((-len(sm) % nants,len(freqs)))]
        sm = np.nanmean(sm.reshape(-1,nants,nfreqs),axis=1)
        df = np.r_[df, np.nan + np.zeros((-len(df) % nants,len(freqs)))]
        df = np.nanmean(df.reshape(-1,nants,nfreqs),axis=1)

        evens = (sm + df)/2
        odds = (sm - df)/2
        rat = np.divide(evens,odds)
        rat = np.nan_to_num(rat)
        for xbox in range(0,8):
            xavg = np.nanmean(rat[:,xbox*192:(xbox+1)*192],axis=1)
            if np.nanmax(xavg)>1.5 or np.nanmin(xavg)<0.5:
                x_status[xbox] = 0
    for ant in ants:
        for pol in ["xx", "yy"]:
            if antCon[ant] is False:
                continue
            spectrum = s.get_data(ant, ant, pol)
            stdev = np.std(spectrum)
            med = np.median(np.abs(spectrum))
            if (initial_source == 'digital_noise_same' or initial_source == 'digital_noise_different') and med < 10:
                antCon[ant] = True
            elif (initial_source == "load" or initial_source == 'noise') and 80000 < stdev <= 4000000 and antCon[ant] is not False:
                antCon[ant] = True
            elif initial_source == "antenna" and stdev > 500000 and med > 950000 and antCon[ant] is not False:
                antCon[ant] = True
            else:
                antCon[ant] = False
            if np.min(np.abs(spectrum)) < 100000:
                antCon[ant] = False
for ant in ants:
    if antCon[ant] is True:
        rightAnts.append(ant)
            
x_status_str = ''
for i,x in enumerate(x_status):
    if x==0:
        x_status_str += '\u274C '
    else:
        x_status_str += '\u2705 '

Build Overall Health DataFrame¶

In [16]:
def comma_sep_paragraph(vals, chars_per_line=40):
    outstrs = []
    for val in vals:
        if (len(outstrs) == 0) or (len(outstrs[-1]) > chars_per_line):
            outstrs.append(str(val))
        else:
            outstrs[-1] += ', ' + str(val)
    return ',<br>'.join(outstrs)
In [17]:
# Time data
to_show = {'JD': [JD]}
to_show['Date'] = f'{utc.month}-{utc.day}-{utc.year}'
to_show['LST Range'] = f'{hd.lsts[0] * 12 / np.pi:.3f} -- {hd_last.lsts[-1] * 12 / np.pi:.3f} hours'

# X-engine status
to_show['X-Engine Status'] = x_status_str

# Files
to_show['Number of Files'] = len(data_files)

# Antenna Calculations
to_show['Total Number of Antennas'] = len(ants)

to_show[' '] = ''
to_show['OPERATIONAL STATUS SUMMARY'] = ''

status_count = {status: 0 for status in status_colors}
for ant, status in a_priori_statuses.items():
    if status in status_count:
        status_count[status] = status_count[status] + 1
    else:
        status_count[status] = 1
to_show['Antenna A Priori Status Count'] = '<br>'.join([f'{status}: {status_count[status]}' for status in status_colors if status in status_count and status_count[status] > 0])

to_show['Commanded Signal Source'] = ', '.join(command_source if hasattr(command_source, '__iter__') else [str(command_source)])
to_show['Antennas in Commanded State (reported)'] = f'{len(right_rep_ant)} / {len(ants)} ({len(right_rep_ant) / len(ants):.1%})'
to_show['Antennas in Commanded State (observed)'] = f'{len(rightAnts)} / {len(ants)} ({len(rightAnts) / len(ants):.1%})'

if use_ant_metrics:
    to_show['Cross-Polarized Antennas'] = ', '.join([str(ant) for ant in ants if (np.max([dead_ant_frac[ap][ant] for ap in antpols]) + crossed_ant_frac[ant] == 1) 
                                                                                 and (crossed_ant_frac[ant] > .5)])

# Node calculations
nodes_used = set([nodes[ant] for ant in ants if np.isfinite(nodes[ant])])
to_show['Total Number of Nodes'] = len(nodes_used)
if use_ant_metrics:
    node_off = {node: True for node in nodes_used}
    not_correlating = {node: True for node in nodes_used}
    for ant in ants:
        for ap in antpols:
            if np.isfinite(nodes[ant]):
                if np.isfinite(average_dead_metrics[ap][ant]):
                    node_off[nodes[ant]] = False
                if dead_ant_frac[ap][ant] < 1:
                    not_correlating[nodes[ant]] = False
    to_show['Nodes Registering 0s'] = ', '.join([f'N{n:02}' for n in sorted([node for node in node_off if node_off[node]])])
    to_show['Nodes Not Correlating'] = ', '.join([f'N{n:02}' for n in sorted([node for node in not_correlating if not_correlating[node] and not node_off[node]])])

# Pipeline calculations    
to_show['  '] = ''
to_show['NIGHTLY ANALYSIS SUMMARY'] = ''
    
all_flagged_ants = []
if use_ant_metrics:
    to_show['Ant Metrics Done?'] = '\u2705'
    ant_metrics_flagged_ants = [ant for ant in ants if ant_metrics_xants_frac_by_ant[ant] > 0]
    all_flagged_ants.extend(ant_metrics_flagged_ants)
    to_show['Ant Metrics Flagged Antennas'] = f'{len(ant_metrics_flagged_ants)} / {len(ants)} ({len(ant_metrics_flagged_ants) / len(ants):.1%})' 
else:
    to_show['Ant Metrics Done?'] = '\u274C'
if use_auto_metrics:
    to_show['Auto Metrics Done?'] = '\u2705'
    auto_metrics_flagged_ants = [ant for ant in ants if ant in auto_ex_ants]
    all_flagged_ants.extend(auto_metrics_flagged_ants)    
    to_show['Auto Metrics Flagged Antennas'] = f'{len(auto_metrics_flagged_ants)} / {len(ants)} ({len(auto_metrics_flagged_ants) / len(ants):.1%})' 
else:
    to_show['Auto Metrics Done?'] = '\u274C'
if use_redcal:
    to_show['Redcal Done?'] = '\u2705'    
    redcal_flagged_ants = [ant for ant in ants if redcal_flagged_frac[ant] > 0]
    all_flagged_ants.extend(redcal_flagged_ants)    
    to_show['Redcal Flagged Antennas'] = f'{len(redcal_flagged_ants)} / {len(ants)} ({len(redcal_flagged_ants) / len(ants):.1%})' 
else:
    to_show['Redcal Done?'] = '\u274C' 
to_show['Never Flagged Antennas'] = f'{len(ants) - len(set(all_flagged_ants))} / {len(ants)} ({(len(ants) - len(set(all_flagged_ants))) / len(ants):.1%})'

# Count bad antennas with good statuses and vice versa
n_apriori_good = len([ant for ant in ants if a_priori_statuses[ant] in good_statuses.split(',')])
apriori_good_flagged = []
aprior_bad_unflagged = []
for ant in ants:
    if ant in set(all_flagged_ants) and a_priori_statuses[ant] in good_statuses.split(','):
        apriori_good_flagged.append(ant)
    elif ant not in set(all_flagged_ants) and a_priori_statuses[ant] not in good_statuses.split(','):
        aprior_bad_unflagged.append(ant)
to_show['A Priori Good Antennas Flagged'] = f'{len(apriori_good_flagged)} / {n_apriori_good} total a priori good antennas:<br>' + \
                                            comma_sep_paragraph(apriori_good_flagged)
to_show['A Priori Bad Antennas Not Flagged'] = f'{len(aprior_bad_unflagged)} / {len(ants) - n_apriori_good} total a priori bad antennas:<br>' + \
                                            comma_sep_paragraph(aprior_bad_unflagged)

# Apply Styling
df = pd.DataFrame(to_show)
divider_cols = [df.columns.get_loc(col) for col in ['NIGHTLY ANALYSIS SUMMARY', 'OPERATIONAL STATUS SUMMARY']]
try:
    to_red_columns = [df.columns.get_loc(col) for col in ['Cross-Polarized Antennas', 'Nodes Registering 0s', 
                                                          'Nodes Not Correlating', 'A Priori Good Antennas Flagged']]
except:
    to_red_columns = []
def red_specific_cells(x):
    df1 = pd.DataFrame('', index=x.index, columns=x.columns)
    for col in to_red_columns:
        df1.iloc[col] = 'color: red'
    return df1

df = df.T
table = df.style.hide_columns().apply(red_specific_cells, axis=None)
for col in divider_cols:
    table = table.set_table_styles([{"selector":f"tr:nth-child({col+1})", "props": [("background-color", "black"), ("color", "white")]}], overwrite=False)

Table 1: Overall Array Health¶

In [18]:
HTML(table.render())
Out[18]:
JD 2459964
Date 1-19-2023
LST Range 2.466 -- 12.417 hours
X-Engine Status ✅ ✅ ✅ ✅ ✅ ✅ ✅ ✅
Number of Files 1849
Total Number of Antennas 196
OPERATIONAL STATUS SUMMARY
Antenna A Priori Status Count dish_maintenance: 9
dish_ok: 1
RF_maintenance: 50
RF_ok: 19
digital_ok: 93
not_connected: 24
Commanded Signal Source None
Antennas in Commanded State (reported) 0 / 196 (0.0%)
Antennas in Commanded State (observed) 0 / 196 (0.0%)
Cross-Polarized Antennas
Total Number of Nodes 19
Nodes Registering 0s
Nodes Not Correlating
NIGHTLY ANALYSIS SUMMARY
Ant Metrics Done? ✅
Ant Metrics Flagged Antennas 64 / 196 (32.7%)
Auto Metrics Done? ✅
Auto Metrics Flagged Antennas 115 / 196 (58.7%)
Redcal Done? ❌
Never Flagged Antennas 81 / 196 (41.3%)
A Priori Good Antennas Flagged 49 / 93 total a priori good antennas:
3, 7, 9, 15, 16, 29, 30, 40, 42, 54, 55, 56,
67, 71, 72, 81, 86, 91, 94, 101, 103, 105,
109, 111, 121, 122, 123, 128, 136, 143, 144,
146, 147, 148, 149, 151, 158, 161, 165, 170,
173, 182, 185, 187, 189, 191, 192, 193, 202
A Priori Bad Antennas Not Flagged 37 / 103 total a priori bad antennas:
8, 22, 35, 43, 46, 48, 49, 61, 62, 64, 73,
74, 82, 89, 90, 95, 102, 120, 125, 132, 137,
139, 205, 207, 211, 220, 221, 223, 229, 237,
238, 239, 245, 261, 324, 325, 333
In [19]:
# write to csv
outpath = os.path.join(nb_outdir, f'array_health_table_{JD}.csv')
print(f'Now saving Table 2 to a csv at {outpath}')
df.replace({'\u2705': 'Y'}, regex=True).replace({'\u274C': 'N'}, regex=True).replace({'<br>': ' '}, regex=True).to_csv(outpath)

Now saving Table 2 to a csv at /home/obs/src/H6C_Notebooks/_rtp_summary_/array_health_table_2459964.csv

Build DataFrame¶

In [20]:
# build dataframe
to_show = {'Ant': [f'<a href="{ant_to_report_url(ant)}" target="_blank">{ant}</a>' for ant in ants],
           'Node': [f'N{nodes[ant]:02}' for ant in ants], 
           'A Priori Status': [a_priori_statuses[ant] for ant in ants]}
           #'Worst Metric': worst_metrics, 'Worst Modified Z-Score': worst_zs}
df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
if use_auto_metrics:
    bar_cols['Auto Metrics Flags'] = [float(ant in auto_ex_ants) for ant in ants]
if use_ant_metrics:
    if np.sum(list(a_priori_flag_frac.values())) > 0:  # only include this col if there are any a priori flags
        bar_cols['A Priori Flag Fraction in Ant Metrics'] = [a_priori_flag_frac[ant] for ant in ants]
    for ap in antpols:
        bar_cols[f'Dead Fraction in Ant Metrics ({ap})'] = [dead_ant_frac[ap][ant] for ant in ants]
    bar_cols['Crossed Fraction in Ant Metrics'] = [crossed_ant_frac[ant] for ant in ants]
if use_redcal:
    bar_cols['Flag Fraction Before Redcal'] = [redcal_prior_flag_frac[ant] for ant in ants]
    bar_cols['Flagged By Redcal chi^2 Fraction'] = [redcal_flagged_frac[ant] for ant in ants]  
for col in bar_cols:
    df[col] = bar_cols[col]

# add auto_metrics
if use_auto_metrics:
    for label, modz in all_modzs.items():
        df[label] = modz
z_score_cols = [col for col in df.columns if 'Modified Z-Score' in col]        
        
# add ant_metrics
ant_metrics_cols = {}
if use_ant_metrics:
    for ap in antpols:
        ant_metrics_cols[f'Average Dead Ant Metric ({ap})'] = [average_dead_metrics[ap][ant] for ant in ants]
    ant_metrics_cols['Average Crossed Ant Metric'] = [average_crossed_metrics[ant] for ant in ants]
    for col in ant_metrics_cols:
        df[col] = ant_metrics_cols[col]   

# add redcal chisq
redcal_cols = []
if use_redcal:
    for ap in antpols:
        col_title = f'Median chi^2 Per Antenna ({ap})'
        df[col_title] = [cspa[ant, ap] for ant in ants]
        redcal_cols.append(col_title)

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=z_score_cols) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=redcal_cols) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=z_score_cols) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=z_score_cols) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format({col: '{:,.2%}'.format for col in bar_cols}) \
          .applymap(lambda val: 'font-weight: bold', subset=['Ant']) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

Table 2: RTP Per-Antenna Metrics Summary Table¶

This admittedly very busy table incorporates summary information about all antennas in the array. Its columns depend on what information is available when the notebook is run (i.e. whether auto_metrics, ant_metrics, and/or redcal is done). These can be divided into 5 sections:

Basic Antenna Info: antenna number, node, and its a priori status.

Flag Fractions: Fraction of the night that an antenna was flagged for various reasons. Note that auto_metrics flags antennas for the whole night, so it'll be 0% or 100%.

auto_metrics Details: If auto_metrics is included, this section shows the modified Z-score signifying how much of an outlier each antenna and polarization is in each of four categories: bandpass shape, overall power, temporal variability, and temporal discontinuities. Bold red text indicates that this is a reason for flagging the antenna. It is reproduced from the auto_metrics_inspect.ipynb nightly notebook, so check that out for more details on the precise metrics.

ant_metrics Details: If ant_metrics is included, this section shows the average correlation-based metrics for antennas over the whole night. Low "dead ant" metrics (nominally below 0.4) indicate antennas not correlating with the rest of the array. Negative "crossed ant" metrics indicate antennas that show stronger correlations in their cross-pols than their same-pols, indicating that the two polarizations are probably swapped. Bold text indicates that the average is below the threshold for flagging.

redcal chi^2 Details: If redcal is included, this shows the median chi^2 per antenna. This would be 1 in an ideal array. Antennas are thrown out when they they are outliers in their median chi^2, usually greater than 4-sigma outliers in modified Z-score.

In [21]:
HTML(table.render())
Out[21]:
Ant Node A Priori Status Auto Metrics Flags Dead Fraction in Ant Metrics (Jee) Dead Fraction in Ant Metrics (Jnn) Crossed Fraction in Ant Metrics ee Shape Modified Z-Score nn Shape Modified Z-Score ee Power Modified Z-Score nn Power Modified Z-Score ee Temporal Variability Modified Z-Score nn Temporal Variability Modified Z-Score ee Temporal Discontinuties Modified Z-Score nn Temporal Discontinuties Modified Z-Score Average Dead Ant Metric (Jee) Average Dead Ant Metric (Jnn) Average Crossed Ant Metric
3 N01 digital_ok 100.00% 100.00% 0.00% 0.00% 9.747998 14.103636 10.699023 -0.272329 9.664246 5.749955 5.708647 5.121126 0.032605 0.404041 0.319875
4 N01 RF_maintenance 100.00% 0.00% 0.00% 0.00% -0.181443 -0.885741 3.043468 0.760896 8.359780 3.117791 13.978632 0.758493 0.675143 0.686979 0.345162
5 N01 digital_ok 0.00% 0.00% 0.00% 0.00% 0.083482 0.069989 0.305649 0.159192 -0.400774 1.355256 0.499758 0.785712 0.685792 0.692226 0.326532
7 N02 digital_ok 100.00% 0.00% 0.00% 0.00% -1.198507 -0.348041 -1.173661 -0.329079 -0.388152 1.011693 3.183640 4.726675 0.682150 0.692739 0.325630
8 N02 RF_maintenance 0.00% 0.00% 0.00% 0.00% 0.128742 -1.391430 -0.627651 -0.033907 -0.416053 0.167697 1.038776 0.838821 0.680627 0.686267 0.323846
9 N02 digital_ok 100.00% 0.00% 0.00% 0.00% 4.241179 -1.137431 8.904580 -0.558573 6.014744 0.101687 0.217029 0.454882 0.533576 0.682260 0.389830
10 N02 digital_ok 0.00% 0.00% 0.00% 0.00% 3.420412 -0.435734 -0.230983 -1.632147 -0.202839 1.278806 1.042085 1.485507 0.661852 0.676941 0.333143
15 N01 digital_ok 100.00% 100.00% 0.00% 0.00% 9.923434 18.037711 10.042408 -0.028771 9.647944 5.479132 5.470255 4.346300 0.032209 0.411974 0.315603
16 N01 digital_ok 100.00% 100.00% 0.00% 0.00% 9.993207 -0.723879 10.661119 0.821393 9.670265 1.229326 5.652601 1.560306 0.032223 0.697673 0.559540
17 N01 digital_ok 0.00% 0.00% 0.00% 0.00% 0.822711 1.484098 0.527271 0.343709 -0.055939 0.380868 0.296240 0.630867 0.689634 0.697067 0.329026
18 N01 RF_maintenance 100.00% 100.00% 9.03% 0.00% 10.642209 16.557492 10.635747 -1.037089 9.758775 7.090939 5.588514 15.571801 0.029059 0.287773 0.220354
19 N02 digital_ok 0.00% 0.00% 0.00% 0.00% -1.243949 -0.774988 -1.291823 -0.908632 -0.790510 1.645209 -0.714394 0.192735 0.688137 0.699383 0.321136
20 N02 digital_ok 0.00% 0.00% 0.00% 0.00% -0.708744 -1.034624 2.730098 -1.211427 0.004497 0.392284 2.693792 -0.766730 0.683871 0.693127 0.326255
21 N02 digital_ok 0.00% 0.00% 0.00% 0.00% 0.377832 0.109057 -0.739995 0.004572 0.349127 1.513661 0.127882 1.204987 0.670511 0.674078 0.318419
22 N06 not_connected 0.00% 0.00% 0.00% 0.00% -1.241231 -0.611501 0.511325 0.358373 1.141443 2.396996 -0.206755 -0.694333 0.643031 0.650604 0.332002
27 N01 RF_maintenance 100.00% 100.00% 100.00% 0.00% 9.252053 11.250882 10.692526 11.078161 9.779773 11.639072 6.259878 6.513291 0.033396 0.036412 0.004064
28 N01 RF_maintenance 100.00% 0.00% 73.72% 0.00% 12.670634 23.725432 -0.306854 1.975182 8.749422 13.695867 4.238868 19.659917 0.417520 0.211504 0.284762
29 N01 digital_ok 100.00% 100.00% 100.00% 0.00% 9.983962 11.776715 10.252207 10.642464 9.732481 11.547249 5.616127 5.852775 0.029889 0.035100 0.005553
30 N01 digital_ok 100.00% 0.00% 0.00% 0.00% -0.336499 0.160552 0.072742 0.515934 0.947524 0.069908 5.814247 -0.355686 0.693678 0.706137 0.319293
31 N02 digital_ok 0.00% 0.00% 0.00% 0.00% 1.052738 -1.659306 1.231707 1.029083 1.242591 -0.073943 1.250098 1.714851 0.703352 0.703404 0.315319
32 N02 RF_maintenance 100.00% 0.00% 0.00% 0.00% -0.319022 22.905722 -0.166716 3.004904 -0.155791 2.210942 1.139244 5.186130 0.690033 0.621076 0.302107
34 N06 not_connected 100.00% 100.00% 100.00% 0.00% 11.564317 13.144138 4.555420 4.813379 9.760938 11.569407 6.225567 6.547441 0.034600 0.047865 0.009049
35 N06 not_connected 0.00% 0.00% 0.00% 0.00% -1.201410 -0.238949 0.726157 -1.218207 -0.097553 -0.693404 0.653751 0.135037 0.648226 0.644534 0.323954
36 N03 RF_maintenance 100.00% 100.00% 100.00% 0.00% 24.706118 24.646234 14.194882 14.036384 9.818429 11.467421 7.185595 7.759129 0.031439 0.029717 0.001324
37 N03 digital_ok 0.00% 0.00% 0.00% 0.00% 0.849335 0.524142 -1.436731 0.878625 0.125443 0.440324 -1.357964 1.838715 0.698017 0.697130 0.324126
38 N03 digital_ok 0.00% 0.00% 0.00% 0.00% -0.055269 -0.016198 0.200090 0.562221 0.104471 0.816832 2.335549 0.639053 0.704771 0.705756 0.319850
40 N04 digital_ok 100.00% 100.00% 0.00% 0.00% 9.177354 -0.486031 10.279955 0.429419 9.827785 -0.441817 6.784315 0.402750 0.037873 0.699089 0.530560
41 N04 digital_ok 0.00% 0.00% 0.00% 0.00% 0.605653 -0.310289 0.014427 0.064566 1.258697 -0.413256 -0.318131 0.424941 0.702937 0.708973 0.308654
42 N04 digital_ok 100.00% 100.00% 100.00% 0.00% 10.663566 12.360523 11.008581 11.649124 9.541387 11.358404 5.616612 6.286870 0.036288 0.040538 0.004349
43 N05 RF_maintenance 0.00% 0.00% 0.00% 0.00% 0.137283 -0.013572 0.007750 0.662016 -0.867281 0.146294 -0.551665 0.357448 0.704238 0.707187 0.315699
44 N05 digital_ok 0.00% 0.00% 0.00% 0.00% -0.772424 -0.331061 -0.488260 -0.672490 0.002721 -0.002721 -1.116551 -0.787719 0.697742 0.713755 0.316960
45 N05 digital_ok 0.00% 0.00% 0.00% 0.00% 0.405966 1.025078 0.183712 0.613792 -0.671629 1.310275 0.122343 1.753996 0.698018 0.702935 0.311176
46 N05 RF_maintenance 0.00% 0.00% 0.00% 0.00% -1.147767 0.445623 -1.003233 -1.024250 0.082801 -0.497541 -0.496428 -1.251833 0.695102 0.706806 0.330190
47 N06 not_connected 100.00% 100.00% 100.00% 0.00% 10.721805 12.726522 4.359525 4.404400 9.684278 11.481972 6.098695 6.105706 0.031170 0.051540 0.013873
48 N06 not_connected 0.00% 0.00% 0.00% 0.00% 0.251362 0.192134 0.673969 2.267004 -0.452453 2.293655 -0.256631 0.660860 0.655808 0.666688 0.333576
49 N06 not_connected 0.00% 0.00% 0.00% 0.00% 0.277039 -0.370533 -0.386370 -0.285146 0.793532 -0.979606 -0.084430 0.600490 0.618102 0.649626 0.328818
50 N03 RF_maintenance 100.00% 0.00% 0.00% 0.00% 1.467428 16.774160 0.273048 1.451827 1.521705 6.702811 6.918798 22.216684 0.675324 0.619576 0.313975
51 N03 dish_maintenance 100.00% 98.05% 0.00% 0.00% 21.930354 4.191362 13.633828 -0.561347 9.944782 5.487159 9.119885 1.550098 0.044226 0.582482 0.439234
52 N03 RF_maintenance 100.00% 0.00% 0.00% 0.00% 8.323700 7.229018 -0.390516 0.508786 1.130731 0.469971 0.219425 0.329356 0.706148 0.704981 0.315678
53 N03 digital_ok 0.00% 0.00% 0.00% 0.00% 0.832448 2.216986 0.164096 0.296333 0.371250 1.056520 0.964703 1.223011 0.714986 0.711361 0.315929
54 N04 digital_ok 100.00% 0.00% 0.00% 0.00% 22.950834 2.371697 5.354288 3.230501 2.231064 1.941107 2.412044 0.852218 0.485390 0.625412 0.248500
55 N04 digital_ok 100.00% 100.00% 100.00% 0.00% 9.966353 12.174372 10.126127 10.612491 9.757807 11.564838 6.330340 7.352122 0.028856 0.031659 0.002788
56 N04 digital_ok 100.00% 100.00% 100.00% 0.00% 10.396492 12.853378 10.759238 11.487475 9.637199 11.455825 5.981761 6.440811 0.036259 0.037070 0.003344
57 N04 RF_maintenance 100.00% 0.00% 0.00% 0.00% 20.961144 5.543382 8.065813 0.438738 6.979959 2.678351 3.954791 1.751742 0.476232 0.714585 0.348379
58 N05 RF_maintenance 100.00% 100.00% 100.00% 0.00% 9.320941 11.724585 10.594019 11.224711 9.717773 11.567548 6.560525 6.678559 0.035214 0.035085 0.001590
59 N05 RF_maintenance 100.00% 100.00% 0.00% 0.00% 10.296578 0.864225 10.111010 1.105797 9.575207 1.202134 6.470758 2.332610 0.049992 0.703279 0.543743
60 N05 RF_maintenance 100.00% 0.00% 91.73% 0.00% 1.235822 11.638429 -0.436131 11.249595 0.002459 11.631573 1.226728 7.940137 0.694702 0.113847 0.521417
61 N06 not_connected 0.00% 0.00% 0.00% 0.00% 2.752823 0.304225 -0.705014 -1.635092 2.071163 -0.731446 -1.032224 -0.760541 0.644127 0.666659 0.316533
62 N06 not_connected 0.00% 0.00% 0.00% 0.00% 0.652320 -0.057164 -0.926067 1.433781 -0.662850 -0.153939 -0.224546 0.103084 0.634928 0.666157 0.324143
63 N06 not_connected 100.00% 0.00% 100.00% 0.00% -0.051329 12.218854 -0.245788 4.838445 -0.197793 11.621767 -1.195550 6.627754 0.654348 0.046099 0.491781
64 N06 not_connected 0.00% 0.00% 0.00% 0.00% 0.546958 0.577751 -1.415799 -1.225657 1.238165 -0.684052 0.522423 0.128469 0.636300 0.631903 0.316381
65 N03 digital_ok 0.00% 0.00% 0.00% 0.00% 0.409670 -0.404541 0.475912 1.169695 0.451959 1.085445 1.262752 0.714802 0.678349 0.679114 0.326940
66 N03 digital_ok 0.00% 0.00% 0.00% 0.00% 0.031813 1.029432 -1.317832 -1.415137 2.214049 -0.017011 -1.110726 -1.133202 0.688737 0.698455 0.327284
67 N03 digital_ok 100.00% 0.00% 0.00% 0.00% 3.157173 11.827339 -0.146530 1.507509 -0.326421 0.731909 0.183015 1.216037 0.703312 0.690163 0.311951
68 N03 dish_maintenance 100.00% 0.00% 100.00% 0.00% 22.503401 25.640801 1.231321 14.866424 6.269062 11.561041 2.929794 8.913559 0.432560 0.030944 0.315527
69 N04 digital_ok 0.00% 0.00% 0.00% 0.00% 0.496223 -0.821732 0.293548 0.847952 -0.532643 0.905461 -0.035715 -0.163008 0.708978 0.715762 0.302659
70 N04 digital_ok 0.00% 0.00% 0.00% 0.00% 0.522586 -0.870210 -0.099911 0.027234 0.774470 0.788031 -0.180888 -0.204988 0.714807 0.721173 0.302258
71 N04 digital_ok 100.00% 0.00% 0.00% 0.00% 8.302597 -0.243300 0.735046 0.928416 0.668621 -0.082645 1.430362 0.882592 0.721460 0.722598 0.298039
72 N04 digital_ok 100.00% 100.00% 100.00% 0.00% 9.860767 12.868113 11.125803 11.673514 9.620157 11.425335 6.407896 6.870837 0.029089 0.029870 0.001915
73 N05 RF_maintenance 0.00% 0.00% 0.00% 0.00% -0.595400 1.698481 -1.479479 -1.287749 0.740702 0.197313 -0.994663 -1.167406 0.714414 0.721450 0.310547
74 N05 RF_maintenance 0.00% 0.00% 0.00% 0.00% -0.470944 -0.367513 -0.067931 -0.690130 0.729997 1.476228 -0.567708 1.183973 0.708571 0.716249 0.312034
77 N06 not_connected 100.00% 0.00% 0.00% 0.00% 58.105662 -0.210323 0.710286 -0.292547 8.805193 -0.807814 7.294694 -1.011887 0.425158 0.667428 0.376488
78 N06 not_connected 100.00% 0.00% 0.00% 0.00% 30.073794 -0.622261 -0.570572 1.415003 2.436319 0.308499 1.917595 1.059837 0.504417 0.672277 0.326069
79 N11 not_connected 100.00% 0.00% 100.00% 0.00% 1.873247 12.477316 -1.601436 4.869656 -0.332747 11.455056 -0.588656 5.906295 0.644202 0.039952 0.477804
80 N11 not_connected 100.00% 0.00% 99.84% 0.00% -0.799347 13.322694 -0.065310 4.759175 0.675576 11.471866 4.170979 6.158201 0.639839 0.049739 0.483633
81 N07 digital_ok 100.00% 0.00% 100.00% 0.00% -0.625046 12.586451 0.196720 9.728044 -0.371366 11.185576 -0.036481 6.421792 0.653964 0.040893 0.479643
82 N07 RF_maintenance 0.00% 0.00% 0.00% 0.00% 1.714922 -0.519361 0.392914 2.272469 -0.423890 -1.013814 0.357970 0.918020 0.676339 0.669623 0.313233
83 N07 digital_ok 0.00% 0.00% 0.00% 0.00% -0.468773 -0.001629 0.290523 0.398379 0.052614 -0.104484 -0.224325 0.070671 0.688905 0.695130 0.313496
84 N08 RF_maintenance 100.00% 29.42% 100.00% 0.00% 18.710263 21.818845 13.830630 14.323043 8.149428 11.413487 4.543283 7.058968 0.297855 0.034848 0.180751
85 N08 digital_ok 0.00% 0.00% 0.00% 0.00% -0.663407 0.353533 2.283491 0.765476 -1.146468 -0.696713 1.018904 -0.557197 0.698827 0.710142 0.303255
86 N08 digital_ok 100.00% 0.00% 0.00% 0.00% 0.419130 -0.468682 0.606186 2.191033 2.892428 7.605783 -0.307710 10.263192 0.699233 0.697927 0.287002
87 N08 RF_maintenance 100.00% 0.00% 0.00% 0.00% 5.385591 9.094158 -0.651781 -0.275981 0.139968 0.008425 -0.845949 -0.217373 0.719659 0.726739 0.298379
88 N09 digital_ok 0.00% 0.00% 0.00% 0.00% 0.319257 -0.305586 0.436064 0.663381 -0.837257 -0.550796 0.174155 -0.485999 0.712196 0.717553 0.295173
89 N09 RF_maintenance 0.00% 0.00% 0.00% 0.00% 1.363154 0.475367 0.040714 0.807281 -0.813130 -0.972086 -0.557294 -0.600600 0.713785 0.716804 0.299963
90 N09 RF_maintenance 0.00% 0.00% 0.00% 0.00% -0.225816 -1.154789 1.026621 0.655325 -0.804299 -1.290214 0.500920 0.852757 0.706260 0.712212 0.301170
91 N09 digital_ok 100.00% 98.81% 98.81% 0.00% nan nan inf inf nan nan nan nan 0.383211 0.403679 0.281645
92 N10 RF_maintenance 100.00% 0.00% 1.03% 0.00% 33.580185 40.078193 0.511623 1.135252 6.521774 6.247163 3.033437 6.659813 0.348236 0.297437 0.080583
93 N10 digital_ok 0.00% 0.00% 0.00% 0.00% 0.961312 -0.018052 2.371483 -1.346684 1.802099 0.179548 2.870440 -1.132047 0.690433 0.700383 0.319906
94 N10 digital_ok 100.00% 100.00% 100.00% 0.00% 10.809681 12.155527 10.890148 11.108632 9.650616 11.564553 5.965061 6.154939 0.031150 0.026208 0.002449
95 N11 not_connected 0.00% 0.00% 0.00% 0.00% 1.430460 -0.697414 -1.209503 1.151610 -0.675875 0.379653 -0.625651 0.451956 0.654070 0.671057 0.333439
96 N11 not_connected 100.00% 100.00% 100.00% 0.00% 11.090598 12.758861 4.404069 4.984603 9.539356 11.356548 5.838352 6.110626 0.033601 0.037826 0.002429
97 N11 not_connected 100.00% 0.00% 0.00% 0.00% 2.602167 5.098325 0.463925 1.127683 0.428916 2.115309 1.057983 5.471044 0.602201 0.595955 0.301184
101 N08 digital_ok 100.00% 0.00% 0.00% 0.00% 8.631648 8.987194 -0.347474 1.251394 0.137143 0.398914 -0.170876 0.529835 0.704290 0.705532 0.310652
102 N08 RF_maintenance 0.00% 0.00% 0.00% 0.00% -0.855583 0.001629 -1.047674 -1.664504 0.445542 0.471702 -1.566243 3.417000 0.710288 0.712822 0.307057
103 N08 digital_ok 100.00% 0.00% 0.00% 0.00% 11.118830 5.508437 10.649775 -1.455209 5.574147 -0.278723 3.019849 3.019782 0.529232 0.717996 0.357630
104 N08 RF_maintenance 100.00% 0.00% 0.00% 0.00% 8.483216 53.461865 -1.112538 7.356728 1.548062 0.359130 -0.860615 3.573885 0.719936 0.703459 0.296865
105 N09 digital_ok 100.00% 98.86% 99.03% 0.00% nan nan inf inf nan nan nan nan 0.482758 0.397039 0.353586
106 N09 digital_ok 0.00% 0.00% 0.00% 0.00% 2.024486 1.107265 -1.574287 -0.716185 0.104272 -1.009421 -1.370597 -1.206177 0.711384 0.718111 0.294373
107 N09 digital_ok 0.00% 0.00% 0.00% 0.00% 1.329657 1.707562 -0.191238 -0.535798 1.122189 -0.036850 1.521670 1.508045 0.713276 0.718764 0.299572
108 N09 RF_maintenance 100.00% 99.19% 99.57% 0.00% nan nan inf inf nan nan nan nan 0.448370 0.268274 0.343648
109 N10 digital_ok 100.00% 100.00% 100.00% 0.00% 9.468833 11.355428 10.088553 10.361748 9.766638 11.571111 5.493712 6.401505 0.027739 0.027588 0.001412
110 N10 RF_maintenance 100.00% 100.00% 100.00% 0.00% 24.596127 24.525720 14.313265 14.598500 9.664484 11.373737 6.251968 6.818357 0.024231 0.026454 0.001229
111 N10 digital_ok 100.00% 0.00% 100.00% 0.00% 0.160627 11.503635 0.657110 11.080365 -0.694236 11.613239 1.555873 6.525805 0.696379 0.038710 0.469983
112 N10 digital_ok 0.00% 0.00% 0.00% 0.00% -0.398129 -0.674085 0.297265 0.275754 0.727457 2.187457 0.465058 0.351483 0.685938 0.692814 0.324053
113 N11 not_connected 100.00% 100.00% 100.00% 0.00% 11.773548 12.988465 4.160048 4.863911 9.579303 11.428507 6.162354 6.113997 0.034999 0.030788 0.002274
114 N11 not_connected 100.00% 100.00% 100.00% 0.00% 8.517168 8.528978 13.222095 11.823578 10.701011 14.119412 129.312402 60.942934 0.025041 0.027301 0.001653
115 N11 not_connected 100.00% 98.92% 99.03% 0.00% nan nan inf inf nan nan nan nan 0.439250 0.350271 0.327539
117 N07 RF_maintenance 100.00% 100.00% 100.00% 0.00% 10.751501 13.242293 10.793276 11.628741 9.578757 11.498438 5.837683 7.155392 0.027987 0.031776 0.002709
118 N07 digital_ok 0.00% 0.00% 0.00% 0.00% -0.097554 1.416729 -0.198675 0.576716 -0.297828 -0.009516 -0.086015 0.483660 0.672970 0.684791 0.322176
120 N08 RF_maintenance 0.00% 0.00% 0.00% 0.00% 3.531368 1.659640 3.035610 -1.089696 -0.645585 0.748823 2.929142 -0.671137 0.693945 0.709583 0.307258
121 N08 digital_ok 100.00% 0.00% 0.00% 0.00% 2.979112 5.690664 -0.511114 2.173735 0.861936 -0.482080 14.096900 8.383253 0.716189 0.717309 0.301299
122 N08 digital_ok 100.00% 0.00% 0.00% 0.00% 9.171273 6.794171 0.146156 1.006533 1.244418 1.146451 -0.087055 -0.000107 0.720700 0.720754 0.296391
123 N08 digital_ok 100.00% 0.00% 0.00% 0.00% 7.026638 9.317470 0.899216 1.044803 0.231939 -0.568599 0.802648 0.595328 0.724420 0.727566 0.298683
124 N09 digital_ok 0.00% 0.00% 0.00% 0.00% 0.069245 -0.041414 0.078445 0.675013 -0.182214 0.260603 0.420499 0.182399 0.721805 0.726131 0.302190
125 N09 RF_maintenance 0.00% 0.00% 0.00% 0.00% 0.556339 -0.438062 -0.419829 0.754001 0.074086 -0.742879 -0.457507 -0.150079 0.716937 0.719662 0.298474
126 N09 RF_maintenance 100.00% 0.00% 0.00% 0.00% 1.210473 6.106444 -1.462699 1.597144 2.221035 -1.076209 0.019170 0.559663 0.711785 0.712573 0.303104
127 N10 digital_ok 0.00% 0.00% 0.00% 0.00% 0.136229 0.153489 0.641307 0.414994 1.032762 0.339797 0.416402 -0.131781 0.709430 0.717654 0.312986
128 N10 digital_ok 100.00% 100.00% 100.00% 0.00% 9.052394 10.782179 10.187070 10.627024 9.589689 11.451901 5.477342 5.967686 0.031156 0.028774 0.001407
131 N11 not_connected 100.00% 98.92% 99.03% 0.00% nan nan inf inf nan nan nan nan 0.455292 0.453829 0.210484
132 N11 not_connected 0.00% 0.00% 0.00% 0.00% -0.582219 1.334259 -0.030051 -1.592321 -1.182832 -0.375682 -1.451628 -1.174401 0.653141 0.654207 0.319642
133 N11 not_connected 100.00% 98.70% 98.81% 0.00% 202.589914 202.223145 inf inf 4358.095047 4233.863714 3819.216677 3521.267043 0.413649 0.419796 0.185959
135 N12 RF_maintenance 100.00% 0.00% 0.00% 0.00% -1.015632 -1.185386 -0.830984 -1.585466 3.249470 0.916450 4.930237 -0.308510 0.641818 0.660962 0.348000
136 N12 digital_ok 100.00% 100.00% 0.00% 0.00% 8.820740 -0.323595 10.212641 -0.922128 9.783466 -0.289795 5.988477 -0.524920 0.044496 0.663732 0.462584
137 N07 RF_maintenance 0.00% 0.00% 0.00% 0.00% 0.263343 -0.405092 0.310388 -1.491618 1.362332 -0.733609 0.206554 -0.873537 0.655239 0.674827 0.331498
139 N13 RF_maintenance 0.00% 0.00% 0.00% 0.00% -0.073616 -0.219416 1.791441 -0.894840 0.831853 -1.382586 0.718918 -0.745709 0.675862 0.678516 0.317029
140 N13 digital_ok 0.00% 0.00% 0.00% 0.00% -0.347424 -0.290673 -1.043405 -0.042403 -0.600568 -0.261567 1.143309 0.553591 0.700206 0.709724 0.314378
141 N13 digital_ok 0.00% 0.00% 0.00% 0.00% -1.540774 -1.036359 -0.197111 0.903274 1.861253 -1.052233 -0.003761 -0.401352 0.706738 0.711415 0.311003
142 N13 RF_maintenance 100.00% 0.00% 98.16% 0.00% 1.267616 11.528390 -0.557854 11.254266 3.519185 11.591284 18.799966 6.510387 0.710086 0.051537 0.545497
143 N14 digital_ok 100.00% 0.00% 100.00% 0.00% -1.212991 12.004494 1.178690 11.343613 1.775731 11.344848 1.030961 6.775475 0.715883 0.042602 0.571734
144 N14 digital_ok 100.00% 0.00% 0.00% 0.00% -0.195243 5.168068 -1.199972 9.740089 -0.430623 6.035760 -0.939718 0.930271 0.719215 0.536357 0.379651
145 N14 RF_maintenance 100.00% 0.00% 0.00% 0.00% -1.318389 0.119837 -1.130717 4.099193 0.382030 7.819175 0.016382 4.079348 0.719178 0.705823 0.297074
146 N14 digital_ok 100.00% 100.00% 0.00% 0.00% 11.172860 -0.355280 4.132945 -0.007512 9.688974 -0.526609 5.401966 -1.318737 0.039954 0.702890 0.535209
147 N15 digital_ok 100.00% 98.92% 99.08% 0.00% nan nan inf inf nan nan nan nan 0.408922 0.290519 0.291392
148 N15 digital_ok 100.00% 98.59% 98.76% 0.00% nan nan inf inf nan nan nan nan 0.413530 0.367366 0.322311
149 N15 digital_ok 100.00% 98.32% 98.76% 0.00% 210.551390 210.475548 inf inf 4141.461407 4058.900595 3393.379497 3089.854313 0.418242 0.339741 0.309050
150 N15 digital_ok 0.00% 0.00% 0.00% 0.00% 0.286767 -0.824910 -1.240417 -1.357793 -0.186408 0.005042 -1.451315 -1.323372 0.691776 0.696398 0.329332
151 N16 digital_ok 100.00% 0.00% 0.00% 0.00% 24.375494 1.491617 -0.123427 0.407995 2.680958 -0.163762 0.831922 -0.883873 0.537612 0.636616 0.286530
155 N12 RF_maintenance 100.00% 98.81% 0.00% 0.00% 9.301996 -0.407474 10.379528 -1.669779 9.824079 1.136510 6.288845 0.595270 0.047094 0.658246 0.475138
156 N12 RF_maintenance 100.00% 0.00% 100.00% 0.00% 3.673347 11.271562 8.938975 10.957988 5.774464 11.649340 1.347888 6.721686 0.489724 0.039369 0.355056
157 N12 digital_ok 0.00% 0.00% 0.00% 0.00% 0.264241 -0.380102 -0.004572 0.650290 -0.472485 -0.169573 0.709473 0.971434 0.660859 0.674154 0.332638
158 N12 digital_ok 100.00% 0.00% 0.00% 0.00% -0.109449 -0.382300 -0.002721 -0.781249 1.649756 0.960365 0.804201 6.811795 0.676517 0.688876 0.332080
159 N13 RF_maintenance 100.00% 0.00% 0.00% 0.00% 0.607248 26.856172 -1.629735 -0.772634 -0.415089 4.386193 -1.183527 19.788884 0.650368 0.556458 0.301911
160 N13 digital_ok 0.00% 0.00% 0.00% 0.00% -1.040839 -1.252349 -0.300611 -0.778894 -0.675539 0.869398 0.577410 0.003761 0.692721 0.700968 0.315009
161 N13 digital_ok 100.00% 0.00% 0.00% 0.00% -1.086129 25.944388 0.070839 -0.698401 -0.067951 2.341768 0.018298 0.902701 0.700375 0.592320 0.281053
162 N13 digital_ok 0.00% 0.00% 0.00% 0.00% -0.292187 -1.383837 -0.983692 -1.279378 1.316083 1.330883 0.599693 -0.737620 0.706694 0.714328 0.312807
163 N14 digital_ok 0.00% 0.00% 0.00% 0.00% 0.764089 0.569229 -0.176372 0.511049 -0.507302 0.304183 -0.346314 0.920643 0.717094 0.716838 0.308450
164 N14 digital_ok 0.00% 0.00% 0.00% 0.00% -1.311114 0.577093 0.666650 -0.070490 3.336477 1.923199 0.799920 0.112082 0.714749 0.719723 0.302440
165 N14 digital_ok 100.00% 0.00% 0.00% 0.00% 28.728013 0.140472 -0.647427 -1.324904 2.234807 -0.126531 1.226988 -0.747207 0.588995 0.720499 0.297692
166 N14 RF_maintenance 100.00% 0.00% 0.00% 0.00% 1.065224 4.521607 -0.912538 -1.385808 1.353127 43.858457 1.486564 34.846778 0.716038 0.716912 0.310930
167 N15 digital_ok 0.00% 0.00% 0.00% 0.00% -0.546644 -1.111082 -1.470031 -0.175399 1.052879 0.637072 -0.064292 2.403101 0.711676 0.716092 0.312451
168 N15 digital_ok 0.00% 0.00% 0.00% 0.00% -0.780741 -1.023235 0.391916 -0.444337 1.126004 -0.566705 0.556265 0.313111 0.705787 0.708378 0.317769
169 N15 digital_ok 0.00% 0.00% 0.00% 0.00% -0.496130 -0.655573 -0.974570 -1.532271 0.356820 0.194122 -0.727774 -1.795162 0.702418 0.707737 0.325042
170 N15 digital_ok 100.00% 99.95% 0.00% 0.00% 10.530374 -1.098492 11.004302 -1.235927 9.599564 1.951988 6.585226 1.494199 0.044585 0.704221 0.539220
171 N16 digital_ok 0.00% 0.00% 0.00% 0.00% 1.207365 3.150210 -1.521205 -0.079461 -0.945416 1.989832 -1.259976 -0.690011 0.645014 0.625196 0.308551
173 N16 digital_ok 100.00% 100.00% 100.00% 0.00% 11.681140 12.227029 3.826509 4.415368 9.888753 11.668443 7.581152 8.331051 0.039429 0.044653 0.004349
179 N12 RF_maintenance 100.00% 0.00% 0.00% 0.00% -0.267271 -0.031126 -0.067765 0.703757 -0.570365 22.403898 0.515590 3.715734 0.673249 0.681266 0.329462
180 N13 RF_maintenance 100.00% 0.00% 98.38% 0.00% -0.614340 12.204181 -1.688311 11.411035 0.831498 11.498748 5.628131 6.638213 0.686679 0.057875 0.544861
181 N13 digital_ok 0.00% 0.00% 0.00% 0.00% 0.223938 -0.569690 0.356570 0.095451 0.168479 -0.408138 0.227716 1.291145 0.697739 0.699732 0.321130
182 N13 digital_ok 100.00% 0.00% 98.65% 0.00% -0.710379 11.267906 -0.422430 10.930739 -1.282380 11.600689 2.972753 6.387457 0.704746 0.053133 0.516886
183 N13 digital_ok 0.00% 0.00% 0.00% 0.00% -0.086328 -0.146289 -0.831061 0.058773 0.964085 -0.547066 -0.084217 -0.162757 0.697306 0.702472 0.304633
184 N14 digital_ok 0.00% 0.00% 0.00% 0.00% -0.892936 -0.984233 -1.637593 -0.589597 -0.276533 -0.311924 -0.362743 0.204017 0.708234 0.717668 0.298536
185 N14 digital_ok 100.00% 0.00% 0.00% 0.00% 34.311133 0.091734 -0.470700 -1.603145 7.638946 -0.482192 1.856811 -0.992338 0.601594 0.714631 0.305425
186 N14 digital_ok 0.00% 0.00% 0.00% 0.00% -0.885385 -1.224407 -1.635768 -0.445065 1.017492 -0.721344 -1.016220 -1.248158 0.717938 0.721153 0.317309
187 N14 digital_ok 100.00% 0.00% 0.00% 0.00% 0.110925 1.341198 -1.255306 2.324521 -0.002364 0.558137 -0.399399 4.637503 0.712600 0.709113 0.321446
189 N15 digital_ok 100.00% 100.00% 100.00% 0.00% 8.686495 11.157133 10.180276 11.067581 9.962162 11.669288 7.514135 6.950313 0.028658 0.031531 0.001373
190 N15 digital_ok 0.00% 0.00% 0.00% 0.00% -1.097452 -1.680785 -0.755926 0.636131 -0.725181 -0.275798 -0.061930 -0.700365 0.698594 0.704600 0.335953
191 N15 digital_ok 100.00% 0.00% 0.00% 0.00% -1.672238 -0.153658 1.454585 -0.567992 0.505718 -0.044755 4.777005 0.060216 0.689831 0.697360 0.324439
192 N16 digital_ok 100.00% 0.00% 0.00% 0.00% 3.915873 5.037966 5.619886 5.753792 7.175214 9.177268 3.732496 4.393056 0.631272 0.638866 0.339350
193 N16 digital_ok 100.00% 0.00% 0.00% 0.00% 4.639347 0.128080 5.876824 2.104706 7.617545 1.963061 4.060313 1.114068 0.615568 0.656575 0.357763
200 N18 RF_maintenance 100.00% 98.32% 98.43% 0.00% 223.267514 223.203259 inf inf 4858.399785 4533.934753 4643.992007 4134.697944 0.383389 0.396428 0.332352
201 N18 RF_maintenance 100.00% 98.05% 98.76% 0.00% nan nan inf inf nan nan nan nan 0.416166 0.294260 0.304466
202 N18 digital_ok 100.00% 98.43% 98.65% 0.00% nan nan inf inf nan nan nan nan 0.347827 0.339928 0.244271
205 N19 RF_ok 0.00% 0.00% 0.00% 0.00% 1.895417 3.243837 0.143845 -0.554345 -0.831062 -0.378892 -0.372971 1.489673 0.688221 0.674657 0.305129
206 N19 RF_ok 100.00% 0.00% 0.00% 0.00% 1.613351 1.086144 2.746875 -0.754138 10.678233 -0.480495 1.987438 1.275661 0.687393 0.686562 0.313590
207 N19 RF_ok 0.00% 0.00% 0.00% 0.00% 2.497014 3.337884 1.658313 0.015677 1.089838 -0.059348 0.579053 -0.675661 0.670636 0.675677 0.296334
208 N20 dish_maintenance 100.00% 100.00% 100.00% 0.00% 4.893678 8.545890 9.735716 11.258310 9.345790 10.303855 8.978959 34.855042 0.034471 0.034400 0.000480
209 N20 dish_maintenance 100.00% 100.00% 100.00% 0.00% 4.872174 6.364833 9.809143 9.887522 8.858795 11.045375 10.955322 11.509447 0.041759 0.040119 0.002673
210 N20 dish_maintenance 100.00% 0.00% 0.00% 0.00% 11.287008 5.671313 -1.086452 -1.070798 -0.919677 -0.373897 -0.696069 -0.895300 0.692522 0.659477 0.329295
211 N20 RF_ok 0.00% 0.00% 0.00% 0.00% 1.526335 2.147298 -1.457172 0.234502 -1.032026 -0.033997 -0.368528 -0.900022 0.651450 0.658954 0.318130
220 N18 RF_maintenance 0.00% 0.00% 0.00% 0.00% -1.060993 -1.347683 0.602452 -0.181658 -0.875731 0.748092 1.265415 -1.235261 0.671643 0.672442 0.324817
221 N18 RF_ok 0.00% 0.00% 0.00% 0.00% 2.105164 -0.034502 -1.148814 -0.608702 -0.450760 -0.850842 0.490225 -1.347312 0.664352 0.680592 0.321372
222 N18 RF_ok 100.00% 0.00% 0.00% 0.00% 1.061090 -0.239468 -0.277104 0.037550 -0.623929 5.797821 -0.395546 -0.903090 0.677639 0.688394 0.318121
223 N19 RF_ok 0.00% 0.00% 0.00% 0.00% 0.992229 1.134074 -1.531539 0.912974 -0.312069 -0.525214 -0.604781 0.397168 0.670878 0.688495 0.319524
224 N19 RF_ok 100.00% 0.00% 0.00% 0.00% 4.680282 4.643126 5.971330 5.738242 7.629192 9.132224 4.120338 4.343883 0.651155 0.660716 0.329594
225 N19 RF_ok 100.00% 0.00% 72.09% 0.00% 0.861728 11.926782 1.036670 4.607006 -0.888387 11.352719 -0.339747 5.879099 0.688341 0.207149 0.520868
226 N19 RF_ok 100.00% 0.00% 0.00% 0.00% -0.111983 16.681521 0.213315 1.646123 -1.415896 4.340448 -0.968350 1.484162 0.683587 0.618960 0.323213
227 N20 RF_ok 100.00% 0.00% 0.00% 0.00% 2.894239 0.862534 -1.584410 0.278527 0.357120 -0.489571 4.751043 -0.505809 0.656875 0.677053 0.308663
228 N20 RF_maintenance 100.00% 0.00% 0.00% 0.00% 6.882332 21.066624 -1.058255 -0.162458 0.335955 3.943063 3.837654 21.927803 0.628663 0.580366 0.276031
229 N20 RF_maintenance 0.00% 0.00% 0.00% 0.00% 0.245475 -0.591682 1.915173 1.728445 0.077673 0.797611 -0.180544 -0.463723 0.664202 0.676034 0.329095
237 N18 RF_ok 0.00% 0.00% 0.00% 0.00% 1.877893 0.114515 -0.099041 -1.531011 -0.103105 -1.026302 -1.200889 -1.601276 0.620540 0.654503 0.334338
238 N18 RF_ok 0.00% 0.00% 0.00% 0.00% -0.931953 -0.108514 1.108312 0.508946 -0.790791 -0.501638 -0.673320 -1.364695 0.666837 0.674140 0.333264
239 N18 RF_ok 0.00% 0.00% 0.00% 0.00% -0.928598 -1.544217 0.035923 0.221282 -0.927910 -0.409123 -0.802465 -0.146130 0.674337 0.678135 0.327607
240 N19 RF_maintenance 100.00% 0.00% 0.00% 0.00% 30.326316 52.873476 -0.636487 1.061051 2.912886 7.960011 2.283855 6.102734 0.540150 0.478557 0.228531
241 N19 RF_ok 100.00% 0.00% 0.00% 0.00% 1.958509 4.912166 -0.842775 0.478060 -0.764902 0.646719 1.612847 8.067582 0.669450 0.643381 0.319241
242 N19 RF_ok 100.00% 0.00% 0.00% 0.00% 35.344964 0.639887 -0.024462 1.661038 8.996320 0.731451 6.866527 0.448822 0.528700 0.685565 0.341667
243 N19 RF_ok 100.00% 3.35% 0.00% 0.00% 62.130555 2.826043 1.161175 -1.452293 6.758837 -0.660685 2.447516 -0.644622 0.361563 0.669360 0.439767
244 N20 RF_maintenance 100.00% 0.00% 0.00% 0.00% 4.961674 2.541419 1.456733 -0.889876 3.013714 2.688541 0.679762 2.345448 0.585783 0.651838 0.311594
245 N20 RF_ok 0.00% 0.00% 0.00% 0.00% 0.674896 2.406077 0.270568 -0.878468 -0.673443 -0.315327 -0.994289 -1.134953 0.664502 0.665137 0.318496
246 N20 RF_maintenance 100.00% 0.00% 0.00% 0.00% 9.889073 8.181514 -0.650021 0.357195 5.848454 5.628730 3.857550 3.466412 0.385125 0.392372 0.146572
261 N20 RF_ok 0.00% 0.00% 0.00% 0.00% 0.852877 1.460939 0.845227 -0.339863 -0.164120 -0.013172 -0.243056 0.265362 0.664207 0.663533 0.321495
262 N20 dish_maintenance 100.00% 100.00% 100.00% 0.00% 4.864658 6.300547 9.967793 10.524209 8.884324 10.830936 12.221494 14.337822 0.031544 0.027192 0.003849
320 N03 dish_maintenance 100.00% 0.00% 100.00% 0.00% 7.998526 12.227245 2.792869 7.189357 1.856455 11.694150 22.813595 7.617397 0.520639 0.046535 0.419788
324 N04 not_connected 0.00% 0.00% 0.00% 0.00% 0.992022 1.797038 1.385823 1.956368 0.903436 1.757777 0.895728 0.284166 0.580660 0.581801 0.323885
325 N09 dish_ok 0.00% 0.00% 0.00% 0.00% -0.442602 -0.948849 1.510194 -1.246457 0.806296 0.188108 -1.054597 2.342759 0.629652 0.621583 0.327909
329 N12 dish_maintenance 100.00% 0.00% 0.00% 0.00% 2.515514 -0.314527 -1.592778 -0.902172 0.713512 0.199357 5.072190 2.485579 0.542008 0.565341 0.335592
333 N12 dish_maintenance 0.00% 0.00% 0.00% 0.00% 3.107728 1.848378 -0.744880 -1.546377 0.811055 0.063520 3.504912 2.478130 0.506787 0.545467 0.331587
In [22]:
# print ex_ants for easy copy-pasting to YAML file
proposed_ex_ants = [ant for i, ant in enumerate(ants) if np.any([col[i] > 0 for col in bar_cols.values()])]
print('ex_ants: [' + ", ".join(str(ant) for ant in proposed_ex_ants) + ']')
print(f'\nunflagged_ants: [{", ".join([str(ant) for ant in ants if ant not in proposed_ex_ants])}]')
# "golden" means no flags and good a priori status
golden_ants = ", ".join([str(ant) for ant in ants if ((ant not in proposed_ex_ants) and (a_priori_statuses[ant] in good_statuses.split(',')))])
print(f'\ngolden_ants: [{golden_ants}]')

ex_ants: [3, 4, 7, 9, 15, 16, 18, 27, 28, 29, 30, 32, 34, 36, 40, 42, 47, 50, 51, 52, 54, 55, 56, 57, 58, 59, 60, 63, 67, 68, 71, 72, 77, 78, 79, 80, 81, 84, 86, 87, 91, 92, 94, 96, 97, 101, 103, 104, 105, 108, 109, 110, 111, 113, 114, 115, 117, 121, 122, 123, 126, 128, 131, 133, 135, 136, 142, 143, 144, 145, 146, 147, 148, 149, 151, 155, 156, 158, 159, 161, 165, 166, 170, 173, 179, 180, 182, 185, 187, 189, 191, 192, 193, 200, 201, 202, 206, 208, 209, 210, 222, 224, 225, 226, 227, 228, 240, 241, 242, 243, 244, 246, 262, 320, 329]

unflagged_ants: [5, 8, 10, 17, 19, 20, 21, 22, 31, 35, 37, 38, 41, 43, 44, 45, 46, 48, 49, 53, 61, 62, 64, 65, 66, 69, 70, 73, 74, 82, 83, 85, 88, 89, 90, 93, 95, 102, 106, 107, 112, 118, 120, 124, 125, 127, 132, 137, 139, 140, 141, 150, 157, 160, 162, 163, 164, 167, 168, 169, 171, 181, 183, 184, 186, 190, 205, 207, 211, 220, 221, 223, 229, 237, 238, 239, 245, 261, 324, 325, 333]

golden_ants: [5, 10, 17, 19, 20, 21, 31, 37, 38, 41, 44, 45, 53, 65, 66, 69, 70, 83, 85, 88, 93, 106, 107, 112, 118, 124, 127, 140, 141, 150, 157, 160, 162, 163, 164, 167, 168, 169, 171, 181, 183, 184, 186, 190]
In [23]:
# write to csv
outpath = os.path.join(nb_outdir, f'rtp_summary_table_{JD}.csv')
print(f'Now saving Table 2 to a csv at {outpath}')
df.to_csv(outpath)

Now saving Table 2 to a csv at /home/obs/src/H6C_Notebooks/_rtp_summary_/rtp_summary_table_2459964.csv
In [24]:
# Load antenna positions
data_list = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}.?????.sum.uvh5')))
hd = io.HERAData(data_list[len(data_list) // 2])

# Figure out where to draw the nodes
node_centers = {}
for node in sorted(set(list(nodes.values()))):
    if np.isfinite(node):
        this_node_ants = [ant for ant in ants + unused_ants if nodes[ant] == node]
        if len(this_node_ants) == 1:
            # put the node label just to the west of the lone antenna 
            node_centers[node] = hd.antpos[ant][node] + np.array([-14.6 / 2, 0, 0])
        else:
            # put the node label between the two antennas closest to the node center
            node_centers[node] = np.mean([hd.antpos[ant] for ant in this_node_ants], axis=0)
            closest_two_pos = sorted([hd.antpos[ant] for ant in this_node_ants], 
                                     key=lambda pos: np.linalg.norm(pos - node_centers[node]))[0:2]
            node_centers[node] = np.mean(closest_two_pos, axis=0)
In [25]:
def Plot_Array(ants, unused_ants, outriggers):
    plt.figure(figsize=(16,16))
    
    plt.scatter(np.array([hd.antpos[ant][0] for ant in hd.data_ants if ant in ants]), 
                np.array([hd.antpos[ant][1] for ant in hd.data_ants if ant in ants]), c='w', s=0)

    # connect every antenna to their node
    for ant in ants:
        if nodes[ant] in node_centers:
            plt.plot([hd.antpos[ant][0], node_centers[nodes[ant]][0]], 
                     [hd.antpos[ant][1], node_centers[nodes[ant]][1]], 'k', zorder=0)

    rc_color = '#0000ff'
    antm_color = '#ffa500'
    autom_color = '#ff1493'

    # Plot 
    unflagged_ants = []
    for i, ant in enumerate(ants):
        ant_has_flag = False
        # plot large blue annuli for redcal flags
        if use_redcal:
            if redcal_flagged_frac[ant] > 0:
                ant_has_flag = True
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=7 * (2 - 1 * float(not outriggers)), fill=True, lw=0,
                                                color=rc_color, alpha=redcal_flagged_frac[ant]))
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=6 * (2 - 1 * float(not outriggers)), fill=True, color='w'))
        
        # plot medium green annuli for ant_metrics flags
        if use_ant_metrics: 
            if ant_metrics_xants_frac_by_ant[ant] > 0:
                ant_has_flag = True
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=6 * (2 - 1 * float(not outriggers)), fill=True, lw=0,
                                                color=antm_color, alpha=ant_metrics_xants_frac_by_ant[ant]))
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=5 * (2 - 1 * float(not outriggers)), fill=True, color='w'))
        
        # plot small red annuli for auto_metrics
        if use_auto_metrics:
            if ant in auto_ex_ants:
                ant_has_flag = True                
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=5 * (2 - 1 * float(not outriggers)), fill=True, lw=0, color=autom_color)) 
        
        # plot black/white circles with black outlines for antennas
        plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=4 * (2 - 1 * float(not outriggers)), fill=True, color=['w', 'k'][ant_has_flag], ec='k'))
        if not ant_has_flag:
            unflagged_ants.append(ant)

        # label antennas, using apriori statuses if available
        try:
            bgc = matplotlib.colors.to_rgb(status_colors[a_priori_statuses[ant]])
            c = 'black' if (bgc[0]*0.299 + bgc[1]*0.587 + bgc[2]*0.114) > 186 / 256 else 'white'
        except:
            c = 'k'
            bgc='white'
        plt.text(hd.antpos[ant][0], hd.antpos[ant][1], str(ant), va='center', ha='center', color=c, backgroundcolor=bgc)

    # label nodes
    for node in sorted(set(list(nodes.values()))):
        if not np.isnan(node) and not np.all(np.isnan(node_centers[node])):
            plt.text(node_centers[node][0], node_centers[node][1], str(node), va='center', ha='center', bbox={'color': 'w', 'ec': 'k'})
    
    # build legend 
    legend_objs = []
    legend_labels = []
    
    # use circles for annuli 
    legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgecolor='k', markerfacecolor='w', markersize=13))
    legend_labels.append(f'{len(unflagged_ants)} / {len(ants)} Total {["Core", "Outrigger"][outriggers]} Antennas Never Flagged')
    legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markerfacecolor='k', markersize=15))
    legend_labels.append(f'{len(ants) - len(unflagged_ants)} Antennas {["Core", "Outrigger"][outriggers]} Flagged for Any Reason')

    if use_auto_metrics:
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgewidth=2, markeredgecolor=autom_color, markersize=15))
        legend_labels.append(f'{len([ant for ant in auto_ex_ants if ant in ants])} {["Core", "Outrigger"][outriggers]} Antennas Flagged by Auto Metrics')
    if use_ant_metrics: 
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgewidth=2, markeredgecolor=antm_color, markersize=15))
        legend_labels.append(f'{np.round(np.sum([frac for ant, frac in ant_metrics_xants_frac_by_ant.items() if ant in ants]), 2)} Antenna-Nights on' 
                             f'\n{np.sum([frac > 0 for ant, frac in ant_metrics_xants_frac_by_ant.items() if ant in ants])} {["Core", "Outrigger"][outriggers]} Antennas '
                             'Flagged by Ant Metrics\n(alpha indicates fraction of time)')        
    if use_redcal:
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgewidth=2, markeredgecolor=rc_color, markersize=15))
        legend_labels.append(f'{np.round(np.sum(list(redcal_flagged_frac.values())), 2)} Antenna-Nights on' 
                             f'\n{np.sum([frac > 0 for ant, frac in redcal_flagged_frac.items() if ant in ants])} {["Core", "Outrigger"][outriggers]} Antennas '
                             'Flagged by Redcal\n(alpha indicates fraction of time)')

    # use rectangular patches for a priori statuses that appear in the array
    for aps in sorted(list(set(list(a_priori_statuses.values())))):
        if aps != 'Not Found':
            legend_objs.append(plt.Circle((0, 0), radius=7, fill=True, color=status_colors[aps]))
            legend_labels.append(f'A Priori Status:\n{aps} ({[status for ant, status in a_priori_statuses.items() if ant in ants].count(aps)} {["Core", "Outrigger"][outriggers]} Antennas)')

    # label nodes as a white box with black outline
    if len(node_centers) > 0:
        legend_objs.append(matplotlib.patches.Patch(facecolor='w', edgecolor='k'))
        legend_labels.append('Node Number')

    if len(unused_ants) > 0:
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markerfacecolor='grey', markersize=15, alpha=.2))
        legend_labels.append(f'Anntenna Not In Data')
        
    
    plt.legend(legend_objs, legend_labels, ncol=2, fontsize='large', framealpha=1)
    
    if outriggers:
        pass
    else:
        plt.xlim([-200, 150])
        plt.ylim([-150, 150])        
       
    # set axis equal and label everything
    plt.axis('equal')
    plt.tight_layout()
    plt.title(f'Summary of {["Core", "Outrigger"][outriggers]} Antenna Statuses and Metrics on {JD}', size=20)    
    plt.xlabel("Antenna East-West Position (meters)", size=12)
    plt.ylabel("Antenna North-South Position (meters)", size=12)
    plt.xticks(fontsize=12)
    plt.yticks(fontsize=12)
    xlim = plt.gca().get_xlim()
    ylim = plt.gca().get_ylim()    
        
    # plot unused antennas
    plt.autoscale(False)    
    for ant in unused_ants:
        if nodes[ant] in node_centers:
            plt.plot([hd.antpos[ant][0], node_centers[nodes[ant]][0]], 
                     [hd.antpos[ant][1], node_centers[nodes[ant]][1]], 'k', alpha=.2, zorder=0)
        
        plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=4, fill=True, color='w', ec=None, alpha=1, zorder=0))
        plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=4, fill=True, color='grey', ec=None, alpha=.2, zorder=0))
        if hd.antpos[ant][0] < xlim[1] and hd.antpos[ant][0] > xlim[0]:
            if hd.antpos[ant][1] < ylim[1] and hd.antpos[ant][1] > ylim[0]:
                plt.text(hd.antpos[ant][0], hd.antpos[ant][1], str(ant), va='center', ha='center', color='k', alpha=.2)

Figure 1: Array Plot of Flags and A Priori Statuses¶

This plot shows all antennas, which nodes they are connected to, and their a priori statuses (as the highlight text of their antenna numbers). It may also show (depending on what is finished running):

  • Whether they were flagged by auto_metrics (red circle) for bandpass shape, overall power, temporal variability, or temporal discontinuities. This is done in a binary fashion for the whole night.
  • Whether they were flagged by ant_metrics (green circle) as either dead (on either polarization) or crossed, with the transparency indicating the fraction of the night (i.e. number of files) that were flagged.
  • Whether they were flagged by redcal (blue circle) for high chi^2, with the transparency indicating the fraction of the night (i.e. number of files) that were flagged.

Note that the last fraction does not include antennas that were flagged before going into redcal due to their a priori status, for example.

In [26]:
core_ants = [ant for ant in ants if ant < 320]
outrigger_ants = [ant for ant in ants if ant >= 320]
Plot_Array(ants=core_ants, unused_ants=unused_ants, outriggers=False)
if len(outrigger_ants) > 0:
    Plot_Array(ants=outrigger_ants, unused_ants=sorted(set(unused_ants + core_ants)), outriggers=True)

Metadata¶

In [27]:
from hera_qm import __version__
print(__version__)
from hera_cal import __version__
print(__version__)

2.0.5.dev13+gd6c757c
3.2.1
In [ ]: