Nightly Per-Antenna Quality Summary Notebook

Josh Dillon, Last Revised February 2021

This notebooks brings together as much information as possible from ant_metrics, auto_metrics and redcal to help figure out which antennas are working properly and summarizes it in a single giant table. It is meant to be lightweight and re-run as often as necessary over the night, so it can be run when any of those is done and then be updated when another one completes.

import numpy as np
import matplotlib.pyplot as plt
import matplotlib
import pandas as pd
pd.set_option('display.max_rows', 1000)
from hera_qm.metrics_io import load_metric_file
from hera_cal import utils, io, redcal
import glob
import os
import h5py
from copy import deepcopy
from IPython.display import display, HTML
from hera_notebook_templates.utils import status_colors

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))

# If you want to run this notebook locally, copy the output of the next cell into the first few lines of this cell.

# JD = "2459122"
# data_path = '/lustre/aoc/projects/hera/H4C/2459122'
# ant_metrics_ext = ".ant_metrics.hdf5"
# redcal_ext = ".maybe_good.omni.calfits"
# nb_outdir = '/lustre/aoc/projects/hera/H4C/h4c_software/H4C_Notebooks/_rtp_summary_'
# os.environ["JULIANDATE"] = JD
# os.environ["DATA_PATH"] = data_path
# os.environ["ANT_METRICS_EXT"] = ant_metrics_ext
# os.environ["REDCAL_EXT"] = redcal_ext
# os.environ["NB_OUTDIR"] = nb_outdir

# Use environment variables to figure out path to data
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
ant_metrics_ext = os.environ['ANT_METRICS_EXT']
redcal_ext = os.environ['REDCAL_EXT']
nb_outdir = os.environ['NB_OUTDIR']
print(f'JD = "{JD}"')
print(f'data_path = "{data_path}"')
print(f'ant_metrics_ext = "{ant_metrics_ext}"')
print(f'redcal_ext = "{redcal_ext}"')
print(f'nb_outdir = "{nb_outdir}"')

JD = "2459563"
data_path = "/mnt/sn1/2459563"
ant_metrics_ext = ".ant_metrics.hdf5"
redcal_ext = ".known_good.omni.calfits"
nb_outdir = "/home/obs/src/H5C_Notebooks/_rtp_summary_"

from astropy.time import Time
utc = Time(JD, format='jd').datetime
print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 12-14-2021

Load Auto Metrics

use_auto_metrics = False

# find the auto_metrics file
glob_str = os.path.join(data_path, f'zen.{JD}*.auto_metrics.h5')
auto_metrics_file = sorted(glob.glob(glob_str))

# if it exists, load and extract relevant information
if len(auto_metrics_file) > 0:
    auto_metrics_file = auto_metrics_file[0]
    print(f'Found auto_metrics results file at {auto_metrics_file}.')
    
    auto_metrics = load_metric_file(auto_metrics_file)
    mean_round_modz_cut = auto_metrics['parameters']['mean_round_modz_cut']
    auto_ex_ants = auto_metrics['ex_ants']['r2_ex_ants']
    
    use_auto_metrics = True
else:
    print(f'No files found matching glob {glob_str}. Skipping auto_metrics.')

Found auto_metrics results file at /mnt/sn1/2459563/zen.2459563.25261.sum.auto_metrics.h5.

Load Ant Metrics

use_ant_metrics = False

# get a list of all ant_metrics files
glob_str = os.path.join(data_path, f'zen.{JD}.?????.sum{ant_metrics_ext}')
ant_metrics_files = sorted(glob.glob(glob_str))

# if they exist, load as many of them as possible
if len(ant_metrics_files) > 0:
    print(f'Found {len(ant_metrics_files)} ant_metrics files matching glob {glob_str}')
    ant_metrics_apriori_exants = {}
    ant_metrics_xants_dict = {}
    ant_metrics_dead_ants_dict = {}
    ant_metrics_crossed_ants_dict = {}
    ant_metrics_dead_metrics = {}
    ant_metrics_crossed_metrics = {}
    dead_cuts = {}
    crossed_cuts = {}
    for amf in ant_metrics_files:
        with h5py.File(amf, "r") as infile: # use h5py directly since it's much faster than load_metric_file
            # get out results for this file
            dead_cuts[amf] = infile['Metrics']['dead_ant_cut'][()]
            crossed_cuts[amf] = infile['Metrics']['cross_pol_cut'][()]
            xants = infile['Metrics']['xants'][:]
            dead_ants = infile['Metrics']['dead_ants'][:]
            crossed_ants = infile['Metrics']['crossed_ants'][:]        
            try:
                # look for ex_ants in history
                ex_ants_string = infile['Header']['history'][()].decode()
                ex_ants_string = ex_ants_string.split('--apriori_xants')[1]
                ex_ants_string = ex_ants_string.split('--')[0].strip()
            except:
                ex_ants_string = ''
                    
            # This only works for the new correlation-matrix-based ant_metrics
            if 'corr' in infile['Metrics']['final_metrics'] and 'corrXPol' in infile['Metrics']['final_metrics']:
                ant_metrics_dead_metrics[amf] = {eval(ant): infile['Metrics']['final_metrics']['corr'][ant][()]
                                                 for ant in infile['Metrics']['final_metrics']['corr']}
                ant_metrics_crossed_metrics[amf] = {eval(ant): infile['Metrics']['final_metrics']['corrXPol'][ant][()]
                                                    for ant in infile['Metrics']['final_metrics']['corrXPol']}                       
            else:
                raise(KeywordError)
        
        # organize results by file
        ant_metrics_xants_dict[amf] = [(int(ant[0]), ant[1].decode()) for ant in xants]
        ant_metrics_dead_ants_dict[amf] = [(int(ant[0]), ant[1].decode()) for ant in dead_ants]
        ant_metrics_crossed_ants_dict[amf] = [(int(ant[0]), ant[1].decode()) for ant in crossed_ants]
        ant_metrics_apriori_exants[amf] = [int(ant) for ant in ex_ants_string.split()]
    
    dead_cut = np.median(list(dead_cuts.values()))
    crossed_cut = np.median(list(crossed_cuts.values()))
        
    use_ant_metrics = True
else:
    print(f'No files found matching glob {glob_str}. Skipping ant_metrics.')

Found 1851 ant_metrics files matching glob /mnt/sn1/2459563/zen.2459563.?????.sum.ant_metrics.hdf5

Load chi^2 info from redcal

use_redcal = False
glob_str = os.path.join(data_path, f'zen.{JD}.?????.sum{redcal_ext}')

redcal_files = sorted(glob.glob(glob_str))
if len(redcal_files) > 0:
    print(f'Found {len(redcal_files)} ant_metrics files matching glob {glob_str}')
    post_redcal_ant_flags_dict = {}
    flagged_by_redcal_dict = {}
    cspa_med_dict = {}
    for cal in redcal_files:
        hc = io.HERACal(cal)
        _, flags, cspa, chisq = hc.read()
        cspa_med_dict[cal] = {ant: np.nanmedian(cspa[ant], axis=1) for ant in cspa}

        post_redcal_ant_flags_dict[cal] = {ant: np.all(flags[ant]) for ant in flags}
        # check history to distinguish antennas flagged going into redcal from ones flagged during redcal
        tossed_antenna_lines =  hc.history.replace('\n','').split('Throwing out antenna ')[1:]
        flagged_by_redcal_dict[cal] = sorted([int(line.split(' ')[0]) for line in tossed_antenna_lines])
        
    use_redcal = True
else:
    print(f'No files found matching glob {glob_str}. Skipping redcal chisq.')

Found 186 ant_metrics files matching glob /mnt/sn1/2459563/zen.2459563.?????.sum.known_good.omni.calfits

Figure out some general properties

# Parse some general array properties, taking into account the fact that we might be missing some of the metrics
ants = []
pols = []
antpol_pairs = []

if use_auto_metrics:
    ants = sorted(set(bl[0] for bl in auto_metrics['modzs']['r2_shape_modzs']))
    pols = sorted(set(bl[2] for bl in auto_metrics['modzs']['r2_shape_modzs']))
if use_ant_metrics:
    antpol_pairs = sorted(set([antpol for dms in ant_metrics_dead_metrics.values() for antpol in dms.keys()]))
    antpols = sorted(set(antpol[1] for antpol in antpol_pairs))
    ants = sorted(set(antpol[0] for antpol in antpol_pairs) | set(ants))
    pols = sorted(set(utils.join_pol(ap, ap) for ap in antpols) | set(pols))
if use_redcal:
    antpol_pairs = sorted(set([ant for cspa in cspa_med_dict.values() for ant in cspa.keys()]) | set(antpol_pairs))
    antpols = sorted(set(antpol[1] for antpol in antpol_pairs))
    ants = sorted(set(antpol[0] for antpol in antpol_pairs) | set(ants))
    pols = sorted(set(utils.join_pol(ap, ap) for ap in antpols) | set(pols))

# Figure out remaining antennas not in data
hd = io.HERAData(sorted(glob.glob(os.path.join(data_path, 'zen.*.sum.uvh5')))[0])
unused_ants = [ant for ant in hd.antpos if ant not in ants]    

Load a priori antenna statuses and node numbers

# try to load a priori antenna statusesm but fail gracefully if this doesn't work.
a_priori_statuses = {ant: 'Not Found' for ant in ants}
nodes = {ant: np.nan for ant in ants + unused_ants}
try:
    from hera_mc import cm_active, cm_hookup

    # get apriori antenna status
    h = cm_active.ActiveData(at_date=JD)
    h.load_apriori()    
    for ant_name in h.apriori:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in a_priori_statuses:
            a_priori_statuses[ant] = h.apriori[ant_name].status

    # get node numbers
    h = cm_hookup.Hookup()
    hookup = h.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in nodes:
            if hookup[ant_name].get_part_from_type('node')['E<ground'] is not None:
                nodes[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])
    
except Exception as err:
    print(f'Could not load node numbers and a priori antenna statuses.\nEncountered {type(err)} with message: {err}')

Summarize auto metrics

if use_auto_metrics:
    # Parse modzs
    modzs_to_check = {'Shape': 'r2_shape_modzs', 'Power': 'r2_power_modzs', 
                      'Temporal Variability': 'r2_temp_var_modzs', 'Temporal Discontinuties': 'r2_temp_diff_modzs'}
    worst_metrics = []
    worst_zs = []
    all_modzs = {}
    binary_flags = {rationale: [] for rationale in modzs_to_check}

    for ant in ants:
        # parse modzs and figure out flag counts
        modzs = {f'{pol} {rationale}': auto_metrics['modzs'][dict_name][(ant, ant, pol)] 
                 for rationale, dict_name in modzs_to_check.items() for pol in pols}
        for pol in pols:
            for rationale, dict_name in modzs_to_check.items():
                binary_flags[rationale].append(auto_metrics['modzs'][dict_name][(ant, ant, pol)] > mean_round_modz_cut)

        # parse out all metrics for dataframe
        for k in modzs:
            col_label = k + ' Modified Z-Score'
            if col_label in all_modzs:
                all_modzs[col_label].append(modzs[k])
            else:
                all_modzs[col_label] = [modzs[k]]
                
    mean_round_modz_cut = auto_metrics['parameters']['mean_round_modz_cut']
else:
    mean_round_modz_cut = 0

Summarize ant metrics

if use_ant_metrics:
    a_priori_flag_frac = {ant: np.mean([ant in apxa for apxa in ant_metrics_apriori_exants.values()]) for ant in ants}
    dead_ant_frac = {ap: {ant: np.mean([(ant, ap) in das for das in ant_metrics_dead_ants_dict.values()])
                                 for ant in ants} for ap in antpols}
    crossed_ant_frac = {ant: np.mean([np.any([(ant, ap) in cas for ap in antpols])
                                      for cas in ant_metrics_crossed_ants_dict.values()]) for ant in ants}
    ant_metrics_xants_frac_by_antpol = {antpol: np.mean([antpol in amx for amx in ant_metrics_xants_dict.values()]) for antpol in antpol_pairs}
    ant_metrics_xants_frac_by_ant = {ant: np.mean([np.any([(ant, ap) in amx for ap in antpols])
                                     for amx in ant_metrics_xants_dict.values()]) for ant in ants}
    average_dead_metrics = {ap: {ant: np.nanmean([dm.get((ant, ap), np.nan) for dm in ant_metrics_dead_metrics.values()]) 
                                 for ant in ants} for ap in antpols}
    average_crossed_metrics = {ant: np.nanmean([cm.get((ant, ap), np.nan) for ap in antpols 
                                                for cm in ant_metrics_crossed_metrics.values()]) for ant in ants}
else:
    dead_cut = 0.4
    crossed_cut = 0.0

Summarize redcal chi^2 metrics

if use_redcal:
    cspa = {ant: np.nanmedian(np.hstack([cspa_med_dict[cal][ant] for cal in redcal_files])) for ant in antpol_pairs}
    redcal_prior_flag_frac = {ant: np.mean([np.any([afd[ant, ap] and not ant in flagged_by_redcal_dict[cal] for ap in antpols])
                                            for cal, afd in post_redcal_ant_flags_dict.items()]) for ant in ants}
    redcal_flagged_frac = {ant: np.mean([ant in fbr for fbr in flagged_by_redcal_dict.values()]) for ant in ants}

Build DataFrame

# build dataframe
to_show = {'Ant': ants, 'Node': [nodes[ant] for ant in ants], 'A Priori Status': [a_priori_statuses[ant] for ant in ants]}
           #'Worst Metric': worst_metrics, 'Worst Modified Z-Score': worst_zs}
df = pd.DataFrame(to_show)

# create bar chart columns for flagging percentages:
bar_cols = {}
if use_auto_metrics:
    bar_cols['Auto Metrics Flags'] = [float(ant in auto_ex_ants) for ant in ants]
if use_ant_metrics:
    if np.sum(list(a_priori_flag_frac.values())) > 0:  # only include this col if there are any a priori flags
        bar_cols['A Priori Flag Fraction in Ant Metrics'] = [a_priori_flag_frac[ant] for ant in ants]
    for ap in antpols:
        bar_cols[f'Dead Fraction in Ant Metrics ({ap})'] = [dead_ant_frac[ap][ant] for ant in ants]
    bar_cols['Crossed Fraction in Ant Metrics'] = [crossed_ant_frac[ant] for ant in ants]
if use_redcal:
    bar_cols['Flag Fraction Before Redcal'] = [redcal_prior_flag_frac[ant] for ant in ants]
    bar_cols['Flagged By Redcal chi^2 Fraction'] = [redcal_flagged_frac[ant] for ant in ants]  
for col in bar_cols:
    df[col] = bar_cols[col]

# add auto_metrics
if use_auto_metrics:
    for label, modz in all_modzs.items():
        df[label] = modz
z_score_cols = [col for col in df.columns if 'Modified Z-Score' in col]        
        
# add ant_metrics
ant_metrics_cols = {}
if use_ant_metrics:
    for ap in antpols:
        ant_metrics_cols[f'Average Dead Ant Metric ({ap})'] = [average_dead_metrics[ap][ant] for ant in ants]
    ant_metrics_cols['Average Crossed Ant Metric'] = [average_crossed_metrics[ant] for ant in ants]
    for col in ant_metrics_cols:
        df[col] = ant_metrics_cols[col]   

# add redcal chisq
redcal_cols = []
if use_redcal:
    for ap in antpols:
        col_title = f'Median chi^2 Per Antenna ({ap})'
        df[col_title] = [cspa[ant, ap] for ant in ants]
        redcal_cols.append(col_title)

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

# style dataframe
table = df.style.hide_index()\
          .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
          .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=z_score_cols) \
          .background_gradient(cmap='bwr_r', vmin=dead_cut-.25, vmax=dead_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .background_gradient(cmap='bwr_r', vmin=crossed_cut-.25, vmax=crossed_cut+.25, axis=0, subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .background_gradient(cmap='plasma', vmax=4, vmin=1, axis=None, subset=redcal_cols) \
          .applymap(lambda val: 'font-weight: bold' if val < dead_cut else '', subset=list([col for col in ant_metrics_cols if 'dead' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val < crossed_cut else '', subset=list([col for col in ant_metrics_cols if 'crossed' in col.lower()])) \
          .applymap(lambda val: 'font-weight: bold' if val > mean_round_modz_cut else '', subset=z_score_cols) \
          .applymap(lambda val: 'color: red' if val > mean_round_modz_cut else '', subset=z_score_cols) \
          .bar(subset=list(bar_cols.keys()), vmin=0, vmax=1) \
          .format({col: '{:,.4f}'.format for col in z_score_cols}) \
          .format({col: '{:,.4f}'.format for col in ant_metrics_cols}) \
          .format({col: '{:,.2%}'.format for col in bar_cols}) \
          .applymap(lambda val: 'font-weight: bold', subset=['Ant']) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])]) \

Table 1: RTP Per-Antenna Metrics Summary Table

This admittedly very busy table incorporates summary information about all antennas in the array. Its columns depend on what information is available when the notebook is run (i.e. whether auto_metrics, ant_metrics, and/or redcal is done). These can be divided into 5 sections:

Basic Antenna Info: antenna number, node, and its a priori status.

Flag Fractions: Fraction of the night that an antenna was flagged for various reasons. Note that auto_metrics flags antennas for the whole night, so it'll be 0% or 100%.

auto_metrics Details: If auto_metrics is included, this section shows the modified Z-score signifying how much of an outlier each antenna and polarization is in each of four categories: bandpass shape, overall power, temporal variability, and temporal discontinuities. Bold red text indicates that this is a reason for flagging the antenna. It is reproduced from the auto_metrics_inspect.ipynb nightly notebook, so check that out for more details on the precise metrics.

ant_metrics Details: If ant_metrics is included, this section shows the average correlation-based metrics for antennas over the whole night. Low "dead ant" metrics (nominally below 0.4) indicate antennas not correlating with the rest of the array. Negative "crossed ant" metrics indicate antennas that show stronger correlations in their cross-pols than their same-pols, indicating that the two polarizations are probably swapped. Bold text indicates that the average is below the threshold for flagging.

redcal chi^2 Details: If redcal is included, this shows the median chi^2 per antenna. This would be 1 in an ideal array. Antennas are thrown out when they they are outliers in their median chi^2, usually greater than 4-sigma outliers in modified Z-score.

HTML(table.render())

Ant	Node	A Priori Status	Auto Metrics Flags	Dead Fraction in Ant Metrics (Jee)	Dead Fraction in Ant Metrics (Jnn)	Crossed Fraction in Ant Metrics	Flag Fraction Before Redcal	Flagged By Redcal chi^2 Fraction	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	Average Dead Ant Metric (Jee)	Average Dead Ant Metric (Jnn)	Average Crossed Ant Metric	Median chi^2 Per Antenna (Jee)	Median chi^2 Per Antenna (Jnn)
0	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	48.072840	93.517532	143.797043	180.162990	41.249406	64.424473	407.125155	691.143004	0.017947	0.016179	0.001654	1.125404	1.108567
1	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	55.200055	75.237394	149.252668	148.504148	35.438656	46.177974	373.270540	373.258904	0.016461	0.016307	0.000349	0.000000	0.000000
2	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	42.270479	68.397206	137.605650	131.858027	35.957280	33.217432	266.404026	237.480235	0.016874	0.016625	0.000431	0.000000	0.000000
3	1	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
4	1	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
5	1	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
7	2	dish_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.811741	5.615050	20.906049	21.692442	9.988611	12.823510	-1.275346	-0.784249	0.663501	0.664174	0.407357	3.626837	3.575106
8	2	dish_ok	100.00%	100.00%	0.00%	0.00%	100.00%	0.00%	16.464635	3.542928	67.118976	14.537650	19.688218	7.989742	2.758971	-0.680182	0.068434	0.662286	0.504399	0.000000	0.000000
9	2	dish_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	5.375607	8.156981	18.212059	23.739625	11.274057	15.902131	9.056070	7.688903	0.652446	0.640652	0.399194	0.000000	0.000000
10	2	dish_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	5.378140	4.716232	18.826481	13.828756	9.263191	5.525678	-1.174176	-0.962883	0.647034	0.659460	0.402186	4.358630	4.261997
11	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	211.150627	74.245051	166.131817	144.510760	40.002527	32.603597	293.725108	299.106837	0.019046	0.017377	0.001751	0.000000	0.000000
12	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	61.576265	66.337670	125.143848	132.153881	22.708145	37.780330	206.008802	241.657409	0.018986	0.018204	0.000646	0.000000	0.000000
13	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	77.407138	61.344762	150.070872	142.824288	38.343583	39.368977	332.483035	308.593412	0.018874	0.017662	0.000745	0.966201	0.957847
14	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	155.436690	80.562358	172.944736	157.062426	42.872935	53.453716	463.654504	483.319821	0.018675	0.017151	0.001062	0.000000	0.000000
15	1	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
16	1	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
17	1	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
18	1	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
19	2	dish_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.223559	0.433345	23.691984	2.521617	12.169485	53.669280	-1.298257	5.393947	0.663467	0.681303	0.406161	0.000000	0.000000
20	2	dish_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.971012	5.437879	9.786916	20.658692	2.297735	11.680060	-0.075806	-1.733389	0.676179	0.671195	0.405249	3.200629	2.818714
21	2	dish_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
23	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	52.494452	74.706906	157.389950	144.688746	58.875142	41.083457	456.905293	353.568041	0.017383	0.016580	0.000630	0.000000	0.000000
24	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	47.439716	97.832947	151.080107	140.072193	50.997838	39.472707	441.704849	254.029914	0.016527	0.016481	0.000341	0.000000	0.000000
25	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	312.382308	309.705539	309.885781	278.310375	30.992816	41.804660	204.562800	282.432598	0.025755	0.017575	0.008412	0.000000	0.000000
26	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	318.528412	318.055139	444.220960	368.166894	184.688124	75.808133	1642.305543	582.449740	nan	0.008700	nan	0.000000	0.000000
27	1	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
28	1	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
29	1	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
30	1	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
31	2	dish_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
32	2	dish_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
33	2	dish_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.808426	7.359330	14.112768	23.862884	5.771596	14.409410	-0.645587	-1.974285	0.672358	0.660726	0.406552	4.872632	4.512327
36	3	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	1.136778	-0.598240	1.069571	0.315278	0.210443	0.712377	-0.716145	1.279786	0.095472	0.083709	0.010442	1.156012	1.153183
37	3	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	-1.001347	16.822069	-0.486419	68.414203	-0.654433	22.333810	-0.541675	1.129090	0.081606	0.038810	0.047320	0.000000	0.000000
38	3	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.116458	17.868622	0.702067	68.334335	-0.350356	22.457530	-0.422861	2.042976	0.083765	0.038294	0.050900	0.000000	0.000000
39	0	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	324.361366	324.556083	inf	inf	275.981504	695.415722	2353.909724	4012.270630	nan	nan	nan	0.000000	0.000000
40	4	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.757788	0.842955	17.607697	9.378994	9.205975	4.119896	-1.473412	0.988282	0.610325	0.612845	0.357042	1.884494	1.971888
41	4	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
42	4	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.246124	3.298838	23.759817	18.198144	11.757405	9.929632	-1.723345	-1.620108	0.606067	0.614605	0.346802	2.501051	2.610600
45	5	RF_maintenance	100.00%	0.00%	100.00%	0.00%	100.00%	0.00%	-0.008775	16.044015	0.041200	67.143897	0.104567	22.451769	-0.506168	1.257612	0.691095	0.083084	0.501523	4.700284	2.697408
46	5	digital_ok	0.00%	0.00%	0.00%	0.00%	7.53%	0.00%	-0.284621	-0.150803	0.237356	-0.662013	1.707667	-0.036069	0.924263	1.434220	0.687650	0.687546	0.406854	-0.090966	-0.076828
50	3	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.183571	12.813850	-0.781100	4.811975	1.930955	8.516604	0.264839	1.998654	0.100822	0.080766	0.012478	1.136277	1.145733
51	3	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	-1.533976	17.649139	-0.928909	68.160744	-0.700148	22.330603	-0.252491	1.587170	0.088374	0.047351	0.036864	0.886171	0.849713
52	3	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	0.918196	3.687850	-0.288663	2.663174	2.125081	5.828313	15.147127	40.563150	0.071830	0.063469	0.005054	1.203154	1.198039
53	3	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	-1.322859	-0.804371	0.680509	0.521093	0.719725	0.813679	-0.010999	0.066750	0.075514	0.072325	0.007027	0.891015	0.889804
54	4	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.824594	5.162333	19.178194	20.626067	8.520492	12.406403	-1.443395	-1.855541	0.610325	0.610512	0.342688	0.000000	0.000000
55	4	RF_maintenance	100.00%	92.92%	100.00%	0.00%	100.00%	0.00%	6.160380	22.192157	24.445879	68.698901	12.528063	21.872858	-1.931005	3.325832	0.295493	0.106978	0.019232	1.924553	1.203422
56	4	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.450163	6.442283	24.063451	24.242722	13.081125	16.402177	-1.435097	-0.411220	0.611203	0.604816	0.333104	5.257109	4.719995
57	4	RF_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
65	3	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	15.475376	16.007834	67.264343	67.730791	19.906804	22.600329	1.625746	2.379296	0.034998	0.035497	0.001618	1.406696	1.413123
66	3	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	14.088690	16.203511	66.851014	68.429858	19.870073	22.560192	1.697289	1.905975	0.033934	0.036820	0.003175	1.251566	1.250190
67	3	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	15.159658	1.613114	66.898661	9.008385	19.531655	2.170233	1.125561	0.599584	0.034840	0.070212	0.050375	1.262674	1.306667
68	3	digital_ok	0.00%	100.00%	100.00%	0.00%	100.00%	0.00%	-0.030879	0.440329	-0.303491	0.921643	-1.243102	-0.635367	-0.731844	-0.771350	0.062831	0.070083	0.006093	1.120098	1.117940
69	4	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.731077	5.136102	25.048953	21.386172	14.405388	12.013245	-1.691865	-1.617705	0.605868	0.618040	0.336749	5.371410	4.948552
70	4	digital_ok	100.00%	0.00%	93.46%	0.00%	100.00%	0.00%	5.199254	7.056703	21.114694	23.754075	10.237398	15.287827	-1.394899	-1.741577	0.622470	0.310535	0.240884	6.266171	2.724369
71	4	RF_maintenance	100.00%	86.44%	0.00%	0.00%	100.00%	0.00%	15.159610	7.181805	19.160525	25.033079	12.961780	16.421702	1.639475	-1.465907	0.326393	0.617186	0.253742	6.224217	2.026718
72	4	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.465742	6.706668	19.722320	24.441187	9.982540	16.241008	-0.523541	-1.696637	0.621523	0.618915	0.326914	4.825451	4.782971
73	5	RF_maintenance	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	-0.266394	0.552296	-0.485803	-0.726737	-0.107233	-0.424961	1.849213	-0.313482	0.708072	0.708379	0.385198	8.997798	8.309537
81	7	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	0.376114	0.387342	1.625770	-0.729848	-0.969003	-0.824688	0.606754	0.035885	0.651638	0.666818	0.416543	1.273137	1.269879
82	7	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	5.996014	1.789335	5.942977	-0.025682	14.719940	8.630283	9.107440	3.640745	0.659812	0.683687	0.409918	4.510666	4.851908
83	7	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	16.834958	16.604997	60.060950	61.350362	19.129215	21.840184	1.427692	1.086725	0.045507	0.048585	0.001112	1.524007	1.756294
84	8	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	7.961862	1.830085	25.426692	8.815631	13.760052	3.245615	-1.729666	-1.291954	0.680345	0.702363	0.381178	5.450240	4.528576
85	8	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
86	8	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
87	8	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.502366	1.411327	20.924242	4.903384	-0.060258	1.857915	0.175411	2.881794	0.702294	0.716417	0.379786	4.879116	4.474995
88	9	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	17.272836	17.262921	59.466196	60.901361	19.098168	21.830153	3.595314	1.653553	0.048819	0.053036	0.002079	1.244747	1.245162
89	9	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.476544	3.807780	-0.638278	0.958076	0.112422	2.044036	3.891596	7.610110	0.708214	0.710932	0.369395	4.980154	4.094267
90	9	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.510824	6.754597	6.778595	2.930548	1.453184	2.286774	1.415591	1.242698	0.704108	0.686758	0.383699	3.458258	2.885930
91	9	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
92	10	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.585188	1.721839	12.338768	10.992723	4.492601	3.542452	0.715055	0.102316	0.656931	0.662629	0.368989	6.253326	5.083167
93	10	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	16.859816	16.995551	67.080360	69.623441	18.573547	21.567933	2.164047	2.704278	0.051841	0.061387	0.003126	1.381655	1.362864
94	10	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.949108	6.315827	20.571699	23.236279	11.171760	14.475359	-0.974509	-1.433159	0.631353	0.626007	0.370503	5.174233	4.199928
98	7	digital_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	16.294064	16.666538	58.008513	58.726702	18.286466	22.005610	11.724932	1.204609	0.041431	0.047347	0.006960	1.300121	1.320324
99	7	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	5.626231	0.142335	1.942020	-0.788957	0.647470	4.464373	0.587994	0.629439	0.665255	0.677574	0.406136	4.096514	3.576472
100	7	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	17.223931	16.978852	55.974470	59.744645	18.815741	21.515873	2.476792	1.907280	0.045486	0.049166	0.004134	1.358611	1.413971
101	8	RF_maintenance	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.186409	0.145678	-0.541381	-0.479303	-0.136235	0.147740	2.398258	-0.626364	0.699217	0.708939	0.386356	4.187993	3.752612
102	8	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
103	8	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	1.114825	0.562406	1.477811	-0.394631	0.896920	1.445011	-0.827635	-0.670272	0.712850	0.717167	0.378915	1.671922	1.595162
104	8	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	-0.816619	-0.322548	-0.221485	-0.793244	0.297370	1.387355	-0.210238	-0.784682	0.718718	0.722608	0.379677	1.729763	1.628788
105	9	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
106	9	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.788214	1.500081	-0.515073	0.027949	5.407333	4.146600	-0.109427	0.094144	0.715739	0.719466	0.372209	3.847238	3.524582
107	9	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.700049	0.145045	0.895744	-0.682848	2.519940	2.697615	8.269591	5.470176	0.705718	0.715180	0.380755	3.985671	3.638494
108	9	digital_ok	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	nan	nan	inf	inf	nan	nan	nan	nan	nan	nan	nan	0.000000	0.000000
109	10	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	1.524318	-0.799377	-0.442144	-0.861557	-0.559199	-0.564336	-0.346460	0.895701	0.662189	0.670115	0.366626	1.710079	1.733124
110	10	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	-0.098711	0.259531	-0.126487	-0.023973	-0.309865	0.075343	2.721537	4.898543	0.658668	0.665559	0.374502	6.227290	6.046282
111	10	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	-0.101269	1.129734	-0.813355	-0.668473	-0.969156	0.323007	-0.519430	-0.313475	0.648370	0.652238	0.373204	1.625158	1.754690
112	10	RF_maintenance	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	-0.017638	-0.784025	-0.182099	0.875692	-1.390805	1.302951	-0.309265	-0.483192	0.631074	0.640448	0.385101	3.545926	3.765083
116	7	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	18.951348	19.107166	60.329105	61.845747	18.743006	21.516281	1.373984	1.988060	0.046783	0.045059	0.002123	1.390110	1.392358
119	7	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.479569	-0.304587	-0.476896	1.063611	-0.849978	-0.589079	-0.373637	4.585066	0.691919	0.704613	0.394016	4.017349	4.096639
120	8	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	0.156996	-0.699482	0.783846	0.077197	3.226977	-0.783707	2.429002	0.682011	0.698640	0.710882	0.381098	1.766578	1.640155
121	8	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.350941	2.828488	15.477377	13.142137	4.845055	0.421151	10.583624	22.040427	0.707173	0.713243	0.373050	5.486639	4.163605
122	8	digital_maintenance	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.147127	0.882543	-0.400358	-0.333714	0.600299	-0.510740	-0.464897	-0.489326	0.716521	0.721822	0.372046	4.958842	4.134153
123	8	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	7.168259	5.190713	25.257841	20.969294	13.696626	13.102760	-1.797034	-1.740687	0.707874	0.717314	0.371674	3.834142	3.772469
124	9	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	1.227900	1.155363	1.190579	-0.360646	-0.622079	-1.013647	-0.270235	-0.313989	0.725582	0.730247	0.373420	1.881285	1.653634
125	9	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	15.998178	15.946219	59.741117	61.953946	18.929580	21.870295	0.182106	2.179745	0.034923	0.039009	0.003165	1.207554	1.208384
126	9	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	18.118376	19.418249	59.289832	62.307320	19.139999	22.009559	1.367017	2.108311	0.038751	0.042141	0.000336	1.206840	1.210240
127	10	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.994327	0.016506	5.204879	4.319251	2.753200	0.427774	3.493725	0.063560	0.668879	0.679843	0.363707	5.031909	5.448128
128	10	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	-1.668574	0.307899	-0.770306	1.899202	-0.308822	0.106067	-0.810033	-0.499757	0.664354	0.671374	0.370338	1.630179	1.622686
129	10	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	0.017638	-0.512846	2.642772	3.072544	0.214334	-0.018377	-0.300609	-0.291288	0.646664	0.659102	0.366177	1.780346	1.780606
130	10	digital_maintenance	100.00%	0.00%	100.00%	0.00%	100.00%	0.00%	-1.704484	16.499349	-1.012944	69.016920	-0.876723	22.207750	-0.140766	4.942112	0.642831	0.062884	0.441637	4.518552	5.804087
135	12	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	-1.783713	-0.259517	-1.013007	-0.813131	-0.250878	-0.704424	1.479824	2.054000	0.643866	0.662438	0.415481	1.339794	1.389602
136	12	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.096726	13.268792	-0.749703	3.122760	0.301727	1.950022	-0.122314	1.975216	0.662641	0.656796	0.383018	4.078416	5.235301
138	7	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	1.265065	2.979437	0.106710	0.980657	-0.650241	0.693100	-0.174708	0.043448	0.685247	0.705304	0.390696	1.660241	1.648712
140	13	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	10.75%	0.352903	0.400702	1.062286	1.067676	0.151605	2.471914	0.948256	3.196690	0.706334	0.712784	0.368526	2.101588	1.949889
141	13	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	4.30%	0.779965	0.156976	-0.642537	-0.309835	2.691864	-0.738602	0.247011	-0.678709	0.710532	0.719207	0.362494	2.158690	1.954778
142	13	RF_maintenance	100.00%	0.00%	100.00%	0.00%	100.00%	0.00%	2.912216	18.358507	-0.327154	68.938114	0.199665	22.468486	2.255688	1.474472	0.718636	0.059788	0.467400	4.229436	1.526913
143	14	RF_maintenance	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.862754	-0.377593	-0.466507	0.478994	0.206060	0.521858	-0.026180	-0.835427	0.723094	0.731466	0.367395	5.299235	4.794542
144	14	RF_maintenance	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	-1.662710	-1.485667	-0.982194	-0.194495	-1.364602	-1.104597	-0.382549	0.261135	0.719503	0.723407	0.377333	5.074911	4.651958
145	14	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	2.15%	-0.823560	-0.218412	1.533886	1.465666	0.969359	-0.198281	1.741533	1.730808	0.708551	0.716632	0.380087	1.836424	1.755393
150	15	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	16.675236	16.359009	67.571206	70.763615	19.584918	22.307662	1.946894	2.452923	0.053715	0.055935	0.001431	1.315398	1.322515
155	12	RF_maintenance	100.00%	100.00%	100.00%	0.00%	100.00%	0.00%	14.483950	15.533860	65.542016	67.498271	19.792249	23.147117	1.810769	3.449229	0.063040	0.063295	0.001448	1.648324	1.755278
156	12	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.089461	4.408065	23.757385	17.681237	-0.375367	11.851656	2.331223	22.952601	0.646568	0.651371	0.390284	3.524617	3.397211
157	12	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.726545	-0.048403	6.977802	16.611337	0.041295	0.649686	1.682628	0.691245	0.675332	0.684846	0.394913	4.227065	4.004896
158	12	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.181217	0.561901	2.292051	15.880347	3.201101	2.575473	2.840513	13.976499	0.684616	0.695933	0.390728	4.212152	4.150024
160	13	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	0.342530	-0.859455	2.336998	0.363287	0.103915	0.276039	2.212244	-0.033021	0.699409	0.712058	0.382452	1.779686	1.613183
161	13	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.983612	34.763241	-0.469166	5.877333	-0.836728	5.793719	-0.359491	0.391925	0.709641	0.610173	0.349290	4.900384	6.279951
162	13	digital_maintenance	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.149444	-0.521755	-0.533540	-0.850619	2.778989	0.824397	1.206923	1.489086	0.715259	0.723846	0.375232	4.266770	4.256831
163	14	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	0.505827	-0.716322	-0.216666	-0.869925	-1.097495	0.231893	0.763347	2.381827	0.719315	0.726057	0.376129	1.886046	1.869003
164	14	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	-0.492344	-1.199854	-0.788065	-0.468307	-0.947086	1.037697	1.231832	1.757402	0.711825	0.719016	0.379924	1.883995	1.820645
165	14	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	-0.649500	0.354827	0.495464	-0.699168	1.900979	-0.655882	2.411611	-0.457120	0.705435	0.715776	0.389511	1.785722	1.833985
166	14	RF_maintenance	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	-0.960451	-0.834359	0.525329	1.429465	0.018377	0.262165	0.010999	0.947732	0.698804	0.710218	0.376698	4.184426	3.792476
167	15	digital_ok	100.00%	11.35%	0.00%	0.00%	100.00%	0.00%	33.943253	23.916929	14.752659	19.365729	18.972464	19.229250	61.414063	19.952393	0.522077	0.527331	0.206735	2.800558	2.616640
168	15	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	4.036223	7.240934	16.021087	23.319387	5.630609	13.890688	-0.519048	-1.702307	0.693232	0.693717	0.391839	8.292630	5.667146
169	15	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.618482	5.714158	22.503877	21.098907	11.391417	12.762249	-1.783579	-1.348122	0.673135	0.681260	0.408392	4.490917	3.482702
170	15	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.297140	3.582965	22.959601	17.509670	11.930142	8.832128	1.312920	1.585653	0.657367	0.676903	0.419255	12.034356	10.944508
176	12	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	2.910306	6.800551	-0.269204	23.481596	-0.365956	14.560558	0.772455	-1.173755	0.656005	0.655738	0.405374	3.617242	3.096087
177	12	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.307978	3.295050	24.055331	16.231544	13.239400	11.501246	-1.447254	1.212332	0.654203	0.677457	0.405366	3.194611	3.290441
178	12	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	5.641812	6.367175	21.972560	22.710191	11.492568	14.222018	-1.462397	-1.872537	0.671240	0.680483	0.395377	3.726835	3.291207
179	12	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.156638	0.210936	0.272809	0.569525	4.046211	-0.340510	5.592535	0.325016	0.689726	0.701891	0.400538	3.729858	3.526019
180	13	RF_maintenance	100.00%	94.00%	100.00%	0.00%	100.00%	0.00%	51.643742	18.467348	7.879643	69.539235	9.463650	22.129740	0.915138	1.823895	0.276385	0.070186	0.131486	3.946972	3.714472
181	13	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	0.142338	1.659687	0.814748	1.217193	-0.840743	-0.408891	-0.489422	2.176315	0.706410	0.710418	0.392708	1.759342	1.586048
182	13	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	3.567959	1.413895	17.010551	2.325130	7.978224	54.677653	-0.430446	13.001901	0.705686	0.707700	0.386311	4.185354	3.981118
183	13	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	-1.792720	-1.718654	0.077720	0.383835	-0.368458	0.368990	0.653818	1.845194	0.713347	0.717452	0.384510	1.790812	1.778375
184	14	RF_maintenance	0.00%	0.00%	0.00%	0.00%	100.00%	0.00%	-0.106200	-0.751256	0.244812	0.311994	0.941766	-0.868543	-0.013332	-0.340401	0.712612	0.713642	0.384139	5.680741	4.856519
185	14	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	0.00%	1.087594	0.118621	-0.316655	-0.077197	-0.171061	-0.518753	-0.111014	1.076474	0.707706	0.712378	0.387193	1.697740	1.723815
186	14	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.907908	0.320073	-0.476819	-0.759371	2.839267	0.394768	3.978653	0.403433	0.700866	0.707727	0.396001	4.446553	4.539300
187	14	RF_maintenance	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	0.003116	0.941681	-0.284502	-0.202887	-0.206411	1.860229	2.760416	27.564333	0.694823	0.708404	0.392838	3.234820	3.045641
189	15	digital_ok	0.00%	0.00%	0.00%	0.00%	5.38%	94.62%	1.650922	1.064608	1.560642	0.886993	-0.779731	1.944344	0.712884	-0.339835	0.679519	0.690529	0.413795	12.020165	11.927929
190	15	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	13.853716	37.372817	2.477278	5.085869	9.359927	8.134918	27.660606	14.560019	0.633830	0.588100	0.366169	2.620464	2.232928
191	15	digital_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	1.149141	1.163686	6.127927	-0.662326	-0.228659	-0.365749	0.684172	2.518014	0.658699	0.670217	0.435236	3.227077	3.010954
220	18	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	7.962633	9.777834	27.746495	29.566964	15.492981	19.451647	-1.958021	-2.593481	0.660379	0.649950	0.399101	4.437261	3.345837
221	18	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	6.814064	9.259342	24.910962	28.603543	12.402123	18.939313	-1.509228	-2.518833	0.668704	0.656615	0.398125	5.018781	3.640594
222	18	RF_ok	100.00%	0.00%	0.00%	0.00%	100.00%	0.00%	8.415518	9.342660	28.677185	28.385035	15.779296	21.686618	-1.538465	-2.246831	0.653283	0.656737	0.393975	3.979976	3.236357

# print ex_ants for easy copy-pasting to YAML file
proposed_ex_ants = [ant for i, ant in enumerate(ants) if np.any([col[i] > .1 for col in bar_cols.values()])]
print('ex_ants: [' + ", ".join(str(ant) for ant in proposed_ex_ants) + ']')

ex_ants: [0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 41, 42, 45, 50, 51, 52, 53, 54, 55, 56, 57, 65, 66, 67, 68, 69, 70, 71, 72, 73, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 98, 99, 100, 101, 102, 105, 106, 107, 108, 110, 112, 116, 119, 121, 122, 123, 125, 126, 127, 130, 136, 140, 142, 143, 144, 150, 155, 156, 157, 158, 161, 162, 166, 167, 168, 169, 170, 176, 177, 178, 179, 180, 182, 184, 186, 187, 189, 190, 191, 220, 221, 222]

# write to csv
outpath = os.path.join(nb_outdir, f'rtp_summary_table_{JD}.csv')
print(f'Now saving Table 1 to a csv at {outpath}')
df.to_csv(outpath)

Now saving Table 1 to a csv at /home/obs/src/H5C_Notebooks/_rtp_summary_/rtp_summary_table_2459563.csv

# Load antenna positions
data_list = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}.?????.sum.uvh5')))
hd = io.HERAData(data_list[len(data_list) // 2])

# Figure out where to draw the nodes
node_centers = {}
for node in sorted(set(list(nodes.values()))):
    if np.isfinite(node):
        this_node_ants = [ant for ant in ants + unused_ants if nodes[ant] == node]
        if len(this_node_ants) == 1:
            # put the node label just to the west of the lone antenna 
            node_centers[node] = hd.antpos[ant][node] + np.array([-14.6 / 2, 0, 0])
        else:
            # put the node label between the two antennas closest to the node center
            node_centers[node] = np.mean([hd.antpos[ant] for ant in this_node_ants], axis=0)
            closest_two_pos = sorted([hd.antpos[ant] for ant in this_node_ants], 
                                     key=lambda pos: np.linalg.norm(pos - node_centers[node]))[0:2]
            node_centers[node] = np.mean(closest_two_pos, axis=0)

def Plot_Array():
    plt.figure(figsize=(16,16))
    
    plt.scatter(np.array([hd.antpos[ant][0] for ant in hd.data_ants]), 
                np.array([hd.antpos[ant][1] for ant in hd.data_ants]), c='w', s=0)

    # connect every antenna to their node
    for ant in ants:
        if nodes[ant] in node_centers:
            plt.plot([hd.antpos[ant][0], node_centers[nodes[ant]][0]], 
                     [hd.antpos[ant][1], node_centers[nodes[ant]][1]], 'k', zorder=0)

    rc_color = '#0000ff'
    antm_color = '#ffa500'
    autom_color = '#ff1493'

    # Plot 
    unflagged_ants = []
    for i, ant in enumerate(ants):
        ant_has_flag = False
        # plot large blue annuli for redcal flags
        if use_redcal:
            if redcal_flagged_frac[ant] > 0:
                ant_has_flag = True
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=7, fill=True, lw=0,
                                                color=rc_color, alpha=redcal_flagged_frac[ant]))
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=6, fill=True, color='w'))
        
        # plot medium green annuli for ant_metrics flags
        if use_ant_metrics: 
            if ant_metrics_xants_frac_by_ant[ant] > 0:
                ant_has_flag = True
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=6, fill=True, lw=0,
                                                color=antm_color, alpha=ant_metrics_xants_frac_by_ant[ant]))
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=5, fill=True, color='w'))
        
        # plot small red annuli for auto_metrics
        if use_auto_metrics:
            if ant in auto_ex_ants:
                ant_has_flag = True                
                plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=5, fill=True, lw=0, color=autom_color)) 
        
        # plot black/white circles with black outlines for antennas
        plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=4, fill=True, color=['w', 'k'][ant_has_flag], ec='k'))
        if not ant_has_flag:
            unflagged_ants.append(ant)

        # label antennas, using apriori statuses if available
        try:
            bgc = matplotlib.colors.to_rgb(status_colors[a_priori_statuses[ant]])
            c = 'black' if (bgc[0]*0.299 + bgc[1]*0.587 + bgc[2]*0.114) > 186 / 256 else 'white'
        except:
            c = 'k'
            bgc='white'
        plt.text(hd.antpos[ant][0], hd.antpos[ant][1], str(ant), va='center', ha='center', color=c, backgroundcolor=bgc)

    # label nodes
    for node in sorted(set(list(nodes.values()))):
        if not np.isnan(node) and not np.all(np.isnan(node_centers[node])):
            plt.text(node_centers[node][0], node_centers[node][1], str(node), va='center', ha='center', bbox={'color': 'w', 'ec': 'k'})
    
    # build legend 
    legend_objs = []
    legend_labels = []
    
    # use circles for annuli 
    legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgecolor='k', markerfacecolor='w', markersize=13))
    legend_labels.append(f'{len(unflagged_ants)} / {len(ants)} Total Antennas Never Flagged')
    legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markerfacecolor='k', markersize=15))
    legend_labels.append(f'{len(ants) - len(unflagged_ants)} Antennas Flagged for Any Reason')

    if use_auto_metrics:
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgewidth=2, markeredgecolor=autom_color, markersize=15))
        legend_labels.append(f'{len(auto_ex_ants)} Antennas Flagged by Auto Metrics')
    if use_ant_metrics: 
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgewidth=2, markeredgecolor=antm_color, markersize=15))
        legend_labels.append(f'{np.round(np.sum(list(ant_metrics_xants_frac_by_ant.values())), 2)} Antenna-Nights on' 
                             f'\n{np.sum([frac > 0 for frac in ant_metrics_xants_frac_by_ant.values()])} Antennas '
                             'Flagged by Ant Metrics\n(alpha indicates fraction of time)')        
    if use_redcal:
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markeredgewidth=2, markeredgecolor=rc_color, markersize=15))
        legend_labels.append(f'{np.round(np.sum(list(redcal_flagged_frac.values())), 2)} Antenna-Nights on' 
                             f'\n{np.sum([frac > 0 for frac in redcal_flagged_frac.values()])} Antennas '
                             'Flagged by Redcal\n(alpha indicates fraction of time)')

    # use rectangular patches for a priori statuses that appear in the array
    for aps in sorted(list(set(list(a_priori_statuses.values())))):
        if aps != 'Not Found':
            legend_objs.append(plt.Circle((0, 0), radius=7, fill=True, color=status_colors[aps]))
            legend_labels.append(f'A Priori Status:\n{aps} ({list(a_priori_statuses.values()).count(aps)} Antennas)')

    # label nodes as a white box with black outline
    if len(node_centers) > 0:
        legend_objs.append(matplotlib.patches.Patch(facecolor='w', edgecolor='k'))
        legend_labels.append('Node Number')

    if len(unused_ants) > 0:
        legend_objs.append(matplotlib.lines.Line2D([0], [0], marker='o', color='w', markerfacecolor='grey', markersize=15, alpha=.2))
        legend_labels.append(f'Anntenna Not In Data')
        
    
    plt.legend(legend_objs, legend_labels, ncol=2, fontsize='large', framealpha=1)
    
    # set axis equal and label everything
    plt.axis('equal')
    plt.tight_layout()
    plt.title(f'Summary of Antenna Statuses and Metrics on {JD}', size=20)    
    plt.xlabel("Antenna East-West Position (meters)", size=12)
    plt.ylabel("Antenna North-South Position (meters)", size=12)
    plt.xticks(fontsize=12)
    plt.yticks(fontsize=12)
    xlim = plt.gca().get_xlim()
    ylim = plt.gca().get_ylim()
    
    # plot unused antennas
    plt.autoscale(False)    
    for ant in unused_ants:
        if nodes[ant] in node_centers:
            plt.plot([hd.antpos[ant][0], node_centers[nodes[ant]][0]], 
                     [hd.antpos[ant][1], node_centers[nodes[ant]][1]], 'k', alpha=.2, zorder=0)
        
        plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=4, fill=True, color='w', ec=None, alpha=1, zorder=0))
        plt.gca().add_artist(plt.Circle(tuple(hd.antpos[ant][0:2]), radius=4, fill=True, color='grey', ec=None, alpha=.2, zorder=0))
        if hd.antpos[ant][0] < xlim[1] and hd.antpos[ant][0] > xlim[0]:
            if hd.antpos[ant][1] < ylim[1] and hd.antpos[ant][1] > ylim[0]:
                plt.text(hd.antpos[ant][0], hd.antpos[ant][1], str(ant), va='center', ha='center', color='k', alpha=.2)
#     plt.gca().set_xlim(*xlim)
#     plt.gca().set_ylim(*ylim)

Figure 1: Array Plot of Flags and A Priori Statuses

This plot shows all antennas, which nodes they are connected to, and their a priori statuses (as the highlight text of their antenna numbers). It may also show (depending on what is finished running):

• Whether they were flagged by auto_metrics (red circle) for bandpass shape, overall power, temporal variability, or temporal discontinuities. This is done in a binary fashion for the whole night.
• Whether they were flagged by ant_metrics (green circle) as either dead (on either polarization) or crossed, with the transparency indicating the fraction of the night (i.e. number of files) that were flagged.
• Whether they were flagged by redcal (blue circle) for high s either dead (on either polarization) or crossed, with the transparency indicating the fraction of the night (i.e. number of files) that were flagged.

Note that the last fraction does not include antennas that were flagged before going into redcal due to their a priori status, for example.

Plot_Array()

Metadata

from hera_qm import version
print(version.construct_version_info())
print(redcal.version.history_string())

{'version': '1.0', 'git_origin': 'https://github.com/HERA-Team/hera_qm.git', 'git_hash': 'd5d91cd9b768b99faeed3e7819ce128639d09330', 'git_description': 'v1.1-547-gd5d91cd', 'git_branch': 'master'}

------------
This file was produced by the function <module>() in 3622974967.py using: 
    git_branch: main
    git_description: v3.0-1283-gbaee107
    git_hash: baee107cc5e73bfa8ae0339672805fd2199448c3
    git_origin: https://github.com/HERA-Team/hera_cal.git
    version: 3.0
------------