Antenna Classification Daily Summary

by Josh Dillon last updated October 17, 2022

This notebook parses and summarizes the output of the file_calibration notebook to produce a report on per-antenna malfunctions on a daily basis.

Quick links:

• Summary of Per Antenna Issues

• Figure 1: Per File Overall Antenna Classification Summary

• Figure 2: Per Classifier Antenna Flagging Summary

• Figure 3: Array Visualization of Overall Daily Classification

import numpy as np
import pandas as pd
import glob
import os
import matplotlib.pyplot as plt
from hera_cal import io, utils
from hera_qm import ant_class
from uvtools.plot import plot_antpos, plot_antclass
%matplotlib inline
from IPython.display import display, HTML

Settings

# Parse settings from environment
ANT_CLASS_FOLDER = os.environ.get("ANT_CLASS_FOLDER", "./")
SUM_FILE = os.environ.get("SUM_FILE", None)
OC_SKIP_OUTRIGGERS = os.environ.get("OC_SKIP_OUTRIGGERS", "TRUE").upper() == "TRUE"

for param in ['ANT_CLASS_FOLDER', 'SUM_FILE', 'OC_SKIP_OUTRIGGERS']:
    print(f"{param} = '{eval(param)}'")

ANT_CLASS_FOLDER = '/mnt/sn1/2460047'
SUM_FILE = '/mnt/sn1/2460047/zen.2460047.46140.sum.uvh5'
OC_SKIP_OUTRIGGERS = 'True'

if SUM_FILE is not None:
    from astropy.time import Time, TimeDelta
    utc = Time(float(SUM_FILE.split('zen.')[-1].split('.sum.uvh5')[0]), format='jd').datetime
    print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 4-12-2023

# set thresholds for fraction of the day
overall_thresh = .1
all_zero_thresh = .1
eo_zeros_thresh = .1
cross_pol_thresh = .5
bad_fem_thresh = .1
high_power_thresh = .1
low_power_thresh = .1
low_corr_thresh = .1
bad_shape_thresh = .5
excess_rfi_thresh = .1
chisq_thresh = .25

Load classifications and other metadata

# Load csvs
csv_files = sorted(glob.glob(os.path.join(ANT_CLASS_FOLDER, '*.ant_class.csv')))
jds = [float(f.split('/')[-1].split('zen.')[-1].split('.sum')[0]) for f in csv_files]
tables = [pd.read_csv(f).dropna(axis=0, how='all') for f in csv_files]
table_cols = tables[0].columns[1::2]
class_cols = tables[0].columns[2::2]
print(f'Found {len(csv_files)} csv files starting with {csv_files[0]}')

Found 361 csv files starting with /mnt/sn1/2460047/zen.2460047.42114.sum.ant_class.csv

# parse ant_strings
ap_strs = np.array(tables[0]['Antenna'])
ants = sorted(set(int(a[:-1]) for a in ap_strs))
translator = ''.maketrans('e', 'n') | ''.maketrans('n', 'e')

# get node numbers
node_dict = {ant: 'Unknown' for ant in ants}
try:
    from hera_mc import cm_hookup
    hookup = cm_hookup.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in node_dict:
            if hookup[ant_name].get_part_from_type('node')['E<ground'] is not None:
                node_dict[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])
except:
    pass
nodes = sorted(set(node_dict.values()))

def classification_array(col):
    class_array = np.vstack([t[col] for t in tables])
    class_array[class_array == 'good'] = 1.7
    class_array[class_array == 'suspect'] = 1
    class_array[class_array == 'bad'] = 0
    return class_array.astype(float)

if SUM_FILE is not None:
    hd = io.HERADataFastReader(SUM_FILE)
    ap_tuples = [(int(ap[:-1]), {'e': 'Jee', 'n': 'Jnn'}[ap[-1]]) for ap in ap_strs]
    bad_bools = np.mean(classification_array('Antenna Class') == 0, axis=0) > overall_thresh
    bad_aps = [ap_tuples[i] for i in np.arange(len(ap_tuples))[bad_bools]]
    suspect_bools = np.mean(classification_array('Antenna Class') == 1, axis=0) > overall_thresh
    suspect_aps = [ap_tuples[i] for i in np.arange(len(ap_tuples))[suspect_bools] if ap_tuples[i] not in bad_aps]
    good_aps = [ap for ap in ap_tuples if ap not in bad_aps and ap not in suspect_aps]
    overall_class = ant_class.AntennaClassification(bad=bad_aps, suspect=suspect_aps, good=good_aps)
    autos, _, _ = hd.read(bls=[bl for bl in hd.bls if utils.split_bl(bl)[0] == utils.split_bl(bl)[1]], read_flags=False, read_nsamples=False)
    avg_unflagged_auto = {pol: np.mean([autos[bl] for bl in autos if bl[2] == pol and overall_class[utils.split_bl(bl)[0]] != 'bad'], axis=(0, 1)) for pol in ['ee', 'nn']}

Figure out and summarize per-antenna issues

def print_issue_summary(bad_ant_strs, title, notes='', plot=False):
    '''Print report for list of bad antenna polarizations strings'''
    unique_bad_antnums = [int(ap[:-1]) for ap in bad_ant_strs]
    display(HTML(f'<h2>{title}: ({len(bad_ant_strs)} antpols across {len(set([ba[:-1] for ba in bad_ant_strs]))} antennas)</h2>'))
    if len(notes) > 0:
        display(HTML(f'<h4>{notes}</h4>'))
    if len(bad_ant_strs) > 0:
        print(f'All Bad Antpols: {", ".join(bad_ant_strs)}\n')
    for node in nodes:
        if np.any([node == node_dict[a] for a in unique_bad_antnums]):
            aps = [ap for ap in bad_ant_strs if node_dict[int(ap[:-1])] == node]
            whole_ants = [str(wa) for wa in set([int(ap[:-1]) for ap in aps if ap.translate(translator) in bad_ant_strs])]
            single_pols =  [ap for ap in aps if ap.translate(translator) not in bad_ant_strs]
            print(f'Node {node}:')
            print(f'\tAntpols ({len(aps)} total): {", ".join(aps)}')
            print(f'\tWhole Ants ({len(whole_ants)} total): {", ".join(whole_ants)}')
            print(f'\tSingle Pols ({len(single_pols)} total): {", ".join(single_pols)}')
            if plot and SUM_FILE is not None:
                fig, axes = plt.subplots(1, 2, figsize=(12,4), dpi=70, sharey=True, gridspec_kw={'wspace': 0})
                for ax, pol in zip(axes, ['ee', 'nn']):                    
                    ax.semilogy(autos.freqs / 1e6, avg_unflagged_auto[pol], 'k--', label='Average\nUnflagged\nAuto')
                    for ap in aps:
                        ant = int(ap[:-1]), utils.comply_pol(ap[-1])
                        auto_bl = utils.join_bl(ant, ant)
                        if auto_bl[2] == pol:
                            ax.semilogy(autos.freqs / 1e6, np.mean(autos[auto_bl], axis=0), label=ap)
                    ax.legend()
                    ax.set_xlim([40, 299])
                    ax.set_title(f'{title} on Node {node} ({pol}-antennas)')
                    ax.set_xlabel('Frequency (MHz)')
                axes[0].set_ylabel('Single File Raw Autocorrelation')
                plt.tight_layout()
                plt.show() 

# precompute various helpful quantities
all_slopes = np.vstack([t['Autocorr Slope'] for t in tables])
median_slope = np.median(all_slopes)
bad_slopes = np.vstack([t['Autocorr Slope Class'] for t in tables]) == 'bad'
suspect_slopes = np.vstack([t['Autocorr Slope Class'] for t in tables]) == 'suspect'
bad_shapes = np.vstack([t['Autocorr Shape Class'] for t in tables]) == 'bad'
suspect_shapes = np.vstack([t['Autocorr Shape Class'] for t in tables]) == 'suspect'
all_powers = np.vstack([t['Autocorr Power'] for t in tables])
median_power = np.median(all_powers)
bad_powers = np.vstack([t['Autocorr Power Class'] for t in tables]) == 'bad'
suspect_powers = np.vstack([t['Autocorr Power Class'] for t in tables]) == 'suspect'
bad_rfi = np.vstack([t['RFI in Autos Class'] for t in tables]) == 'bad'
suspect_rfi = np.vstack([t['RFI in Autos Class'] for t in tables]) == 'suspect'

# find all zeros
all_zeros_strs = ap_strs[np.mean(np.vstack([t['Dead? Class'] for t in tables]) == 'bad', axis=0) > all_zero_thresh]

# find even/odd zeros
eo_zeros_strs = ap_strs[np.mean(np.vstack([t['Even/Odd Zeros Class'] for t in tables]) == 'bad', axis=0) > eo_zeros_thresh]
eo_zeros_strs = [ap for ap in eo_zeros_strs if ap not in all_zeros_strs] 

# find cross-polarized antennas
cross_pol_strs = ap_strs[np.mean(np.vstack([t['Cross-Polarized Class'] for t in tables]) == 'bad', axis=0) > cross_pol_thresh]

# find FEM power issues: must be low power, high slope, and bad or suspect in power, slope, rfi, and shape
fem_off_prod = (bad_powers + .5 * suspect_powers) * (bad_slopes + .5 * suspect_slopes)
fem_off_prod *= (bad_rfi + .5 * suspect_rfi) * (bad_shapes + .5 * suspect_shapes)
fem_off_strs = ap_strs[np.mean(fem_off_prod * (all_powers < median_power) * (all_slopes > median_slope), axis=0) > .1]

# find high power issues
high_power_strs = ap_strs[np.mean(bad_powers & (all_powers > median_power), axis=0) > high_power_thresh]

# find other low power issues
low_power_strs = ap_strs[np.mean(bad_powers & (all_powers < median_power), axis=0) > low_power_thresh]
low_power_strs = [ap for ap in low_power_strs if ap not in all_zeros_strs and ap not in fem_off_strs] 

# find low correlation (but not low power)
low_corr_strs = ap_strs[np.mean(np.vstack([t['Low Correlation Class'] for t in tables]) == 'bad', axis=0) > low_corr_thresh]
low_corr_strs = [ap for ap in low_corr_strs if ap not in (set(low_power_strs) | set(all_zeros_strs) | set(fem_off_strs))] 

# find bad bandpasses
bad_bandpass_strs = ap_strs[np.mean(bad_shapes, axis=0) > bad_shape_thresh]
bad_bandpass_strs = [ap for ap in bad_bandpass_strs if ap not in (set(low_power_strs) | set(all_zeros_strs) | set(high_power_strs) | set(fem_off_strs))]

# find antennas with excess RFI
excess_rfi_strs = ap_strs[np.mean(np.vstack([t['RFI in Autos Class'] for t in tables]) == 'bad', axis=0) > excess_rfi_thresh]
excess_rfi_strs = [ap for ap in excess_rfi_strs if ap not in (set(low_power_strs) | set(all_zeros_strs) |  set(fem_off_strs) |
                                                              set(bad_bandpass_strs) | set(high_power_strs))] 

# find antennas with high redcal chi^2
chisq_strs = ap_strs[np.mean(np.vstack([t['Redcal chi^2 Class'] for t in tables]) == 'bad', axis=0) > chisq_thresh]
chisq_strs = [ap for ap in chisq_strs if ap not in (set(bad_bandpass_strs) | set(low_power_strs) | set(excess_rfi_strs) | set(low_corr_strs) |
                                                    set(all_zeros_strs) | set(high_power_strs) | set(fem_off_strs) | set(eo_zeros_strs))]
if OC_SKIP_OUTRIGGERS:
    chisq_strs = [ap for ap in chisq_strs if int(ap[:-1]) < 320]

# collect all results
to_print = [(all_zeros_strs, 'All-Zeros', 'These antennas have visibilities that are more than half zeros.'),
            (eo_zeros_strs, 'Excess Zeros in Either Even or Odd Spectra', 
             'These antennas are showing evidence of packet loss or X-engine failure.', True),
            (cross_pol_strs, 'Cross-Polarized', 'These antennas have their east and north cables swapped.'),
            (fem_off_strs, 'Likely FEM Power Issue', 'These antennas have low power, anomolously high slopes, and extra channels identified as RFI.', True),
            (high_power_strs, 'High Power', 'These antennas have high median power.', True),
            (low_power_strs, 'Other Low Power Issues', 'These antennas have low power, but are not all-zeros and not FEM off.', True),
            (low_corr_strs, 'Low Correlation, But Not Low Power', 'These antennas are low correlation, but their autocorrelation power levels look OK.'),
            (bad_bandpass_strs, 'Bad Bandpass Shapes, But Not Bad Power', 
             'These antennas have unusual bandpass shapes, but are not all-zeros, high power, low power, or FEM off.', True),
            (excess_rfi_strs, 'Excess RFI', 'These antennas have excess strucutre (identified as possible RFI) in their bandpassed relative to the ' + \
             'median antenna, but not low or high power or a bad bandpass.', True),
            (chisq_strs, 'Redcal chi^2', 'These antennas have been idenfied as not redundantly calibrating well, even after passing the above checks.')]

def print_high_level_summary():
    for tp in sorted(to_print, key=lambda x: len(x[0]), reverse=True):
        print(f'{len(tp[0])} antpols (on {len(set([ap[:-1] for ap in tp[0]]))} antennas) frequently flagged for {tp[1]}.')
        
def print_all_issue_summaries():
    for tp in to_print:
        print_issue_summary(*tp)

Summary of Per-Antenna Issues

print_high_level_summary()

37 antpols (on 30 antennas) frequently flagged for Likely FEM Power Issue.
34 antpols (on 17 antennas) frequently flagged for All-Zeros.
17 antpols (on 11 antennas) frequently flagged for High Power.
16 antpols (on 10 antennas) frequently flagged for Other Low Power Issues.
12 antpols (on 11 antennas) frequently flagged for Bad Bandpass Shapes, But Not Bad Power.
12 antpols (on 9 antennas) frequently flagged for Redcal chi^2.
11 antpols (on 6 antennas) frequently flagged for Excess Zeros in Either Even or Odd Spectra.
8 antpols (on 4 antennas) frequently flagged for Cross-Polarized.
8 antpols (on 7 antennas) frequently flagged for Excess RFI.
3 antpols (on 3 antennas) frequently flagged for Low Correlation, But Not Low Power.

print_all_issue_summaries()

All-Zeros: (34 antpols across 17 antennas)

These antennas have visibilities that are more than half zeros.

All Bad Antpols: 135e, 135n, 136e, 136n, 147e, 147n, 148e, 148n, 149e, 149n, 150e, 150n, 155e, 155n, 167e, 167n, 168e, 168n, 169e, 169n, 170e, 170n, 189e, 189n, 190e, 190n, 191e, 191n, 208e, 208n, 209e, 209n, 210e, 210n

Node 12:
	Antpols (6 total): 135e, 135n, 136e, 136n, 155e, 155n
	Whole Ants (3 total): 136, 155, 135
	Single Pols (0 total): 
Node 15:
	Antpols (22 total): 147e, 147n, 148e, 148n, 149e, 149n, 150e, 150n, 167e, 167n, 168e, 168n, 169e, 169n, 170e, 170n, 189e, 189n, 190e, 190n, 191e, 191n
	Whole Ants (11 total): 167, 168, 169, 170, 147, 148, 149, 150, 189, 190, 191
	Single Pols (0 total): 
Node 20:
	Antpols (6 total): 208e, 208n, 209e, 209n, 210e, 210n
	Whole Ants (3 total): 208, 209, 210
	Single Pols (0 total): 

Excess Zeros in Either Even or Odd Spectra: (11 antpols across 6 antennas)

These antennas are showing evidence of packet loss or X-engine failure.

All Bad Antpols: 81e, 81n, 82e, 82n, 83e, 83n, 117e, 117n, 118n, 137e, 137n

Node 7:
	Antpols (11 total): 81e, 81n, 82e, 82n, 83e, 83n, 117e, 117n, 118n, 137e, 137n
	Whole Ants (5 total): 137, 81, 82, 83, 117
	Single Pols (1 total): 118n

Cross-Polarized: (8 antpols across 4 antennas)

These antennas have their east and north cables swapped.

All Bad Antpols: 40e, 40n, 42e, 42n, 70e, 70n, 112e, 112n

Node 4:
	Antpols (6 total): 40e, 40n, 42e, 42n, 70e, 70n
	Whole Ants (3 total): 40, 42, 70
	Single Pols (0 total): 
Node 10:
	Antpols (2 total): 112e, 112n
	Whole Ants (1 total): 112
	Single Pols (0 total): 

Likely FEM Power Issue: (37 antpols across 30 antennas)

These antennas have low power, anomolously high slopes, and extra channels identified as RFI.

All Bad Antpols: 27e, 27n, 28e, 28n, 34e, 37n, 47e, 55n, 58e, 58n, 59e, 60n, 63n, 65e, 65n, 66n, 68e, 84n, 92e, 93e, 93n, 94e, 109e, 109n, 111n, 124e, 134n, 142n, 143e, 143n, 156n, 160e, 180n, 182n, 200e, 211n, 225n

Node 1:
	Antpols (4 total): 27e, 27n, 28e, 28n
	Whole Ants (2 total): 27, 28
	Single Pols (0 total): 

Node 3:
	Antpols (5 total): 37n, 65e, 65n, 66n, 68e
	Whole Ants (1 total): 65
	Single Pols (3 total): 37n, 66n, 68e

Node 4:
	Antpols (1 total): 55n
	Whole Ants (0 total): 
	Single Pols (1 total): 55n

Node 5:
	Antpols (4 total): 58e, 58n, 59e, 60n
	Whole Ants (1 total): 58
	Single Pols (2 total): 59e, 60n

Node 6:
	Antpols (3 total): 34e, 47e, 63n
	Whole Ants (0 total): 
	Single Pols (3 total): 34e, 47e, 63n

Node 8:
	Antpols (1 total): 84n
	Whole Ants (0 total): 
	Single Pols (1 total): 84n

Node 9:
	Antpols (1 total): 124e
	Whole Ants (0 total): 
	Single Pols (1 total): 124e

Node 10:
	Antpols (7 total): 92e, 93e, 93n, 94e, 109e, 109n, 111n
	Whole Ants (2 total): 109, 93
	Single Pols (3 total): 92e, 94e, 111n

Node 11:
	Antpols (1 total): 134n
	Whole Ants (0 total): 
	Single Pols (1 total): 134n

Node 12:
	Antpols (1 total): 156n
	Whole Ants (0 total): 
	Single Pols (1 total): 156n

Node 13:
	Antpols (4 total): 142n, 160e, 180n, 182n
	Whole Ants (0 total): 
	Single Pols (4 total): 142n, 160e, 180n, 182n

Node 14:
	Antpols (2 total): 143e, 143n
	Whole Ants (1 total): 143
	Single Pols (0 total): 

Node 18:
	Antpols (1 total): 200e
	Whole Ants (0 total): 
	Single Pols (1 total): 200e

Node 19:
	Antpols (1 total): 225n
	Whole Ants (0 total): 
	Single Pols (1 total): 225n

Node 20:
	Antpols (1 total): 211n
	Whole Ants (0 total): 
	Single Pols (1 total): 211n

High Power: (17 antpols across 11 antennas)

These antennas have high median power.

All Bad Antpols: 8e, 8n, 80n, 113e, 113n, 114e, 123e, 172e, 173e, 173n, 192e, 192n, 193e, 193n, 201n, 224e, 224n

Node 2:
	Antpols (2 total): 8e, 8n
	Whole Ants (1 total): 8
	Single Pols (0 total): 

Node 8:
	Antpols (1 total): 123e
	Whole Ants (0 total): 
	Single Pols (1 total): 123e

Node 11:
	Antpols (4 total): 80n, 113e, 113n, 114e
	Whole Ants (1 total): 113
	Single Pols (2 total): 80n, 114e

Node 16:
	Antpols (7 total): 172e, 173e, 173n, 192e, 192n, 193e, 193n
	Whole Ants (3 total): 192, 193, 173
	Single Pols (1 total): 172e

Node 18:
	Antpols (1 total): 201n
	Whole Ants (0 total): 
	Single Pols (1 total): 201n

Node 19:
	Antpols (2 total): 224e, 224n
	Whole Ants (1 total): 224
	Single Pols (0 total): 

Other Low Power Issues: (16 antpols across 10 antennas)

These antennas have low power, but are not all-zeros and not FEM off.

All Bad Antpols: 17n, 72n, 81e, 81n, 82e, 82n, 83e, 83n, 112n, 117e, 117n, 118e, 118n, 127e, 137e, 137n

Node 1:
	Antpols (1 total): 17n
	Whole Ants (0 total): 
	Single Pols (1 total): 17n

Node 4:
	Antpols (1 total): 72n
	Whole Ants (0 total): 
	Single Pols (1 total): 72n

Node 7:
	Antpols (12 total): 81e, 81n, 82e, 82n, 83e, 83n, 117e, 117n, 118e, 118n, 137e, 137n
	Whole Ants (6 total): 137, 81, 82, 83, 117, 118
	Single Pols (0 total): 

Node 10:
	Antpols (2 total): 112n, 127e
	Whole Ants (0 total): 
	Single Pols (2 total): 112n, 127e

Low Correlation, But Not Low Power: (3 antpols across 3 antennas)

These antennas are low correlation, but their autocorrelation power levels look OK.

All Bad Antpols: 47n, 134e, 246n

Node 6:
	Antpols (1 total): 47n
	Whole Ants (0 total): 
	Single Pols (1 total): 47n
Node 11:
	Antpols (1 total): 134e
	Whole Ants (0 total): 
	Single Pols (1 total): 134e
Node 20:
	Antpols (1 total): 246n
	Whole Ants (0 total): 
	Single Pols (1 total): 246n

Bad Bandpass Shapes, But Not Bad Power: (12 antpols across 11 antennas)

These antennas have unusual bandpass shapes, but are not all-zeros, high power, low power, or FEM off.

All Bad Antpols: 4n, 15e, 32e, 32n, 77e, 78e, 96n, 104n, 110e, 161n, 165e, 200n

Node 1:
	Antpols (2 total): 4n, 15e
	Whole Ants (0 total): 
	Single Pols (2 total): 4n, 15e

Node 2:
	Antpols (2 total): 32e, 32n
	Whole Ants (1 total): 32
	Single Pols (0 total): 

Node 6:
	Antpols (2 total): 77e, 78e
	Whole Ants (0 total): 
	Single Pols (2 total): 77e, 78e

Node 8:
	Antpols (1 total): 104n
	Whole Ants (0 total): 
	Single Pols (1 total): 104n

Node 10:
	Antpols (1 total): 110e
	Whole Ants (0 total): 
	Single Pols (1 total): 110e

Node 11:
	Antpols (1 total): 96n
	Whole Ants (0 total): 
	Single Pols (1 total): 96n

Node 13:
	Antpols (1 total): 161n
	Whole Ants (0 total): 
	Single Pols (1 total): 161n

Node 14:
	Antpols (1 total): 165e
	Whole Ants (0 total): 
	Single Pols (1 total): 165e

Node 18:
	Antpols (1 total): 200n
	Whole Ants (0 total): 
	Single Pols (1 total): 200n

Excess RFI: (8 antpols across 7 antennas)

These antennas have excess strucutre (identified as possible RFI) in their bandpassed relative to the median antenna, but not low or high power or a bad bandpass.

All Bad Antpols: 18e, 18n, 29n, 51e, 72e, 108e, 202n, 329e

Node 1:
	Antpols (3 total): 18e, 18n, 29n
	Whole Ants (1 total): 18
	Single Pols (1 total): 29n

Node 3:
	Antpols (1 total): 51e
	Whole Ants (0 total): 
	Single Pols (1 total): 51e

Node 4:
	Antpols (1 total): 72e
	Whole Ants (0 total): 
	Single Pols (1 total): 72e

Node 9:
	Antpols (1 total): 108e
	Whole Ants (0 total): 
	Single Pols (1 total): 108e

Node 12:
	Antpols (1 total): 329e
	Whole Ants (0 total): 
	Single Pols (1 total): 329e

Node 18:
	Antpols (1 total): 202n
	Whole Ants (0 total): 
	Single Pols (1 total): 202n

Redcal chi^2: (12 antpols across 9 antennas)

These antennas have been idenfied as not redundantly calibrating well, even after passing the above checks.

All Bad Antpols: 40e, 40n, 42e, 42n, 54n, 70e, 70n, 112e, 151e, 184e, 242e, 243e

Node 4:
	Antpols (7 total): 40e, 40n, 42e, 42n, 54n, 70e, 70n
	Whole Ants (3 total): 40, 42, 70
	Single Pols (1 total): 54n
Node 10:
	Antpols (1 total): 112e
	Whole Ants (0 total): 
	Single Pols (1 total): 112e
Node 14:
	Antpols (1 total): 184e
	Whole Ants (0 total): 
	Single Pols (1 total): 184e
Node 16:
	Antpols (1 total): 151e
	Whole Ants (0 total): 
	Single Pols (1 total): 151e
Node 19:
	Antpols (2 total): 242e, 243e
	Whole Ants (0 total): 
	Single Pols (2 total): 242e, 243e

Full-Day Visualizations

def classification_plot(col):
    class_array = classification_array(col)
    plt.figure(figsize=(12, len(ants) / 10), dpi=100)
    plt.imshow(class_array.T, aspect='auto', interpolation='none', cmap='RdYlGn', vmin=0, vmax=2,
               extent=[jds[0] - np.floor(jds[0]), jds[-1] - np.floor(jds[0]), len(ants), 0])
    plt.xlabel(f'JD - {int(jds[0])}')
    plt.yticks(ticks=np.arange(.5, len(ants)+.5), labels=[ant for ant in ants], fontsize=6)
    plt.ylabel('Antenna Number (East First, Then North)')
    plt.gca().tick_params(right=True, top=True, labelright=True, labeltop=True)
    plt.tight_layout()
    plt.title(f'{col}: Green is "good", Yellow is "suspect", Red is "bad"')

Figure 1: Per-File Overall Antenna Classification Summary

This "big green board" shows the overall (i.e. after redundant calibration) classification of antennas on a per-file basis. This is useful for looking at time-dependent effects across the array. While only antenna numbers are labeled, both polarizations are shown, first East then North going down, above and below the antenna's tick mark.

classification_plot('Antenna Class')

# compute flag fractions for all classifiers and antennas
frac_flagged = []
for col in class_cols[1:]:
    class_array = np.vstack([t[col] for t in tables])
    class_array[class_array == 'good'] = False
    class_array[class_array == 'suspect'] = False
    class_array[class_array == 'bad'] = True
    frac_flagged.append(np.sum(class_array, axis=0))

def plot_flag_frac_all_classifiers():
    ticks = []
    for i, col in enumerate(list(class_cols[1:])):
        ticks.append(f'{col} ({np.nanmean(np.array(frac_flagged).astype(float)[i]) / len(csv_files):.2%})')
    plt.figure(figsize=(8, len(ants) / 10), dpi=100)
    plt.imshow(np.array(frac_flagged).astype(float).T, aspect='auto', interpolation='none', cmap='viridis')
    plt.xticks(ticks=np.arange(len(list(class_cols[1:]))), labels=ticks, rotation=-45, ha='left')
    plt.yticks(ticks=np.arange(.5, len(ap_strs)+.5, 2), labels=[ant for ant in ants], fontsize=6)
    plt.ylabel('Antenna Number (East First, Then North)')
    plt.gca().tick_params(right=True, labelright=True,)
    ax2 = plt.gca().twiny()
    ax2.set_xticks(ticks=np.arange(len(list(class_cols[1:]))), labels=ticks, rotation=45, ha='left')
    plt.colorbar(ax=plt.gca(), label=f'Number of Files Flagged Out of {len(csv_files)}', aspect=50)
    plt.tight_layout()

Figure 2: Per-Classifier Antenna Flagging Summary

This plot shows the fraction of files flagged for each reason for each antenna. It's useful for seeing which problems are transitory and which ones are more common. Note that not all flags are independent and in particular redcal chi^2 takes an OR of other classifications as an input. Also note that only antenna numbers are labeled, both polarizations are shown, first East then North going down, above and below the antenna's tick mark.

plot_flag_frac_all_classifiers()

def array_class_plot():
    fig, axes = plt.subplots(1, 2, figsize=(14, 6), dpi=100, gridspec_kw={'width_ratios': [2, 1]})
    plot_antclass(hd.antpos, overall_class, ax=axes[0], ants=[ant for ant in hd.data_ants if ant < 320], legend=False, 
                  title=f'HERA Core: Overall Flagging Based on {overall_thresh:.1%} Daily Threshold')
    plot_antclass(hd.antpos, overall_class, ax=axes[1], ants=[ant for ant in hd.data_ants if ant >= 320], radius=50, title='Outriggers')

Figure 3: Array Visualization of Overall Daily Classification

Overall classification of antenna-polarizations shown on the array layout. If any antenna is marked bad for any reason more than the threshold (default 10%), it is marked bad here. Likewise, if any antenna is marked suspect for more than 10% of the night (but not bad), it's suspect here.

if SUM_FILE is not None: array_class_plot()