import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin  # REQUIRED to have the compression plugins available
import numpy as np
import pandas as pd
import glob
import os
import matplotlib.pyplot as plt
from hera_cal import io, utils
from hera_qm import ant_class
from uvtools.plot import plot_antpos, plot_antclass
%matplotlib inline
from IPython.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
_ = np.seterr(all='ignore')  # get rid of red warnings
%config InlineBackend.figure_format = 'retina'

# Parse settings from environment
ANT_CLASS_FOLDER = os.environ.get("ANT_CLASS_FOLDER", "./")
SUM_FILE = os.environ.get("SUM_FILE", None)
# ANT_CLASS_FOLDER = "/mnt/sn1/2460330"
# SUM_FILE = "/mnt/sn1/2460330/zen.2460330.25463.sum.uvh5"
OC_SKIP_OUTRIGGERS = os.environ.get("OC_SKIP_OUTRIGGERS", "TRUE").upper() == "TRUE"

for param in ['ANT_CLASS_FOLDER', 'SUM_FILE', 'OC_SKIP_OUTRIGGERS']:
    print(f"{param} = '{eval(param)}'")

ANT_CLASS_FOLDER = '/mnt/sn1/data2/2461156'
SUM_FILE = '/mnt/sn1/data2/2461156/zen.2461156.42814.sum.uvh5'
OC_SKIP_OUTRIGGERS = 'True'

if SUM_FILE is not None:
    from astropy.time import Time, TimeDelta
    utc = Time(float(SUM_FILE.split('zen.')[-1].split('.sum.uvh5')[0]), format='jd').datetime
    print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 4-25-2026

# set thresholds for fraction of the day
overall_thresh = .1
all_zero_thresh = .1
eo_zeros_thresh = .1
xengine_diff_thresh = .1
cross_pol_thresh = .5
bad_fem_thresh = .1
high_power_thresh = .1
low_power_thresh = .1
low_corr_thresh = .1
bad_shape_thresh = .5
excess_rfi_thresh = .1
chisq_thresh = .25

# Load csvs
csv_files = sorted(glob.glob(os.path.join(ANT_CLASS_FOLDER, '*.ant_class.csv')))
jds = [float(f.split('/')[-1].split('zen.')[-1].split('.sum')[0]) for f in csv_files]
tables = [pd.read_csv(f).dropna(axis=0, how='all') for f in csv_files]
table_cols = tables[0].columns[1::2]
class_cols = tables[0].columns[2::2]
print(f'Found {len(csv_files)} csv files starting with {csv_files[0]}')

Found 2130 csv files starting with /mnt/sn1/data2/2461156/zen.2461156.18990.sum.ant_class.csv

# parse ant_strings
ap_strs = np.array(tables[0]['Antenna'])
ants = sorted(set(int(a[:-1]) for a in ap_strs))
translator = ''.maketrans('e', 'n') | ''.maketrans('n', 'e')

# get node numbers
node_dict = {ant: 'Unknown' for ant in ants}
try:
    from hera_mc import cm_hookup
    hookup = cm_hookup.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in node_dict:
            if hookup[ant_name].get_part_from_type('node')['E<ground'] is not None:
                node_dict[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])
except:
    pass
nodes = sorted(set(node_dict.values()))

def classification_array(col):
    class_array = np.vstack([t[col] for t in tables])
    class_array[class_array == 'good'] = 1.7
    class_array[class_array == 'suspect'] = 1
    class_array[class_array == 'bad'] = 0
    return class_array.astype(float)

if SUM_FILE is not None:
    hd = io.HERADataFastReader(SUM_FILE)
    ap_tuples = [(int(ap[:-1]), {'e': 'Jee', 'n': 'Jnn'}[ap[-1]]) for ap in ap_strs]
    bad_bools = np.mean(classification_array('Antenna Class') == 0, axis=0) > overall_thresh
    bad_aps = [ap_tuples[i] for i in np.arange(len(ap_tuples))[bad_bools]]
    suspect_bools = np.mean(classification_array('Antenna Class') == 1, axis=0) > overall_thresh
    suspect_aps = [ap_tuples[i] for i in np.arange(len(ap_tuples))[suspect_bools] if ap_tuples[i] not in bad_aps]
    good_aps = [ap for ap in ap_tuples if ap not in bad_aps and ap not in suspect_aps]
    overall_class = ant_class.AntennaClassification(bad=bad_aps, suspect=suspect_aps, good=good_aps)
    autos, _, _ = hd.read(bls=[bl for bl in hd.bls if utils.split_bl(bl)[0] == utils.split_bl(bl)[1]], read_flags=False, read_nsamples=False)
    avg_unflagged_auto = {}
    for pol in ['ee', 'nn']:
        unflagged_autos = [autos[bl] for bl in autos if bl[2] == pol and overall_class[utils.split_bl(bl)[0]] != 'bad']
        if len(unflagged_autos) > 0:
            avg_unflagged_auto[pol] = np.mean(unflagged_autos, axis=(0, 1))
        else:
            avg_unflagged_auto[pol] = np.zeros(len(hd.freqs), dtype=complex)

def print_issue_summary(bad_ant_strs, title, notes='', plot=False):
    '''Print report for list of bad antenna polarizations strings'''
    unique_bad_antnums = [int(ap[:-1]) for ap in bad_ant_strs]
    display(HTML(f'<h2>{title}: ({len(bad_ant_strs)} antpols across {len(set([ba[:-1] for ba in bad_ant_strs]))} antennas)</h2>'))
    if len(notes) > 0:
        display(HTML(f'<h4>{notes}</h4>'))
    if len(bad_ant_strs) > 0:
        print(f'All Bad Antpols: {", ".join(bad_ant_strs)}\n')
    for node in nodes:
        if np.any([node == node_dict[a] for a in unique_bad_antnums]):
            aps = [ap for ap in bad_ant_strs if node_dict[int(ap[:-1])] == node]
            whole_ants = [str(wa) for wa in set([int(ap[:-1]) for ap in aps if ap.translate(translator) in bad_ant_strs])]
            single_pols =  [ap for ap in aps if ap.translate(translator) not in bad_ant_strs]
            print(f'Node {node}:')
            print(f'\tAntpols ({len(aps)} total): {", ".join(aps)}')
            print(f'\tWhole Ants ({len(whole_ants)} total): {", ".join(whole_ants)}')
            print(f'\tSingle Pols ({len(single_pols)} total): {", ".join(single_pols)}')
            if plot and SUM_FILE is not None:
                fig, axes = plt.subplots(1, 2, figsize=(12,4), dpi=70, sharey=True, gridspec_kw={'wspace': 0})
                for ax, pol in zip(axes, ['ee', 'nn']):                    
                    ax.semilogy(autos.freqs / 1e6, avg_unflagged_auto[pol], 'k--', label='Average\nUnflagged\nAuto')
                    for ap in aps:
                        ant = int(ap[:-1]), utils.comply_pol(ap[-1])
                        auto_bl = utils.join_bl(ant, ant)
                        if auto_bl[2] == pol:
                            ax.semilogy(autos.freqs / 1e6, np.mean(autos[auto_bl], axis=0), label=ap)
                    ax.legend()
                    ax.set_xlim([40, 299])
                    ax.set_title(f'{title} on Node {node} ({pol}-antennas)')
                    ax.set_xlabel('Frequency (MHz)')
                axes[0].set_ylabel('Single File Raw Autocorrelation')
                plt.tight_layout()
                plt.show()

# precompute various helpful quantities
all_slopes = np.vstack([t['Autocorr Slope'] for t in tables])
median_slope = np.median(all_slopes)
bad_slopes = np.vstack([t['Autocorr Slope Class'] for t in tables]) == 'bad'
suspect_slopes = np.vstack([t['Autocorr Slope Class'] for t in tables]) == 'suspect'
bad_shapes = np.vstack([t['Autocorr Shape Class'] for t in tables]) == 'bad'
suspect_shapes = np.vstack([t['Autocorr Shape Class'] for t in tables]) == 'suspect'
all_powers = np.vstack([t['Autocorr Power'] for t in tables])
median_power = np.median(all_powers)
bad_powers = np.vstack([t['Autocorr Power Class'] for t in tables]) == 'bad'
suspect_powers = np.vstack([t['Autocorr Power Class'] for t in tables]) == 'suspect'
bad_rfi = np.vstack([t['Auto RFI RMS Class'] for t in tables]) == 'bad'
suspect_rfi = np.vstack([t['Auto RFI RMS Class'] for t in tables]) == 'suspect'

# find all zeros
all_zeros_strs = ap_strs[np.mean(np.vstack([t['Dead? Class'] for t in tables]) == 'bad', axis=0) > all_zero_thresh]

# find even/odd zeros
eo_zeros_strs = ap_strs[np.mean(np.vstack([t['Even/Odd Zeros Class'] for t in tables]) == 'bad', axis=0) > eo_zeros_thresh]
eo_zeros_strs = [ap for ap in eo_zeros_strs if ap not in all_zeros_strs]

# find cross-polarized antennas
cross_pol_strs = ap_strs[np.mean(np.vstack([t['Cross-Polarized Class'] for t in tables]) == 'bad', axis=0) > cross_pol_thresh]
cross_pol_strs = [ap for ap in cross_pol_strs if ap not in all_zeros_strs]

# find FEM power issues: must be low power, high slope, and bad or suspect in power, slope, rfi, and shape
fem_off_prod = (bad_powers + .5 * suspect_powers) * (bad_slopes + .5 * suspect_slopes)
fem_off_prod *= (bad_rfi + .5 * suspect_rfi) * (bad_shapes + .5 * suspect_shapes)
fem_off_strs = ap_strs[np.mean(fem_off_prod * (all_powers < median_power) * (all_slopes > median_slope), axis=0) > .1]

# find high power issues
high_power_strs = ap_strs[np.mean(bad_powers & (all_powers > median_power), axis=0) > high_power_thresh]

# find other low power issues
low_power_strs = ap_strs[np.mean(bad_powers & (all_powers < median_power), axis=0) > low_power_thresh]
low_power_strs = [ap for ap in low_power_strs if ap not in all_zeros_strs and ap not in fem_off_strs]

# find low correlation (but not low power)
low_corr_strs = ap_strs[np.mean(np.vstack([t['Low Correlation Class'] for t in tables]) == 'bad', axis=0) > low_corr_thresh]
low_corr_strs = [ap for ap in low_corr_strs if ap not in (set(low_power_strs) | set(all_zeros_strs) | set(fem_off_strs))]

# find bad bandpasses
bad_bandpass_strs = ap_strs[np.mean(bad_shapes, axis=0) > bad_shape_thresh]
bad_bandpass_strs = [ap for ap in bad_bandpass_strs if ap not in (set(low_power_strs) | set(all_zeros_strs) | set(high_power_strs) | set(fem_off_strs))]

# find antennas with excess RFI
excess_rfi_strs = ap_strs[np.mean(np.vstack([t['Auto RFI RMS Class'] for t in tables]) == 'bad', axis=0) > excess_rfi_thresh]
excess_rfi_strs = [ap for ap in excess_rfi_strs if ap not in (set(low_power_strs) | set(all_zeros_strs) |  set(fem_off_strs) |
                                                              set(bad_bandpass_strs) | set(high_power_strs))]

# find bad x-engine diffs
xengine_diff_strs = ap_strs[np.mean(np.vstack([t['Bad Diff X-Engines Class'] for t in tables]) == 'bad', axis=0) > xengine_diff_thresh]
xengine_diff_strs = [ap for ap in xengine_diff_strs if ap not in (set(bad_bandpass_strs) | set(low_power_strs) | set(excess_rfi_strs) | set(low_corr_strs) |
                                                                  set(all_zeros_strs) | set(high_power_strs) | set(fem_off_strs) | set(eo_zeros_strs))]

# find antennas with high redcal chi^2
chisq_strs = ap_strs[np.mean(np.vstack([t['Redcal chi^2 Class'] for t in tables]) == 'bad', axis=0) > chisq_thresh]
chisq_strs = [ap for ap in chisq_strs if ap not in (set(bad_bandpass_strs) | set(low_power_strs) | set(excess_rfi_strs) | set(low_corr_strs) |
                                                    set(all_zeros_strs) | set(high_power_strs) | set(fem_off_strs) | set(eo_zeros_strs) | set(xengine_diff_strs))]
if OC_SKIP_OUTRIGGERS:
    chisq_strs = [ap for ap in chisq_strs if int(ap[:-1]) < 320]

# collect all results
to_print = [(all_zeros_strs, 'All-Zeros', 'These antennas have visibilities that are more than half zeros.'),
            (eo_zeros_strs, 'Excess Zeros in Either Even or Odd Spectra', 
             'These antennas are showing evidence of packet loss or X-engine failure.', True),
            (xengine_diff_strs, 'Excess Power in X-Engine Diffs', 
             'These antennas are showing evidence of mis-written packets in either the evens or the odds.', True),            
            (cross_pol_strs, 'Cross-Polarized', 'These antennas have their east and north cables swapped.'),
            (fem_off_strs, 'Likely FEM Power Issue', 'These antennas have low power and anomolously high slopes.', True),
            (high_power_strs, 'High Power', 'These antennas have high median power.', True),
            (low_power_strs, 'Other Low Power Issues', 'These antennas have low power, but are not all-zeros and not FEM off.', True),
            (low_corr_strs, 'Low Correlation, But Not Low Power', 'These antennas are low correlation, but their autocorrelation power levels look OK.'),
            (bad_bandpass_strs, 'Bad Bandpass Shapes, But Not Bad Power', 
             'These antennas have unusual bandpass shapes, but are not all-zeros, high power, low power, or FEM off.', True),
            (excess_rfi_strs, 'Excess RFI', 'These antennas have excess RMS after DPSS filtering (likely RFI), but not low or high power or a bad bandpass.', True),
            (chisq_strs, 'Redcal chi^2', 'These antennas have been idenfied as not redundantly calibrating well, even after passing the above checks.')]

def print_high_level_summary():
    for tp in sorted(to_print, key=lambda x: len(x[0]), reverse=True):
        print(f'{len(tp[0])} antpols (on {len(set([ap[:-1] for ap in tp[0]]))} antennas) frequently flagged for {tp[1]}.')
        
def print_all_issue_summaries():
    for tp in to_print:
        print_issue_summary(*tp)

print_high_level_summary()

100 antpols (on 50 antennas) frequently flagged for All-Zeros.
81 antpols (on 60 antennas) frequently flagged for Redcal chi^2.
76 antpols (on 47 antennas) frequently flagged for Low Correlation, But Not Low Power.
66 antpols (on 53 antennas) frequently flagged for Excess Power in X-Engine Diffs.
41 antpols (on 33 antennas) frequently flagged for High Power.
40 antpols (on 31 antennas) frequently flagged for Excess RFI.
38 antpols (on 28 antennas) frequently flagged for Likely FEM Power Issue.
16 antpols (on 10 antennas) frequently flagged for Other Low Power Issues.
9 antpols (on 8 antennas) frequently flagged for Bad Bandpass Shapes, But Not Bad Power.
6 antpols (on 3 antennas) frequently flagged for Cross-Polarized.
0 antpols (on 0 antennas) frequently flagged for Excess Zeros in Either Even or Odd Spectra.

print_all_issue_summaries()

All Bad Antpols: 10e, 10n, 19e, 19n, 20e, 20n, 21e, 21n, 31e, 31n, 32e, 32n, 37e, 37n, 38e, 38n, 66e, 66n, 88e, 88n, 90e, 90n, 107e, 107n, 112e, 112n, 127e, 127n, 128e, 128n, 143e, 143n, 151e, 151n, 163e, 163n, 164e, 164n, 171e, 171n, 172e, 172n, 211e, 211n, 212e, 212n, 225e, 225n, 226e, 226n, 231e, 231n, 232e, 232n, 240e, 240n, 244e, 244n, 245e, 245n, 250e, 250n, 254e, 254n, 255e, 255n, 256e, 256n, 266e, 266n, 269e, 269n, 270e, 270n, 271e, 271n, 277e, 277n, 278e, 278n, 287e, 287n, 292e, 292n, 299e, 299n, 300e, 300n, 301e, 301n, 302e, 302n, 311e, 311n, 312e, 312n, 339e, 339n, 345e, 345n

Node 2:
	Antpols (12 total): 10e, 10n, 19e, 19n, 20e, 20n, 21e, 21n, 31e, 31n, 32e, 32n
	Whole Ants (6 total): 32, 10, 19, 20, 21, 31
	Single Pols (0 total): 
Node 3:
	Antpols (6 total): 37e, 37n, 38e, 38n, 66e, 66n
	Whole Ants (3 total): 66, 37, 38
	Single Pols (0 total): 
Node 9:
	Antpols (6 total): 88e, 88n, 90e, 90n, 107e, 107n
	Whole Ants (3 total): 88, 90, 107
	Single Pols (0 total): 
Node 10:
	Antpols (6 total): 112e, 112n, 127e, 127n, 128e, 128n
	Whole Ants (3 total): 112, 128, 127
	Single Pols (0 total): 
Node 14:
	Antpols (6 total): 143e, 143n, 163e, 163n, 164e, 164n
	Whole Ants (3 total): 163, 164, 143
	Single Pols (0 total): 
Node 16:
	Antpols (6 total): 151e, 151n, 171e, 171n, 172e, 172n
	Whole Ants (3 total): 171, 172, 151
	Single Pols (0 total): 
Node 19:
	Antpols (6 total): 225e, 225n, 226e, 226n, 240e, 240n
	Whole Ants (3 total): 240, 225, 226
	Single Pols (0 total): 
Node 20:
	Antpols (6 total): 211e, 211n, 244e, 244n, 245e, 245n
	Whole Ants (3 total): 211, 244, 245
	Single Pols (0 total): 
Node 21:
	Antpols (6 total): 212e, 212n, 231e, 231n, 232e, 232n
	Whole Ants (3 total): 232, 212, 231
	Single Pols (0 total): 
Node 22:
	Antpols (6 total): 250e, 250n, 266e, 266n, 269e, 269n
	Whole Ants (3 total): 250, 266, 269
	Single Pols (0 total): 
Node 23:
	Antpols (12 total): 254e, 254n, 255e, 255n, 256e, 256n, 270e, 270n, 271e, 271n, 287e, 287n
	Whole Ants (6 total): 256, 287, 270, 271, 254, 255
	Single Pols (0 total): 
Node 27:
	Antpols (12 total): 299e, 299n, 300e, 300n, 301e, 301n, 302e, 302n, 311e, 311n, 312e, 312n
	Whole Ants (6 total): 299, 300, 301, 302, 311, 312
	Single Pols (0 total): 
Node 29:
	Antpols (10 total): 277e, 277n, 278e, 278n, 292e, 292n, 339e, 339n, 345e, 345n
	Whole Ants (5 total): 292, 339, 277, 278, 345
	Single Pols (0 total):

All Bad Antpols: 3e, 28e, 29n, 30e, 40e, 46n, 53e, 59e, 59n, 60e, 60n, 74e, 74n, 75n, 76n, 80n, 81n, 82e, 83e, 83n, 84e, 89n, 91n, 96e, 96n, 97e, 98e, 98n, 106n, 108n, 115n, 116e, 116n, 124e, 124n, 131e, 136n, 137e, 140n, 165e, 165n, 169e, 175e, 176n, 177e, 181e, 182e, 184e, 184n, 185e, 185n, 189n, 209n, 215n, 216n, 233e, 233n, 253e, 261e, 261n, 262e, 267n, 284e, 315n, 325e, 342e

Node 1:
	Antpols (4 total): 3e, 28e, 29n, 30e
	Whole Ants (0 total): 
	Single Pols (4 total): 3e, 28e, 29n, 30e

Casting complex values to real discards the imaginary part
Casting complex values to real discards the imaginary part

Node 3:
	Antpols (1 total): 53e
	Whole Ants (0 total): 
	Single Pols (1 total): 53e

Node 4:
	Antpols (1 total): 40e
	Whole Ants (0 total): 
	Single Pols (1 total): 40e

Node 5:
	Antpols (9 total): 46n, 59e, 59n, 60e, 60n, 74e, 74n, 75n, 76n
	Whole Ants (3 total): 74, 59, 60
	Single Pols (3 total): 46n, 75n, 76n

Node 7:
	Antpols (9 total): 81n, 82e, 83e, 83n, 98e, 98n, 116e, 116n, 137e
	Whole Ants (3 total): 98, 83, 116
	Single Pols (3 total): 81n, 82e, 137e

Node 8:
	Antpols (1 total): 84e
	Whole Ants (0 total): 
	Single Pols (1 total): 84e

Node 9:
	Antpols (7 total): 89n, 91n, 106n, 108n, 124e, 124n, 325e
	Whole Ants (1 total): 124
	Single Pols (5 total): 89n, 91n, 106n, 108n, 325e

Node 11:
	Antpols (6 total): 80n, 96e, 96n, 97e, 115n, 131e
	Whole Ants (1 total): 96
	Single Pols (4 total): 80n, 97e, 115n, 131e

def classification_plot(col):
    class_array = classification_array(col)
    plt.figure(figsize=(12, len(ants) / 10), dpi=100)
    plt.imshow(class_array.T, aspect='auto', interpolation='none', cmap='RdYlGn', vmin=0, vmax=2,
               extent=[jds[0] - np.floor(jds[0]), jds[-1] - np.floor(jds[0]), len(ants), 0])
    plt.xlabel(f'JD - {int(jds[0])}')
    plt.yticks(ticks=np.arange(.5, len(ants)+.5), labels=[ant for ant in ants], fontsize=6)
    plt.ylabel('Antenna Number (East First, Then North)')
    plt.gca().tick_params(right=True, top=True, labelright=True, labeltop=True)
    plt.tight_layout()
    plt.title(f'{col}: Green is "good", Yellow is "suspect", Red is "bad"')

classification_plot('Antenna Class')

# compute flag fractions for all classifiers and antennas
frac_flagged = []
for col in class_cols[1:]:
    class_array = np.vstack([t[col] for t in tables])
    class_array[class_array == 'good'] = False
    class_array[class_array == 'suspect'] = False
    class_array[class_array == 'bad'] = True
    frac_flagged.append(np.sum(class_array, axis=0))

def plot_flag_frac_all_classifiers():
    ticks = []
    for i, col in enumerate(list(class_cols[1:])):
        ticks.append(f'{col} ({np.nanmean(np.array(frac_flagged).astype(float)[i]) / len(csv_files):.2%})')
    plt.figure(figsize=(8, len(ants) / 10), dpi=100)
    plt.imshow(np.array(frac_flagged).astype(float).T, aspect='auto', interpolation='none', cmap='viridis')
    plt.xticks(ticks=np.arange(len(list(class_cols[1:]))), labels=ticks, rotation=-45, ha='left')
    plt.yticks(ticks=np.arange(.5, len(ap_strs)+.5, 2), labels=[ant for ant in ants], fontsize=6)
    plt.ylabel('Antenna Number (East First, Then North)')
    plt.gca().tick_params(right=True, labelright=True,)
    ax2 = plt.gca().twiny()
    ax2.set_xticks(ticks=np.arange(len(list(class_cols[1:]))), labels=ticks, rotation=45, ha='left')
    plt.colorbar(ax=plt.gca(), label=f'Number of Files Flagged Out of {len(csv_files)}', aspect=50)
    plt.tight_layout()

plot_flag_frac_all_classifiers()

def array_class_plot():
    fig, axes = plt.subplots(1, 2, figsize=(14, 6), dpi=100, gridspec_kw={'width_ratios': [2, 1]})
    if len([ant for ant in hd.data_ants if ant < 320]) > 0:
        plot_antclass(hd.antpos, overall_class, ax=axes[0], ants=[ant for ant in hd.data_ants if ant < 320], legend=False, 
                      title=f'HERA Core: Overall Flagging Based on {overall_thresh:.1%} Daily Threshold')
    if len([ant for ant in hd.data_ants if ant >= 320]) > 0:
        plot_antclass(hd.antpos, overall_class, ax=axes[1], ants=[ant for ant in hd.data_ants if ant >= 320], radius=50, title='Outriggers')

if SUM_FILE is not None: array_class_plot()

WARNING:matplotlib.axes._base:Ignoring fixed x limits to fulfill fixed data aspect with adjustable data limits.

WARNING:matplotlib.axes._base:Ignoring fixed x limits to fulfill fixed data aspect with adjustable data limits.

WARNING:matplotlib.axes._base:Ignoring fixed y limits to fulfill fixed data aspect with adjustable data limits.

WARNING:matplotlib.axes._base:Ignoring fixed x limits to fulfill fixed data aspect with adjustable data limits.

WARNING:matplotlib.axes._base:Ignoring fixed x limits to fulfill fixed data aspect with adjustable data limits.

for repo in ['pyuvdata', 'hera_cal', 'hera_qm', 'hera_notebook_templates']:
    exec(f'from {repo} import __version__')
    print(f'{repo}: {__version__}')

pyuvdata: 3.2.5.dev1+g5a985ae31
hera_cal: 3.7.7.dev97+gc2668d3f7
hera_qm: 2.2.1.dev4+gf6d02113b

hera_notebook_templates: 0.0.1.dev1313+g92178a09c

Antenna Classification Daily Summary¶

• Summary of Per Antenna Issues ¶

• Figure 1: Per File Overall Antenna Classification Summary ¶

• Figure 2: Per Classifier Antenna Flagging Summary ¶

• Figure 3: Array Visualization of Overall Daily Classification ¶

Settings¶

Load classifications and other metadata¶

Figure out and summarize per-antenna issues¶

Summary of Per-Antenna Issues¶

All-Zeros: (100 antpols across 50 antennas)

These antennas have visibilities that are more than half zeros.

Excess Zeros in Either Even or Odd Spectra: (0 antpols across 0 antennas)

These antennas are showing evidence of packet loss or X-engine failure.

Excess Power in X-Engine Diffs: (66 antpols across 53 antennas)

These antennas are showing evidence of mis-written packets in either the evens or the odds.

Cross-Polarized: (6 antpols across 3 antennas)

These antennas have their east and north cables swapped.

Likely FEM Power Issue: (38 antpols across 28 antennas)

These antennas have low power and anomolously high slopes.

High Power: (41 antpols across 33 antennas)

These antennas have high median power.

Other Low Power Issues: (16 antpols across 10 antennas)

These antennas have low power, but are not all-zeros and not FEM off.

Low Correlation, But Not Low Power: (76 antpols across 47 antennas)

These antennas are low correlation, but their autocorrelation power levels look OK.

Bad Bandpass Shapes, But Not Bad Power: (9 antpols across 8 antennas)

These antennas have unusual bandpass shapes, but are not all-zeros, high power, low power, or FEM off.

Excess RFI: (40 antpols across 31 antennas)

These antennas have excess RMS after DPSS filtering (likely RFI), but not low or high power or a bad bandpass.

Redcal chi^2: (81 antpols across 60 antennas)

These antennas have been idenfied as not redundantly calibrating well, even after passing the above checks.

Full-Day Visualizations¶

Figure 1: Per-File Overall Antenna Classification Summary¶

Figure 2: Per-Classifier Antenna Flagging Summary¶

Figure 3: Array Visualization of Overall Daily Classification¶

Antenna Classification Daily Summary¶

• Summary of Per Antenna Issues¶

• Figure 1: Per File Overall Antenna Classification Summary¶

• Figure 2: Per Classifier Antenna Flagging Summary¶

• Figure 3: Array Visualization of Overall Daily Classification¶

Settings¶

Load classifications and other metadata¶

Figure out and summarize per-antenna issues¶

Summary of Per-Antenna Issues¶

All-Zeros: (100 antpols across 50 antennas)

These antennas have visibilities that are more than half zeros.

Excess Zeros in Either Even or Odd Spectra: (0 antpols across 0 antennas)

These antennas are showing evidence of packet loss or X-engine failure.

Excess Power in X-Engine Diffs: (66 antpols across 53 antennas)

These antennas are showing evidence of mis-written packets in either the evens or the odds.

Cross-Polarized: (6 antpols across 3 antennas)

These antennas have their east and north cables swapped.

Likely FEM Power Issue: (38 antpols across 28 antennas)

These antennas have low power and anomolously high slopes.

High Power: (41 antpols across 33 antennas)

These antennas have high median power.

Other Low Power Issues: (16 antpols across 10 antennas)

These antennas have low power, but are not all-zeros and not FEM off.

Low Correlation, But Not Low Power: (76 antpols across 47 antennas)

These antennas are low correlation, but their autocorrelation power levels look OK.

Bad Bandpass Shapes, But Not Bad Power: (9 antpols across 8 antennas)

These antennas have unusual bandpass shapes, but are not all-zeros, high power, low power, or FEM off.

Excess RFI: (40 antpols across 31 antennas)

These antennas have excess RMS after DPSS filtering (likely RFI), but not low or high power or a bad bandpass.

Redcal chi^2: (81 antpols across 60 antennas)

These antennas have been idenfied as not redundantly calibrating well, even after passing the above checks.

Full-Day Visualizations¶

Figure 1: Per-File Overall Antenna Classification Summary¶

Figure 2: Per-Classifier Antenna Flagging Summary¶

Figure 3: Array Visualization of Overall Daily Classification¶

• Summary of Per Antenna Issues ¶

• Figure 1: Per File Overall Antenna Classification Summary ¶

• Figure 2: Per Classifier Antenna Flagging Summary ¶

• Figure 3: Array Visualization of Overall Daily Classification ¶