Single File Calibration

by Josh Dillon, Aaron Parsons, Tyler Cox, and Zachary Martinot, last updated August 11, 2023

This notebook is designed to infer as much information about the array from a single file, including pushing the calibration and RFI mitigation as far as possible. Calibration includes redundant-baseline calibration, RFI-based calibration of delay slopes, model-based calibration of overall amplitudes, and a full per-frequency phase gradient absolute calibration if abscal model files are available.

Here's a set of links to skip to particular figures and tables:

• Figure 1: RFI Flagging

• Figure 2: Plot of autocorrelations with classifications

• Figure 3: Summary of antenna classifications prior to calibration

• Figure 4: Redundant calibration of a single baseline group

• Figure 5: Absolute calibration of redcal degeneracies

• Figure 6: chi^2 per antenna across the array

• Figure 7: Summary of antenna classifications after redundant calibration

• Table 1: Complete summary of per antenna classifications

import time
tstart = time.time()

import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin  # REQUIRED to have the compression plugins available
import numpy as np
from scipy import constants, interpolate
import copy
import glob
import re
import matplotlib
import matplotlib.pyplot as plt
import pandas as pd
pd.set_option('display.max_rows', 1000)
from uvtools.plot import plot_antpos, plot_antclass
from hera_qm import ant_metrics, ant_class, xrfi
from hera_cal import io, utils, redcal, apply_cal, datacontainer, abscal
from hera_notebook_templates.data import DATA_PATH as HNBT_DATA
from IPython.display import display, HTML
import linsolve
display(HTML("<style>.container { width:100% !important; }</style>"))
_ = np.seterr(all='ignore')  # get rid of red warnings
%config InlineBackend.figure_format = 'retina'

# this enables better memory management on linux
import ctypes
def malloc_trim():
    try:
        ctypes.CDLL('libc.so.6').malloc_trim(0) 
    except OSError:
        pass

Parse inputs and outputs

To use this notebook interactively, you will have to provide a sum filename path if none exists as an environment variable. All other parameters have reasonable default values.

# figure out whether to save results
SAVE_RESULTS = os.environ.get("SAVE_RESULTS", "TRUE").upper() == "TRUE"
SAVE_OMNIVIS_FILE = os.environ.get("SAVE_OMNIVIS_FILE", "FALSE").upper() == "TRUE"

# get infile names
SUM_FILE = os.environ.get("SUM_FILE", None)
# SUM_FILE = '/lustre/aoc/projects/hera/h6c-analysis/IDR2/2459866/zen.2459866.33010.sum.uvh5'  # If sum_file is not defined in the environment variables, define it here.
DIFF_FILE = SUM_FILE.replace('sum', 'diff')

# get outfilenames
AM_FILE = (SUM_FILE.replace('.uvh5', '.ant_metrics.hdf5') if SAVE_RESULTS else None)
ANTCLASS_FILE = (SUM_FILE.replace('.uvh5', '.ant_class.csv') if SAVE_RESULTS else None)
OMNICAL_FILE = (SUM_FILE.replace('.uvh5', '.omni.calfits') if SAVE_RESULTS else None)
OMNIVIS_FILE = (SUM_FILE.replace('.uvh5', '.omni_vis.uvh5') if SAVE_RESULTS else None)

for fname in ['SUM_FILE', 'DIFF_FILE', 'AM_FILE', 'ANTCLASS_FILE', 'OMNICAL_FILE', 'OMNIVIS_FILE', 'SAVE_RESULTS', 'SAVE_OMNIVIS_FILE']:
    print(f"{fname} = '{eval(fname)}'")

SUM_FILE = '/mnt/sn1/2460234/zen.2460234.44960.sum.uvh5'
DIFF_FILE = '/mnt/sn1/2460234/zen.2460234.44960.diff.uvh5'
AM_FILE = '/mnt/sn1/2460234/zen.2460234.44960.sum.ant_metrics.hdf5'
ANTCLASS_FILE = '/mnt/sn1/2460234/zen.2460234.44960.sum.ant_class.csv'
OMNICAL_FILE = '/mnt/sn1/2460234/zen.2460234.44960.sum.omni.calfits'
OMNIVIS_FILE = '/mnt/sn1/2460234/zen.2460234.44960.sum.omni_vis.uvh5'
SAVE_RESULTS = 'True'
SAVE_OMNIVIS_FILE = 'False'

Parse settings

Load settings relating to the operation of the notebook, then print what was loaded (or default).

# parse plotting settings
PLOT = os.environ.get("PLOT", "TRUE").upper() == "TRUE"
if PLOT:
    %matplotlib inline

# parse omnical settings
OC_MAX_DIMS = int(os.environ.get("OC_MAX_DIMS", 4))
OC_MIN_DIM_SIZE = int(os.environ.get("OC_MIN_DIM_SIZE", 8))
OC_SKIP_OUTRIGGERS = os.environ.get("OC_SKIP_OUTRIGGERS", "TRUE").upper() == "TRUE"
OC_MIN_BL_LEN = float(os.environ.get("OC_MIN_BL_LEN", 1))
OC_MAX_BL_LEN = float(os.environ.get("OC_MAX_BL_LEN", 1e100))
OC_MAXITER = int(os.environ.get("OC_MAXITER", 50))
OC_MAX_RERUN = int(os.environ.get("OC_MAX_RERUN", 4))
OC_RERUN_MAXITER = int(os.environ.get("OC_MAXITER", 25))
OC_USE_PRIOR_SOL = os.environ.get("OC_USE_PRIOR_SOL", "FALSE").upper() == "TRUE"
OC_PRIOR_SOL_FLAG_THRESH = float(os.environ.get("OC_PRIOR_SOL_FLAG_THRESH", .95))
OC_USE_GPU = os.environ.get("SAVE_RESULTS", "FALSE").upper() == "TRUE"

# parse RFI settings
RFI_DPSS_HALFWIDTH = float(os.environ.get("RFI_DPSS_HALFWIDTH", 300e-9))
RFI_NSIG = float(os.environ.get("RFI_NSIG", 6))

# parse abscal settings
ABSCAL_MODEL_FILES_GLOB = os.environ.get("ABSCAL_MODEL_FILES_GLOB", None)
ABSCAL_MIN_BL_LEN = float(os.environ.get("ABSCAL_MIN_BL_LEN", 1.0))
ABSCAL_MAX_BL_LEN = float(os.environ.get("ABSCAL_MAX_BL_LEN", 140.0))

# print settings
for setting in ['PLOT', 'SAVE_RESULTS', 'OC_MAX_DIMS', 'OC_MIN_DIM_SIZE', 'OC_SKIP_OUTRIGGERS', 
                'OC_MIN_BL_LEN', 'OC_MAX_BL_LEN', 'OC_MAXITER', 'OC_MAX_RERUN', 'OC_RERUN_MAXITER', 
                'OC_USE_PRIOR_SOL', 'OC_PRIOR_SOL_FLAG_THRESH', 'OC_USE_GPU', 'RFI_DPSS_HALFWIDTH', 'RFI_NSIG',
                'ABSCAL_MODEL_FILES_GLOB', 'ABSCAL_MIN_BL_LEN', 'ABSCAL_MAX_BL_LEN']:
    print(f'{setting} = {eval(setting)}')

PLOT = True
SAVE_RESULTS = True
OC_MAX_DIMS = 4
OC_MIN_DIM_SIZE = 8
OC_SKIP_OUTRIGGERS = True
OC_MIN_BL_LEN = 1.0
OC_MAX_BL_LEN = 1e+100
OC_MAXITER = 50
OC_MAX_RERUN = 4
OC_RERUN_MAXITER = 50
OC_USE_PRIOR_SOL = True
OC_PRIOR_SOL_FLAG_THRESH = 0.95
OC_USE_GPU = False
RFI_DPSS_HALFWIDTH = 3e-07
RFI_NSIG = 6.0
ABSCAL_MODEL_FILES_GLOB = None
ABSCAL_MIN_BL_LEN = 1.0
ABSCAL_MAX_BL_LEN = 140.0

Parse bounds

Load settings related to classifying antennas as good, suspect, or bad, then print what was loaded (or default).

# ant_metrics bounds for low correlation / dead antennas
am_corr_bad = (0, float(os.environ.get("AM_CORR_BAD", 0.3)))
am_corr_suspect = (float(os.environ.get("AM_CORR_BAD", 0.3)), float(os.environ.get("AM_CORR_SUSPECT", 0.5)))

# ant_metrics bounds for cross-polarized antennas
am_xpol_bad = (-1, float(os.environ.get("AM_XPOL_BAD", -0.1)))
am_xpol_suspect = (float(os.environ.get("AM_XPOL_BAD", -0.1)), float(os.environ.get("AM_XPOL_SUSPECT", 0)))

# bounds on solar altitude (in degrees)
good_solar_altitude = (-90, float(os.environ.get("SUSPECT_SOLAR_ALTITUDE", 0)))
suspect_solar_altitude = (float(os.environ.get("SUSPECT_SOLAR_ALTITUDE", 0)), 90)

# bounds on zeros in spectra
good_zeros_per_eo_spectrum = (0, int(os.environ.get("MAX_ZEROS_PER_EO_SPEC_GOOD", 2)))
suspect_zeros_per_eo_spectrum = (0, int(os.environ.get("MAX_ZEROS_PER_EO_SPEC_SUSPECT", 8)))

# bounds on autocorrelation power
auto_power_good = (float(os.environ.get("AUTO_POWER_GOOD_LOW", 5)), float(os.environ.get("AUTO_POWER_GOOD_HIGH", 30)))
auto_power_suspect = (float(os.environ.get("AUTO_POWER_SUSPECT_LOW", 1)), float(os.environ.get("AUTO_POWER_SUSPECT_HIGH", 60)))

# bounds on autocorrelation slope
auto_slope_good = (float(os.environ.get("AUTO_SLOPE_GOOD_LOW", -0.4)), float(os.environ.get("AUTO_SLOPE_GOOD_HIGH", 0.4)))
auto_slope_suspect = (float(os.environ.get("AUTO_SLOPE_SUSPECT_LOW", -0.6)), float(os.environ.get("AUTO_SLOPE_SUSPECT_HIGH", 0.6)))

# bounds on autocorrelation RFI
auto_rfi_good = (0, float(os.environ.get("AUTO_RFI_GOOD", 0.015)))
auto_rfi_suspect = (0, float(os.environ.get("AUTO_RFI_SUSPECT", 0.03)))

# bounds on autocorrelation shape
auto_shape_good = (0, float(os.environ.get("AUTO_SHAPE_GOOD", 0.1)))
auto_shape_suspect = (0, float(os.environ.get("AUTO_SHAPE_SUSPECT", 0.2)))

# bounds on chi^2 per antenna in omnical
oc_cspa_good = (0, float(os.environ.get("OC_CSPA_GOOD", 2)))
oc_cspa_suspect = (0, float(os.environ.get("OC_CSPA_SUSPECT", 3)))

# print bounds
for bound in ['am_corr_bad', 'am_corr_suspect', 'am_xpol_bad', 'am_xpol_suspect', 
              'good_solar_altitude', 'suspect_solar_altitude',
              'good_zeros_per_eo_spectrum', 'suspect_zeros_per_eo_spectrum',
              'auto_power_good', 'auto_power_suspect', 'auto_slope_good', 'auto_slope_suspect',
              'auto_rfi_good', 'auto_rfi_suspect', 'auto_shape_good', 'auto_shape_suspect',
              'oc_cspa_good', 'oc_cspa_suspect']:
    print(f'{bound} = {eval(bound)}')

am_corr_bad = (0, 0.2)
am_corr_suspect = (0.2, 0.4)
am_xpol_bad = (-1, -0.1)
am_xpol_suspect = (-0.1, 0.0)
good_solar_altitude = (-90, 0.0)
suspect_solar_altitude = (0.0, 90)
good_zeros_per_eo_spectrum = (0, 2)
suspect_zeros_per_eo_spectrum = (0, 8)
auto_power_good = (5.0, 30.0)
auto_power_suspect = (1.0, 60.0)
auto_slope_good = (-0.4, 0.4)
auto_slope_suspect = (-0.6, 0.6)
auto_rfi_good = (0, 0.015)
auto_rfi_suspect = (0, 0.03)
auto_shape_good = (0, 0.1)
auto_shape_suspect = (0, 0.2)
oc_cspa_good = (0, 2.0)
oc_cspa_suspect = (0, 3.0)

Load sum and diff data

hd = io.HERADataFastReader(SUM_FILE)
data, _, _ = hd.read(read_flags=False, read_nsamples=False)
hd_diff = io.HERADataFastReader(DIFF_FILE)
diff_data, _, _ = hd_diff.read(read_flags=False, read_nsamples=False)

ants = sorted(set([ant for bl in hd.bls for ant in utils.split_bl(bl)]))
auto_bls = [bl for bl in data if (bl[0] == bl[1]) and (utils.split_pol(bl[2])[0] == utils.split_pol(bl[2])[1])]
antpols = sorted(set([ant[1] for ant in ants]))

# print basic information about the file
print(f'File: {SUM_FILE}')
print(f'JDs: {hd.times} ({np.median(np.diff(hd.times)) * 24 * 3600:.5f} s integrations)')
print(f'LSTS: {hd.lsts * 12 / np.pi } hours')
print(f'Frequencies: {len(hd.freqs)} {np.median(np.diff(hd.freqs)) / 1e6:.5f} MHz channels from {hd.freqs[0] / 1e6:.5f} to {hd.freqs[-1] / 1e6:.5f} MHz')
print(f'Antennas: {len(hd.data_ants)}')
print(f'Polarizations: {hd.pols}')

File: /mnt/sn1/2460234/zen.2460234.44960.sum.uvh5
JDs: [2460234.4495415  2460234.44965335] (9.66368 s integrations)
LSTS: [1.8969873  1.89967901] hours
Frequencies: 1536 0.12207 MHz channels from 46.92078 to 234.29871 MHz
Antennas: 251
Polarizations: ['nn', 'ee', 'ne', 'en']

Classify good, suspect, and bad antpols

Run ant_metrics

This classifies antennas as cross-polarized, low-correlation, or dead. Such antennas are excluded from any calibration.

am = ant_metrics.AntennaMetrics(SUM_FILE, DIFF_FILE, sum_data=data, diff_data=diff_data)
am.iterative_antenna_metrics_and_flagging(crossCut=am_xpol_bad[1], deadCut=am_corr_bad[1])
am.all_metrics = {}  # this saves time and disk by getting rid of per-iteration information we never use
if SAVE_RESULTS:
    am.save_antenna_metrics(AM_FILE, overwrite=True)

# Turn ant metrics into classifications
totally_dead_ants = [ant for ant, i in am.xants.items() if i == -1]
am_totally_dead = ant_class.AntennaClassification(good=[ant for ant in ants if ant not in totally_dead_ants], bad=totally_dead_ants)
am_corr = ant_class.antenna_bounds_checker(am.final_metrics['corr'], bad=[am_corr_bad], suspect=[am_corr_suspect], good=[(0, 1)])
am_xpol = ant_class.antenna_bounds_checker(am.final_metrics['corrXPol'], bad=[am_xpol_bad], suspect=[am_xpol_suspect], good=[(-1, 1)])
ant_metrics_class = am_totally_dead + am_corr + am_xpol

Mark sun-up (or high solar altitude) data as suspect

min_sun_alt = np.min(utils.get_sun_alt(hd.times))
solar_class = ant_class.antenna_bounds_checker({ant: min_sun_alt for ant in ants}, good=[good_solar_altitude], suspect=[suspect_solar_altitude])

Classify antennas responsible for 0s in visibilities as bad:

This classifier looks for X-engine failure or packet loss specific to an antenna which causes either the even visibilities (or the odd ones, or both) to be 0s.

zeros_class = ant_class.even_odd_zeros_checker(data, diff_data, good=good_zeros_per_eo_spectrum, suspect=suspect_zeros_per_eo_spectrum)

Examine and classify autocorrelation power, slope, and RFI occpancy

These classifiers look for antennas with too high or low power, to steep a slope, or too much excess RFI.

auto_power_class = ant_class.auto_power_checker(data, good=auto_power_good, suspect=auto_power_suspect)
auto_slope_class = ant_class.auto_slope_checker(data, good=auto_slope_good, suspect=auto_slope_suspect, edge_cut=100, filt_size=17)
cache = {}
auto_rfi_class = ant_class.auto_rfi_checker(data, good=auto_rfi_good, suspect=auto_rfi_suspect, 
                                            filter_half_widths=[RFI_DPSS_HALFWIDTH], nsig=RFI_NSIG, cache=cache)
auto_class = auto_power_class + auto_slope_class + auto_rfi_class

del cache
malloc_trim()

Find and flag RFI

# Compute int_count for all unflagged autocorrelations averaged together
int_time = 24 * 3600 * np.median(np.diff(data.times_by_bl[auto_bls[0][0:2]]))
chan_res = np.median(np.diff(data.freqs))
final_class = ant_metrics_class + zeros_class + auto_class
int_count = int(int_time * chan_res) * (len(final_class.good_ants) + len(final_class.suspect_ants))
avg_auto = {(-1, -1, 'ee'): np.mean([data[bl] for bl in auto_bls if final_class[utils.split_bl(bl)[0]] != 'bad'], axis=0)}
# Flag RFI first with channel differences and then with DPSS
antenna_flags, _ = xrfi.flag_autos(avg_auto, int_count=int_count, nsig=(RFI_NSIG * 5))
_, rfi_flags = xrfi.flag_autos(avg_auto, int_count=int_count, flag_method='dpss_flagger',
                               flags=antenna_flags, freqs=data.freqs, filter_centers=[0],
                               filter_half_widths=[RFI_DPSS_HALFWIDTH], eigenval_cutoff=[1e-9], nsig=RFI_NSIG)
malloc_trim()

def rfi_plot():
    plt.figure(figsize=(12, 5), dpi=100)
    plt.semilogy(hd.freqs / 1e6, np.where(rfi_flags, np.nan, avg_auto[(-1, -1, 'ee')])[1], label = 'Average Good or Suspect Autocorrelation', zorder=100)
    plt.semilogy(hd.freqs / 1e6, np.where(False, np.nan, avg_auto[(-1, -1, 'ee')])[1], 'r', lw=.5, label=f'{np.sum(rfi_flags[0])} Channels Flagged for RFI')
    plt.legend()
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Uncalibrated Autocorrelation')
    plt.tight_layout()

Figure 1: RFI Flagging

This figure shows RFI identified using the average of all autocorrelations---excluding bad antennas---for the first integration in the file.

rfi_plot()

def autocorr_plot(cls):    
    fig, axes = plt.subplots(1, 2, figsize=(14, 5), dpi=100, sharey=True, gridspec_kw={'wspace': 0})
    labels = []
    colors = ['darkgreen', 'goldenrod', 'maroon']
    for ax, pol in zip(axes, antpols):
        for ant in cls.ants:
            if ant[1] == pol:
                color = colors[cls.quality_classes.index(cls[ant])]
                ax.semilogy(np.mean(data[utils.join_bl(ant, ant)], axis=0), color=color, lw=.5)
        ax.set_xlabel('Channel', fontsize=12)
        ax.set_title(f'{utils.join_pol(pol, pol)}-Polarized Autos')

    axes[0].set_ylabel('Raw Autocorrelation', fontsize=12)
    axes[1].legend([matplotlib.lines.Line2D([0], [0], color=color) for color in colors], 
                   [cls.capitalize() for cls in auto_class.quality_classes], ncol=1, fontsize=12, loc='upper right', framealpha=1)
    plt.tight_layout()

Classify antennas based on shapes, excluding RFI-contamined channels

auto_shape_class = ant_class.auto_shape_checker(data, good=auto_shape_good, suspect=auto_shape_suspect,
                                                flag_spectrum=np.sum(rfi_flags, axis=0).astype(bool), antenna_class=final_class)
auto_class += auto_shape_class

final_class = ant_metrics_class + solar_class + zeros_class + auto_class

Figure 2: Plot of autocorrelations with classifications

This figure shows a plot of all autocorrelations in the array, split by polarization. Antennas are classified based on their autocorrelations into good, suspect, and bad, by examining power, slope, and RFI-occupancy.

if PLOT: autocorr_plot(auto_class)

Classify antennas based on non-noiselike diffs

xengine_diff_class = ant_class.non_noiselike_diff_by_xengine_checker(data, diff_data, flag_waterfall=rfi_flags, 
                                                                     antenna_class=final_class, 
                                                                     xengine_chans=96, bad_xengine_zcut=10)
final_class += xengine_diff_class

# delete diffs to save memory
del diff_data, hd_diff
malloc_trim()

Summarize antenna classification prior to redundant-baseline calibration

def array_class_plot(cls, extra_label=""):
    outriggers = [ant for ant in hd.data_ants if ant >= 320]

    if len(outriggers) > 0:
        fig, axes = plt.subplots(1, 2, figsize=(14, 6), dpi=100, gridspec_kw={'width_ratios': [2, 1]})
        plot_antclass(hd.antpos, cls, ax=axes[0], ants=[ant for ant in hd.data_ants if ant < 320], legend=False, title=f'HERA Core{extra_label}')
        plot_antclass(hd.antpos, cls, ax=axes[1], ants=outriggers, radius=50, title='Outriggers')
    else:
        fig, axes = plt.subplots(1, 1, figsize=(9, 6), dpi=100)
        plot_antclass(hd.antpos, cls, ax=axes, ants=[ant for ant in hd.data_ants if ant < 320], legend=False, title=f'HERA Core{extra_label}')

Figure 3: Summary of antenna classifications prior to calibration

This figure shows the location and classification of all antennas prior to calibration. Antennas are split along the diagonal, with ee-polarized antpols represented by the southeast half of each antenna and nn-polarized antpols represented by the northwest half. Outriggers are split from the core and shown at exaggerated size in the right-hand panel. This classification includes ant_metrics, a count of the zeros in the even or odd visibilities, and autocorrelation power, slope, and RFI occupancy. An antenna classified as bad in any classification will be considered bad. An antenna marked as suspect any in any classification will be considered suspect unless it is also classified as bad elsewhere.

if PLOT: array_class_plot(final_class)

Perform redundant-baseline calibration

def classify_off_grid(reds, all_ants):
    '''Returns AntennaClassification of all_ants where good ants are in reds while bad ants are not.'''
    ants_in_reds = set([ant for red in reds for bl in red for ant in utils.split_bl(bl)])
    on_grid = [ant for ant in all_ants if ant in ants_in_reds]
    off_grid = [ant for ant in all_ants if ant not in ants_in_reds]
    return ant_class.AntennaClassification(good=on_grid, bad=off_grid)

def per_pol_filter_reds(reds, pols=['nn', 'ee'], **kwargs):
    '''Performs redcal filtering separately on polarizations (which might have different min_dim_size issues).'''
    return [red for pol in pols for red in redcal.filter_reds(copy.deepcopy(reds), pols=[pol], **kwargs)]

def check_if_whole_pol_flagged(redcal_class, pols=['Jee', 'Jnn']):
    '''Checks if an entire polarization is flagged. If it is, returns True and marks all antennas as bad in redcal_class.'''
    if np.logical_or(*[np.all([redcal_class[ant] == 'bad' for ant in redcal_class.ants if ant[1] == pol]) for pol in pols]):
        print('An entire polarization has been flagged. Stopping redcal.')
        for ant in redcal_class:
            redcal_class[ant] = 'bad'
        return True
    return False

Perform iterative redcal

# figure out and filter reds and classify antennas based on whether or not they are on the main grid
fr_settings = {'max_dims': OC_MAX_DIMS, 'min_dim_size': OC_MIN_DIM_SIZE, 'min_bl_cut': OC_MIN_BL_LEN, 'max_bl_cut': OC_MAX_BL_LEN}
reds = redcal.get_reds(hd.data_antpos, pols=['ee', 'nn'], pol_mode='2pol')
reds = per_pol_filter_reds(reds, ex_ants=final_class.bad_ants, antpos=hd.data_antpos, **fr_settings)
if OC_SKIP_OUTRIGGERS:
    reds = redcal.filter_reds(reds, ex_ants=[ant for ant in ants if ant[0] >= 320])
redcal_class = classify_off_grid(reds, ants)

if OC_USE_PRIOR_SOL:
    # Find closest omnical file
    omnical_files = sorted(glob.glob('.'.join(OMNICAL_FILE.split('.')[:-5]) + '.*.' + '.'.join(OMNICAL_FILE.split('.')[-3:])))
    if len(omnical_files) == 0:
        OC_USE_PRIOR_SOL = False
    else:
        omnical_jds = np.array([float(re.findall("\d+\.\d+", ocf)[-1]) for ocf in omnical_files])
        closest_omnical = omnical_files[np.argmin(np.abs(omnical_jds - data.times[0]))]

        # Load closest omnical file and use it if the antenna flagging is not too dissimilar
        hc = io.HERACal(closest_omnical)
        prior_gains, prior_flags, _, _ = hc.read()
        not_bad_not_prior_flagged = [ant for ant in final_class if not ant in redcal_class.bad_ants and not np.all(prior_flags[ant])]
        if (len(redcal_class.bad_ants) == len(redcal_class.ants)):
            OC_USE_PRIOR_SOL = False  # all antennas flagged
        elif (len(not_bad_not_prior_flagged) / (len(redcal_class.ants) - len(redcal_class.bad_ants))) < OC_PRIOR_SOL_FLAG_THRESH:
            OC_USE_PRIOR_SOL = False  # too many antennas unflaged that were flagged in the prior sol
        else:
            print(f'Using {closest_omnical} as a starting point for redcal.')

redcal_start = time.time()
rc_settings = {'max_dims': OC_MAX_DIMS, 'oc_conv_crit': 1e-10, 'gain': 0.4, 'run_logcal': False,
               'oc_maxiter': OC_MAXITER, 'check_after': OC_MAXITER, 'use_gpu': OC_USE_GPU}

if check_if_whole_pol_flagged(redcal_class):
    # skip redcal, initialize empty sol and meta 
    sol = redcal.RedSol(reds)
    meta = {'chisq': None, 'chisq_per_ant': None}
else:    
    if OC_USE_PRIOR_SOL:
        # use prior gains and data to create starting point for next step
        ants_in_reds = set([ant for red in reds for bl in red for ant in utils.split_bl(bl)])
        sol = redcal.RedSol(reds=reds, gains={ant: prior_gains[ant] for ant in ants_in_reds})
        sol.update_vis_from_data(data)
    else:
        # perform first stage of redundant calibration, 
        meta, sol = redcal.redundantly_calibrate(data, reds, **rc_settings)
        max_dly = np.max(np.abs(list(meta['fc_meta']['dlys'].values())))  # Needed for RFI delay-slope cal
    malloc_trim()
    
# iteratively rerun redundant calibration
redcal_done = False
rc_settings['oc_maxiter'] = rc_settings['check_after'] = OC_RERUN_MAXITER
for i in range(OC_MAX_RERUN + 1):
    # refilter reds and update classification to reflect new off-grid ants, if any
    reds = per_pol_filter_reds(reds, ex_ants=(final_class + redcal_class).bad_ants, antpos=hd.data_antpos, **fr_settings)
    reds = sorted(reds, key=len, reverse=True)
    redcal_class += classify_off_grid(reds, ants)
    ants_in_reds = set([ant for red in reds for bl in red for ant in utils.split_bl(bl)])
    
    # check to see whether we're done
    if redcal_done or (i == OC_MAX_RERUN) or check_if_whole_pol_flagged(redcal_class):
        break

    # re-run redundant calibration using previous solution, updating bad and suspicious antennas
    meta, sol = redcal.redundantly_calibrate(data, reds, sol0=sol, **rc_settings)
    malloc_trim()
    
    # recompute chi^2 for bad antennas without bad antennas to make sure they are actually bad
    mean_cspa = {ant: np.nanmean(np.where(rfi_flags, np.nan, meta['chisq_per_ant'][ant])) for ant in meta['chisq_per_ant']}
    sol2 = redcal.RedSol(sol.reds, gains={ant: sol[ant] for ant in mean_cspa if mean_cspa[ant] <= oc_cspa_suspect[1]}, vis=sol.vis)
    new_chisq_per_ant = {ant: np.array(meta['chisq_per_ant'][ant]) for ant in sol2.gains}
    if len(set([bl[2] for red in per_pol_filter_reds(reds, ants=sol2.gains.keys(), antpos=hd.data_antpos, **fr_settings) for bl in red])) >= 2:
        redcal.expand_omni_gains(sol2, reds, data, chisq_per_ant=new_chisq_per_ant)
    for ant in mean_cspa:
        if ant in new_chisq_per_ant:
            if np.any(np.isfinite(new_chisq_per_ant[ant])):
                if not np.all(np.isclose(new_chisq_per_ant[ant], 0)):
                    new_mean_cspa = np.nanmean(np.where(rfi_flags, np.nan, meta['chisq_per_ant'][ant]))
                    if new_mean_cspa > 0:
                        mean_cspa[ant] = np.min([mean_cspa[ant], new_mean_cspa])
    
    # remove bad antennas
    cspa_class = ant_class.antenna_bounds_checker(mean_cspa, good=oc_cspa_good, suspect=oc_cspa_suspect, bad=[(-np.inf, np.inf)])
    redcal_class += cspa_class
    print(f'Removing {cspa_class.bad_ants} for high chi^2.')
    if len(cspa_class.bad_ants) == 0:
        redcal_done = True  # no new antennas to flag

print(f'Finished redcal in {(time.time() - redcal_start) / 60:.2f} minutes.')

Removing set() for high chi^2.
Finished redcal in 7.52 minutes.

final_class += redcal_class

Expand solution to include calibratable baselines excluded from redcal (e.g. because they were too long)

expanded_reds = redcal.get_reds(hd.data_antpos, pols=['ee', 'nn'], pol_mode='2pol')
expanded_reds = per_pol_filter_reds(expanded_reds, ex_ants=(ant_metrics_class + solar_class + zeros_class + auto_class + xengine_diff_class).bad_ants,
                                    max_dims=OC_MAX_DIMS, min_dim_size=OC_MIN_DIM_SIZE)
if OC_SKIP_OUTRIGGERS:
    expanded_reds = redcal.filter_reds(expanded_reds, ex_ants=[ant for ant in ants if ant[0] >= 320])
if len(sol.gains) > 0:
    redcal.expand_omni_vis(sol, expanded_reds, data, chisq=meta['chisq'], chisq_per_ant=meta['chisq_per_ant'])

# now figure out flags, nsamples etc.
omni_flags = {ant: (~np.isfinite(g)) | (ant in final_class.bad_ants) for ant, g in sol.gains.items()}
vissol_flags = datacontainer.RedDataContainer({bl: ~np.isfinite(v) for bl, v in sol.vis.items()}, reds=sol.vis.reds)
single_nsamples_array = np.ones((len(hd.times), len(hd.freqs)), dtype=float)
nsamples = datacontainer.DataContainer({bl: single_nsamples_array for bl in data})
vissol_nsamples = redcal.count_redundant_nsamples(nsamples, [red for red in expanded_reds if red[0] in vissol_flags], 
                                                  good_ants=[ant for ant in final_class if ant not in final_class.bad_ants])
for bl in vissol_flags:
    vissol_flags[bl][vissol_nsamples[bl] == 0] = True
sol.make_sol_finite()        

Fix the firstcal delay slope degeneracy using RFI transmitters

if not OC_USE_PRIOR_SOL:
    # find channels clearly contaminated by RFI
    not_bad_ants = [ant for ant in final_class.ants if final_class[ant] != 'bad']
    if len(not_bad_ants) > 0:
        chan_flags = np.mean([xrfi.detrend_medfilt(data[utils.join_bl(ant, ant)], Kf=8, Kt=2) for ant in not_bad_ants], axis=(0, 1)) > 5

        # hardcoded RFI transmitters and their headings
        # channel: frequency (Hz), heading (rad), chi^2
        phs_sol = {359: ( 90744018.5546875, 0.7853981, 23.3),
                   360: ( 90866088.8671875, 0.7853981, 10.8),
                   385: ( 93917846.6796875, 0.7853981, 27.3),
                   386: ( 94039916.9921875, 0.7853981, 18.1),
                   400: ( 95748901.3671875, 6.0632738, 24.0),
                   441: (100753784.1796875, 0.7853981, 21.7),
                   442: (100875854.4921875, 0.7853981, 19.4),
                   455: (102462768.5546875, 6.0632738, 18.8),
                   456: (102584838.8671875, 6.0632738,  8.8),
                   471: (104415893.5546875, 0.7853981, 13.3),
                   484: (106002807.6171875, 6.0632738, 21.2),
                   485: (106124877.9296875, 6.0632738,  4.0),
                  1181: (191085815.4296875, 0.7853981, 26.3),
                  1182: (191207885.7421875, 0.7853981, 27.0),
                  1183: (191329956.0546875, 0.7853981, 25.6),
                  1448: (223678588.8671875, 2.6075219, 25.7),
                  1449: (223800659.1796875, 2.6075219, 22.6),
                  1450: (223922729.4921875, 2.6075219, 11.6),
                  1451: (224044799.8046875, 2.6075219,  5.9),
                  1452: (224166870.1171875, 2.6075219, 22.6),
                  1510: (231246948.2421875, 0.1068141, 23.9)}

        if not np.isclose(hd.freqs[0], 46920776.3671875, atol=0.001) or len(hd.freqs) != 1536:
            # We have less frequencies than usual (maybe testing)
            phs_sol = {np.argmin(np.abs(hd.freqs - freq)): (freq, heading, chisq) for chan, (freq, heading, chisq) in phs_sol.items() if hd.freqs[0] <= freq <= hd.freqs[-1]}


        rfi_chans = [chan for chan in phs_sol if chan_flags[chan]]
        print('Channels used for delay-slope calibration with RFI:', rfi_chans)
        rfi_angles = np.array([phs_sol[chan][1] for chan in rfi_chans])
        rfi_headings = np.array([np.cos(rfi_angles), np.sin(rfi_angles), np.zeros_like(rfi_angles)])
        rfi_chisqs = np.array([phs_sol[chan][2] for chan in rfi_chans])

        # resolve firstcal degeneracy with delay slopes set by RFI transmitters, update cal
        RFI_dly_slope_gains = abscal.RFI_delay_slope_cal([red for red in expanded_reds if red[0] in sol.vis], hd.antpos, sol.vis, hd.freqs, rfi_chans, rfi_headings, rfi_wgts=rfi_chisqs**-1,
                                                         min_tau=-max_dly, max_tau=max_dly, delta_tau=0.1e-9, return_gains=True, gain_ants=sol.gains.keys())
        sol.gains = {ant: g * RFI_dly_slope_gains[ant] for ant, g in sol.gains.items()}
        apply_cal.calibrate_in_place(sol.vis, RFI_dly_slope_gains)
        malloc_trim()

Channels used for delay-slope calibration with RFI: [359, 360, 385, 386, 400, 441, 442, 455, 456, 471, 484, 485]

Perform absolute amplitude calibration using a model of autocorrelations

# Load simulated and then downsampled model of autocorrelations that includes receiver noise, then interpolate to upsample
hd_model = io.HERADataFastReader(f'{HNBT_DATA}/SSM_autocorrelations_downsampled.uvh5')
model, _, _ = hd_model.read(read_flags=False, read_nsamples=False)
per_pol_interpolated_model = {}
for bl in model:
    sorted_lsts, lst_indices = np.unique(model.lsts, return_index=True)
    periodic_model = np.vstack([model[bl][lst_indices, :], model[bl][lst_indices[0], :]])
    periodic_lsts = np.append(sorted_lsts, sorted_lsts[0] + 2 * np.pi)
    lst_interpolated = interpolate.CubicSpline(periodic_lsts, periodic_model, axis=0, bc_type='periodic')(data.lsts)
    per_pol_interpolated_model[bl[2]] = interpolate.CubicSpline(model.freqs, lst_interpolated, axis=1)(data.freqs)
model = {bl: per_pol_interpolated_model[bl[2]] for bl in auto_bls if utils.split_bl(bl)[0] not in final_class.bad_ants}

# Run abscal and update omnical gains with abscal gains
if len(model) > 0:
    redcaled_autos = {bl: sol.calibrate_bl(bl, data[bl]) for bl in auto_bls if utils.split_bl(bl)[0] not in final_class.bad_ants}
    g_abscal = abscal.abs_amp_logcal(model, redcaled_autos, verbose=False, return_gains=True, gain_ants=sol.gains)
    sol.gains = {ant: g * g_abscal[ant] for ant, g in sol.gains.items()}
    apply_cal.calibrate_in_place(sol.vis, g_abscal)
    del redcaled_autos, g_abscal

del hd_model, model
malloc_trim()

Full absolute calibration of phase gradients

If an ABSCAL_MODEL_FILES_GLOB is provided, try to perform a full absolute calibration of tip-tilt phase gradients across the array using that those model files. Specifically, this step calibrates omnical visbility solutions using unique baselines simulated with a model of the sky and HERA's beam.

if ABSCAL_MODEL_FILES_GLOB is not None:
    abscal_model_files = sorted(glob.glob(ABSCAL_MODEL_FILES_GLOB))
else:
    # try to find files on site
    abscal_model_files = sorted(glob.glob('/mnt/sn1/abscal_models/H4C_1/abscal_files_unique_baselines/zen.2458894.?????.uvh5'))
    if len(abscal_model_files) == 0:
        # try to find files at NRAO
        abscal_model_files = sorted(glob.glob('/lustre/aoc/projects/hera/zmartino/hera_calib_model/H4C_1/abscal_files_unique_baselines/zen.2458894.?????.uvh5'))
print(f'Found {len(abscal_model_files)} abscal model files{" in " + os.path.dirname(abscal_model_files[0]) if len(abscal_model_files) > 0 else ""}.')

Found 425 abscal model files in /mnt/sn1/abscal_models/H4C_1/abscal_files_unique_baselines.

# Try to perform a full abscal of phase
if len(abscal_model_files) == 0:
    DO_FULL_ABSCAL = False
    print('No model files found... not performing full absolute calibration of phase gradients.')
elif np.all([ant in final_class.bad_ants for ant in ants]):
    DO_FULL_ABSCAL = False
    print('All antennas classified as bad... skipping absolute calibration of phase gradients.')
else:
    abscal_start = time.time()
    # figure out which model files match the LSTs of the data
    matched_model_files = sorted(set(abscal.match_times(SUM_FILE, abscal_model_files, filetype='uvh5')))
    if len(matched_model_files) == 0:
        DO_FULL_ABSCAL = False
        print(f'No model files found matching the LSTs of this file after searching for {(time.time() - abscal_start) / 60:.2f} minutes. '
              'Not performing full absolute calibration of phase gradients.')
    else:
        DO_FULL_ABSCAL = True
        # figure out appropriate model times to load
        hdm = io.HERAData(matched_model_files)
        all_model_times, all_model_lsts = abscal.get_all_times_and_lsts(hdm, unwrap=True)
        d2m_time_map = abscal.get_d2m_time_map(data.times, np.unwrap(data.lsts), all_model_times, all_model_lsts, extrap_limit=.5)

if DO_FULL_ABSCAL:
    abscal_meta = {}
    for pol in ['ee', 'nn']:
        print(f'Performing absolute phase gradient calibration of {pol}-polarized visibility solutions...')
        
        # load matching times and baselines
        unflagged_data_bls = [bl for bl in vissol_flags if not np.all(vissol_flags[bl]) and bl[2] == pol]
        model_bls = copy.deepcopy(hdm.bls)
        model_antpos = hdm.data_antpos
        if len(matched_model_files) > 1:  # in this case, it's a dictionary
            model_bls = list(set([bl for bls in list(hdm.bls.values()) for bl in bls]))
            model_antpos = {ant: pos for antpos in hdm.data_antpos.values() for ant, pos in antpos.items()}
        data_bls, model_bls, data_to_model_bl_map = abscal.match_baselines(unflagged_data_bls, model_bls, data.antpos, model_antpos=model_antpos, 
                                                                         pols=[pol], data_is_redsol=True, model_is_redundant=True, tol=1.0,
                                                                         min_bl_cut=ABSCAL_MIN_BL_LEN, max_bl_cut=ABSCAL_MAX_BL_LEN, verbose=True)
        model, model_flags, _ = io.partial_time_io(hdm, np.unique([d2m_time_map[time] for time in data.times]), bls=model_bls)
        model_bls = [data_to_model_bl_map[bl] for bl in data_bls]
        
        # rephase model to match in lsts
        model_blvecs = {bl: model.antpos[bl[0]] - model.antpos[bl[1]] for bl in model.keys()}
        utils.lst_rephase(model, model_blvecs, model.freqs, data.lsts - model.lsts,
                          lat=hdm.telescope_location_lat_lon_alt_degrees[0], inplace=True)

        # run abscal and apply 
        abscal_meta[pol], delta_gains = abscal.complex_phase_abscal(sol.vis, model, sol.reds, data_bls, model_bls)
        
        # apply gains
        sol.gains = {antpol : g * delta_gains.get(antpol, 1) for antpol, g in sol.gains.items()}
        apply_cal.calibrate_in_place(sol.vis, delta_gains)            
    
    del hdm, model, model_flags, delta_gains
    malloc_trim()
    
    print(f'Finished absolute calibration of tip-tilt phase slopes in {(time.time() - abscal_start) / 60:.2f} minutes.')

Performing absolute phase gradient calibration of ee-polarized visibility solutions...
Selected 480 data baselines and 480 model baselines to load.

WARNING:jax._src.lib.xla_bridge:No GPU/TPU found, falling back to CPU. (Set TF_CPP_MIN_LOG_LEVEL=0 and rerun for more info.)

Performing absolute phase gradient calibration of nn-polarized visibility solutions...
Selected 480 data baselines and 480 model baselines to load.
Finished absolute calibration of tip-tilt phase slopes in 0.39 minutes.

def redundant_group_plot():
    if np.all([ant in final_class.bad_ants for ant in ants]):
        print('All antennas classified as bad. Nothing to plot.')
        return
    
    fig, axes = plt.subplots(2, 2, figsize=(14, 6), dpi=100, sharex='col', sharey='row', gridspec_kw={'hspace': 0, 'wspace': 0})
    for i, pol in enumerate(['ee', 'nn']):
        reds_here = redcal.get_reds(hd.data_antpos, pols=[pol], pol_mode='1pol')
        red = sorted(redcal.filter_reds(reds_here, ex_ants=final_class.bad_ants), key=len, reverse=True)[0]
        rc_data = {bl: sol.calibrate_bl(bl, data[bl]) for bl in red}
        for bl in red:
            axes[0, i].plot(hd.freqs/1e6, np.angle(rc_data[bl][0]), alpha=.5, lw=.5)
            axes[1, i].semilogy(hd.freqs/1e6, np.abs(rc_data[bl][0]), alpha=.5, lw=.5)
        axes[0, i].plot(hd.freqs / 1e6, np.angle(sol.vis[red[0]][0]), lw=1, c='k')
        axes[1, i].semilogy(hd.freqs / 1e6, np.abs(sol.vis[red[0]][0]), lw=1, c='k', label=f'Baseline Group:\n{red[0]}')
        axes[1, i].set_xlabel('Frequency (MHz)')
        axes[1, i].legend(loc='upper right')
    axes[0, 0].set_ylabel('Visibility Phase (radians)')
    axes[1, 0].set_ylabel('Visibility Amplitude (Jy)')
    plt.tight_layout()

def abscal_degen_plot():
    if DO_FULL_ABSCAL:
        fig, axes = plt.subplots(3, 1, figsize=(14, 6), dpi=100, sharex=True, gridspec_kw={'hspace': .05})

        for ax, pol in zip(axes[:2], ['ee', 'nn']):
            for kk in range(abscal_meta[pol]['Lambda_sol'].shape[-1]):
                ax.plot(hd.freqs[~rfi_flags[0]] * 1e-6, abscal_meta[pol]['Lambda_sol'][0, ~rfi_flags[0], kk], '.', ms=1, label=f"Component {kk}")

            ax.set_ylim(-np.pi-0.5, np.pi+0.5)
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('Phase Gradient\nVector Component')
            ax.legend(markerscale=20, title=f'{pol}-polarization', loc='lower right')
            ax.grid()
            
        for pol, color in zip(['ee', 'nn'], ['b', 'r']):
            axes[2].plot(hd.freqs[~rfi_flags[0]]*1e-6, abscal_meta[pol]['Z_sol'].real[0, ~rfi_flags[0]], '.', ms=1, label=pol, color=color)
        axes[2].set_ylim(-.25, 1.05)
        axes[2].set_ylabel('Re[Z($\\nu$)]')
        axes[2].legend(markerscale=20, loc='lower right')
        axes[2].grid()            
        plt.tight_layout()

Figure 4: Redundant calibration of a single baseline group

The results of a redundant-baseline calibration of a single integration and a single group, the one with the highest redundancy in each polarization after antenna classification and excision based on the above, plus the removal of antennas with high chi^2 per antenna. The black line is the redundant visibility solution. Each thin colored line is a different baseline group. Phases are shown in the top row, amplitudes in the bottom, ee-polarized visibilities in the left column, and nn-polarized visibilities in the right.

if PLOT: redundant_group_plot()

Figure 5: Absolute calibration of redcal degeneracies

This figure shows the per-frequency phase gradient solutions across the array for both polarizations and all components of the degenerate subspace of redundant-baseline calibraton. While full HERA only has two such tip-tilt degeneracies, a subset of HERA can have up to OC_MAX_DIMS (depending on antenna flagging). In addition to the absolute amplitude, this is the full set of the calibration degrees of freedom not constrainted by redcal. This figure also includes a plot of $Re[Z(\nu)]$, the complex objective function which varies from -1 to 1 and indicates how well the data and the absolute calibration model have been made to agree. Perfect agreement is 1.0 and good agreement is anything above $\sim$0.5 Decorrelation yields values closer to 0, where anything below $\sim$0.3 is suspect.

if PLOT: abscal_degen_plot()

Attempt to calibrate some flagged antennas

This attempts to calibrate bad antennas using information from good or suspect antennas without allowing bad antennas to affect their calibration. However, introducing 0s in gains or infs/nans in gains or visibilities can create problems down the line, so those are removed.

expanded_reds = redcal.get_reds(hd.data_antpos, pols=['ee', 'nn'], pol_mode='2pol')
sol.vis.build_red_keys(expanded_reds)
redcal.expand_omni_gains(sol, expanded_reds, data, chisq_per_ant=meta['chisq_per_ant'])
if not np.all([ant in final_class.bad_ants for ant in ants]):
    redcal.expand_omni_vis(sol, expanded_reds, data)

# Replace near-zeros in gains and infs/nans in gains/sols
for ant in sol.gains:
    zeros_in_gains = np.isclose(sol.gains[ant], 0)
    if ant in omni_flags:
        omni_flags[ant][zeros_in_gains] = True
    sol.gains[ant][zeros_in_gains] = 1.0 + 0.0j
sol.make_sol_finite()
malloc_trim()

def array_chisq_plot():
    if np.all([ant in final_class.bad_ants for ant in ants]):
        print('All antennas classified as bad. Nothing to plot.')
        return    
    
    fig, axes = plt.subplots(1, 2, figsize=(14, 5), dpi=100)
    for ax, pol in zip(axes, ['ee', 'nn']):
        ants_to_plot = set([ant for ant in meta['chisq_per_ant'] if utils.join_pol(ant[1], ant[1]) == pol])
        cspas = np.array([np.nanmean(np.where(rfi_flags, np.nan, meta['chisq_per_ant'][ant])) for ant in ants_to_plot])
        xpos = [hd.antpos[ant[0]][0] for ant in ants_to_plot]
        ypos = [hd.antpos[ant[0]][1] for ant in ants_to_plot]
        scatter = ax.scatter(xpos, ypos, s=300, c=cspas, lw=.25, edgecolors=np.where(np.isfinite(cspas) & (cspas > 0), 'none', 'k'), 
                             norm=matplotlib.colors.LogNorm(vmin=1, vmax=oc_cspa_suspect[1]))
        for ant in ants_to_plot:
            ax.text(hd.antpos[ant[0]][0], hd.antpos[ant[0]][1], ant[0], va='center', ha='center', fontsize=8,
                    c=('r' if ant in final_class.bad_ants else 'w'))
        plt.colorbar(scatter, ax=ax, extend='both')
        ax.axis('equal')
        ax.set_xlabel('East-West Position (meters)')
        ax.set_ylabel('North-South Position (meters)')
        ax.set_title(f'{pol}-pol $\\chi^2$ / Antenna (Red is Flagged)')
    plt.tight_layout()

Figure 6: chi^2 per antenna across the array

This plot shows median (taken over time and frequency) of the normalized chi^2 per antenna. The expectation value for this quantity when the array is perfectly redundant is 1.0. Antennas that are classified as bad for any reason have their numbers shown in red. Some of those antennas were classified as bad during redundant calibration for high chi^2. Some of those antennas were originally excluded from redundant calibration because they were classified as bad earlier for some reason. See here for more details. Note that the color scale saturates at below 1 and above 10.

if PLOT: array_chisq_plot()

Figure 7: Summary of antenna classifications after redundant calibration

This figure is the same as Figure 2, except that it now includes additional suspect or bad antennas based on redundant calibration. This can include antennas with high chi^2, but it can also include antennas classified as "bad" because they would add extra degeneracies to calibration.

if PLOT: array_class_plot(final_class, extra_label=", Post-Redcal")

to_show = {'Antenna': [f'{ant[0]}{ant[1][-1]}' for ant in ants]}
classes = {'Antenna': [final_class[ant] if ant in final_class else '-' for ant in ants]}
to_show['Dead?'] = [{'good': 'No', 'bad': 'Yes'}[am_totally_dead[ant]] if (ant in am_totally_dead) else '' for ant in ants]
classes['Dead?'] = [am_totally_dead[ant] if (ant in am_totally_dead) else '' for ant in ants]
for title, ac in [('Low Correlation', am_corr),
                  ('Cross-Polarized', am_xpol),
                  ('Solar Alt', solar_class),
                  ('Even/Odd Zeros', zeros_class),
                  ('Autocorr Power', auto_power_class),
                  ('Autocorr Slope', auto_slope_class),
                  ('RFI in Autos', auto_rfi_class),
                  ('Autocorr Shape', auto_shape_class),
                  ('Bad Diff X-Engines', xengine_diff_class)]:
    to_show[title] = [f'{ac._data[ant]:.2G}' if (ant in ac._data) else '' for ant in ants]
    classes[title] = [ac[ant] if ant in ac else 'bad' for ant in ants]
    
to_show['Redcal chi^2'] = [f'{np.nanmean(np.where(rfi_flags, np.nan, meta["chisq_per_ant"][ant])):.3G}' \
                           if (meta['chisq_per_ant'] is not None and ant in meta['chisq_per_ant']) else '' for ant in ants]
classes['Redcal chi^2'] = [redcal_class[ant] if ant in redcal_class else '' for ant in ants]

df = pd.DataFrame(to_show)
df_classes = pd.DataFrame(classes)
colors = {'good': 'darkgreen', 'suspect': 'goldenrod', 'bad': 'maroon'}
df_colors = df_classes.applymap(lambda x: f'background-color: {colors.get(x, None)}')

table = df.style.hide() \
                .apply(lambda x: pd.DataFrame(df_colors.values, columns=x.columns), axis=None) \
                .set_properties(subset=['Antenna'], **{'font-weight': 'bold', 'border-right': "3pt solid black"}) \
                .set_properties(subset=df.columns[1:], **{'border-left': "1pt solid black"}) \
                .set_properties(**{'text-align': 'center', 'color': 'white'})

Table 1: Complete summary of per-antenna classifications

This table summarizes the results of the various classifications schemes detailed above. As before, green is good, yellow is suspect, and red is bad. The color for each antenna (first column) is the final summary of all other classifications. Antennas missing from redcal $\chi^2$ were excluded redundant-baseline calibration, either because they were flagged by ant_metrics or the even/odd zeros check, or because they would add unwanted extra degeneracies.

HTML(table.to_html())

Antenna	Dead?	Low Correlation	Cross-Polarized	Solar Alt	Even/Odd Zeros	Autocorr Power	Autocorr Slope	RFI in Autos	Autocorr Shape	Bad Diff X-Engines	Redcal chi^2
3e	No	0.72	0.62	-50	0	14	0.17	0.0072	0.047	0	1.4
3n	No	0.038	0.62	-50	0	0.61	0.61	0.077	0.13		1.13
4e	No	0.72	0.51	-50	0	13	0.14	0.0026	0.045	0	1.46
4n	No	0.51	0.51	-50	0	15	1.4	0.0039	0.34		1.96
5e	No	0.72	0.52	-50	0	14	0.1	0.0042	0.041	0	1.43
5n	No	0.7	0.52	-50	0	15	0.12	0.00033	0.027	0	1.23
7e	No	0.71	0.5	-50	0	12	0.14	0.0016	0.033	0	1.43
7n	No	0.7	0.5	-50	0	14	0.16	0.00065	0.022	0	1.23
8e	No	0.71	0.5	-50	0	14	0.16	0.0039	0.037	0	1.37
8n	No	0.7	0.5	-50	0	12	0.14	0.00033	0.022	0	1.25
9e	No	0.64	0.51	-50	0	2.1	0.32	0.017	0.036	0	1.39
9n	No	0.69	0.51	-50	0	14	0.16	0.00065	0.019	0	1.27
10e	No	0.71	0.51	-50	0	12	0.2	0.0026	0.03	0	1.39
10n	No	0.69	0.51	-50	0	15	0.15	0.00033	0.02	0	1.24
15e	No	0.54	0.5	-50	0	17	1.2	0.00033	0.29		2.34
15n	No	0.71	0.5	-50	0	14	0.11	0	0.025	0	1.19
16e	No	0.2	-0.45	-50	0	10	0.29	0	0.027		4.02
16n	No	0.2	-0.45	-50	0	32	0.11	0.0026	0.027		3.02
17e	No	0.72	0.51	-50	0	13	0.16	0.00065	0.029	0	1.31
17n	No	0.7	0.51	-50	0	9.3	0.19	0.00033	0.035	0	1.2
18e	No	0.68	0.55	-50	0	16	0.13	0.37	0.2		1.46
18n	No	0.45	0.55	-50	0	14	0.39	0.25	0.15		1.23
19e	No	0.71	0.5	-50	0	14	0.18	0.0039	0.03	0	1.32
19n	No	0.7	0.5	-50	0	44	0.085	0	0.048	0	1.23
20e	No	0.69	0.48	-50	0	15	0.42	0.0039	0.15	0	1.36
20n	No	0.69	0.48	-50	0	6	0.17	0.00065	0.015	0	1.25
21e	No	0.6	0.52	-50	0	1.4	0.33	0.022	0.05	0	1.32
21n	No	0.69	0.52	-50	0	27	0.14	0	0.022	0	1.3
22e	No	0.64	0.49	-50	0	11	0.27	0.00098	0.019	0	1.55
22n	No	0.67	0.49	-50	0	22	0.18	0.00098	0.017	0	1.34
27e	No	0.11	0.32	-50	0	7	1.1	0.39	0.39		1.14
27n	No	0.39	0.32	-50	0	12	0.5	0.28	0.19		1.19
28e	No	0.72	0.61	-50	0	8.2	0.24	0.0039	0.028	0	1.33
28n	No	0.28	0.61	-50	0	5.1	0.85	0.34	0.29		1.16
29e	No	0.68	0.48	-50	0	17	0.43	0.0013	0.08	0	1.4
29n	No	0.71	0.48	-50	0	15	0.21	0	0.037	0	1.21
30e	No	0.52	0.36	-50	0	8.4	1.1	0.002	0.28		2.99
30n	No	0.29	0.36	-50	0	18	2.4	0	0.55		1.77
31e	No	0.71	0.55	-50	0	3	0.14	0.00065	0.031	0	1.29
31n	No	0.54	0.55	-50	0	1.2	0.42	0.02	0.08	0	1.19
32e	No	0.57	0.36	-50	0	13	0.9	0.0049	0.21		2.2
32n	No	0.61	0.36	-50	0	21	0.68	0.0023	0.16		1.85
33e	No	0.66	0.51	-50	0	2.6	0.33	0.012	0.033	0	1.31
33n	No	0.7	0.51	-50	0	9.7	0.18	0.00033	0.032	0	1.26
34e	No	0.041	0.56	-50	0	2.9	0.57	0.035	0.12		1.19
34n	No	0.67	0.56	-50	0	15	0.31	0.0013	0.035	0	1.27
35e	No	0.69	0.49	-50	0	28	0.14	0	0.029	0	1.38
35n	No	0.67	0.49	-50	0	19	0.22	0.00033	0.017	0	1.28
36e	No	0.7	0.5	-50	0	14	-0.035	0.0016	0.091	0	1.37
36n	No	0.7	0.5	-50	0	13	-0.16	0.00033	0.1	0	1.27
37e	No	0.71	0.51	-50	1	2.2	-0.22	0.0049	0.16	0	1.27
37n	No	0.7	0.51	-50	0	1.7	-0.22	0.00033	0.15	0	1.21
38e	No	0.73	0.51	-50	0	12	0.21	0.0023	0.046	0	1.23
38n	No	0.71	0.51	-50	0	13	0.17	0.00033	0.029	0	1.19
40e	No	0.72	0.5	-50	0	12	0.24	0.0016	0.029	0	1.35
40n	No	0.71	0.5	-50	0	13	0.16	0.073	0.037		1.4
41e	No	0.72	0.5	-50	0	15	0.19	0.0029	0.026	0	1.35
41n	No	0.71	0.5	-50	0	13	0.13	0	0.03	0	1.23
42e	No	0.73	0.52	-50	0	12	0.21	0.00065	0.04	0	1.36
42n	No	0.65	0.52	-50	0	1.8	0.21	0.015	0.037	0	1.24
45e	Yes			-50	1.5E+03	0	0	0	INF		0
45n	Yes			-50	1.5E+03	0	0	0	INF		0
46e	Yes			-50	1.5E+03	0	0	0	INF		0
46n	Yes			-50	1.5E+03	0	0	0	INF		0
47e	No	0.67	0.47	-50	0	13	0.32	0.017	0.027	0	1.3
47n	No	0.67	0.47	-50	0	16	0.27	0	0.025	0	1.27
48e	No	0.68	0.48	-50	0	21	0.32	0.0042	0.03	0	1.4
48n	No	0.69	0.48	-50	0	36	0.16	0	0.029	0	1.27
49e	No	0.65	0.48	-50	0	15	0.27	0.0039	0.02	0	1.41
49n	No	0.67	0.48	-50	0	26	0.21	0.00033	0.022	0	1.27
50e	No	0.71	0.51	-50	0	10	0.2	0.0013	0.047	0	1.47
50n	No	0.7	0.51	-50	0	13	0.092	0.00065	0.047	0	1.32
51e	No	0.69	0.5	-50	0	7.9	0.49	0.43	0.11		1.44
51n	No	0.71	0.5	-50	0	13	0.093	0.00033	0.048	0	1.24
52e	No	0.72	0.5	-50	0	12	-0.08	0.0029	0.097	0	1.28
52n	No	0.71	0.5	-50	0	13	-0.067	0	0.08	0	1.28
53e	No	0.72	0.5	-50	0	15	0.23	0.0033	0.039	0	1.27
53n	No	0.72	0.5	-50	0	15	0.023	0	0.062	0	1.27
54e	No	0.63	0.49	-50	0	16	0.78	0.00033	0.17		1.84
54n	No	0.72	0.49	-50	0	12	0.16	0	0.023	0	1.33
55e	No	0.71	0.49	-50	0	12	0.2	0	0.029	0	1.31
55n	No	0.7	0.49	-50	0	16	0.12	0	0.029	0	1.28
56e	No	0.73	0.5	-50	0	12	0.22	0.00033	0.028	0	1.39
56n	No	0.69	0.5	-50	0	13	0.18	0.019	0.093	0	1.31
57e	No	0.73	0.5	-50	0	14	0.2	0.00065	0.028	0	1.32
57n	No	0.71	0.5	-50	0	15	0.2	0.0016	0.038	0	1.25
61e	No	0.04	0.56	-50	0	3.2	0.56	0.032	0.12		1.15
61n	No	0.68	0.56	-50	0	18	0.25	0.00033	0.021	0	1.23
62e	No	0.66	0.48	-50	0	12	0.31	0.0052	0.029	0	1.29
62n	No	0.68	0.48	-50	0	33	0.14	0	0.036	0	1.23
63e	Yes			-50	1.5E+03	0	0	0	INF		0
63n	Yes			-50	1.5E+03	0	0	0	INF		0
64e	Yes			-50	1.5E+03	0	0	0	INF		0
64n	Yes			-50	1.5E+03	0	0	0	INF		0
65e	No	0.72	0.51	-50	0	12	0.16	0.00098	0.056	0	1.44
65n	No	0.71	0.51	-50	0	15	0.087	0.0039	0.044	0	1.31
66e	No	0.72	0.51	-50	0	13	0.2	0.00033	0.045	0	1.35
66n	No	0.71	0.51	-50	0	12	0.096	0.00033	0.039	0	1.26
67e	No	0.72	0.5	-50	0	15	0.19	0.00065	0.034	0	1.31
67n	No	0.71	0.5	-50	0	9.3	0.2	0.00033	0.027	0	1.28
68e	No	0.72	0.5	-50	0	12	0.23	0	0.053	0	1.29
68n	No	0.71	0.5	-50	0	6.9	0.29	0	0.037	0	1.29
69e	No	0.72	0.49	-50	0	11	0.22	0.0029	0.031	0	1.3
69n	No	0.71	0.49	-50	0	12	0.25	0.00033	0.05	0	1.35
70e	No	0.71	0.48	-50	0	12	0.26	0.00033	0.032	0	1.28
70n	No	0.71	0.48	-50	0	15	0.24	0	0.031	0	1.26
71e	No	0.72	0.48	-50	0	13	0.28	0	0.032	0	1.34
71n	No	0.72	0.48	-50	1	12	0.18	0	0.021	0	1.27
72e	No	0.73	0.49	-50	0	13	0.32	0	0.034	0	1.33
72n	No	0.72	0.49	-50	0	14	0.21	0	0.033	0	1.25
73e	Yes			-50	1.5E+03	0	0	0	INF		0
73n	Yes			-50	1.5E+03	0	0	0	INF		0
77e	No	0.48	0.41	-50	0	24	1.7	0.0033	0.38		1.84
77n	No	0.6	0.41	-50	0	23	0.76	0.00033	0.16		1.72
78e	Yes			-50	1.5E+03	0	0	0	INF		0
78n	Yes			-50	1.5E+03	0	0	0	INF		0
79e	No	0.66	0.47	-50	0	12	0.38	0.0029	0.039	0	1.32
79n	No	0.67	0.47	-50	0	17	0.25	0.00033	0.024	0	1.21
80e	No	0.68	0.49	-50	0	21	0.25	0.0033	0.016	0	1.47
80n	No	0.67	0.49	-50	0	13	0.31	0.0013	0.033	0	1.22
81e	No	0.7	0.49	-50	0	3	-0.2	0.0078	0.13	0	1.45
81n	No	0.68	0.49	-50	1	1.5	-0.26	0.0013	0.14	0	1.4
82e	No	0.7	0.49	-50	0	2.5	-0.062	0.0039	0.12	0	1.59
82n	No	0.69	0.49	-50	0	1.9	-0.28	0	0.14	0	1.29
83e	No	0.71	0.5	-50	0	2.6	-0.12	0.0033	0.12	0	1.28
83n	No	0.65	0.5	-50	0	1.8	0.13	0	0.079	0	1.26
84e	Yes			-50	1.5E+03	0	0	0	INF		0
84n	Yes			-50	1.5E+03	0	0	0	INF		0
85e	Yes			-50	1.5E+03	0	0	0	INF		0
85n	Yes			-50	1.5E+03	0	0	0	INF		0
86e	Yes			-50	1.5E+03	0	0	0	INF		0
86n	Yes			-50	1.5E+03	0	0	0	INF		0
87e	Yes			-50	1.5E+03	0	0	0	INF		0
87n	Yes			-50	1.5E+03	0	0	0	INF		0
88e	No	0.71	0.49	-50	0	14	0.19	0.002	0.041	0	1.34
88n	No	0.69	0.49	-50	0	11	0.29	0.00033	0.04	0	1.25
89e	No	0.72	0.48	-50	0	11	0.16	0.002	0.038	0	1.37
89n	No	0.71	0.48	-50	0	12	0.17	0.0016	0.027	0	1.33
90e	No	0.72	0.49	-50	0	14	0.18	0.0026	0.034	0	1.32
90n	No	0.71	0.49	-50	0	12	0.19	0	0.019	0	1.27
91e	No	0.69	0.48	-50	0	13	0.52	0.00033	0.079	0	1.36
91n	No	0.71	0.48	-50	0	14	0.17	0	0.03	0	1.3
92e	No	0.72	0.49	-50	0	14	0.23	0.00098	0.027	0	1.31
92n	No	0.71	0.49	-50	0	13	0.19	0	0.026	0	1.26
93e	No	0.22	-0.26	-50	0	9.8	0.26	0	0.038		2.37
93n	No	0.11	-0.26	-50	1	0.73	0.5	0.067	0.1		1.38
94e	No	0.72	0.5	-50	0	14	0.24	0.00065	0.027	0	1.27
94n	No	0.71	0.5	-50	0	15	0.27	0.00033	0.031	0	1.2
95e	No	0.67	0.48	-50	0	14	0.39	0	0.042	0	1.29
95n	No	0.69	0.48	-50	0	26	0.2	0	0.016	0	1.21
96e	No	0.69	0.48	-50	0	34	0.22	0	0.025	0	1.35
96n	No	0.53	0.48	-50	0	15	1.1	0.0029	0.28		1.86
97e	No	0.68	0.49	-50	0	19	0.21	0.00098	0.014	0	1.58
97n	No	0.66	0.49	-50	0	12	0.34	0.019	0.047	0	1.27
98e	No	0.68	0.48	-50	0	1.7	-0.21	0.0046	0.14	0	1.55
98n	No	0.64	0.48	-50	0	0.77	-0.24	0.0036	0.13		1.57
99e	No	0.63	0.46	-50	0	0.6	-0.27	0.038	0.14		1.48
99n	No	0.56	0.46	-50	0	0.34	-0.27	0.012	0.14		1.27
100e	No	0.71	0.5	-50	0	2.4	-0.055	0.0052	0.11	0	1.38
100n	No	0.67	0.5	-50	0	0.95	-0.13	0.00098	0.11		1.33
101e	Yes			-50	1.5E+03	0	0	0	INF		0
101n	Yes			-50	1.5E+03	0	0	0	INF		0
102e	Yes			-50	1.5E+03	0	0	0	INF		0
102n	Yes			-50	1.5E+03	0	0	0	INF		0
103e	Yes			-50	1.5E+03	0	0	0	INF		0
103n	Yes			-50	1.5E+03	0	0	0	INF		0
104e	Yes			-50	1.5E+03	0	0	0	INF		0
104n	Yes			-50	1.5E+03	0	0	0	INF		0
105e	No	0.72	0.48	-50	0	12	0.18	0.0013	0.041	0	1.45
105n	No	0.7	0.48	-50	0	12	0.13	0	0.032	0	1.34
106e	No	0.045	0.57	-50	0	1.1	0.51	0.092	0.12		1.11
106n	No	0.71	0.57	-50	0	13	0.071	0	0.042	0	1.33
107e	No	0.72	0.47	-50	0	15	0.26	0.0072	0.053	0	1.37
107n	No	0.71	0.47	-50	0	15	0.18	0.0026	0.052	0	1.36
108e	No	0.72	0.48	-50	0	14	0.23	0	0.033	0	1.38
108n	No	0.72	0.48	-50	0	13	0.11	0	0.052	0	1.35
109e	No	0.72	0.55	-50	0	24	0.2	0	0.015	0	1.33
109n	No	0.039	0.55	-50	0	0.67	0.57	0.06	0.13		1.12
110e	No	0.59	0.48	-50	0	17	1.1	0.00033	0.25		2.12
110n	No	0.72	0.48	-50	0	13	0.21	0	0.026	0	1.32
111e	No	0.56	0.5	-50	0	3.8	1.2	0.0013	0.27		1.71
111n	No	0.72	0.5	-50	0	13	0.17	0	0.02	0	1.28
112e	No	0.042	0.57	-50	0	0.71	0.52	0.035	0.12		1.1
112n	No	0.69	0.57	-50	0	9.9	0.31	0	0.041	0	1.24
113e	No	0.69	0.47	-50	0	24	0.28	0	0.019	0	1.35
113n	No	0.69	0.47	-50	0	27	0.22	0	0.016	0	1.23
114e	No	0.64	0.47	-50	0	15	0.48	0.0013	0.065	0	1.37
114n	No	0.67	0.47	-50	0	17	0.22	0	0.027	0	1.25
115e	No	0.045	0.56	-50	0	3.3	0.53	0.067	0.12		1.14
115n	No	0.67	0.56	-50	0	18	0.24	0	0.018	0	1.27
116e	No	0.61	0.48	-50	0	1.2	0.34	0.0013	0.078	0	1.57
116n	No	0.67	0.48	-50	0	1.9	-0.21	0.0052	0.13	0	1.48
117e	No	0.71	0.48	-50	0	14	0.29	0.0026	0.029	0	1.46
117n	No	0.7	0.48	-50	0	13	0.2	0.00033	0.022	0	1.38
118e	No	0.71	0.49	-50	0	12	0.2	0.00065	0.04	0	1.4
118n	No	0.71	0.49	-50	1	13	0.13	0	0.058	0	1.32
119e	No	0.72	0.52	-50	0	4.2	-0.2	0.0039	0.14	0	1.4
119n	No	0.61	0.52	-50	1	0.47	-0.21	0.0013	0.12		1.3
120e	Yes			-50	1.5E+03	0	0	0	INF		0
120n	Yes			-50	1.5E+03	0	0	0	INF		0
121e	Yes			-50	1.5E+03	0	0	0	INF		0
121n	Yes			-50	1.5E+03	0	0	0	INF		0
122e	Yes			-50	1.5E+03	0	0	0	INF		0
122n	Yes			-50	1.5E+03	0	0	0	INF		0
123e	Yes			-50	1.5E+03	0	0	0	INF		0
123n	Yes			-50	1.5E+03	0	0	0	INF		0
124e	No	0.73	0.49	-50	0	12	0.18	0.00033	0.033	0	1.39
124n	No	0.72	0.49	-50	0	15	0.22	0	0.034	0	1.33
125e	No	0.08	0.57	-50	0	0.98	0.56	0.072	0.12		1.12
125n	No	0.71	0.57	-50	0	12	0.23	0	0.046	0	1.41
126e	No	0.72	0.47	-50	0	11	0.15	0.00033	0.036	0	1.38
126n	No	0.71	0.47	-50	0	13	-0.04	0.00065	0.064	0	1.43
127e	No	0.72	0.48	-50	0	9.6	0.21	0.0016	0.04	0	1.38
127n	No	0.72	0.48	-50	0	13	0.17	0	0.027	0	1.32
128e	No	0.73	0.49	-50	0	34	0.14	0.00033	0.03	0	1.33
128n	No	0.73	0.49	-50	0	28	0.2	0	0.033	0	1.29
129e	No	0.72	0.49	-50	0	19	0.21	0.00033	0.03	0	1.41
129n	No	0.72	0.49	-50	0	18	0.15	0.00065	0.029	0	1.34
130e	No	0.72	0.49	-50	0	14	0.29	0	0.025	0	1.28
130n	No	0.7	0.49	-50	0	15	0.48	0.14	0.085		1.38
131e	No	0.69	0.51	-50	0	22	0.25	0.00033	0.011	0	1.59
131n	No	0.46	0.51	-50	0	3.8	0.5	0.0036	0.098	0	1.27
132e	No	0.68	0.48	-50	0	21	0.25	0	0.017	0	1.46
132n	No	0.67	0.48	-50	0	13	0.27	0.0023	0.029	0	1.28
133e	No	0.66	0.47	-50	0	14	0.29	0.0072	0.022	0	1.4
133n	No	0.66	0.47	-50	0	13	0.28	0.0036	0.028	0	1.28
134e	No	0.61	0.48	-50	0	7.3	0.38	0.0094	0.051	0	1.4
134n	No	0.66	0.48	-50	0	12	0.33	0.00065	0.042	0	1.32
135e	No	0.68	0.49	-50	0	13	0.27	0.002	0.079	0	1.66
135n	No	0.69	0.49	-50	0	15	0.2	0.0039	0.024	0	1.54
136e	No	0.7	0.48	-50	0	14	0.23	0.002	0.028	0	1.41
136n	No	0.67	0.48	-50	0	16	0.49	0.026	0.15	0	1.84
137e	No	0.69	0.47	-50	0	2.5	-0.068	0.0042	0.11	0	1.45
137n	No	0.68	0.47	-50	0	1.9	-0.2	0.002	0.13	0	1.37
138e	No	0.71	0.48	-50	0	15	0.24	0	0.052	0	1.37
138n	No	0.71	0.48	-50	0	16	0.18	0	0.039	0	1.31
139e	No	0.7	0.48	-50	0	33	0.22	0	0.04	0	1.43
139n	No	0.69	0.48	-50	0	18	0.22	0	0.031	0	1.36
140e	No	0.71	0.48	-50	0	11	0.27	0.002	0.041	0	1.34
140n	No	0.71	0.48	-50	0	12	0.15	0	0.026	0	1.27
141e	No	0.71	0.48	-50	0	14	0.22	0	0.021	0	1.32
141n	No	0.71	0.48	-50	0	12	0.19	0	0.02	0	1.25
142e	No	0.71	0.48	-50	0	13	0.27	0.0016	0.03	0	1.35
142n	No	0.71	0.48	-50	0	13	0.29	0	0.041	0	1.27
143e	No	0.73	0.49	-50	0	13	0.25	0.00033	0.025	0	1.38
143n	No	0.73	0.49	-50	0	14	0.18	0	0.021	0	1.31
144e	No	0.72	0.49	-50	0	13	0.23	0.00033	0.027	0	1.48
144n	No	0.71	0.49	-50	0	11	0.26	0	0.043	0	1.43
145e	No	0.72	0.48	-50	0	14	0.22	0.00098	0.033	0	1.46
145n	No	0.71	0.48	-50	0	7.4	0.24	0	0.027	0	1.4
146e	No	0.67	0.48	-50	0	12	0.34	0	0.032	0	1.47
146n	No	0.7	0.48	-50	0	27	0.21	0	0.017	0	1.44
147e	No	0.74	0.49	-50	0	15	0.2	0.00033	0.025	0	1.31
147n	No	0.72	0.49	-50	0	12	0.18	0	0.013	0	1.28
148e	No	0.72	0.49	-50	0	14	0.2	0	0.021	0	1.31
148n	No	0.73	0.49	-50	0	15	0.15	0	0.027	0	1.26
149e	No	0.72	0.49	-50	0	11	0.26	0.00065	0.033	0	1.37
149n	No	0.72	0.49	-50	0	14	0.18	0	0.021	0	1.25
150e	No	0.72	0.49	-50	1	12	0.2	0	0.024	0	1.32
150n	No	0.72	0.49	-50	1	14	0.19	0	0.029	0	1.26
151e	No	0.53	0.46	-50	0	29	0.87	0.00065	0.21		2.57
151n	No	0.68	0.46	-50	0	14	0.2	0.00033	0.011	0	1.24
152e	No	0.67	0.48	-50	0	16	0.26	0.0036	0.019	0	1.42
152n	No	0.68	0.48	-50	0	16	0.25	0.00033	0.021	0	1.25
153e	No	0.64	0.48	-50	0	11	0.31	0.00098	0.025	0	1.31
153n	No	0.56	0.48	-50	0	5	0.44	0.0042	0.074	0	1.22
154e	No	0.66	0.49	-50	0	22	0.18	0.00033	0.022	0	1.36
154n	No	0.67	0.49	-50	0	19	0.24	0.00033	0.018	0	1.26
155e	No	0.69	0.49	-50	0	12	0.18	0.0085	0.027	0	1.59
155n	No	0.7	0.49	-50	0	13	0.14	0.011	0.018	0	1.57
156e	No	0.7	0.48	-50	0	17	0.2	0.0013	0.024	0	1.49
156n	No	0.68	0.48	-50	0	11	0.37	0.0036	0.05	0	1.41
157e	No	0.69	0.48	-50	0	9.9	0.25	0	0.031	0	1.49
157n	No	0.7	0.48	-50	0	11	0.18	0.00033	0.022	0	1.36
158e	No	0.7	0.48	-50	0	9.3	0.22	0.0046	0.046	0	1.41
158n	No	0.7	0.48	-50	0	9.3	0.21	0.0075	0.028	0	1.35
159e	No	0.66	0.46	-50	0	11	0.33	0.00065	0.035	0	1.4
159n	No	0.52	0.46	-50	0	13	1.1	0.0059	0.25		1.91
160e	No	0.72	0.48	-50	0	13	0.18	0.0026	0.024	0	1.41
160n	No	0.72	0.48	-50	0	14	0.19	0	0.025	0	1.29
161e	No	0.72	0.47	-50	0	13	0.23	0.00065	0.025	0	1.4
161n	No	0.54	0.47	-50	0	13	1.1	0.00033	0.26		2.4
162e	No	0.71	0.48	-50	0	13	0.27	0.00033	0.03	0	1.43
162n	No	0.72	0.48	-50	0	13	0.22	0	0.027	0	1.3
163e	No	0.72	0.49	-50	0	14	0.19	0.0023	0.027	0	1.37
163n	No	0.72	0.49	-50	0	14	0.086	0	0.037	0	1.26
164e	No	0.72	0.48	-50	0	13	0.25	0	0.021	0	1.38
164n	No	0.71	0.48	-50	0	15	0.14	0.00033	0.029	0	1.33
165e	No	0.54	0.47	-50	0	14	1.2	0.00065	0.29		2.55
165n	No	0.71	0.47	-50	0	13	0.13	0	0.026	0	1.36
166e	No	0.71	0.46	-50	0	14	0.21	0.00033	0.032	0	1.5
166n	No	0.71	0.46	-50	0	11	0.17	0	0.025	0	1.44
167e	No	0.71	0.49	-50	0	16	0.38	0	0.045	0	1.36
167n	No	0.72	0.49	-50	0	13	0.18	0	0.018	0	1.33
168e	No	0.73	0.49	-50	0	15	0.23	0.00033	0.018	0	1.32
168n	No	0.72	0.49	-50	0	13	0.15	0	0.029	0	1.28
169e	No	0.73	0.49	-50	0	12	0.19	0.0016	0.021	0	1.31
169n	No	0.73	0.49	-50	1	13	0.15	0	0.025	0	1.26
170e	No	0.04	0.58	-50	0	0.65	0.55	0.071	0.12		1.1
170n	No	0.72	0.58	-50	0	12	0.25	0	0.031	0	1.25
171e	No	0.67	0.47	-50	0	14	0.29	0	0.021	0	1.3
171n	No	0.67	0.47	-50	0	15	0.26	0	0.022	0	1.26
172e	No	0.69	0.5	-50	0	26	0.21	0	0.017	0	1.46
172n	No	0.67	0.5	-50	0	14	0.31	0.00098	0.035	0	1.32
173e	No	0.67	0.48	-50	0	15	0.29	0.00098	0.025	0	1.36
173n	No	0.68	0.48	-50	0	19	0.29	0	0.036	0	1.29
174e	No	0.031	0.00032	-50	0	3.1	0.57	0.049	0.12		1.14
174n	No	0.031	0.00032	-50	0	3.1	0.6	0.033	0.13		1.14
175e	No	0.64	0.5	-50	0	12	0.32	0.0016	0.036	0	1.35
175n	No	0.67	0.5	-50	0	20	0.25	0.00033	0.03	0	1.27
176e	No	0.045	0.56	-50	0	0.69	0.51	0.045	0.12		1.15
176n	No	0.7	0.56	-50	0	14	0.12	0.011	0.028	0	1.71
177e	No	0.7	0.49	-50	0	11	0.083	0.00098	0.075	0	1.5
177n	No	0.67	0.49	-50	0	13	-0.034	0.00098	0.17	0	1.85
178e	No	0.71	0.48	-50	0	12	0.31	0.0023	0.053	0	1.4
178n	No	0.7	0.48	-50	0	9.1	0.16	0.00065	0.017	0	1.5
179e	No	0.71	0.48	-50	0	12	0.28	0.00033	0.028	0	1.39
179n	No	0.71	0.48	-50	0	27	0.18	0	0.013	0	1.33
180e	No	0.72	0.48	-50	0	12	0.28	0.0094	0.032	0	1.35
180n	No	0.56	0.48	-50	0	14	0.92	0.00065	0.22		2.25
181e	No	0.72	0.48	-50	0	8.2	0.2	0.00065	0.034	0	1.38
181n	No	0.72	0.48	-50	0	10	0.23	0.00033	0.03	0	1.31
182e	No	0.71	0.57	-50	0	6.7	0.28	0.0026	0.025	0	1.39
182n	No	0.045	0.57	-50	1	0.66	0.55	0.024	0.13		1.15
183e	No	0.71	0.54	-50	0	13	0.18	0.00033	0.031	0	1.36
183n	No	0.042	0.54	-50	0	0.65	0.61	0.068	0.13		1.11
184e	No	0.05	0.57	-50	0	0.71	0.51	0.081	0.12		1.12
184n	No	0.72	0.57	-50	0	13	0.16	0.00033	0.024	0	1.26
185e	No	0.72	0.48	-50	0	14	0.22	0.0023	0.023	0	1.3
185n	No	0.71	0.48	-50	0	12	0.2	0	0.027	0	1.27
186e	No	0.71	0.48	-50	0	15	0.28	0.0016	0.023	0	1.38
186n	No	0.71	0.48	-50	0	12	0.22	0	0.029	0	1.36
187e	No	0.71	0.48	-50	0	12	0.31	0.00033	0.042	0	1.54
187n	No	0.71	0.48	-50	0	14	0.23	0.00033	0.025	0	1.35
189e	No	0.72	0.49	-50	0	14	0.22	0.00033	0.026	0	1.34
189n	No	0.71	0.49	-50	0	9.8	0.18	0.00065	0.016	0	1.24
190e	No	0.72	0.5	-50	0	14	0.3	0	0.029	0	1.52
190n	No	0.72	0.5	-50	0	11	0.19	0.00033	0.023	0	1.26
191e	No	0.72	0.5	-50	0	13	0.22	0.00033	0.019	0	1.38
191n	No	0.71	0.5	-50	0	13	0.28	0.00033	0.033	0	1.27
192e	No	0.68	0.5	-50	0	20	0.35	0.0016	0.039	0	1.39
192n	No	0.7	0.5	-50	0	26	0.19	0	0.012	0	1.28
193e	No	0.67	0.48	-50	0	22	0.35	0	0.037	0	1.65
193n	No	0.67	0.48	-50	0	15	0.28	0.00033	0.028	0	1.32
194e	No	0.032	0.0023	-50	0	3.8	0.53	0.07	0.12		1.15
194n	No	0.034	0.0023	-50	0	3.3	0.59	0.084	0.13		1.14
195e	No	0.67	0.5	-50	0	23	0.2	0.00033	0.016	0	1.34
195n	No	0.65	0.5	-50	0	12	0.27	0.00065	0.026	0	1.3
196e	No	0.67	0.48	-50	0	20	0.26	0.00033	0.016	0	1.36
196n	No	0.65	0.48	-50	0	14	0.26	0.0065	0.025	0	1.41
197e	No	0.65	0.46	-50	0	11	0.34	0.00065	0.034	0	1.42
197n	No	0.67	0.46	-50	0	19	0.25	0.00065	0.026	0	1.41
198e	No	0.71	0.5	-50	0	28	0.16	0	0.025	0	1.34
198n	No	0.65	0.5	-50	0	14	0.42	0.0026	0.064	0	1.26
199e	No	0.68	0.49	-50	0	14	0.36	0.0068	0.038	0	1.34
199n	No	0.61	0.49	-50	0	5.9	0.44	0.03	0.074	0	1.24
200e	No	0.046	0.59	-50	0	3.1	0.57	0.063	0.12		1.12
200n	No	0.68	0.59	-50	0	15	0.23	0.00065	0.034	0	1.22
201e	No	0.67	0.47	-50	0	12	0.37	0.0042	0.037	0	1.3
201n	No	0.68	0.47	-50	0	13	0.26	0	0.027	0	1.22
202e	No	0.71	0.5	-50	0	33	0.21	0	0.025	0	1.29
202n	No	0.68	0.5	-50	0	14	0.33	0.0023	0.038	0	1.22
203e	Yes			-50	1.5E+03	0	0	0	INF		0
203n	Yes			-50	1.5E+03	0	0	0	INF		0
204e	No	0.71	0.49	-50	0	12	-0.075	0.00065	0.11	0	1.52
204n	No	0.71	0.49	-50	0	13	-0.22	0.0036	0.14	0	1.41
205e	No	0.39	0.55	-50	0	4.4	0.53	0.026	0.1	0	1.21
205n	No	0.69	0.55	-50	0	16	0.25	0.15	0.022		1.43
206e	No	0.66	0.46	-50	0	12	0.34	0.00065	0.034	0	1.42
206n	No	0.67	0.46	-50	0	13	0.27	0.0055	0.025	0	1.36
207e	No	0.7	0.48	-50	0	25	0.26	0.0016	0.016	0	1.29
207n	No	0.68	0.48	-50	0	14	0.3	0.0036	0.032	0	1.26
208e	No	0.65	0.56	-50	1	1.7	-0.0086	0.0065	0.092	0	1.29
208n	No	0.042	0.56	-50	0	0.64	0.016	0.15	0.16		1.11
209e	No	0.033	0.00037	-50	0	0.85	0.11	0.043	0.14		1.11
209n	No	0.034	0.00037	-50	0	0.7	0.057	0.046	0.14		1.11
210e	No	0.71	0.49	-50	0	13	-0.16	0	0.12	0	1.28
210n	No	0.7	0.49	-50	0	14	-0.0012	0.00033	0.088	0	1.23
211e	No	0.7	0.49	-50	0	30	0.21	0.00098	0.015	0	1.29
211n	No	0.69	0.49	-50	0	22	0.18	0	0.011	0	1.22
212e	No	0.68	0.5	-50	0	27	0.28	0.0016	0.034	0	1.41
212n	No	0.67	0.5	-50	0	15	0.27	0.11	0.038		1.49
213e	No	0.65	0.5	-50	0	16	0.3	0.0055	0.03	0	1.34
213n	No	0.68	0.5	-50	0	18	0.2	0.0013	0.023	0	1.28
214e	No	0.042	0.56	-50	0	3	0.55	0.062	0.12		1.17
214n	No	0.63	0.56	-50	0	9.1	0.34	0.011	0.045	0	1.25
215e	No	0.69	0.49	-50	0	27	0.24	0.011	0.023	0	1.36
215n	No	0.66	0.49	-50	0	14	0.22	0.011	0.026	0	1.35
216e	No	0.3	0.21	-50	0	10	1.9	0.036	0.43		1.77
216n	No	0.059	0.21	-50	0	2.9	0.57	0.11	0.13		1.36
217e	No	0.44	0.52	-50	0	4.1	0.52	0.098	0.097		1.23
217n	No	0.67	0.52	-50	0	13	0.3	0.0094	0.03	0	1.29
218e	No	0.67	0.57	-50	0	15	0.29	0.0023	0.026	0	1.28
218n	No	0.076	0.57	-50	0	0.49	0.0017	0.28	0.22		1.1
219e	Yes			-50	1.5E+03	0	0	0	INF		0
219n	Yes			-50	1.5E+03	0	0	0	INF		0
220e	No	0.7	0.49	-50	0	22	0.26	0.0023	0.0096	0	1.29
220n	No	0.69	0.49	-50	0	18	0.18	0	0.019	0	1.29
221e	No	0.69	0.48	-50	0	19	0.28	0.00033	0.02	0	1.24
221n	No	0.69	0.48	-50	0	19	0.25	0	0.017	0	1.22
222e	No	0.69	0.49	-50	0	20	0.31	0.00033	0.025	0	1.27
222n	No	0.69	0.49	-50	0	24	0.2	0	0.022	0	1.22
223e	No	0.67	0.47	-50	0	15	0.34	0.00033	0.031	0	1.25
223n	No	0.66	0.47	-50	0	10	0.23	0.0013	0.023	0	1.22
224e	No	0.71	0.5	-50	0	35	0.17	0	0.032	0	1.28
224n	No	0.69	0.5	-50	0	22	0.21	0	0.023	0	1.26
225e	No	0.71	0.5	-50	0	27	0.2	0	0.016	0	1.29
225n	No	0.68	0.5	-50	0	22	0.28	0	0.029	0	1.25
226e	No	0.69	0.48	-50	0	20	0.22	0	0.012	0	1.26
226n	No	0.58	0.48	-50	0	31	0.86	0	0.2		1.91
227e	No	0.67	0.47	-50	0	12	0.29	0.002	0.024	0	1.26
227n	No	0.67	0.47	-50	0	12	0.32	0	0.041	0	1.23
228e	No	0.69	0.49	-50	0	25	0.26	0	0.028	0	1.28
228n	No	0.67	0.49	-50	0	15	0.27	0	0.029	0	1.23
229e	No	0.69	0.49	-50	0	29	0.28	0	0.027	0	1.28
229n	No	0.69	0.49	-50	0	34	0.13	0	0.03	0	1.24
231e	No	0.66	0.5	-50	0	24	0.34	0.00033	0.058	0	1.55
231n	No	0.67	0.5	-50	0	29	0.13	0.00033	0.052	0	1.45
232e	No	0.21	-0.18	-50	0	74	0.26	0	0.08		2.22
232n	No	0.16	-0.18	-50	0	6.3	0.92	0	0.2		1.71
233e	No	0.67	0.49	-50	0	20	0.23	0	0.024	0	1.27
233n	No	0.68	0.49	-50	0	34	0.13	0	0.04	0	1.3
234e	No	0.66	0.49	-50	0	13	0.37	0.011	0.042	0	1.25
234n	No	0.69	0.49	-50	0	27	0.18	0.00033	0.02	0	1.24
235e	No	0.67	0.48	-50	0	13	0.33	0.011	0.031	0	1.26
235n	No	0.66	0.48	-50	0	12	0.21	0.0052	0.022	0	1.22
237e	No	0.67	0.48	-50	0	12	0.31	0.00033	0.025	0	1.23
237n	No	0.68	0.48	-50	0	16	0.25	0	0.022	0	1.22
238e	No	0.7	0.5	-50	0	30	0.18	0	0.031	0	1.25
238n	No	0.7	0.5	-50	0	28	0.22	0	0.014	0	1.21
239e	No	0.7	0.49	-50	0	21	0.23	0	0.014	0	1.23
239n	No	0.69	0.49	-50	0	22	0.19	0	0.008	0	1.2
240e	No	0.64	0.51	-50	0	10	0.43	0.0023	0.055	0	1.23
240n	No	0.69	0.51	-50	0	19	0.25	0	0.021	0	1.19
241e	Yes			-50	1.5E+03	0	0	0	INF		0
241n	Yes			-50	1.5E+03	0	0	0	INF		0
242e	Yes			-50	1.5E+03	0	0	0	INF		0
242n	Yes			-50	1.5E+03	0	0	0	INF		0
243e	Yes			-50	1.5E+03	0	0	0	INF		0
243n	Yes			-50	1.5E+03	0	0	0	INF		0
244e	No	0.66	0.47	-50	0	13	0.38	0.0078	0.043	0	1.26
244n	No	0.66	0.47	-50	0	10	0.31	0	0.032	0	1.22
245e	No	0.69	0.5	-50	0	27	0.28	0	0.022	0	1.24
245n	No	0.51	0.5	-50	0	14	1	0.0039	0.25		1.56
246e	Yes			-50	1.5E+03	0	0	0	INF		0
246n	Yes			-50	1.5E+03	0	0	0	INF		0
250e	No	0.68	0.49	-50	0	21	0.23	0	0.021	0	1.23
250n	No	0.67	0.49	-50	0	23	0.2	0	0.018	0	1.26
251e	No	0.043	0.6	-50	0	0.88	0.13	0.051	0.13		1.12
251n	No	0.66	0.6	-50	0	2.3	-0.21	0.0029	0.13	0	1.25
252e	No	0.61	0.47	-50	0	7.7	0.39	0.016	0.051	0	1.3
252n	No	0.64	0.47	-50	0	11	0.28	0.0029	0.04	0	1.27
253e	Yes			-50	1.5E+03	0	0	0	INF		0
253n	Yes			-50	1.5E+03	0	0	0	INF		0
261e	Yes			-50	1.5E+03	0	0	0	INF		0
261n	Yes			-50	1.5E+03	0	0	0	INF		0
262e	Yes			-50	1.5E+03	0	0	0	INF		0
262n	Yes			-50	1.5E+03	0	0	0	INF		0
266e	No	0.59	0.51	-50	0	11	0.81	0.00033	0.18		1.58
266n	No	0.68	0.51	-50	0	13	-0.3	0.00098	0.14	0	1.42
267e	No	0.67	0.5	-50	0	27	0.018	0	0.11	0	1.36
267n	No	0.6	0.5	-50	0	12	0.096	0	0.16	0	1.6
268e	No	0.68	0.5	-50	0	18	0.23	0.00033	0.028	0	1.33
268n	No	0.64	0.5	-50	0	12	0.29	0.013	0.039	0	1.42
269e	No	0.64	0.55	-50	0	10	0.36	0.0059	0.044	0	1.31
269n	No	0.038	0.55	-50	0	3.1	0.58	0.1	0.13		1.14
281e	No	0.68	0.5	-50	0	32	0.22	0	0.041	0	1.3
281n	No	0.65	0.5	-50	0	15	0.23	0.002	0.023	0	1.21
282e	Yes			-50	1.5E+03	0	0	0	INF		0
282n	Yes			-50	1.5E+03	0	0	0	INF		0
283e	Yes			-50	1.5E+03	0	0	0	INF		0
283n	Yes			-50	1.5E+03	0	0	0	INF		0
295e	No	0.037	0.56	-50	0	0.8	0.11	0.049	0.14		1.15
295n	No	0.64	0.56	-50	0	2.4	-0.19	0.0062	0.13	0	1.22
320e	No	0.61	0.48	-50	0	13	0.14	0.0062	0.044	0	-1.77E+10
320n	No	0.61	0.48	-50	0	14	0.13	0.0036	0.05	0	1.33E+11
321e	No	0.59	0.5	-50	0	13	0.21	0.0023	0.054	0	6.06E+11
321n	No	0.62	0.5	-50	0	26	0.092	0.00033	0.042	0	9.28E+10
323e	No	0.45	0.47	-50	0	16	0.94	0.0052	0.21
323n	No	0.6	0.47	-50	0	27	0.15	0.00033	0.038	0
324e	No	0.62	0.49	-50	0	32	0.17	0.0016	0.048	0	0
324n	No	0.63	0.49	-50	0	36	0.15	0.00033	0.052	0	0
325e	No	0.65	0.5	-50	0	31	0.13	0.0026	0.032	0	0
325n	No	0.62	0.5	-50	0	16	0.21	0	0.015	0	0
326e	Yes			-50	1.5E+03	0	0	0	INF		0
326n	Yes			-50	1.5E+03	0	0	0	INF		0
327e	Yes			-50	1.5E+03	0	0	0	INF		0
327n	Yes			-50	1.5E+03	0	0	0	INF		0
328e	No	0.044	0.0025	-50	0	3.3	0.55	0.0023	0.12		0
328n	No	0.042	0.0025	-50	0	3.1	0.61	0.055	0.14		0
329e	No	0.043	0.00088	-50	0	3.1	0.56	0.057	0.12
329n	No	0.041	0.00088	-50	0	2.7	0.58	0.076	0.13
331e	Yes			-50	1.5E+03	0	0	0	INF		0
331n	Yes			-50	1.5E+03	0	0	0	INF		0
332e	No	0.039	0.002	-50	0	3.3	0.57	0.051	0.12		0
332n	No	0.039	0.002	-50	0	2.9	0.58	0.086	0.13		0
333e	No	0.57	0.46	-50	0	12	0.27	0.011	0.034	0
333n	No	0.59	0.46	-50	0	13	0.3	0.0039	0.046	0
336e	No	0.55	0.45	-50	0	13	0.32	0.0046	0.047	0	-2.74E-17
336n	No	0.57	0.45	-50	0	13	0.25	0.0013	0.046	0	-2.75E-17
340e	No	0.6	0.48	-50	0	40	0.13	0.00033	0.063	0	2.12
340n	No	0.62	0.48	-50	0	26	0.13	0.00098	0.05	0	1.93

# Save antenna classification table as a csv
if SAVE_RESULTS:
    for ind, col in zip(np.arange(len(df.columns), 0, -1), df_classes.columns[::-1]):
        df.insert(int(ind), col + ' Class', df_classes[col])
    df.to_csv(ANTCLASS_FILE)    

print('Final Ant-Pol Classification:\n\n', final_class)

Final Ant-Pol Classification:

 Jee:
----------
good (142 antpols):
3, 4, 5, 7, 8, 10, 17, 19, 22, 28, 35, 36, 38, 40, 41, 42, 48, 49, 50, 52, 53, 55, 56, 57, 62, 65, 66, 67, 68, 69, 70, 71, 72, 79, 80, 88, 89, 90, 92, 94, 95, 97, 105, 107, 108, 109, 113, 117, 118, 124, 126, 127, 129, 130, 131, 132, 133, 134, 135, 136, 138, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 166, 167, 168, 169, 171, 172, 173, 175, 177, 178, 179, 180, 181, 182, 183, 185, 186, 187, 189, 190, 191, 192, 193, 195, 196, 197, 198, 199, 201, 206, 207, 211, 212, 213, 215, 218, 220, 221, 222, 223, 225, 226, 227, 228, 229, 231, 233, 234, 235, 237, 238, 239, 244, 245, 250, 268, 269

suspect (31 antpols):
9, 20, 21, 29, 31, 33, 37, 47, 81, 82, 83, 91, 96, 98, 100, 114, 116, 119, 128, 137, 139, 202, 204, 205, 208, 210, 224, 240, 252, 267, 281

bad (78 antpols):
15, 16, 18, 27, 30, 32, 34, 45, 46, 51, 54, 61, 63, 64, 73, 77, 78, 84, 85, 86, 87, 93, 99, 101, 102, 103, 104, 106, 110, 111, 112, 115, 120, 121, 122, 123, 125, 151, 165, 170, 174, 176, 184, 194, 200, 203, 209, 214, 216, 217, 219, 232, 241, 242, 243, 246, 251, 253, 261, 262, 266, 282, 283, 295, 320, 321, 323, 324, 325, 326, 327, 328, 329, 331, 332, 333, 336, 340


Jnn:
----------
good (146 antpols):
5, 7, 8, 9, 10, 15, 17, 20, 21, 22, 29, 33, 34, 35, 38, 41, 47, 49, 50, 51, 52, 53, 54, 55, 57, 61, 65, 66, 67, 68, 69, 70, 71, 72, 79, 80, 88, 89, 90, 91, 92, 94, 95, 105, 106, 107, 108, 110, 111, 112, 113, 114, 115, 117, 118, 124, 125, 126, 127, 128, 129, 132, 133, 134, 135, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 154, 155, 156, 157, 158, 160, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 175, 176, 178, 179, 181, 184, 185, 186, 187, 189, 190, 191, 192, 193, 195, 196, 197, 200, 201, 202, 206, 207, 210, 211, 213, 214, 215, 217, 220, 221, 222, 223, 224, 225, 227, 228, 231, 234, 235, 237, 238, 239, 240, 244, 250, 252, 268, 281

suspect (27 antpols):
19, 31, 36, 37, 42, 48, 56, 62, 81, 82, 83, 97, 116, 131, 136, 137, 153, 177, 198, 199, 204, 229, 233, 251, 266, 267, 295

bad (78 antpols):
3, 4, 16, 18, 27, 28, 30, 32, 40, 45, 46, 63, 64, 73, 77, 78, 84, 85, 86, 87, 93, 96, 98, 99, 100, 101, 102, 103, 104, 109, 119, 120, 121, 122, 123, 130, 159, 161, 174, 180, 182, 183, 194, 203, 205, 208, 209, 212, 216, 218, 219, 226, 232, 241, 242, 243, 245, 246, 253, 261, 262, 269, 282, 283, 320, 321, 323, 324, 325, 326, 327, 328, 329, 331, 332, 333, 336, 340

Save calibration solutions

# update flags in omnical gains and visibility solutions
for ant in omni_flags:
    omni_flags[ant] |= rfi_flags
for bl in vissol_flags:
    vissol_flags[bl] |= rfi_flags

if SAVE_RESULTS:
    add_to_history = 'Produced by file_calibration notebook with the following environment:\n' + '=' * 65 + '\n' + os.popen('conda env export').read() + '=' * 65    
    
    hd_vissol = io.HERAData(SUM_FILE)
    hc_omni = hd_vissol.init_HERACal(gain_convention='divide', cal_style='redundant')
    hc_omni.update(gains=sol.gains, flags=omni_flags, quals=meta['chisq_per_ant'], total_qual=meta['chisq'])
    hc_omni.history += add_to_history
    hc_omni.write_calfits(OMNICAL_FILE, clobber=True)
    del hc_omni
    malloc_trim()
    
    if SAVE_OMNIVIS_FILE:
        # output results, harmonizing keys over polarizations
        all_reds = redcal.get_reds(hd.data_antpos, pols=['ee', 'nn'], pol_mode='2pol')
        bl_to_red_map = {bl: red[0] for red in all_reds for bl in red}
        hd_vissol.read(bls=[bl_to_red_map[bl] for bl in sol.vis], return_data=False)
        hd_vissol.empty_arrays()
        hd_vissol.history += add_to_history
        hd_vissol.update(data={bl_to_red_map[bl]: sol.vis[bl] for bl in sol.vis}, 
                         flags={bl_to_red_map[bl]: vissol_flags[bl] for bl in vissol_flags}, 
                         nsamples={bl_to_red_map[bl]: vissol_nsamples[bl] for bl in vissol_nsamples})
        hd_vissol.write_uvh5(OMNIVIS_FILE, clobber=True)
    del hd_vissol
    malloc_trim()    

Output fully flagged calibration file if OMNICAL_FILE is not written

if SAVE_RESULTS and not os.path.exists(OMNICAL_FILE):
    print(f'WARNING: No calibration file produced at {OMNICAL_FILE}. Creating a fully-flagged placeholder calibration file.')
    hd_writer = io.HERAData(SUM_FILE)
    io.write_cal(OMNICAL_FILE, freqs=hd_writer.freqs, times=hd_writer.times,
                 gains={ant: np.ones((hd_writer.Ntimes, hd_writer.Nfreqs), dtype=np.complex64) for ant in ants},
                 flags={ant: np.ones((len(data.times), len(data.freqs)), dtype=bool) for ant in ants},
                 quality=None, total_qual=None, outdir='', overwrite=True, history=utils.history_string(add_to_history), 
                 x_orientation=hd_writer.x_orientation, telescope_location=hd_writer.telescope_location, lst_array=np.unique(hd_writer.lsts),
                 antenna_positions=np.array([hd_writer.antenna_positions[hd_writer.antenna_numbers == antnum].flatten() for antnum in set(ant[0] for ant in ants)]),
                 antnums2antnames=dict(zip(hd_writer.antenna_numbers, hd_writer.antenna_names)))

Output empty visibility file if OMNIVIS_FILE is not written

if SAVE_RESULTS and not os.path.exists(OMNIVIS_FILE):
    print(f'WARNING: No omnivis file produced at {OMNIVIS_FILE}. Creating an empty visibility solution file.')
    hd_writer = io.HERAData(SUM_FILE)
    hd_writer.initialize_uvh5_file(OMNIVIS_FILE, clobber=True)

WARNING: No omnivis file produced at /mnt/sn1/2460234/zen.2460234.44960.sum.omni_vis.uvh5. Creating an empty visibility solution file.

TODO: Perform nucal

Metadata

for repo in ['pyuvdata', 'hera_cal', 'hera_filters', 'hera_qm', 'hera_notebook_templates']:
    exec(f'from {repo} import __version__')
    print(f'{repo}: {__version__}')

pyuvdata: 2.4.0
hera_cal: 3.4.0
hera_filters: 0.1.0
hera_qm: 2.1.3.dev4+g50af006
hera_notebook_templates: 0.1.dev486+gfb8560a

print(f'Finished execution in {(time.time() - tstart) / 60:.2f} minutes.')

Finished execution in 12.32 minutes.