Single File Calibration¶

by Josh Dillon, Aaron Parsons, Tyler Cox, and Zachary Martinot, last updated August 11, 2023

This notebook is designed to infer as much information about the array from a single file, including pushing the calibration and RFI mitigation as far as possible. Calibration includes redundant-baseline calibration, RFI-based calibration of delay slopes, model-based calibration of overall amplitudes, and a full per-frequency phase gradient absolute calibration if abscal model files are available.

Here's a set of links to skip to particular figures and tables:

• Figure 1: RFI Flagging¶

• Figure 2: Plot of autocorrelations with classifications¶

• Figure 3: Summary of antenna classifications prior to calibration¶

• Figure 4: Redundant calibration of a single baseline group¶

• Figure 5: Absolute calibration of redcal degeneracies¶

• Figure 6: chi^2 per antenna across the array¶

• Figure 7: Summary of antenna classifications after redundant calibration¶

• Table 1: Complete summary of per antenna classifications¶

In [1]:
import time
tstart = time.time()
In [2]:
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin  # REQUIRED to have the compression plugins available
import numpy as np
from scipy import constants, interpolate
import copy
import glob
import re
import matplotlib
import matplotlib.pyplot as plt
import pandas as pd
pd.set_option('display.max_rows', 1000)
from uvtools.plot import plot_antpos, plot_antclass
from hera_qm import ant_metrics, ant_class, xrfi
from hera_cal import io, utils, redcal, apply_cal, datacontainer, abscal
from hera_notebook_templates.data import DATA_PATH as HNBT_DATA
from IPython.display import display, HTML
import linsolve
display(HTML("<style>.container { width:100% !important; }</style>"))
_ = np.seterr(all='ignore')  # get rid of red warnings
%config InlineBackend.figure_format = 'retina'
In [3]:
# this enables better memory management on linux
import ctypes
def malloc_trim():
    try:
        ctypes.CDLL('libc.so.6').malloc_trim(0) 
    except OSError:
        pass

Parse inputs and outputs¶

To use this notebook interactively, you will have to provide a sum filename path if none exists as an environment variable. All other parameters have reasonable default values.

In [4]:
# figure out whether to save results
SAVE_RESULTS = os.environ.get("SAVE_RESULTS", "TRUE").upper() == "TRUE"
SAVE_OMNIVIS_FILE = os.environ.get("SAVE_OMNIVIS_FILE", "FALSE").upper() == "TRUE"

# get infile names
SUM_FILE = os.environ.get("SUM_FILE", None)
# SUM_FILE = '/lustre/aoc/projects/hera/h6c-analysis/IDR2/2459866/zen.2459866.33010.sum.uvh5'  # If sum_file is not defined in the environment variables, define it here.
DIFF_FILE = SUM_FILE.replace('sum', 'diff')

# get outfilenames
AM_FILE = (SUM_FILE.replace('.uvh5', '.ant_metrics.hdf5') if SAVE_RESULTS else None)
ANTCLASS_FILE = (SUM_FILE.replace('.uvh5', '.ant_class.csv') if SAVE_RESULTS else None)
OMNICAL_FILE = (SUM_FILE.replace('.uvh5', '.omni.calfits') if SAVE_RESULTS else None)
OMNIVIS_FILE = (SUM_FILE.replace('.uvh5', '.omni_vis.uvh5') if SAVE_RESULTS else None)

for fname in ['SUM_FILE', 'DIFF_FILE', 'AM_FILE', 'ANTCLASS_FILE', 'OMNICAL_FILE', 'OMNIVIS_FILE', 'SAVE_RESULTS', 'SAVE_OMNIVIS_FILE']:
    print(f"{fname} = '{eval(fname)}'")
SUM_FILE = '/mnt/sn1/2460214/zen.2460214.46140.sum.uvh5'
DIFF_FILE = '/mnt/sn1/2460214/zen.2460214.46140.diff.uvh5'
AM_FILE = '/mnt/sn1/2460214/zen.2460214.46140.sum.ant_metrics.hdf5'
ANTCLASS_FILE = '/mnt/sn1/2460214/zen.2460214.46140.sum.ant_class.csv'
OMNICAL_FILE = '/mnt/sn1/2460214/zen.2460214.46140.sum.omni.calfits'
OMNIVIS_FILE = '/mnt/sn1/2460214/zen.2460214.46140.sum.omni_vis.uvh5'
SAVE_RESULTS = 'True'
SAVE_OMNIVIS_FILE = 'False'

Parse settings¶

Load settings relating to the operation of the notebook, then print what was loaded (or default).

In [5]:
# parse plotting settings
PLOT = os.environ.get("PLOT", "TRUE").upper() == "TRUE"
if PLOT:
    %matplotlib inline

# parse omnical settings
OC_MAX_DIMS = int(os.environ.get("OC_MAX_DIMS", 4))
OC_MIN_DIM_SIZE = int(os.environ.get("OC_MIN_DIM_SIZE", 8))
OC_SKIP_OUTRIGGERS = os.environ.get("OC_SKIP_OUTRIGGERS", "TRUE").upper() == "TRUE"
OC_MIN_BL_LEN = float(os.environ.get("OC_MIN_BL_LEN", 1))
OC_MAX_BL_LEN = float(os.environ.get("OC_MAX_BL_LEN", 1e100))
OC_MAXITER = int(os.environ.get("OC_MAXITER", 50))
OC_MAX_RERUN = int(os.environ.get("OC_MAX_RERUN", 4))
OC_RERUN_MAXITER = int(os.environ.get("OC_MAXITER", 25))
OC_USE_PRIOR_SOL = os.environ.get("OC_USE_PRIOR_SOL", "FALSE").upper() == "TRUE"
OC_PRIOR_SOL_FLAG_THRESH = float(os.environ.get("OC_PRIOR_SOL_FLAG_THRESH", .95))
OC_USE_GPU = os.environ.get("SAVE_RESULTS", "FALSE").upper() == "TRUE"

# parse RFI settings
RFI_DPSS_HALFWIDTH = float(os.environ.get("RFI_DPSS_HALFWIDTH", 300e-9))
RFI_NSIG = float(os.environ.get("RFI_NSIG", 6))

# parse abscal settings
ABSCAL_MODEL_FILES_GLOB = os.environ.get("ABSCAL_MODEL_FILES_GLOB", None)
ABSCAL_MIN_BL_LEN = float(os.environ.get("ABSCAL_MIN_BL_LEN", 1.0))
ABSCAL_MAX_BL_LEN = float(os.environ.get("ABSCAL_MAX_BL_LEN", 140.0))

# print settings
for setting in ['PLOT', 'SAVE_RESULTS', 'OC_MAX_DIMS', 'OC_MIN_DIM_SIZE', 'OC_SKIP_OUTRIGGERS', 
                'OC_MIN_BL_LEN', 'OC_MAX_BL_LEN', 'OC_MAXITER', 'OC_MAX_RERUN', 'OC_RERUN_MAXITER', 
                'OC_USE_PRIOR_SOL', 'OC_PRIOR_SOL_FLAG_THRESH', 'OC_USE_GPU', 'RFI_DPSS_HALFWIDTH', 'RFI_NSIG',
                'ABSCAL_MODEL_FILES_GLOB', 'ABSCAL_MIN_BL_LEN', 'ABSCAL_MAX_BL_LEN']:
    print(f'{setting} = {eval(setting)}')
PLOT = True
SAVE_RESULTS = True
OC_MAX_DIMS = 4
OC_MIN_DIM_SIZE = 8
OC_SKIP_OUTRIGGERS = True
OC_MIN_BL_LEN = 1.0
OC_MAX_BL_LEN = 1e+100
OC_MAXITER = 50
OC_MAX_RERUN = 4
OC_RERUN_MAXITER = 50
OC_USE_PRIOR_SOL = True
OC_PRIOR_SOL_FLAG_THRESH = 0.95
OC_USE_GPU = False
RFI_DPSS_HALFWIDTH = 3e-07
RFI_NSIG = 6.0
ABSCAL_MODEL_FILES_GLOB = None
ABSCAL_MIN_BL_LEN = 1.0
ABSCAL_MAX_BL_LEN = 140.0

Parse bounds¶

Load settings related to classifying antennas as good, suspect, or bad, then print what was loaded (or default).

In [6]:
# ant_metrics bounds for low correlation / dead antennas
am_corr_bad = (0, float(os.environ.get("AM_CORR_BAD", 0.3)))
am_corr_suspect = (float(os.environ.get("AM_CORR_BAD", 0.3)), float(os.environ.get("AM_CORR_SUSPECT", 0.5)))

# ant_metrics bounds for cross-polarized antennas
am_xpol_bad = (-1, float(os.environ.get("AM_XPOL_BAD", -0.1)))
am_xpol_suspect = (float(os.environ.get("AM_XPOL_BAD", -0.1)), float(os.environ.get("AM_XPOL_SUSPECT", 0)))

# bounds on solar altitude (in degrees)
good_solar_altitude = (-90, float(os.environ.get("SUSPECT_SOLAR_ALTITUDE", 0)))
suspect_solar_altitude = (float(os.environ.get("SUSPECT_SOLAR_ALTITUDE", 0)), 90)

# bounds on zeros in spectra
good_zeros_per_eo_spectrum = (0, int(os.environ.get("MAX_ZEROS_PER_EO_SPEC_GOOD", 2)))
suspect_zeros_per_eo_spectrum = (0, int(os.environ.get("MAX_ZEROS_PER_EO_SPEC_SUSPECT", 8)))

# bounds on autocorrelation power
auto_power_good = (float(os.environ.get("AUTO_POWER_GOOD_LOW", 5)), float(os.environ.get("AUTO_POWER_GOOD_HIGH", 30)))
auto_power_suspect = (float(os.environ.get("AUTO_POWER_SUSPECT_LOW", 1)), float(os.environ.get("AUTO_POWER_SUSPECT_HIGH", 60)))

# bounds on autocorrelation slope
auto_slope_good = (float(os.environ.get("AUTO_SLOPE_GOOD_LOW", -0.4)), float(os.environ.get("AUTO_SLOPE_GOOD_HIGH", 0.4)))
auto_slope_suspect = (float(os.environ.get("AUTO_SLOPE_SUSPECT_LOW", -0.6)), float(os.environ.get("AUTO_SLOPE_SUSPECT_HIGH", 0.6)))

# bounds on autocorrelation RFI
auto_rfi_good = (0, float(os.environ.get("AUTO_RFI_GOOD", 0.015)))
auto_rfi_suspect = (0, float(os.environ.get("AUTO_RFI_SUSPECT", 0.03)))

# bounds on autocorrelation shape
auto_shape_good = (0, float(os.environ.get("AUTO_SHAPE_GOOD", 0.1)))
auto_shape_suspect = (0, float(os.environ.get("AUTO_SHAPE_SUSPECT", 0.2)))

# bounds on chi^2 per antenna in omnical
oc_cspa_good = (0, float(os.environ.get("OC_CSPA_GOOD", 2)))
oc_cspa_suspect = (0, float(os.environ.get("OC_CSPA_SUSPECT", 3)))

# print bounds
for bound in ['am_corr_bad', 'am_corr_suspect', 'am_xpol_bad', 'am_xpol_suspect', 
              'good_solar_altitude', 'suspect_solar_altitude',
              'good_zeros_per_eo_spectrum', 'suspect_zeros_per_eo_spectrum',
              'auto_power_good', 'auto_power_suspect', 'auto_slope_good', 'auto_slope_suspect',
              'auto_rfi_good', 'auto_rfi_suspect', 'auto_shape_good', 'auto_shape_suspect',
              'oc_cspa_good', 'oc_cspa_suspect']:
    print(f'{bound} = {eval(bound)}')
am_corr_bad = (0, 0.2)
am_corr_suspect = (0.2, 0.4)
am_xpol_bad = (-1, -0.1)
am_xpol_suspect = (-0.1, 0.0)
good_solar_altitude = (-90, 0.0)
suspect_solar_altitude = (0.0, 90)
good_zeros_per_eo_spectrum = (0, 2)
suspect_zeros_per_eo_spectrum = (0, 8)
auto_power_good = (5.0, 30.0)
auto_power_suspect = (1.0, 60.0)
auto_slope_good = (-0.4, 0.4)
auto_slope_suspect = (-0.6, 0.6)
auto_rfi_good = (0, 0.015)
auto_rfi_suspect = (0, 0.03)
auto_shape_good = (0, 0.1)
auto_shape_suspect = (0, 0.2)
oc_cspa_good = (0, 2.0)
oc_cspa_suspect = (0, 3.0)

Load sum and diff data¶

In [7]:
hd = io.HERADataFastReader(SUM_FILE)
data, _, _ = hd.read(read_flags=False, read_nsamples=False)
hd_diff = io.HERADataFastReader(DIFF_FILE)
diff_data, _, _ = hd_diff.read(read_flags=False, read_nsamples=False)
In [8]:
ants = sorted(set([ant for bl in hd.bls for ant in utils.split_bl(bl)]))
auto_bls = [bl for bl in data if (bl[0] == bl[1]) and (utils.split_pol(bl[2])[0] == utils.split_pol(bl[2])[1])]
antpols = sorted(set([ant[1] for ant in ants]))
In [9]:
# print basic information about the file
print(f'File: {SUM_FILE}')
print(f'JDs: {hd.times} ({np.median(np.diff(hd.times)) * 24 * 3600:.5f} s integrations)')
print(f'LSTS: {hd.lsts * 12 / np.pi } hours')
print(f'Frequencies: {len(hd.freqs)} {np.median(np.diff(hd.freqs)) / 1e6:.5f} MHz channels from {hd.freqs[0] / 1e6:.5f} to {hd.freqs[-1] / 1e6:.5f} MHz')
print(f'Antennas: {len(hd.data_ants)}')
print(f'Polarizations: {hd.pols}')
File: /mnt/sn1/2460214/zen.2460214.46140.sum.uvh5
JDs: [2460214.46134739 2460214.46145924] (9.66368 s integrations)
LSTS: [0.86692517 0.86961687] hours
Frequencies: 1536 0.12207 MHz channels from 46.92078 to 234.29871 MHz
Antennas: 245
Polarizations: ['nn', 'ee', 'ne', 'en']

Classify good, suspect, and bad antpols¶

Run ant_metrics¶

This classifies antennas as cross-polarized, low-correlation, or dead. Such antennas are excluded from any calibration.

In [10]:
am = ant_metrics.AntennaMetrics(SUM_FILE, DIFF_FILE, sum_data=data, diff_data=diff_data)
am.iterative_antenna_metrics_and_flagging(crossCut=am_xpol_bad[1], deadCut=am_corr_bad[1])
am.all_metrics = {}  # this saves time and disk by getting rid of per-iteration information we never use
if SAVE_RESULTS:
    am.save_antenna_metrics(AM_FILE, overwrite=True)
In [11]:
# Turn ant metrics into classifications
totally_dead_ants = [ant for ant, i in am.xants.items() if i == -1]
am_totally_dead = ant_class.AntennaClassification(good=[ant for ant in ants if ant not in totally_dead_ants], bad=totally_dead_ants)
am_corr = ant_class.antenna_bounds_checker(am.final_metrics['corr'], bad=[am_corr_bad], suspect=[am_corr_suspect], good=[(0, 1)])
am_xpol = ant_class.antenna_bounds_checker(am.final_metrics['corrXPol'], bad=[am_xpol_bad], suspect=[am_xpol_suspect], good=[(-1, 1)])
ant_metrics_class = am_totally_dead + am_corr + am_xpol

Mark sun-up (or high solar altitude) data as suspect¶

In [12]:
min_sun_alt = np.min(utils.get_sun_alt(hd.times))
solar_class = ant_class.antenna_bounds_checker({ant: min_sun_alt for ant in ants}, good=[good_solar_altitude], suspect=[suspect_solar_altitude])

Classify antennas responsible for 0s in visibilities as bad:¶

This classifier looks for X-engine failure or packet loss specific to an antenna which causes either the even visibilities (or the odd ones, or both) to be 0s.

In [13]:
zeros_class = ant_class.even_odd_zeros_checker(data, diff_data, good=good_zeros_per_eo_spectrum, suspect=suspect_zeros_per_eo_spectrum)

Examine and classify autocorrelation power, slope, and RFI occpancy¶

These classifiers look for antennas with too high or low power, to steep a slope, or too much excess RFI.

In [14]:
auto_power_class = ant_class.auto_power_checker(data, good=auto_power_good, suspect=auto_power_suspect)
auto_slope_class = ant_class.auto_slope_checker(data, good=auto_slope_good, suspect=auto_slope_suspect, edge_cut=100, filt_size=17)
cache = {}
auto_rfi_class = ant_class.auto_rfi_checker(data, good=auto_rfi_good, suspect=auto_rfi_suspect, 
                                            filter_half_widths=[RFI_DPSS_HALFWIDTH], nsig=RFI_NSIG, cache=cache)
auto_class = auto_power_class + auto_slope_class + auto_rfi_class
In [15]:
del cache
malloc_trim()

Find and flag RFI¶

In [16]:
# Compute int_count for all unflagged autocorrelations averaged together
int_time = 24 * 3600 * np.median(np.diff(data.times_by_bl[auto_bls[0][0:2]]))
chan_res = np.median(np.diff(data.freqs))
final_class = ant_metrics_class + zeros_class + auto_class
int_count = int(int_time * chan_res) * (len(final_class.good_ants) + len(final_class.suspect_ants))
avg_auto = {(-1, -1, 'ee'): np.mean([data[bl] for bl in auto_bls if final_class[utils.split_bl(bl)[0]] != 'bad'], axis=0)}
# Flag RFI first with channel differences and then with DPSS
antenna_flags, _ = xrfi.flag_autos(avg_auto, int_count=int_count, nsig=(RFI_NSIG * 5))
_, rfi_flags = xrfi.flag_autos(avg_auto, int_count=int_count, flag_method='dpss_flagger',
                               flags=antenna_flags, freqs=data.freqs, filter_centers=[0],
                               filter_half_widths=[RFI_DPSS_HALFWIDTH], eigenval_cutoff=[1e-9], nsig=RFI_NSIG)
malloc_trim()
In [17]:
def rfi_plot():
    plt.figure(figsize=(12, 5), dpi=100)
    plt.semilogy(hd.freqs / 1e6, np.where(rfi_flags, np.nan, avg_auto[(-1, -1, 'ee')])[1], label = 'Average Good or Suspect Autocorrelation', zorder=100)
    plt.semilogy(hd.freqs / 1e6, np.where(False, np.nan, avg_auto[(-1, -1, 'ee')])[1], 'r', lw=.5, label=f'{np.sum(rfi_flags[0])} Channels Flagged for RFI')
    plt.legend()
    plt.xlabel('Frequency (MHz)')
    plt.ylabel('Uncalibrated Autocorrelation')
    plt.tight_layout()

Figure 1: RFI Flagging¶

This figure shows RFI identified using the average of all autocorrelations---excluding bad antennas---for the first integration in the file.

In [18]:
rfi_plot()
In [19]:
def autocorr_plot(cls):    
    fig, axes = plt.subplots(1, 2, figsize=(14, 5), dpi=100, sharey=True, gridspec_kw={'wspace': 0})
    labels = []
    colors = ['darkgreen', 'goldenrod', 'maroon']
    for ax, pol in zip(axes, antpols):
        for ant in cls.ants:
            if ant[1] == pol:
                color = colors[cls.quality_classes.index(cls[ant])]
                ax.semilogy(np.mean(data[utils.join_bl(ant, ant)], axis=0), color=color, lw=.5)
        ax.set_xlabel('Channel', fontsize=12)
        ax.set_title(f'{utils.join_pol(pol, pol)}-Polarized Autos')

    axes[0].set_ylabel('Raw Autocorrelation', fontsize=12)
    axes[1].legend([matplotlib.lines.Line2D([0], [0], color=color) for color in colors], 
                   [cls.capitalize() for cls in auto_class.quality_classes], ncol=1, fontsize=12, loc='upper right', framealpha=1)
    plt.tight_layout()

Classify antennas based on shapes, excluding RFI-contamined channels¶

In [20]:
auto_shape_class = ant_class.auto_shape_checker(data, good=auto_shape_good, suspect=auto_shape_suspect,
                                                flag_spectrum=np.sum(rfi_flags, axis=0).astype(bool), antenna_class=final_class)
auto_class += auto_shape_class
In [21]:
final_class = ant_metrics_class + solar_class + zeros_class + auto_class

Figure 2: Plot of autocorrelations with classifications¶

This figure shows a plot of all autocorrelations in the array, split by polarization. Antennas are classified based on their autocorrelations into good, suspect, and bad, by examining power, slope, and RFI-occupancy.

In [22]:
if PLOT: autocorr_plot(auto_class)

Classify antennas based on non-noiselike diffs¶

In [23]:
xengine_diff_class = ant_class.non_noiselike_diff_by_xengine_checker(data, diff_data, flag_waterfall=rfi_flags, 
                                                                     antenna_class=final_class, 
                                                                     xengine_chans=96, bad_xengine_zcut=10)
final_class += xengine_diff_class
In [24]:
# delete diffs to save memory
del diff_data, hd_diff
malloc_trim()

Summarize antenna classification prior to redundant-baseline calibration¶

In [25]:
def array_class_plot(cls, extra_label=""):
    outriggers = [ant for ant in hd.data_ants if ant >= 320]

    if len(outriggers) > 0:
        fig, axes = plt.subplots(1, 2, figsize=(14, 6), dpi=100, gridspec_kw={'width_ratios': [2, 1]})
        plot_antclass(hd.antpos, cls, ax=axes[0], ants=[ant for ant in hd.data_ants if ant < 320], legend=False, title=f'HERA Core{extra_label}')
        plot_antclass(hd.antpos, cls, ax=axes[1], ants=outriggers, radius=50, title='Outriggers')
    else:
        fig, axes = plt.subplots(1, 1, figsize=(9, 6), dpi=100)
        plot_antclass(hd.antpos, cls, ax=axes, ants=[ant for ant in hd.data_ants if ant < 320], legend=False, title=f'HERA Core{extra_label}')

Figure 3: Summary of antenna classifications prior to calibration¶

This figure shows the location and classification of all antennas prior to calibration. Antennas are split along the diagonal, with ee-polarized antpols represented by the southeast half of each antenna and nn-polarized antpols represented by the northwest half. Outriggers are split from the core and shown at exaggerated size in the right-hand panel. This classification includes ant_metrics, a count of the zeros in the even or odd visibilities, and autocorrelation power, slope, and RFI occupancy. An antenna classified as bad in any classification will be considered bad. An antenna marked as suspect any in any classification will be considered suspect unless it is also classified as bad elsewhere.

In [26]:
if PLOT: array_class_plot(final_class)

Perform redundant-baseline calibration¶

In [27]:
def classify_off_grid(reds, all_ants):
    '''Returns AntennaClassification of all_ants where good ants are in reds while bad ants are not.'''
    ants_in_reds = set([ant for red in reds for bl in red for ant in utils.split_bl(bl)])
    on_grid = [ant for ant in all_ants if ant in ants_in_reds]
    off_grid = [ant for ant in all_ants if ant not in ants_in_reds]
    return ant_class.AntennaClassification(good=on_grid, bad=off_grid)
In [28]:
def per_pol_filter_reds(reds, pols=['nn', 'ee'], **kwargs):
    '''Performs redcal filtering separately on polarizations (which might have different min_dim_size issues).'''
    return [red for pol in pols for red in redcal.filter_reds(copy.deepcopy(reds), pols=[pol], **kwargs)]
In [29]:
def check_if_whole_pol_flagged(redcal_class, pols=['Jee', 'Jnn']):
    '''Checks if an entire polarization is flagged. If it is, returns True and marks all antennas as bad in redcal_class.'''
    if np.logical_or(*[np.all([redcal_class[ant] == 'bad' for ant in redcal_class.ants if ant[1] == pol]) for pol in pols]):
        print('An entire polarization has been flagged. Stopping redcal.')
        for ant in redcal_class:
            redcal_class[ant] = 'bad'
        return True
    return False

Perform iterative redcal¶

In [30]:
# figure out and filter reds and classify antennas based on whether or not they are on the main grid
fr_settings = {'max_dims': OC_MAX_DIMS, 'min_dim_size': OC_MIN_DIM_SIZE, 'min_bl_cut': OC_MIN_BL_LEN, 'max_bl_cut': OC_MAX_BL_LEN}
reds = redcal.get_reds(hd.data_antpos, pols=['ee', 'nn'], pol_mode='2pol')
reds = per_pol_filter_reds(reds, ex_ants=final_class.bad_ants, antpos=hd.data_antpos, **fr_settings)
if OC_SKIP_OUTRIGGERS:
    reds = redcal.filter_reds(reds, ex_ants=[ant for ant in ants if ant[0] >= 320])
redcal_class = classify_off_grid(reds, ants)
In [31]:
if OC_USE_PRIOR_SOL:
    # Find closest omnical file
    omnical_files = sorted(glob.glob('.'.join(OMNICAL_FILE.split('.')[:-5]) + '.*.' + '.'.join(OMNICAL_FILE.split('.')[-3:])))
    if len(omnical_files) == 0:
        OC_USE_PRIOR_SOL = False
    else:
        omnical_jds = np.array([float(re.findall("\d+\.\d+", ocf)[-1]) for ocf in omnical_files])
        closest_omnical = omnical_files[np.argmin(np.abs(omnical_jds - data.times[0]))]

        # Load closest omnical file and use it if the antenna flagging is not too dissimilar
        hc = io.HERACal(closest_omnical)
        prior_gains, prior_flags, _, _ = hc.read()
        not_bad_not_prior_flagged = [ant for ant in final_class if not ant in redcal_class.bad_ants and not np.all(prior_flags[ant])]
        if (len(redcal_class.bad_ants) == len(redcal_class.ants)):
            OC_USE_PRIOR_SOL = False  # all antennas flagged
        elif (len(not_bad_not_prior_flagged) / (len(redcal_class.ants) - len(redcal_class.bad_ants))) < OC_PRIOR_SOL_FLAG_THRESH:
            OC_USE_PRIOR_SOL = False  # too many antennas unflaged that were flagged in the prior sol
        else:
            print(f'Using {closest_omnical} as a starting point for redcal.')
In [32]:
redcal_start = time.time()
rc_settings = {'max_dims': OC_MAX_DIMS, 'oc_conv_crit': 1e-10, 'gain': 0.4, 'run_logcal': False,
               'oc_maxiter': OC_MAXITER, 'check_after': OC_MAXITER, 'use_gpu': OC_USE_GPU}

if check_if_whole_pol_flagged(redcal_class):
    # skip redcal, initialize empty sol and meta 
    sol = redcal.RedSol(reds)
    meta = {'chisq': None, 'chisq_per_ant': None}
else:    
    if OC_USE_PRIOR_SOL:
        # use prior gains and data to create starting point for next step
        ants_in_reds = set([ant for red in reds for bl in red for ant in utils.split_bl(bl)])
        sol = redcal.RedSol(reds=reds, gains={ant: prior_gains[ant] for ant in ants_in_reds})
        sol.update_vis_from_data(data)
    else:
        # perform first stage of redundant calibration, 
        meta, sol = redcal.redundantly_calibrate(data, reds, **rc_settings)
        max_dly = np.max(np.abs(list(meta['fc_meta']['dlys'].values())))  # Needed for RFI delay-slope cal
    malloc_trim()
    
# iteratively rerun redundant calibration
redcal_done = False
rc_settings['oc_maxiter'] = rc_settings['check_after'] = OC_RERUN_MAXITER
for i in range(OC_MAX_RERUN + 1):
    # refilter reds and update classification to reflect new off-grid ants, if any
    reds = per_pol_filter_reds(reds, ex_ants=(final_class + redcal_class).bad_ants, antpos=hd.data_antpos, **fr_settings)
    reds = sorted(reds, key=len, reverse=True)
    redcal_class += classify_off_grid(reds, ants)
    ants_in_reds = set([ant for red in reds for bl in red for ant in utils.split_bl(bl)])
    
    # check to see whether we're done
    if redcal_done or (i == OC_MAX_RERUN) or check_if_whole_pol_flagged(redcal_class):
        break

    # re-run redundant calibration using previous solution, updating bad and suspicious antennas
    meta, sol = redcal.redundantly_calibrate(data, reds, sol0=sol, **rc_settings)
    malloc_trim()
    
    # recompute chi^2 for bad antennas without bad antennas to make sure they are actually bad
    mean_cspa = {ant: np.nanmean(np.where(rfi_flags, np.nan, meta['chisq_per_ant'][ant])) for ant in meta['chisq_per_ant']}
    sol2 = redcal.RedSol(sol.reds, gains={ant: sol[ant] for ant in mean_cspa if mean_cspa[ant] <= oc_cspa_suspect[1]}, vis=sol.vis)
    new_chisq_per_ant = {ant: np.array(meta['chisq_per_ant'][ant]) for ant in sol2.gains}
    if len(set([bl[2] for red in per_pol_filter_reds(reds, ants=sol2.gains.keys(), antpos=hd.data_antpos, **fr_settings) for bl in red])) >= 2:
        redcal.expand_omni_gains(sol2, reds, data, chisq_per_ant=new_chisq_per_ant)
    for ant in mean_cspa:
        if ant in new_chisq_per_ant:
            if np.any(np.isfinite(new_chisq_per_ant[ant])):
                if not np.all(np.isclose(new_chisq_per_ant[ant], 0)):
                    new_mean_cspa = np.nanmean(np.where(rfi_flags, np.nan, meta['chisq_per_ant'][ant]))
                    if new_mean_cspa > 0:
                        mean_cspa[ant] = np.min([mean_cspa[ant], new_mean_cspa])
    
    # remove bad antennas
    cspa_class = ant_class.antenna_bounds_checker(mean_cspa, good=oc_cspa_good, suspect=oc_cspa_suspect, bad=[(-np.inf, np.inf)])
    redcal_class += cspa_class
    print(f'Removing {cspa_class.bad_ants} for high chi^2.')
    if len(cspa_class.bad_ants) == 0:
        redcal_done = True  # no new antennas to flag

print(f'Finished redcal in {(time.time() - redcal_start) / 60:.2f} minutes.')
An entire polarization has been flagged. Stopping redcal.
An entire polarization has been flagged. Stopping redcal.
Finished redcal in 0.00 minutes.
In [33]:
final_class += redcal_class

Expand solution to include calibratable baselines excluded from redcal (e.g. because they were too long)¶

In [34]:
expanded_reds = redcal.get_reds(hd.data_antpos, pols=['ee', 'nn'], pol_mode='2pol')
expanded_reds = per_pol_filter_reds(expanded_reds, ex_ants=(ant_metrics_class + solar_class + zeros_class + auto_class + xengine_diff_class).bad_ants,
                                    max_dims=OC_MAX_DIMS, min_dim_size=OC_MIN_DIM_SIZE)
if OC_SKIP_OUTRIGGERS:
    expanded_reds = redcal.filter_reds(expanded_reds, ex_ants=[ant for ant in ants if ant[0] >= 320])
if len(sol.gains) > 0:
    redcal.expand_omni_vis(sol, expanded_reds, data, chisq=meta['chisq'], chisq_per_ant=meta['chisq_per_ant'])
In [35]:
# now figure out flags, nsamples etc.
omni_flags = {ant: (~np.isfinite(g)) | (ant in final_class.bad_ants) for ant, g in sol.gains.items()}
vissol_flags = datacontainer.RedDataContainer({bl: ~np.isfinite(v) for bl, v in sol.vis.items()}, reds=sol.vis.reds)
single_nsamples_array = np.ones((len(hd.times), len(hd.freqs)), dtype=float)
nsamples = datacontainer.DataContainer({bl: single_nsamples_array for bl in data})
vissol_nsamples = redcal.count_redundant_nsamples(nsamples, [red for red in expanded_reds if red[0] in vissol_flags], 
                                                  good_ants=[ant for ant in final_class if ant not in final_class.bad_ants])
for bl in vissol_flags:
    vissol_flags[bl][vissol_nsamples[bl] == 0] = True
sol.make_sol_finite()        

Fix the firstcal delay slope degeneracy using RFI transmitters¶

In [36]:
if not OC_USE_PRIOR_SOL:
    # find channels clearly contaminated by RFI
    not_bad_ants = [ant for ant in final_class.ants if final_class[ant] != 'bad']
    if len(not_bad_ants) > 0:
        chan_flags = np.mean([xrfi.detrend_medfilt(data[utils.join_bl(ant, ant)], Kf=8, Kt=2) for ant in not_bad_ants], axis=(0, 1)) > 5

        # hardcoded RFI transmitters and their headings
        # channel: frequency (Hz), heading (rad), chi^2
        phs_sol = {359: ( 90744018.5546875, 0.7853981, 23.3),
                   360: ( 90866088.8671875, 0.7853981, 10.8),
                   385: ( 93917846.6796875, 0.7853981, 27.3),
                   386: ( 94039916.9921875, 0.7853981, 18.1),
                   400: ( 95748901.3671875, 6.0632738, 24.0),
                   441: (100753784.1796875, 0.7853981, 21.7),
                   442: (100875854.4921875, 0.7853981, 19.4),
                   455: (102462768.5546875, 6.0632738, 18.8),
                   456: (102584838.8671875, 6.0632738,  8.8),
                   471: (104415893.5546875, 0.7853981, 13.3),
                   484: (106002807.6171875, 6.0632738, 21.2),
                   485: (106124877.9296875, 6.0632738,  4.0),
                  1181: (191085815.4296875, 0.7853981, 26.3),
                  1182: (191207885.7421875, 0.7853981, 27.0),
                  1183: (191329956.0546875, 0.7853981, 25.6),
                  1448: (223678588.8671875, 2.6075219, 25.7),
                  1449: (223800659.1796875, 2.6075219, 22.6),
                  1450: (223922729.4921875, 2.6075219, 11.6),
                  1451: (224044799.8046875, 2.6075219,  5.9),
                  1452: (224166870.1171875, 2.6075219, 22.6),
                  1510: (231246948.2421875, 0.1068141, 23.9)}

        if not np.isclose(hd.freqs[0], 46920776.3671875, atol=0.001) or len(hd.freqs) != 1536:
            # We have less frequencies than usual (maybe testing)
            phs_sol = {np.argmin(np.abs(hd.freqs - freq)): (freq, heading, chisq) for chan, (freq, heading, chisq) in phs_sol.items() if hd.freqs[0] <= freq <= hd.freqs[-1]}


        rfi_chans = [chan for chan in phs_sol if chan_flags[chan]]
        print('Channels used for delay-slope calibration with RFI:', rfi_chans)
        rfi_angles = np.array([phs_sol[chan][1] for chan in rfi_chans])
        rfi_headings = np.array([np.cos(rfi_angles), np.sin(rfi_angles), np.zeros_like(rfi_angles)])
        rfi_chisqs = np.array([phs_sol[chan][2] for chan in rfi_chans])

        # resolve firstcal degeneracy with delay slopes set by RFI transmitters, update cal
        RFI_dly_slope_gains = abscal.RFI_delay_slope_cal([red for red in expanded_reds if red[0] in sol.vis], hd.antpos, sol.vis, hd.freqs, rfi_chans, rfi_headings, rfi_wgts=rfi_chisqs**-1,
                                                         min_tau=-max_dly, max_tau=max_dly, delta_tau=0.1e-9, return_gains=True, gain_ants=sol.gains.keys())
        sol.gains = {ant: g * RFI_dly_slope_gains[ant] for ant, g in sol.gains.items()}
        apply_cal.calibrate_in_place(sol.vis, RFI_dly_slope_gains)
        malloc_trim()

Perform absolute amplitude calibration using a model of autocorrelations¶

In [37]:
# Load simulated and then downsampled model of autocorrelations that includes receiver noise, then interpolate to upsample
hd_model = io.HERADataFastReader(f'{HNBT_DATA}/SSM_autocorrelations_downsampled.uvh5')
model, _, _ = hd_model.read(read_flags=False, read_nsamples=False)
per_pol_interpolated_model = {}
for bl in model:
    sorted_lsts, lst_indices = np.unique(model.lsts, return_index=True)
    periodic_model = np.vstack([model[bl][lst_indices, :], model[bl][lst_indices[0], :]])
    periodic_lsts = np.append(sorted_lsts, sorted_lsts[0] + 2 * np.pi)
    lst_interpolated = interpolate.CubicSpline(periodic_lsts, periodic_model, axis=0, bc_type='periodic')(data.lsts)
    per_pol_interpolated_model[bl[2]] = interpolate.CubicSpline(model.freqs, lst_interpolated, axis=1)(data.freqs)
model = {bl: per_pol_interpolated_model[bl[2]] for bl in auto_bls if utils.split_bl(bl)[0] not in final_class.bad_ants}
In [38]:
# Run abscal and update omnical gains with abscal gains
if len(model) > 0:
    redcaled_autos = {bl: sol.calibrate_bl(bl, data[bl]) for bl in auto_bls if utils.split_bl(bl)[0] not in final_class.bad_ants}
    g_abscal = abscal.abs_amp_logcal(model, redcaled_autos, verbose=False, return_gains=True, gain_ants=sol.gains)
    sol.gains = {ant: g * g_abscal[ant] for ant, g in sol.gains.items()}
    apply_cal.calibrate_in_place(sol.vis, g_abscal)
    del redcaled_autos, g_abscal
In [39]:
del hd_model, model
malloc_trim()

Full absolute calibration of phase gradients¶

If an ABSCAL_MODEL_FILES_GLOB is provided, try to perform a full absolute calibration of tip-tilt phase gradients across the array using that those model files. Specifically, this step calibrates omnical visbility solutions using unique baselines simulated with a model of the sky and HERA's beam.

In [40]:
if ABSCAL_MODEL_FILES_GLOB is not None:
    abscal_model_files = sorted(glob.glob(ABSCAL_MODEL_FILES_GLOB))
else:
    # try to find files on site
    abscal_model_files = sorted(glob.glob('/mnt/sn1/abscal_models/H4C_1/abscal_files_unique_baselines/zen.2458894.?????.uvh5'))
    if len(abscal_model_files) == 0:
        # try to find files at NRAO
        abscal_model_files = sorted(glob.glob('/lustre/aoc/projects/hera/zmartino/hera_calib_model/H4C_1/abscal_files_unique_baselines/zen.2458894.?????.uvh5'))
print(f'Found {len(abscal_model_files)} abscal model files{" in " + os.path.dirname(abscal_model_files[0]) if len(abscal_model_files) > 0 else ""}.')
Found 425 abscal model files in /mnt/sn1/abscal_models/H4C_1/abscal_files_unique_baselines.
In [41]:
# Try to perform a full abscal of phase
if len(abscal_model_files) == 0:
    DO_FULL_ABSCAL = False
    print('No model files found... not performing full absolute calibration of phase gradients.')
elif np.all([ant in final_class.bad_ants for ant in ants]):
    DO_FULL_ABSCAL = False
    print('All antennas classified as bad... skipping absolute calibration of phase gradients.')
else:
    abscal_start = time.time()
    # figure out which model files match the LSTs of the data
    matched_model_files = sorted(set(abscal.match_times(SUM_FILE, abscal_model_files, filetype='uvh5')))
    if len(matched_model_files) == 0:
        DO_FULL_ABSCAL = False
        print(f'No model files found matching the LSTs of this file after searching for {(time.time() - abscal_start) / 60:.2f} minutes. '
              'Not performing full absolute calibration of phase gradients.')
    else:
        DO_FULL_ABSCAL = True
        # figure out appropriate model times to load
        hdm = io.HERAData(matched_model_files)
        all_model_times, all_model_lsts = abscal.get_all_times_and_lsts(hdm, unwrap=True)
        d2m_time_map = abscal.get_d2m_time_map(data.times, np.unwrap(data.lsts), all_model_times, all_model_lsts, extrap_limit=.5)
All antennas classified as bad... skipping absolute calibration of phase gradients.
In [42]:
if DO_FULL_ABSCAL:
    abscal_meta = {}
    for pol in ['ee', 'nn']:
        print(f'Performing absolute phase gradient calibration of {pol}-polarized visibility solutions...')
        
        # load matching times and baselines
        unflagged_data_bls = [bl for bl in vissol_flags if not np.all(vissol_flags[bl]) and bl[2] == pol]
        model_bls = copy.deepcopy(hdm.bls)
        model_antpos = hdm.data_antpos
        if len(matched_model_files) > 1:  # in this case, it's a dictionary
            model_bls = list(set([bl for bls in list(hdm.bls.values()) for bl in bls]))
            model_antpos = {ant: pos for antpos in hdm.data_antpos.values() for ant, pos in antpos.items()}
        data_bls, model_bls, data_to_model_bl_map = abscal.match_baselines(unflagged_data_bls, model_bls, data.antpos, model_antpos=model_antpos, 
                                                                         pols=[pol], data_is_redsol=True, model_is_redundant=True, tol=1.0,
                                                                         min_bl_cut=ABSCAL_MIN_BL_LEN, max_bl_cut=ABSCAL_MAX_BL_LEN, verbose=True)
        model, model_flags, _ = io.partial_time_io(hdm, np.unique([d2m_time_map[time] for time in data.times]), bls=model_bls)
        model_bls = [data_to_model_bl_map[bl] for bl in data_bls]
        
        # rephase model to match in lsts
        model_blvecs = {bl: model.antpos[bl[0]] - model.antpos[bl[1]] for bl in model.keys()}
        utils.lst_rephase(model, model_blvecs, model.freqs, data.lsts - model.lsts,
                          lat=hdm.telescope_location_lat_lon_alt_degrees[0], inplace=True)

        # run abscal and apply 
        abscal_meta[pol], delta_gains = abscal.complex_phase_abscal(sol.vis, model, sol.reds, data_bls, model_bls)
        
        # apply gains
        sol.gains = {antpol : g * delta_gains.get(antpol, 1) for antpol, g in sol.gains.items()}
        apply_cal.calibrate_in_place(sol.vis, delta_gains)            
    
    del hdm, model, model_flags, delta_gains
    malloc_trim()
    
    print(f'Finished absolute calibration of tip-tilt phase slopes in {(time.time() - abscal_start) / 60:.2f} minutes.')
In [43]:
def redundant_group_plot():
    if np.all([ant in final_class.bad_ants for ant in ants]):
        print('All antennas classified as bad. Nothing to plot.')
        return
    
    fig, axes = plt.subplots(2, 2, figsize=(14, 6), dpi=100, sharex='col', sharey='row', gridspec_kw={'hspace': 0, 'wspace': 0})
    for i, pol in enumerate(['ee', 'nn']):
        reds_here = redcal.get_reds(hd.data_antpos, pols=[pol], pol_mode='1pol')
        red = sorted(redcal.filter_reds(reds_here, ex_ants=final_class.bad_ants), key=len, reverse=True)[0]
        rc_data = {bl: sol.calibrate_bl(bl, data[bl]) for bl in red}
        for bl in red:
            axes[0, i].plot(hd.freqs/1e6, np.angle(rc_data[bl][0]), alpha=.5, lw=.5)
            axes[1, i].semilogy(hd.freqs/1e6, np.abs(rc_data[bl][0]), alpha=.5, lw=.5)
        axes[0, i].plot(hd.freqs / 1e6, np.angle(sol.vis[red[0]][0]), lw=1, c='k')
        axes[1, i].semilogy(hd.freqs / 1e6, np.abs(sol.vis[red[0]][0]), lw=1, c='k', label=f'Baseline Group:\n{red[0]}')
        axes[1, i].set_xlabel('Frequency (MHz)')
        axes[1, i].legend(loc='upper right')
    axes[0, 0].set_ylabel('Visibility Phase (radians)')
    axes[1, 0].set_ylabel('Visibility Amplitude (Jy)')
    plt.tight_layout()
In [44]:
def abscal_degen_plot():
    if DO_FULL_ABSCAL:
        fig, axes = plt.subplots(3, 1, figsize=(14, 6), dpi=100, sharex=True, gridspec_kw={'hspace': .05})

        for ax, pol in zip(axes[:2], ['ee', 'nn']):
            for kk in range(abscal_meta[pol]['Lambda_sol'].shape[-1]):
                ax.plot(hd.freqs[~rfi_flags[0]] * 1e-6, abscal_meta[pol]['Lambda_sol'][0, ~rfi_flags[0], kk], '.', ms=1, label=f"Component {kk}")

            ax.set_ylim(-np.pi-0.5, np.pi+0.5)
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('Phase Gradient\nVector Component')
            ax.legend(markerscale=20, title=f'{pol}-polarization', loc='lower right')
            ax.grid()
            
        for pol, color in zip(['ee', 'nn'], ['b', 'r']):
            axes[2].plot(hd.freqs[~rfi_flags[0]]*1e-6, abscal_meta[pol]['Z_sol'].real[0, ~rfi_flags[0]], '.', ms=1, label=pol, color=color)
        axes[2].set_ylim(-.25, 1.05)
        axes[2].set_ylabel('Re[Z($\\nu$)]')
        axes[2].legend(markerscale=20, loc='lower right')
        axes[2].grid()            
        plt.tight_layout()

Figure 4: Redundant calibration of a single baseline group¶

The results of a redundant-baseline calibration of a single integration and a single group, the one with the highest redundancy in each polarization after antenna classification and excision based on the above, plus the removal of antennas with high chi^2 per antenna. The black line is the redundant visibility solution. Each thin colored line is a different baseline group. Phases are shown in the top row, amplitudes in the bottom, ee-polarized visibilities in the left column, and nn-polarized visibilities in the right.

In [45]:
if PLOT: redundant_group_plot()
All antennas classified as bad. Nothing to plot.

Figure 5: Absolute calibration of redcal degeneracies¶

This figure shows the per-frequency phase gradient solutions across the array for both polarizations and all components of the degenerate subspace of redundant-baseline calibraton. While full HERA only has two such tip-tilt degeneracies, a subset of HERA can have up to OC_MAX_DIMS (depending on antenna flagging). In addition to the absolute amplitude, this is the full set of the calibration degrees of freedom not constrainted by redcal. This figure also includes a plot of $Re[Z(\nu)]$, the complex objective function which varies from -1 to 1 and indicates how well the data and the absolute calibration model have been made to agree. Perfect agreement is 1.0 and good agreement is anything above $\sim$0.5 Decorrelation yields values closer to 0, where anything below $\sim$0.3 is suspect.

In [46]:
if PLOT: abscal_degen_plot()

Attempt to calibrate some flagged antennas¶

This attempts to calibrate bad antennas using information from good or suspect antennas without allowing bad antennas to affect their calibration. However, introducing 0s in gains or infs/nans in gains or visibilities can create problems down the line, so those are removed.

In [47]:
expanded_reds = redcal.get_reds(hd.data_antpos, pols=['ee', 'nn'], pol_mode='2pol')
sol.vis.build_red_keys(expanded_reds)
redcal.expand_omni_gains(sol, expanded_reds, data, chisq_per_ant=meta['chisq_per_ant'])
if not np.all([ant in final_class.bad_ants for ant in ants]):
    redcal.expand_omni_vis(sol, expanded_reds, data)

# Replace near-zeros in gains and infs/nans in gains/sols
for ant in sol.gains:
    zeros_in_gains = np.isclose(sol.gains[ant], 0)
    if ant in omni_flags:
        omni_flags[ant][zeros_in_gains] = True
    sol.gains[ant][zeros_in_gains] = 1.0 + 0.0j
sol.make_sol_finite()
malloc_trim()
In [48]:
def array_chisq_plot():
    if np.all([ant in final_class.bad_ants for ant in ants]):
        print('All antennas classified as bad. Nothing to plot.')
        return    
    
    fig, axes = plt.subplots(1, 2, figsize=(14, 5), dpi=100)
    for ax, pol in zip(axes, ['ee', 'nn']):
        ants_to_plot = set([ant for ant in meta['chisq_per_ant'] if utils.join_pol(ant[1], ant[1]) == pol])
        cspas = np.array([np.nanmean(np.where(rfi_flags, np.nan, meta['chisq_per_ant'][ant])) for ant in ants_to_plot])
        xpos = [hd.antpos[ant[0]][0] for ant in ants_to_plot]
        ypos = [hd.antpos[ant[0]][1] for ant in ants_to_plot]
        scatter = ax.scatter(xpos, ypos, s=300, c=cspas, lw=.25, edgecolors=np.where(np.isfinite(cspas) & (cspas > 0), 'none', 'k'), 
                             norm=matplotlib.colors.LogNorm(vmin=1, vmax=oc_cspa_suspect[1]))
        for ant in ants_to_plot:
            ax.text(hd.antpos[ant[0]][0], hd.antpos[ant[0]][1], ant[0], va='center', ha='center', fontsize=8,
                    c=('r' if ant in final_class.bad_ants else 'w'))
        plt.colorbar(scatter, ax=ax, extend='both')
        ax.axis('equal')
        ax.set_xlabel('East-West Position (meters)')
        ax.set_ylabel('North-South Position (meters)')
        ax.set_title(f'{pol}-pol $\\chi^2$ / Antenna (Red is Flagged)')
    plt.tight_layout()

Figure 6: chi^2 per antenna across the array¶

This plot shows median (taken over time and frequency) of the normalized chi^2 per antenna. The expectation value for this quantity when the array is perfectly redundant is 1.0. Antennas that are classified as bad for any reason have their numbers shown in red. Some of those antennas were classified as bad during redundant calibration for high chi^2. Some of those antennas were originally excluded from redundant calibration because they were classified as bad earlier for some reason. See here for more details. Note that the color scale saturates at below 1 and above 10.

In [49]:
if PLOT: array_chisq_plot()
All antennas classified as bad. Nothing to plot.

Figure 7: Summary of antenna classifications after redundant calibration¶

This figure is the same as Figure 2, except that it now includes additional suspect or bad antennas based on redundant calibration. This can include antennas with high chi^2, but it can also include antennas classified as "bad" because they would add extra degeneracies to calibration.

In [50]:
if PLOT: array_class_plot(final_class, extra_label=", Post-Redcal")
In [51]:
to_show = {'Antenna': [f'{ant[0]}{ant[1][-1]}' for ant in ants]}
classes = {'Antenna': [final_class[ant] if ant in final_class else '-' for ant in ants]}
to_show['Dead?'] = [{'good': 'No', 'bad': 'Yes'}[am_totally_dead[ant]] if (ant in am_totally_dead) else '' for ant in ants]
classes['Dead?'] = [am_totally_dead[ant] if (ant in am_totally_dead) else '' for ant in ants]
for title, ac in [('Low Correlation', am_corr),
                  ('Cross-Polarized', am_xpol),
                  ('Solar Alt', solar_class),
                  ('Even/Odd Zeros', zeros_class),
                  ('Autocorr Power', auto_power_class),
                  ('Autocorr Slope', auto_slope_class),
                  ('RFI in Autos', auto_rfi_class),
                  ('Autocorr Shape', auto_shape_class),
                  ('Bad Diff X-Engines', xengine_diff_class)]:
    to_show[title] = [f'{ac._data[ant]:.2G}' if (ant in ac._data) else '' for ant in ants]
    classes[title] = [ac[ant] if ant in ac else 'bad' for ant in ants]
    
to_show['Redcal chi^2'] = [f'{np.nanmean(np.where(rfi_flags, np.nan, meta["chisq_per_ant"][ant])):.3G}' \
                           if (meta['chisq_per_ant'] is not None and ant in meta['chisq_per_ant']) else '' for ant in ants]
classes['Redcal chi^2'] = [redcal_class[ant] if ant in redcal_class else '' for ant in ants]

df = pd.DataFrame(to_show)
df_classes = pd.DataFrame(classes)
colors = {'good': 'darkgreen', 'suspect': 'goldenrod', 'bad': 'maroon'}
df_colors = df_classes.applymap(lambda x: f'background-color: {colors.get(x, None)}')

table = df.style.hide() \
                .apply(lambda x: pd.DataFrame(df_colors.values, columns=x.columns), axis=None) \
                .set_properties(subset=['Antenna'], **{'font-weight': 'bold', 'border-right': "3pt solid black"}) \
                .set_properties(subset=df.columns[1:], **{'border-left': "1pt solid black"}) \
                .set_properties(**{'text-align': 'center', 'color': 'white'})

Table 1: Complete summary of per-antenna classifications¶

This table summarizes the results of the various classifications schemes detailed above. As before, green is good, yellow is suspect, and red is bad. The color for each antenna (first column) is the final summary of all other classifications. Antennas missing from redcal $\chi^2$ were excluded redundant-baseline calibration, either because they were flagged by ant_metrics or the even/odd zeros check, or because they would add unwanted extra degeneracies.

In [52]:
HTML(table.to_html())
Out[52]:
Antenna Dead? Low Correlation Cross-Polarized Solar Alt Even/Odd Zeros Autocorr Power Autocorr Slope RFI in Autos Autocorr Shape Bad Diff X-Engines Redcal chi^2
3e No 0.5 0.32 -57 0 55 0.062 0.0059 0.025 1
3n No 0.47 0.32 -57 0 54 0.025 0.00033 0.021 3
4e No 0.51 0.32 -57 0 49 0.053 0.0098 0.015 0
4n No 0.46 0.32 -57 0 57 0.13 0 0.05 2
5e No 0.5 0.32 -57 0 52 0.049 0.029 0.019 1
5n No 0.47 0.32 -57 0 52 0.029 0.00033 0.015 2
7e No 0.5 0.32 -57 0 53 0.058 0.0052 0.012 2
7n No 0.48 0.32 -57 0 36 0.069 0.00033 0.031 0
8e No 0.48 0.31 -57 0 62 0.052 0.02 0.037
8n No 0.45 0.31 -57 0 61 0.031 0 0.032
9e No 0.41 0.28 -57 0 2 0.28 0.02 0.062 0
9n No 0.43 0.28 -57 0 66 0.028 0 0.044
10e No 0.47 0.32 -57 0 47 0.075 0.012 0.011 2
10n No 0.44 0.32 -57 0 49 0.049 0.00033 0.009 0
15e No 0.37 0.29 -57 0 40 0.67 0 0.19
15n No 0.47 0.29 -57 0 54 0.025 0 0.022 0
16e No 0.48 0.4 -57 0 68 0.048 0.067 0.045
16n No 0.065 0.4 -57 1 0.62 0.62 0.098 0.15
17e No 0.5 0.31 -57 0 57 0.042 0.002 0.024 2
17n No 0.49 0.31 -57 0 10 0.12 0.0033 0.078 3
18e No 0.45 0.38 -57 0 44 0.077 0.14 0.075
18n No 0.033 0.38 -57 1 0.61 0.61 0.15 0.14
19e No 0.49 0.31 -57 0 61 0.052 0.029 0.029
19n No 0.49 0.31 -57 0 39 0.061 0 0.022 4
20e No 0.49 0.31 -57 0 13 0.17 0.002 0.073 3
20n No 0.47 0.31 -57 0 54 0.041 0 0.017 3
21e No 0.48 0.3 -57 0 13 0.2 0.0026 0.074 3
21n No 0.46 0.3 -57 0 35 0.099 0.00033 0.034 1
22e No 0.36 0.24 -57 0 58 0.049 0 0.027 0
22n No 0.33 0.24 -57 0 77 0.019 0 0.068
27e No 0.042 0.046 -57 1 0.92 0.77 0.31 0.31
27n No 0.085 0.046 -57 0 9.4 0.97 0.28 0.34
28e No 0.49 0.41 -57 0 60 0.06 0.002 0.028
28n No 0.17 0.41 -57 1 4.6 0.81 0.24 0.3
29e No 0.4 0.31 -57 0 63 0.29 0.00065 0.06
29n No 0.48 0.31 -57 0 57 0.051 0 0.021 0
30e No 0.5 0.32 -57 0 48 0.08 0.0039 0.009 0
30n No 0.5 0.32 -57 0 39 0.05 0 0.021 2
31e No 0.5 0.31 -57 0 18 0.13 0.0023 0.06 1
31n No 0.49 0.31 -57 0 11 0.33 0.0036 0.091 3
32e No 0.37 0.27 -57 0 56 0.49 0 0.12 2
32n No 0.45 0.27 -57 0 47 0.23 0 0.063 4
33e No 0.48 0.3 -57 0 46 0.14 0.0068 0.016 0
33n No 0.48 0.3 -57 0 59 0.045 0 0.026 2
34e No 0.035 0.27 -57 0 2.9 0.57 0.072 0.13
34n No 0.36 0.27 -57 0 68 0.053 0 0.046
35e No 0.35 0.23 -57 0 82 0.017 0.038 0.08
35n No 0.35 0.23 -57 0 78 0.022 0 0.068
36e No 0.51 0.29 -57 0 50 -0.03 0.0042 0.045 0
36n No 0.51 0.29 -57 0 58 -0.084 0 0.05 4
37e No 0.51 0.29 -57 0 2.4 -0.28 0.0072 0.15 2
37n No 0.51 0.29 -57 0 1.9 -0.28 0.017 0.14 3
38e No 0.51 0.3 -57 0 56 0.061 0.0016 0.026 0
38n No 0.52 0.3 -57 0 48 0.038 0 0.018 2
40e No 0.5 0.3 -57 0 32 0.16 0.0029 0.046 0
40n No 0.51 0.3 -57 0 33 0.083 0.048 0.043
41e No 0.48 0.31 -57 0 65 0.049 0.0023 0.039
41n No 0.51 0.31 -57 0 29 0.04 0.00065 0.047 0
42e No 0.5 0.31 -57 0 14 0.15 0.0013 0.074 3
42n No 0.45 0.31 -57 0 2 0.15 0.011 0.065 3
45e No 0.44 0.27 -57 0 48 0.061 0.0062 0.011 6
45n No 0.45 0.27 -57 0 36 0.069 0.00033 0.028 6
46e No 0.43 0.28 -57 0 57 0.063 0.0059 0.023 4
46n No 0.45 0.28 -57 0 43 0.038 0 0.021 3
47e No 0.39 0.25 -57 0 29 0.17 0.013 0.042 2
47n No 0.38 0.25 -57 0 74 0.03 0 0.061
48e No 0.36 0.22 -57 0 79 0.042 0 0.069
48n No 0.36 0.22 -57 0 85 0.018 0 0.082
49e No 0.37 0.24 -57 0 70 0.038 0 0.053
49n No 0.35 0.24 -57 0 81 0.02 0 0.076
50e No 0.49 0.31 -57 0 66 0.041 0.0013 0.044
50n No 0.53 0.31 -57 0 54 5.8E-05 0 0.03 0
51e No 0.5 0.3 -57 0 11 0.32 0.0062 0.15 3
51n No 0.53 0.3 -57 0 53 0.013 0 0.029 4
52e No 0.52 0.29 -57 0 43 -0.063 0.00098 0.054 0
52n No 0.52 0.29 -57 0 61 -0.039 0 0.044
53e No 0.51 0.29 -57 0 48 0.1 0 0.016 1
53n No 0.52 0.29 -57 0 57 -0.0076 0 0.035 3
54e No 0.38 0.3 -57 0 42 0.73 0 0.19
54n No 0.53 0.3 -57 0 39 0.062 0 0.022 6
55e No 0.48 0.29 -57 0 63 0.048 0.013 0.034
55n No 0.5 0.29 -57 0 58 0.028 0 0.025 2
56e No 0.5 0.32 -57 0 64 0.038 0.0023 0.039
56n No 0.53 0.32 -57 0 25 0.07 0.00065 0.064 6
57e No 0.5 0.3 -57 0 53 0.063 0.0029 0.023 0
57n No 0.51 0.3 -57 0 50 0.076 0.0013 0.0097 1
61e No 0.037 0.29 -57 0 3.2 0.56 0.14 0.13
61n No 0.39 0.29 -57 0 75 0.031 0 0.063
62e No 0.38 0.23 -57 0 58 0.072 0.00033 0.029 0
62n No 0.37 0.23 -57 0 84 0.014 0 0.083
63e No 0.14 -0.2 -57 0 81 0.024 0 0.077
63n No 0.14 -0.2 -57 0 82 0.018 0.037 0.078
64e No 0.36 0.23 -57 0 71 0.049 0.002 0.056
64n No 0.36 0.23 -57 0 79 0.024 0 0.07
65e No 0.5 0.3 -57 0 59 0.042 0.00033 0.03 0
65n No 0.53 0.3 -57 0 60 0.0051 0 0.034 2
66e No 0.5 0.32 -57 0 63 0.051 0.0062 0.038
66n No 0.55 0.32 -57 0 34 0.024 0 0.035 1
67e No 0.51 0.32 -57 0 63 0.049 0.00033 0.039
67n No 0.56 0.32 -57 0 11 0.12 0.0065 0.066 2
68e No 0.52 0.3 -57 0 52 0.083 0.00033 0.022 3
68n No 0.54 0.3 -57 0 11 0.2 0.0042 0.072 5
69e No 0.5 0.29 -57 0 64 0.057 0.0036 0.036
69n No 0.52 0.29 -57 0 52 0.096 0 0.027 0
70e No 0.51 0.29 -57 0 48 0.1 0.0016 0.0084 0
70n No 0.52 0.29 -57 0 60 0.059 0 0.026
71e No 0.5 0.31 -57 0 61 0.075 0 0.03
71n No 0.54 0.31 -57 0 44 0.062 0 0.012 1
72e No 0.52 0.3 -57 0 52 0.12 0.0026 0.014 0
72n No 0.53 0.3 -57 0 46 0.081 0 0.012 5
73e No 0.46 0.27 -57 0 53 0.093 0.0016 0.012 6
73n No 0.47 0.27 -57 0 8.2 0.15 0.0033 0.077 6
77e No 0.24 0.21 -57 0 80 0.35 0 0.084
77n No 0.33 0.21 -57 0 79 0.12 0 0.067
78e No 0.27 0.22 -57 0 74 0.29 0 0.079
78n No 0.38 0.22 -57 0 85 0.011 0 0.082
79e No 0.4 0.26 -57 0 72 0.059 0 0.054
79n No 0.42 0.26 -57 0 76 0.027 0 0.067
80e No 0.37 0.26 -57 32 78 0.033 0.01 0.07
80n No 0.41 0.26 -57 32 64 0.049 0.00033 0.038
81e No 0.49 0.3 -57 0 3.7 -0.25 0.012 0.12 1
81n No 0.53 0.3 -57 0 1.8 -0.36 0.02 0.14 3
82e No 0.5 0.31 -57 0 2.9 -0.088 0.0036 0.11 2
82n No 0.54 0.31 -57 0 2.3 -0.38 0.0052 0.15 2
83e No 0.52 0.29 -57 0 3 -0.18 0.0042 0.12 0
83n No 0.53 0.29 -57 0 2.2 -0.1 0.019 0.093 1
84e No 0.52 0.28 -57 0 51 0.086 0.00065 0.0095 0
84n No 0.51 0.28 -57 0 63 0.12 0 0.037
85e No 0.52 0.3 -57 0 59 0.033 0.0016 0.034 1
85n No 0.54 0.3 -57 0 57 0.032 0 0.029 3
86e No 0.43 0.32 -57 0 2.2 0.3 0.014 0.058 0
86n No 0.043 0.32 -57 0 0.97 0.57 0.13 0.14
87e No 0.36 0.31 -57 0 6.6 1.4 0.0029 0.33
87n No 0.56 0.31 -57 0 38 0.052 0 0.031 4
88e No 0.5 0.29 -57 0 68 0.035 0.041 0.051
88n No 0.53 0.29 -57 0 48 0.1 0 0.016 2
89e No 0.51 0.31 -57 0 68 0.024 0.045 0.049
89n No 0.54 0.31 -57 0 55 0.031 0 0.021 0
90e No 0.52 0.32 -57 0 33 0.091 0.00033 0.039 3
90n No 0.55 0.32 -57 0 23 0.1 0.0013 0.048 1
91e No 0.46 0.32 -57 0 56 0.18 0.00033 0.034 0
91n No 0.52 0.32 -57 0 58 0.033 0 0.027 1
92e No 0.021 0.0016 -57 0 0.68 0.5 0.074 0.11
92n No 0.025 0.0016 -57 0 0.64 0.56 0.11 0.13
93e No 0.2 -0.06 -57 0 35 0.12 0.00033 0.056
93n No 0.032 -0.06 -57 0 0.61 0.59 0.16 0.14
94e No 0.039 0.0098 -57 0 0.65 0.57 0.091 0.13
94n No 0.023 0.0098 -57 0 0.64 0.6 0.13 0.14
95e No 0.41 0.25 -57 0 73 0.06 0 0.057
95n No 0.41 0.25 -57 0 82 0.024 0 0.077
96e No 0.37 0.22 -57 0 85 0.026 0.036 0.083
96n No 0.33 0.22 -57 0 56 0.44 0.00033 0.13 0
97e No 0.38 0.26 -57 0 78 0.028 0.018 0.07
97n No 0.41 0.26 -57 0 14 0.27 0.027 0.072 0
98e No 0.47 0.3 -57 0 1.9 -0.26 0.0033 0.12 1
98n No 0.52 0.3 -57 0 1.8 -0.34 0.025 0.17 3
99e No 0.45 0.27 -57 0 0.72 -0.14 0.013 0.088
99n No 0.39 0.27 -57 0 0.28 -0.36 0.17 0.16
100e No 0.51 0.29 -57 0 3 -0.12 0.013 0.1 1
100n No 0.52 0.29 -57 0 1.1 -0.22 0.0042 0.11 2
101e No 0.49 0.29 -57 0 73 0.032 0.062 0.062
101n No 0.52 0.29 -57 0 69 0.011 0 0.052
102e No 0.52 0.3 -57 0 43 0.099 0.00065 0.016 4
102n No 0.54 0.3 -57 0 55 0.07 0 0.017 7
103e No 0.51 0.29 -57 0 60 0.079 0.00065 0.029
103n No 0.54 0.29 -57 0 59 0.043 0 0.028 1
104e No 0.52 0.31 -57 0 12 0.17 0.0023 0.059 2
104n No 0.48 0.31 -57 1 1.3 2.1 0.038 0.52
105e No 0.52 0.29 -57 0 53 0.053 0.002 0.021 1
105n No 0.53 0.29 -57 0 66 0.014 0 0.046
106e No 0.045 0.39 -57 0 1.1 0.51 0.087 0.12
106n No 0.54 0.39 -57 0 63 0.0032 0 0.04
107e No 0.52 0.3 -57 0 52 0.11 0.0033 0.019 3
107n No 0.54 0.3 -57 0 52 0.041 0.0068 0.039 1
108e No 0.52 0.31 -57 0 29 0.14 0.00065 0.047 1
108n No 0.55 0.31 -57 0 40 0.019 0.00065 0.044 3
109e No 0.024 0.0059 -57 0 0.67 0.52 0.13 0.12
109n No 0.033 0.0059 -57 0 0.66 0.57 0.1 0.13
110e No 0.34 0.29 -57 0 51 0.61 0 0.15
110n No 0.49 0.29 -57 0 54 0.066 0 0.013 0
111e No 0.36 0.25 -57 0 30 0.91 0.00033 0.23
111n No 0.041 0.25 -57 0 0.64 0.57 0.043 0.14
112e No 0.15 0.33 -57 0 0.79 0.48 0.063 0.11
112n No 0.45 0.33 -57 0 65 0.077 0 0.039
113e No 0.4 0.26 -57 0 80 0.036 0 0.073
113n No 0.42 0.26 -57 0 82 0.025 0 0.076
114e No 0.36 0.25 -57 0 74 0.082 0 0.058
114n No 0.4 0.25 -57 0 77 0.022 0 0.073
115e No 0.042 0.29 -57 0 3.3 0.54 0.065 0.12
115n No 0.4 0.29 -57 0 77 0.024 0 0.066
116e No 0.4 0.33 -57 0 1.3 0.29 0.0081 0.072 2
116n No 0.52 0.33 -57 0 2.4 -0.33 0.029 0.13 3
117e No 0.47 0.31 -57 0 63 0.073 0.00098 0.034
117n No 0.52 0.31 -57 0 58 0.049 0 0.024 0
118e No 0.5 0.32 -57 0 29 0.14 0.00065 0.049 0
118n No 0.53 0.32 -57 0 57 0.044 0 0.026 2
119e No 0.51 0.31 -57 0 4.3 -0.24 0.0039 0.13 0
119n No 0.43 0.31 -57 1 0.38 -0.3 0.24 0.13
120e No 0.5 0.3 -57 1 12 0.27 0.002 0.098 1
120n No 0.54 0.3 -57 0 53 0.055 0.0026 0.012 2
121e No 0.51 0.3 -57 0 56 0.082 0.00098 0.02 2
121n No 0.55 0.3 -57 0 11 0.14 0.0046 0.072 7
122e No 0.53 0.29 -57 0 41 0.051 0.00065 0.025 0
122n No 0.55 0.29 -57 0 60 0.018 0 0.032
123e No 0.51 0.31 -57 0 70 0.027 0.049 0.055
123n No 0.56 0.31 -57 0 46 0.03 0 0.021 0
124e No 0.53 0.3 -57 0 41 0.079 0 0.018 4
124n No 0.55 0.3 -57 0 49 0.079 0 0.014 2
125e No 0.055 0.39 -57 0 0.99 0.56 0.075 0.13
125n No 0.54 0.39 -57 0 65 0.025 0 0.041
126e No 0.51 0.32 -57 0 45 0.049 0.00065 0.019 1
126n No 0.56 0.32 -57 0 47 -0.077 0 0.044 0
127e No 0.47 0.29 -57 0 9.4 0.14 0.00033 0.078 2
127n No 0.51 0.29 -57 0 53 0.055 0 0.014 3
128e No 0.48 0.3 -57 0 31 0.12 0.0068 0.041 1
128n No 0.51 0.3 -57 0 41 0.1 0.00033 0.024 3
129e No 0.041 0.38 -57 0 0.68 0.5 0.1 0.11
129n No 0.52 0.38 -57 0 50 0.038 0 0.012 0
130e No 0.46 0.31 -57 0 42 0.16 0.00033 0.025 4
130n No 0.5 0.31 -57 0 53 0.086 0 0.013 3
131e No 0.4 0.27 -57 0 79 0.032 0 0.072
131n No 0.27 0.27 -57 0 3.8 0.49 0.035 0.11
132e No 0.4 0.27 -57 0 79 0.031 0 0.071
132n No 0.43 0.27 -57 0 73 0.032 0 0.059
133e No 0.4 0.26 -57 0 33 0.15 0.0016 0.036 4
133n No 0.42 0.26 -57 0 72 0.032 0 0.054
134e No 0.33 0.28 -57 0 7.9 0.36 0.044 0.087
134n No 0.41 0.28 -57 0 67 0.05 0 0.045
135e No 0.46 0.32 -57 0 53 0.1 0.0046 0.043 1
135n No 0.51 0.32 -57 0 53 0.042 0 0.014 3
136e No 0.49 0.29 -57 0 19 0.16 0.002 0.06 2
136n No 0.5 0.29 -57 0 45 0.27 0.00098 0.11 0
137e No 0.49 0.32 -57 0 3.1 -0.15 0.0049 0.1 0
137n No 0.52 0.32 -57 0 2.1 -0.29 0.035 0.13
138e No 0.51 0.31 -57 0 30 0.14 0.00065 0.06 1
138n No 0.54 0.31 -57 0 50 0.054 0 0.016 3
139e No 0.46 0.3 -57 0 84 0.029 0.00033 0.081
139n No 0.5 0.3 -57 0 76 0.027 0 0.065
140e No 0.45 0.3 -57 0 59 0.37 0 0.085 2
140n No 0.54 0.3 -57 0 55 0.082 0 0.016 1
141e No 0.52 0.3 -57 0 55 0.077 0 0.016 1
141n No 0.54 0.3 -57 0 64 0.037 0 0.037
142e No 0.54 0.31 -57 0 27 0.21 0.00098 0.055 1
142n No 0.57 0.31 -57 0 19 0.2 0.0023 0.068 6
143e No 0.52 0.32 -57 0 58 0.078 0 0.027 1
143n No 0.57 0.32 -57 0 32 0.09 0.00098 0.037 3
144e No 0.52 0.28 -57 0 28 0.18 0.00065 0.071 2
144n No 0.53 0.28 -57 0 25 0.43 0.0013 0.15 4
145e No 0.53 0.32 -57 0 51 0.08 0.0039 0.0078 0
145n No 0.57 0.32 -57 0 18 0.15 0.00098 0.069 3
146e No 0.5 0.29 -57 0 58 0.084 0 0.024 0
146n No 0.52 0.29 -57 0 81 0.026 0 0.075
147e No 0.51 0.31 -57 0 42 0.096 0 0.02 5
147n No 0.54 0.31 -57 0 60 0.04 0 0.032
148e No 0.035 0.41 -57 0 0.68 0.52 0.38 0.12
148n No 0.54 0.41 -57 0 44 0.052 0 0.013 3
149e No 0.49 0.33 -57 0 53 0.084 0.00033 0.011 2
149n No 0.54 0.33 -57 0 43 0.058 0.00033 0.013 2
150e No 0.49 0.31 -57 0 31 0.13 0 0.042 0
150n No 0.51 0.31 -57 0 64 0.046 0 0.035
151e No 0.27 0.25 -57 0 82 0.13 0 0.049
151n No 0.41 0.25 -57 0 74 0.018 0 0.061
152e No 0.33 0.23 -57 0 74 0.035 0 0.063
152n No 0.37 0.23 -57 0 75 0.026 0 0.064
153e No 0.33 0.22 -57 0 56 0.075 0 0.022 0
153n No 0.3 0.22 -57 0 5.3 0.37 0.042 0.082
154e No 0.3 0.23 -57 0 80 0.025 0 0.074
154n No 0.35 0.23 -57 0 78 0.025 0 0.068
155e No 0.46 0.32 -57 0 53 0.054 0.0049 0.015 2
155n No 0.5 0.32 -57 0 53 0.025 0.0072 0.019 0
156e Yes -57 1.5E+03 0 0 0 INF
156n Yes -57 1.5E+03 0 0 0 INF
157e Yes -57 1.5E+03 0 0 0 INF
157n Yes -57 1.5E+03 0 0 0 INF
158e Yes -57 1.5E+03 0 0 0 INF
158n Yes -57 1.5E+03 0 0 0 INF
159e No 0.48 0.27 -57 0 66 0.054 0 0.046
159n No 0.39 0.27 -57 0 65 0.31 0 0.077
160e No 0.51 0.31 -57 0 57 0.052 0.00098 0.021 2
160n No 0.55 0.31 -57 0 29 0.12 0.00065 0.047 0
161e No 0.52 0.27 -57 0 59 0.067 0.00033 0.024 2
161n No 0.43 0.27 -57 0 53 0.54 0 0.13 1
162e No 0.52 0.29 -57 0 61 0.08 0.00098 0.033
162n No 0.54 0.29 -57 0 66 0.044 0 0.039
163e No 0.52 0.31 -57 0 53 0.051 0.00033 0.019 0
163n No 0.56 0.31 -57 0 31 0.0071 0.00065 0.044 1
164e No 0.52 0.3 -57 0 51 0.087 0 0.0065 0
164n No 0.55 0.3 -57 0 39 0.052 0 0.023 3
165e No 0.4 0.29 -57 0 18 1 0.016 0.28
165n No 0.54 0.29 -57 0 56 0.026 0 0.025 1
166e No 0.51 0.3 -57 0 53 0.093 0 0.011 3
166n No 0.55 0.3 -57 0 54 0.051 0 0.016 1
167e No 0.48 0.32 -57 0 57 0.14 0 0.023 1
167n No 0.55 0.32 -57 0 44 0.073 0.00033 0.011 4
168e No 0.49 0.3 -57 0 60 0.045 0 0.039 0
168n No 0.52 0.3 -57 0 60 -0.023 0 0.058 2
169e No 0.49 0.3 -57 0 51 0.046 0 0.022 3
169n No 0.51 0.3 -57 0 59 -0.033 0 0.056 2
170e No 0.04 0.42 -57 0 0.65 0.55 0.089 0.12
170n No 0.54 0.42 -57 0 46 0.092 0.00033 0.013 0
171e No 0.38 0.25 -57 0 66 0.047 0 0.045
171n No 0.42 0.25 -57 0 69 0.025 0 0.052
172e No 0.36 0.25 -57 0 81 0.027 0 0.074
172n No 0.4 0.25 -57 0 70 0.036 0 0.051
173e No 0.33 0.22 -57 0 73 0.042 0 0.058
173n No 0.34 0.22 -57 0 77 0.058 0 0.067
174e No 0.022 0.0017 -57 0 3.1 0.57 0.11 0.13
174n No 0.021 0.0017 -57 0 3.1 0.6 0.053 0.14
175e Yes -57 1.5E+03 0 0 0 INF
175n Yes -57 1.5E+03 0 0 0 INF
176e No 0.042 0.39 -57 0 0.69 0.51 0.073 0.12
176n No 0.51 0.39 -57 0 36 0.024 0.012 0.029 0
177e No 0.47 0.32 -57 0 60 0.059 0.00098 0.026
177n No 0.5 0.32 -57 0 54 0.032 0.00098 0.018 2
178e No 0.49 0.31 -57 0 18 0.25 0.0036 0.073 1
178n No 0.51 0.31 -57 0 46 0.043 0.0013 0.0094 3
179e No 0.5 0.31 -57 0 47 0.11 0 0.0073 0
179n No 0.52 0.31 -57 0 46 0.085 0 0.0097 1
180e No 0.49 0.28 -57 0 60 0.08 0.00098 0.026 0
180n No 0.39 0.28 -57 0 49 0.52 0 0.14 2
181e No 0.52 0.3 -57 0 41 0.098 0 0.022 2
181n No 0.54 0.3 -57 0 42 0.092 0 0.017 0
182e No 0.53 0.39 -57 0 27 0.18 0.0029 0.046 0
182n No 0.046 0.39 -57 0 0.65 0.55 0.055 0.13
183e No 0.5 0.35 -57 0 55 0.078 0 0.024 0
183n No 0.044 0.35 -57 0 0.64 0.62 0.094 0.15
184e No 0.04 0.4 -57 0 0.7 0.51 0.1 0.12
184n No 0.54 0.4 -57 0 35 0.062 0.00033 0.028 1
185e No 0.5 0.3 -57 0 62 0.051 0 0.039
185n No 0.53 0.3 -57 0 55 0.051 0 0.02 4
186e No 0.5 0.29 -57 0 37 0.2 0 0.033 0
186n No 0.53 0.29 -57 0 53 0.059 0 0.016 2
187e No 0.52 0.3 -57 0 18 0.24 0.00065 0.074 2
187n No 0.54 0.3 -57 0 53 0.068 0 0.017 0
189e No 0.48 0.31 -57 0 53 0.082 0 0.022 3
189n No 0.52 0.31 -57 0 49 0.047 0.00065 0.0099 3
190e No 0.48 0.33 -57 0 59 0.094 0 0.024 4
190n No 0.53 0.33 -57 0 58 0.052 0 0.023 2
191e No 0.49 0.33 -57 0 49 0.085 0 0.0062 0
191n No 0.52 0.33 -57 0 49 0.093 0.00033 0.012 2
192e No 0.33 0.23 -57 0 77 0.06 0 0.066
192n No 0.37 0.23 -57 0 82 0.019 0 0.076
193e No 0.32 0.23 -57 0 79 0.045 0 0.072
193n No 0.37 0.23 -57 0 73 0.029 0 0.059
194e No 0.023 0.0027 -57 0 3.2 0.52 0.11 0.12
194n No 0.027 0.0027 -57 0 2.8 0.59 0.15 0.14
195e Yes -57 1.5E+03 0 0 0 INF
195n Yes -57 1.5E+03 0 0 0 INF
196e No 0.43 0.31 -57 0 77 0.036 0 0.067
196n No 0.46 0.31 -57 0 73 0.019 0.0013 0.058
197e No 0.46 0.29 -57 0 51 0.094 0 0.013 1
197n No 0.45 0.29 -57 0 79 0.025 0 0.07
198e No 0.45 0.29 -57 0 82 0.019 0 0.078
198n No 0.46 0.29 -57 0 65 0.085 0 0.041
199e No 0.46 0.29 -57 0 73 0.054 0 0.059
199n No 0.43 0.29 -57 0 7.3 0.35 0.049 0.08
200e No 0.044 0.4 -57 0 3 0.57 0.21 0.13
200n No 0.48 0.4 -57 0 75 0.025 0 0.064
201e No 0.5 0.3 -57 0 54 0.11 0 0.017 0
201n No 0.51 0.3 -57 0 69 0.031 0 0.05
202e No 0.48 0.31 -57 0 84 0.027 0 0.081
202n No 0.52 0.31 -57 0 65 0.06 0.0016 0.039
203e Yes -57 1.5E+03 0 0 0 INF
203n Yes -57 1.5E+03 0 0 0 INF
204e No 0.5 0.3 -57 0 12 -0.12 0.0013 0.11 1
204n No 0.52 0.3 -57 0 14 -0.29 0.0059 0.14 3
205e No 0.27 0.37 -57 0 4 0.49 0.038 0.1
205n No 0.52 0.37 -57 0 55 0.043 0 0.016 0
206e No 0.5 0.3 -57 0 21 0.23 0.0042 0.059 4
206n No 0.53 0.3 -57 0 46 0.084 0.00065 0.0088 3
207e No 0.47 0.31 -57 0 81 0.035 0 0.074
207n No 0.52 0.31 -57 0 62 0.061 0 0.033
208e No 0.41 0.34 -57 0 1.7 -0.038 0.17 0.069
208n No 0.03 0.34 -57 0 0.64 0.014 0.34 0.13
209e No 0.023 0.0012 -57 0 0.79 0.11 0.36 0.099
209n No 0.028 0.0012 -57 0 0.7 0.064 0.34 0.11
210e No 0.45 0.28 -57 0 24 -0.13 0.00065 0.09 2
210n No 0.47 0.28 -57 0 16 -0.1 0.002 0.093 0
211e No 0.41 0.34 -57 0 71 0.057 0 0.052
211n No 0.036 0.34 -57 0 2.8 0.58 0.15 0.14
212e Yes -57 1.5E+03 0 0 0 INF
212n Yes -57 1.5E+03 0 0 0 INF
213e No 0.32 0.23 -57 0 74 0.032 0.00098 0.069
213n No 0.36 0.23 -57 0 78 0.017 0 0.07
214e Yes -57 1.5E+03 0 0 0 INF
214n Yes -57 1.5E+03 0 0 0 INF
215e No 0.43 0.31 -57 0 81 0.033 0 0.074
215n No 0.45 0.31 -57 0 71 0.027 0 0.041
216e No 0.22 0.14 -57 0 35 1.9 0.024 0.45
216n No 0.056 0.14 -57 0 2.9 0.57 0.24 0.14
217e No 0.29 0.33 -57 0 4.3 0.5 0.21 0.1
217n No 0.46 0.33 -57 0 72 0.039 0 0.056
218e No 0.45 0.37 -57 0 74 0.036 0 0.063
218n No 0.063 0.37 -57 0 0.64 -0.0089 0.37 0.21
219e Yes -57 1.5E+03 0 0 0 INF
219n Yes -57 1.5E+03 0 0 0 INF
220e No 0.47 0.29 -57 0 79 0.032 0 0.07
220n No 0.48 0.29 -57 0 77 0.016 0 0.067
221e No 0.47 0.3 -57 0 79 0.033 0 0.072
221n No 0.49 0.3 -57 0 78 0.026 0 0.068
222e No 0.48 0.29 -57 0 78 0.042 0 0.068
222n No 0.49 0.29 -57 0 81 0.022 0 0.076
223e No 0.46 0.3 -57 0 71 0.059 0 0.055
223n No 0.5 0.3 -57 0 60 0.026 0 0.028
224e No 0.46 0.28 -57 0 85 0.022 0 0.084
224n No 0.48 0.28 -57 0 80 0.022 0 0.072
225e No 0.47 0.27 -57 0 82 0.026 0 0.077
225n No 0.47 0.27 -57 0 79 0.034 0 0.07
226e No 0.47 0.26 -57 0 78 0.031 0 0.068
226n No 0.42 0.26 -57 0 82 0.12 0 0.061
227e No 0.44 0.25 -57 0 53 0.094 0 0.018 0
227n No 0.45 0.25 -57 0 71 0.047 0 0.052
228e No 0.4 0.26 -57 0 81 0.035 0 0.075
228n No 0.43 0.26 -57 0 75 0.028 0 0.064
229e No 0.39 0.26 -57 0 82 0.037 0 0.077
229n No 0.42 0.26 -57 0 85 0.013 0 0.082
231e Yes -57 1.5E+03 0 0 0 INF
231n Yes -57 1.5E+03 0 0 0 INF
232e Yes -57 1.5E+03 0 0 0 INF
232n Yes -57 1.5E+03 0 0 0 INF
233e No 0.43 0.3 -57 0 77 0.027 0 0.071
233n No 0.42 0.3 -57 0 85 0.011 0 0.084
234e No 0.46 0.32 -57 0 53 0.11 0 0.014 0
234n No 0.42 0.32 -57 0 82 0.017 0 0.076
235e No 0.45 0.3 -57 0 68 0.053 0 0.05
235n No 0.46 0.3 -57 0 57 0.0084 0.00033 0.035 0
237e No 0.47 0.29 -57 0 68 0.05 0 0.047
237n No 0.46 0.29 -57 0 76 0.026 0 0.064
238e No 0.46 0.29 -57 0 83 0.02 0 0.081
238n No 0.46 0.29 -57 0 83 0.022 0 0.078
239e No 0.47 0.29 -57 0 77 0.033 0 0.068
239n No 0.47 0.29 -57 0 80 0.018 0 0.073
240e No 0.46 0.28 -57 0 71 0.078 0 0.05
240n No 0.47 0.28 -57 0 78 0.026 0 0.066
241e No 0.45 0.29 -57 0 79 0.058 0 0.069
241n No 0.47 0.29 -57 0 81 0.016 0 0.074
242e No 0.4 0.29 -57 0 81 0.1 0 0.07
242n No 0.46 0.29 -57 0 84 0.015 0 0.082
243e No 0.47 0.29 -57 0 86 0.023 0 0.084
243n No 0.48 0.29 -57 0 76 0.021 0 0.065
244e No 0.44 0.27 -57 0 30 0.21 0.00098 0.052 1
244n No 0.47 0.27 -57 0 56 0.058 0 0.019 3
245e No 0.41 0.26 -57 0 80 0.041 0 0.073
245n No 0.43 0.26 -57 0 66 0.12 0 0.041
246e No 0.44 0.36 -57 0 24 0.26 0.0055 0.053 0
246n No 0.035 0.36 -57 0 3.1 0.57 0.15 0.14
252e Yes -57 1.5E+03 0 0 0 INF
252n Yes -57 1.5E+03 0 0 0 INF
253e Yes -57 1.5E+03 0 0 0 INF
253n Yes -57 1.5E+03 0 0 0 INF
261e No 0.43 0.26 -57 0 78 0.032 0 0.07
261n No 0.44 0.26 -57 0 79 0.02 0 0.071
262e No 0.45 0.28 -57 0 13 -0.11 0.0091 0.12 2
262n No 0.46 0.28 -57 0 13 -0.31 0.0026 0.15 0
267e Yes -57 1.5E+03 0 0 0 INF
267n Yes -57 1.5E+03 0 0 0 INF
268e Yes -57 1.5E+03 0 0 0 INF
268n Yes -57 1.5E+03 0 0 0 INF
282e Yes -57 1.5E+03 0 0 0 INF
282n Yes -57 1.5E+03 0 0 0 INF
283e Yes -57 1.5E+03 0 0 0 INF
283n Yes -57 1.5E+03 0 0 0 INF
320e No 0.38 0.28 -57 0 67 0.021 0 0.052
320n No 0.35 0.28 -57 0 51 -0.004 0.00098 0.04 0
321e No 0.36 0.27 -57 0 69 0.027 0 0.052
321n No 0.31 0.27 -57 0 82 0.0029 0 0.077
323e No 0.25 0.23 -57 0 73 0.2 0 0.046
323n No 0.31 0.23 -57 0 83 0.014 0 0.078
324e No 0.33 0.24 -57 0 84 0.022 0.021 0.083
324n No 0.31 0.24 -57 0 86 0.014 0 0.085
325e No 0.37 0.28 -57 0 84 0.016 0.065 0.082
325n No 0.34 0.28 -57 0 76 0.013 0 0.067
326e Yes -57 1.5E+03 0 0 0 INF
326n Yes -57 1.5E+03 0 0 0 INF
327e Yes -57 1.5E+03 0 0 0 INF
327n Yes -57 1.5E+03 0 0 0 INF
328e No 0.038 0.00074 -57 0 3.3 0.54 0.082 0.12
328n No 0.038 0.00074 -57 0 3.1 0.6 0.1 0.14
329e No 0.038 0.00096 -57 0 3.1 0.56 0.096 0.13
329n No 0.038 0.00096 -57 0 2.7 0.57 0.12 0.14
331e Yes -57 1.5E+03 0 0 0 INF
331n Yes -57 1.5E+03 0 0 0 INF
332e Yes -57 1.5E+03 0 0 0 INF
332n Yes -57 1.5E+03 0 0 0 INF
333e No 0.35 0.26 -57 0 16 0.2 0.0059 0.05 0
333n No 0.34 0.26 -57 0 73 0.028 0 0.062
336e Yes -57 1.5E+03 0 0 0 INF
336n Yes -57 1.5E+03 0 0 0 INF
340e Yes -57 1.5E+03 0 0 0 INF
340n Yes -57 1.5E+03 0 0 0 INF
In [53]:
# Save antenna classification table as a csv
if SAVE_RESULTS:
    for ind, col in zip(np.arange(len(df.columns), 0, -1), df_classes.columns[::-1]):
        df.insert(int(ind), col + ' Class', df_classes[col])
    df.to_csv(ANTCLASS_FILE)    
In [54]:
print('Final Ant-Pol Classification:\n\n', final_class)
Final Ant-Pol Classification:

 Jee:
----------
bad (245 antpols):
3, 4, 5, 7, 8, 9, 10, 15, 16, 17, 18, 19, 20, 21, 22, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 40, 41, 42, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 231, 232, 233, 234, 235, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 252, 253, 261, 262, 267, 268, 282, 283, 320, 321, 323, 324, 325, 326, 327, 328, 329, 331, 332, 333, 336, 340


Jnn:
----------
bad (245 antpols):
3, 4, 5, 7, 8, 9, 10, 15, 16, 17, 18, 19, 20, 21, 22, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 40, 41, 42, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, 231, 232, 233, 234, 235, 237, 238, 239, 240, 241, 242, 243, 244, 245, 246, 252, 253, 261, 262, 267, 268, 282, 283, 320, 321, 323, 324, 325, 326, 327, 328, 329, 331, 332, 333, 336, 340

Save calibration solutions¶

In [55]:
# update flags in omnical gains and visibility solutions
for ant in omni_flags:
    omni_flags[ant] |= rfi_flags
for bl in vissol_flags:
    vissol_flags[bl] |= rfi_flags
In [56]:
if SAVE_RESULTS:
    add_to_history = 'Produced by file_calibration notebook with the following environment:\n' + '=' * 65 + '\n' + os.popen('conda env export').read() + '=' * 65    
    
    hd_vissol = io.HERAData(SUM_FILE)
    hc_omni = hd_vissol.init_HERACal(gain_convention='divide', cal_style='redundant')
    hc_omni.update(gains=sol.gains, flags=omni_flags, quals=meta['chisq_per_ant'], total_qual=meta['chisq'])
    hc_omni.history += add_to_history
    hc_omni.write_calfits(OMNICAL_FILE, clobber=True)
    del hc_omni
    malloc_trim()
    
    if SAVE_OMNIVIS_FILE:
        # output results, harmonizing keys over polarizations
        all_reds = redcal.get_reds(hd.data_antpos, pols=['ee', 'nn'], pol_mode='2pol')
        bl_to_red_map = {bl: red[0] for red in all_reds for bl in red}
        hd_vissol.read(bls=[bl_to_red_map[bl] for bl in sol.vis], return_data=False)
        hd_vissol.empty_arrays()
        hd_vissol.history += add_to_history
        hd_vissol.update(data={bl_to_red_map[bl]: sol.vis[bl] for bl in sol.vis}, 
                         flags={bl_to_red_map[bl]: vissol_flags[bl] for bl in vissol_flags}, 
                         nsamples={bl_to_red_map[bl]: vissol_nsamples[bl] for bl in vissol_nsamples})
        hd_vissol.write_uvh5(OMNIVIS_FILE, clobber=True)
    del hd_vissol
    malloc_trim()    

Output fully flagged calibration file if OMNICAL_FILE is not written¶

In [57]:
if SAVE_RESULTS and not os.path.exists(OMNICAL_FILE):
    print(f'WARNING: No calibration file produced at {OMNICAL_FILE}. Creating a fully-flagged placeholder calibration file.')
    hd_writer = io.HERAData(SUM_FILE)
    io.write_cal(OMNICAL_FILE, freqs=hd_writer.freqs, times=hd_writer.times,
                 gains={ant: np.ones((hd_writer.Ntimes, hd_writer.Nfreqs), dtype=np.complex64) for ant in ants},
                 flags={ant: np.ones((len(data.times), len(data.freqs)), dtype=bool) for ant in ants},
                 quality=None, total_qual=None, outdir='', overwrite=True, history=utils.history_string(add_to_history), 
                 x_orientation=hd_writer.x_orientation, telescope_location=hd_writer.telescope_location, lst_array=np.unique(hd_writer.lsts),
                 antenna_positions=np.array([hd_writer.antenna_positions[hd_writer.antenna_numbers == antnum].flatten() for antnum in set(ant[0] for ant in ants)]),
                 antnums2antnames=dict(zip(hd_writer.antenna_numbers, hd_writer.antenna_names)))

Output empty visibility file if OMNIVIS_FILE is not written¶

In [58]:
if SAVE_RESULTS and not os.path.exists(OMNIVIS_FILE):
    print(f'WARNING: No omnivis file produced at {OMNIVIS_FILE}. Creating an empty visibility solution file.')
    hd_writer = io.HERAData(SUM_FILE)
    hd_writer.initialize_uvh5_file(OMNIVIS_FILE, clobber=True)
WARNING: No omnivis file produced at /mnt/sn1/2460214/zen.2460214.46140.sum.omni_vis.uvh5. Creating an empty visibility solution file.

TODO: Perform nucal¶

Metadata¶

In [59]:
for repo in ['pyuvdata', 'hera_cal', 'hera_filters', 'hera_qm', 'hera_notebook_templates']:
    exec(f'from {repo} import __version__')
    print(f'{repo}: {__version__}')
pyuvdata: 2.4.0
hera_cal: 3.4.0
hera_filters: 0.1.0
hera_qm: 2.1.3.dev4+g50af006
hera_notebook_templates: 0.1.dev486+gfb8560a
In [60]:
print(f'Finished execution in {(time.time() - tstart) / 60:.2f} minutes.')
Finished execution in 2.30 minutes.