Calibration Smoothing¶

by Josh Dillon, last updated December 20, 2025

This notebook runs calibration smoothing to the gains coming out of file_calibration notebook. It removes any flags founds on by that notebook and replaces them with flags generated from full_day_rfi and full_day_antenna_flagging. It flags antennas with high relative difference between the original gains and smoothed gains. It also plots the results for a couple of antennas.

Here's a set of links to skip to particular figures and tables:

• Figure 1: Identifying and Blacklisting abscal Failures¶

• Figure 2: Antenna Phases with Identified Phase Flips¶

• Figure 3: Full-Day Gain Amplitudes Before and After smooth_cal¶

• Figure 4: Full-Day Gain Phases Before and After smooth_cal¶

• Figure 5: Full-Day $\chi^2$ / DoF Waterfall from Redundant-Baseline Calibration¶

• Figure 6: Average $\chi^2$ per Antenna¶

• Figure 7: Relative Difference Before and After Smoothing¶

In [1]:
import time
tstart = time.time()
In [2]:
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin  # REQUIRED to have the compression plugins available
import numpy as np
import glob
import copy
import warnings
import matplotlib
import matplotlib.pyplot as plt
from hera_cal import io, utils, smooth_cal
from hera_qm.time_series_metrics import true_stretches
%matplotlib inline
from IPython.display import display, HTML

Parse inputs¶

In [3]:
# get files
SUM_FILE = os.environ.get("SUM_FILE", None)
# SUM_FILE = "/lustre/aoc/projects/hera/h6c-analysis/IDR3/2459893/zen.2459893.25258.sum.uvh5"
SUM_SUFFIX = os.environ.get("SUM_SUFFIX", 'sum.uvh5')
CAL_SUFFIX = os.environ.get("CAL_SUFFIX", 'sum.omni.calfits')
SMOOTH_CAL_SUFFIX = os.environ.get("SMOOTH_CAL_SUFFIX", 'sum.smooth.calfits')
ANT_FLAG_SUFFIX = os.environ.get("ANT_FLAG_SUFFIX", 'sum.antenna_flags.h5')
RFI_FLAG_SUFFIX = os.environ.get("RFI_FLAG_SUFFIX", 'sum.flag_waterfall.h5')
FREQ_SMOOTHING_SCALE = float(os.environ.get("FREQ_SMOOTHING_SCALE", 30.0)) # MHz
TIME_SMOOTHING_SCALE = float(os.environ.get("TIME_SMOOTHING_SCALE", 1e4)) # seconds
EIGENVAL_CUTOFF = float(os.environ.get("EIGENVAL_CUTOFF", 1e-12))
PER_POL_REFANT = os.environ.get("PER_POL_REFANT", "False").upper() == "TRUE"
BLACKLIST_TIMESCALE_FACTOR = float(os.environ.get("BLACKLIST_TIMESCALE_FACTOR", 4.0))
BLACKLIST_RELATIVE_ERROR_THRESH = float(os.environ.get("BLACKLIST_RELATIVE_ERROR_THRESH", 1))
BLACKLIST_RELATIVE_WEIGHT = float(os.environ.get("BLACKLIST_RELATIVE_WEIGHT", 0.1))
FM_LOW_FREQ = float(os.environ.get("FM_LOW_FREQ", 87.5)) # in MHz
FM_HIGH_FREQ = float(os.environ.get("FM_HIGH_FREQ", 108.0)) # in MHz
SC_RELATIVE_DIFF_CUTOFF = float(os.environ.get("SC_RELATIVE_DIFF_CUTOFF", 0.2))

for setting in ['SUM_FILE', 'SUM_SUFFIX', 'CAL_SUFFIX', 'SMOOTH_CAL_SUFFIX', 'ANT_FLAG_SUFFIX',
                'RFI_FLAG_SUFFIX', 'FREQ_SMOOTHING_SCALE', 'TIME_SMOOTHING_SCALE', 'EIGENVAL_CUTOFF', 
                'PER_POL_REFANT', 'BLACKLIST_TIMESCALE_FACTOR', 'BLACKLIST_RELATIVE_ERROR_THRESH', 
                'BLACKLIST_RELATIVE_WEIGHT', 'FM_LOW_FREQ', 'FM_HIGH_FREQ', 'SC_RELATIVE_DIFF_CUTOFF']:
    if issubclass(type(eval(setting)), str):
        print(f'{setting} = "{eval(setting)}"')
    else:
        print(f'{setting} = {eval(setting)}')

SUM_FILE = "/mnt/sn1/data2/2461046/zen.2461046.25246.sum.uvh5"
SUM_SUFFIX = "sum.uvh5"
CAL_SUFFIX = "sum.omni.calfits"
SMOOTH_CAL_SUFFIX = "sum.smooth.calfits"
ANT_FLAG_SUFFIX = "sum.antenna_flags.h5"
RFI_FLAG_SUFFIX = "sum.flag_waterfall.h5"
FREQ_SMOOTHING_SCALE = 10.0
TIME_SMOOTHING_SCALE = 600000.0
EIGENVAL_CUTOFF = 1e-12
PER_POL_REFANT = False
BLACKLIST_TIMESCALE_FACTOR = 4.0
BLACKLIST_RELATIVE_ERROR_THRESH = 1.0
BLACKLIST_RELATIVE_WEIGHT = 0.1
FM_LOW_FREQ = 87.5
FM_HIGH_FREQ = 108.0
SC_RELATIVE_DIFF_CUTOFF = 0.2

Load files and select reference antenna(s)¶

In [4]:
hd = io.HERAData(SUM_FILE)
sum_glob = '.'.join(SUM_FILE.split('.')[:-3]) + '.*.' + SUM_SUFFIX
cal_files_glob = sum_glob.replace(SUM_SUFFIX, CAL_SUFFIX)
cal_files = sorted(glob.glob(cal_files_glob))
print(f'Found {len(cal_files)} *.{CAL_SUFFIX} files starting with {cal_files[0]}.')

Found 1758 *.sum.omni.calfits files starting with /mnt/sn1/data2/2461046/zen.2461046.25246.sum.omni.calfits.
In [5]:
rfi_flag_files_glob = sum_glob.replace(SUM_SUFFIX, RFI_FLAG_SUFFIX)
rfi_flag_files = sorted(glob.glob(rfi_flag_files_glob))
print(f'Found {len(rfi_flag_files)} *.{RFI_FLAG_SUFFIX} files starting with {rfi_flag_files[0]}.')

Found 1758 *.sum.flag_waterfall.h5 files starting with /mnt/sn1/data2/2461046/zen.2461046.25246.sum.flag_waterfall.h5.
In [6]:
ant_flag_files_glob = sum_glob.replace(SUM_SUFFIX, ANT_FLAG_SUFFIX)
ant_flag_files = sorted(glob.glob(ant_flag_files_glob))
print(f'Found {len(ant_flag_files)} *.{ANT_FLAG_SUFFIX} files starting with {ant_flag_files[0]}.')

Found 1758 *.sum.antenna_flags.h5 files starting with /mnt/sn1/data2/2461046/zen.2461046.25246.sum.antenna_flags.h5.
In [7]:
cs = smooth_cal.CalibrationSmoother(cal_files, flag_file_list=(ant_flag_files + rfi_flag_files),
                                    ignore_calflags=True, pick_refant=False, load_chisq=True, load_cspa=True)

invalid value encountered in multiply
In [8]:
# Pick reference antenna(s) but don't let ants known to flip phases get picked as reference antennas
banned_refants = [(144, 'Jnn'), (121, 'Jee'), (71, 'Jnn')]
cs.refant = smooth_cal.pick_reference_antenna({ant: cs.gain_grids[ant] for ant in cs.gain_grids if ant not in banned_refants},
                                              {ant: cs.flag_grids[ant] for ant in cs.gain_grids if ant not in banned_refants},
                                              cs.freqs, per_pol=True, acceptable_candidate_frac=0.25, antpos=hd.antpos)
for pol in cs.refant:
    print(f'Reference antenna {cs.refant[pol][0]} selected for smoothing {pol} gains.')

if not PER_POL_REFANT:
    # in this case, rephase both pols separately before smoothing, but also smooth the relative polarization calibration phasor
    overall_refant = smooth_cal.pick_reference_antenna({ant: cs.gain_grids[ant] for ant in cs.refant.values()}, 
                                                       {ant: cs.flag_grids[ant] for ant in cs.refant.values()}, 
                                                       cs.freqs, per_pol=False)
    print(f'Overall reference antenna {overall_refant} selected.')
    other_refant = [ant for ant in cs.refant.values() if ant != overall_refant][0]

    relative_pol_phasor = cs.gain_grids[overall_refant] * cs.gain_grids[other_refant].conj() # TODO: is this conjugation right?
    relative_pol_phasor /= np.abs(relative_pol_phasor)

abscal_refants = {cs.refant[pol]: cs.gain_grids[cs.refant[pol]] for pol in ['Jee', 'Jnn']}

Reference antenna 201 selected for smoothing Jnn gains.
Reference antenna 237 selected for smoothing Jee gains.

Overall reference antenna (np.int64(201), 'Jnn') selected.
In [9]:
cs.rephase_to_refant(propagate_refant_flags=True)
In [10]:
lst_grid = utils.JD2LST(cs.time_grid) * 12 / np.pi
lst_grid[lst_grid > lst_grid[-1]] -= 24

Find consistent outliers in relative error after a coarse smoothing¶

These are typically a sign of failures of abscal.

In [11]:
relative_error_samples = {pol: np.zeros_like(cs.gain_grids[cs.refant[pol]], dtype=float) for pol in ['Jee', 'Jnn']}
sum_relative_error = {pol: np.zeros_like(cs.gain_grids[cs.refant[pol]], dtype=float) for pol in ['Jee', 'Jnn']}
per_ant_avg_relative_error = {} 

# perform a 2D DPSS filter with a BLACKLIST_TIMESCALE_FACTOR longer timescale, averaging the results per-pol
for ant in cs.gain_grids:
    if np.all(cs.flag_grids[ant]):
        continue
    filtered, _ = smooth_cal.time_freq_2D_filter(gains=cs.gain_grids[ant], 
                                                 wgts=(~cs.flag_grids[ant]).astype(float),
                                                 freqs=cs.freqs,
                                                 times=cs.time_grid,
                                                 freq_scale=FREQ_SMOOTHING_SCALE,
                                                 time_scale=TIME_SMOOTHING_SCALE * BLACKLIST_TIMESCALE_FACTOR,
                                                 eigenval_cutoff=EIGENVAL_CUTOFF,
                                                 method='DPSS', 
                                                 fit_method='lu_solve', 
                                                 fix_phase_flips=True, 
                                                 phase_flip_time_scale = TIME_SMOOTHING_SCALE / 2,
                                                 flag_phase_flip_ints=True,
                                                 skip_flagged_edges=True, 
                                                 freq_cuts=[(FM_LOW_FREQ + FM_HIGH_FREQ) * .5e6],
                                                ) 
    relative_error = np.where(cs.flag_grids[ant], 0, np.abs(cs.gain_grids[ant] - filtered) / np.abs(filtered))
    per_ant_avg_relative_error[ant] = np.nanmean(np.where(cs.flag_grids[ant], np.nan, relative_error))
    relative_error_samples[ant[1]] += (~cs.flag_grids[ant]).astype(float)
    sum_relative_error[ant[1]] += relative_error

# figure out per-antpol cuts for where to set weights to 0 for the main smooth_cal (but not necessarily flags)
cs.blacklist_wgt = BLACKLIST_RELATIVE_WEIGHT
for pol in ['Jee', 'Jnn']:
    avg_rel_error = sum_relative_error[pol] / relative_error_samples[pol]
    to_blacklist = np.where(relative_error_samples[pol] > 0, avg_rel_error > BLACKLIST_RELATIVE_ERROR_THRESH, False)
    for ant in cs.ants:
        if ant[1] == pol:
            cs.waterfall_blacklist[ant] = to_blacklist

invalid value encountered in divide
In [12]:
def plot_relative_error():
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")

        fig, axes = plt.subplots(1, 3, figsize=(14, 7))
        extent = [cs.freqs[0] / 1e6, cs.freqs[-1] / 1e6, lst_grid[-1], lst_grid[0]]
        cmap = plt.get_cmap('Greys', 256)
        cmap.set_over('red')
        for ax, pol in zip(axes[0:2], ['Jee', 'Jnn']):
            to_plot = sum_relative_error[pol] / relative_error_samples[pol]
            im = ax.imshow(np.where(np.isfinite(to_plot), to_plot, np.nan), aspect='auto', interpolation='none', 
                           vmin=0, vmax=BLACKLIST_RELATIVE_ERROR_THRESH, extent=extent, cmap=cmap)
            ax.set_title(pol)
            ax.set_yticklabels(ax.get_yticks() % 24)
            ax.set_ylabel('LST (hours)')
            ax.set_xlabel('Frequency (MHz)')
        plt.colorbar(im, ax=axes[0:2], location='top', extend='max', label='Average Relative Error on Initial Smoothing')
        
        for pol in ['Jee', 'Jnn']:
            axes[2].hist((sum_relative_error[pol] / relative_error_samples[pol]).ravel(), bins=np.arange(0,2,.01), alpha=.5, label=pol)
        axes[2].set_yscale('log')
        axes[2].set_ylabel('Number of Waterfall Pixels')
        axes[2].set_xlabel('Relative Error')
        axes[2].axvline(BLACKLIST_RELATIVE_ERROR_THRESH, ls='--', c='r', label='Blacklist Threshold')
        axes[2].legend()

Figure 1: Identifying and Blacklisting abscal Failures¶

This plot highlights regions of the waterfall that are per-polarization blacklisted (i.e. given 0 weight in the main smooth_cal fit, but not necessarily flagged). This is usually a sign of problems with abscal and often occurs because

In [13]:
plot_relative_error()
No description has been provided for this image
In [14]:
# duplicate a small number of abscal gains for plotting
antnums = set([ant[0] for ant in cs.ants])
flags_per_antnum = [np.sum(cs.flag_grids[ant, 'Jnn']) + np.sum(cs.flag_grids[ant, 'Jee']) for ant in antnums]
larger_relative_error = np.array([np.max([per_ant_avg_relative_error.get((ant, pol), np.inf) for pol in ['Jee', 'Jnn']]) for ant in antnums])
refant_nums = [ant[0] for ant in cs.refant.values()]

# pick candidates that don't exhibit too many flags or non-smooth structure on first pass
candidate_ants = []
rel_error_factor = 1
while len(candidate_ants) < 6:  # Select more candidates to ensure we have enough after potential flagging
    candidate_ants = [ant for ant, nflags, rel_err in zip(antnums, flags_per_antnum, larger_relative_error) 
                      if (ant not in refant_nums) and (nflags <= np.percentile(flags_per_antnum, 25))
                      and (rel_err <= SC_RELATIVE_DIFF_CUTOFF * rel_error_factor)
                      and not np.all(cs.flag_grids[ant, 'Jee']) and not np.all(cs.flag_grids[ant, 'Jnn'])]
    rel_error_factor += .1

# choose antennas to plot: select up to 6 candidates, prioritizing diversity across antenna numbers
candidate_ants_sorted = sorted(candidate_ants)
step = max(1, len(candidate_ants_sorted) // 6)  # spread them out
_candidates = sorted(candidate_ants_sorted[::step][:6])
ants_to_plot_candidates = [_candidates[i//2] if i % 2 == 0 else _candidates[-(i//2)-1] for i in range(len(_candidates))]

# Store abscal gains for all candidates
abscal_gains = {}
for pol in ['Jee', 'Jnn']:
    for antnum in ants_to_plot_candidates:
        if PER_POL_REFANT:
            abscal_gains[antnum, pol] = cs.gain_grids[(antnum, pol)] * np.abs(abscal_refants[cs.refant[pol]]) / abscal_refants[cs.refant[pol]]
        else:
            abscal_gains[antnum, pol] = cs.gain_grids[(antnum, pol)] / np.abs(abscal_refants[cs.refant[pol]]) * abscal_refants[cs.refant[pol]]
            abscal_gains[antnum, pol] *= np.abs(abscal_refants[overall_refant]) / abscal_refants[overall_refant]

Perform smoothing¶

In [15]:
if not PER_POL_REFANT:
    # treat the relative_pol_phasor as if it were antenna -1
    cs.gain_grids[(-1, other_refant[1])] = relative_pol_phasor
    cs.flag_grids[(-1, other_refant[1])] = cs.flag_grids[overall_refant] | cs.flag_grids[other_refant]
    cs.waterfall_blacklist[(-1, other_refant[1])] = cs.waterfall_blacklist[cs.ants[0][0], 'Jee'] | cs.waterfall_blacklist[cs.ants[0][0], 'Jnn']
In [16]:
meta = cs.time_freq_2D_filter(freq_scale=FREQ_SMOOTHING_SCALE,
                              time_scale=TIME_SMOOTHING_SCALE,
                              eigenval_cutoff=EIGENVAL_CUTOFF,
                              method='DPSS', 
                              fit_method='lu_solve',
                              fix_phase_flips=True,
                              phase_flip_time_scale = TIME_SMOOTHING_SCALE / 2,
                              flag_phase_flip_ints=True,
                              skip_flagged_edges=True,
                              freq_cuts=[(FM_LOW_FREQ + FM_HIGH_FREQ) * .5e6],)

4 phase flips detected on antenna (np.int64(193), 'Jnn'). A total of 365 integrations were phase-flipped relative to the 0th integration between 2461046.464298853 and 2461046.5051234155.

4 phase flips detected on antenna (np.int64(316), 'Jnn'). A total of 2 integrations were phase-flipped relative to the 0th integration between 2461046.4821945517 and 2461046.4972940474.

4 phase flips detected on antenna (np.int64(315), 'Jnn'). A total of 2 integrations were phase-flipped relative to the 0th integration between 2461046.4821945517 and 2461046.4972940474.

4 phase flips detected on antenna (np.int64(131), 'Jnn'). A total of 265 integrations were phase-flipped relative to the 0th integration between 2461046.472911158 and 2461046.502662757.

14 phase flips detected on antenna (np.int64(130), 'Jee'). A total of 99 integrations were phase-flipped relative to the 0th integration between 2461046.4811879187 and 2461046.493267515.

2 phase flips detected on antenna (np.int64(132), 'Jnn'). A total of 349 integrations were phase-flipped relative to the 0th integration between 2461046.465976575 and 2461046.5048997193.

8 phase flips detected on antenna (np.int64(131), 'Jee'). A total of 292 integrations were phase-flipped relative to the 0th integration between 2461046.466983208 and 2461046.5014324277.

8 phase flips detected on antenna (np.int64(130), 'Jnn'). A total of 125 integrations were phase-flipped relative to the 0th integration between 2461046.48051683 and 2461046.498636225.

2 phase flips detected on antenna (np.int64(93), 'Jnn'). A total of 1 integrations were phase-flipped relative to the 0th integration between 2461046.485549995 and 2461046.485549995.

8 phase flips detected on antenna (np.int64(319), 'Jnn'). A total of 234 integrations were phase-flipped relative to the 0th integration between 2461046.476825842 and 2461046.5032219975.

6 phase flips detected on antenna (np.int64(307), 'Jnn'). A total of 277 integrations were phase-flipped relative to the 0th integration between 2461046.4732467025 and 2461046.504564175.

2 phase flips detected on antenna (np.int64(115), 'Jee'). A total of 447 integrations were phase-flipped relative to the 0th integration between 2461046.456245789 and 2461046.5061300485.

2 phase flips detected on antenna (np.int64(134), 'Jee'). A total of 443 integrations were phase-flipped relative to the 0th integration between 2461046.4566931813 and 2461046.5061300485.

4 phase flips detected on antenna (np.int64(97), 'Jee'). A total of 432 integrations were phase-flipped relative to the 0th integration between 2461046.4571405738 and 2461046.50545896.

8 phase flips detected on antenna (np.int64(129), 'Jnn'). A total of 58 integrations were phase-flipped relative to the 0th integration between 2461046.4823064 and 2461046.489129135.

2 phase flips detected on antenna (np.int64(152), 'Jnn'). A total of 352 integrations were phase-flipped relative to the 0th integration between 2461046.4657528787 and 2461046.5050115674.

6 phase flips detected on antenna (np.int64(194), 'Jnn'). A total of 366 integrations were phase-flipped relative to the 0th integration between 2461046.4645225494 and 2461046.505570808.

8 phase flips detected on antenna (np.int64(132), 'Jee'). A total of 372 integrations were phase-flipped relative to the 0th integration between 2461046.461278954 and 2461046.503893086.

6 phase flips detected on antenna (np.int64(192), 'Jnn'). A total of 327 integrations were phase-flipped relative to the 0th integration between 2461046.466983208 and 2461046.504564175.

6 phase flips detected on antenna (np.int64(133), 'Jnn'). A total of 378 integrations were phase-flipped relative to the 0th integration between 2461046.461278954 and 2461046.505570808.

4 phase flips detected on antenna (np.int64(149), 'Jnn'). A total of 107 integrations were phase-flipped relative to the 0th integration between 2461046.480740526 and 2461046.4927082746.

12 phase flips detected on antenna (np.int64(277), 'Jnn'). A total of 108 integrations were phase-flipped relative to the 0th integration between 2461046.4820827036 and 2461046.498636225.

6 phase flips detected on antenna (np.int64(152), 'Jee'). A total of 402 integrations were phase-flipped relative to the 0th integration between 2461046.4598249285 and 2461046.5048997193.

12 phase flips detected on antenna (np.int64(306), 'Jee'). A total of 135 integrations were phase-flipped relative to the 0th integration between 2461046.4692201703 and 2461046.498636225.

2 phase flips detected on antenna (np.int64(174), 'Jee'). A total of 448 integrations were phase-flipped relative to the 0th integration between 2461046.4566931813 and 2461046.506689289.

4 phase flips detected on antenna (np.int64(133), 'Jee'). A total of 417 integrations were phase-flipped relative to the 0th integration between 2461046.458482751 and 2461046.5051234155.

16 phase flips detected on antenna (np.int64(150), 'Jee'). A total of 210 integrations were phase-flipped relative to the 0th integration between 2461046.4692201703 and 2461046.498971769.

16 phase flips detected on antenna (np.int64(228), 'Jee'). A total of 34 integrations were phase-flipped relative to the 0th integration between 2461046.4827537923 and 2461046.491254249.

2 phase flips detected on antenna (np.int64(150), 'Jnn'). A total of 229 integrations were phase-flipped relative to the 0th integration between 2461046.4759310572 and 2461046.5014324277.

26 phase flips detected on antenna (np.int64(94), 'Jee'). A total of 60 integrations were phase-flipped relative to the 0th integration between 2461046.4810760706 and 2461046.493267515.

4 phase flips detected on antenna (np.int64(195), 'Jee'). A total of 450 integrations were phase-flipped relative to the 0th integration between 2461046.456357637 and 2461046.506689289.

2 phase flips detected on antenna (np.int64(214), 'Jnn'). A total of 443 integrations were phase-flipped relative to the 0th integration between 2461046.458147207 and 2461046.507584074.

6 phase flips detected on antenna (np.int64(190), 'Jnn'). A total of 123 integrations were phase-flipped relative to the 0th integration between 2461046.4808523743 and 2461046.498636225.

6 phase flips detected on antenna (np.int64(192), 'Jee'). A total of 325 integrations were phase-flipped relative to the 0th integration between 2461046.4664239674 and 2461046.502886453.

2 phase flips detected on antenna (np.int64(173), 'Jnn'). A total of 354 integrations were phase-flipped relative to the 0th integration between 2461046.4657528787 and 2461046.5052352636.

6 phase flips detected on antenna (np.int64(214), 'Jee'). A total of 440 integrations were phase-flipped relative to the 0th integration between 2461046.4571405738 and 2461046.506577441.

4 phase flips detected on antenna (np.int64(153), 'Jee'). A total of 429 integrations were phase-flipped relative to the 0th integration between 2461046.4579235106 and 2461046.5059063523.

2 phase flips detected on antenna (np.int64(232), 'Jnn'). A total of 385 integrations were phase-flipped relative to the 0th integration between 2461046.4636277645 and 2461046.506577441.

8 phase flips detected on antenna (np.int64(231), 'Jee'). A total of 367 integrations were phase-flipped relative to the 0th integration between 2461046.461278954 and 2461046.504564175.

8 phase flips detected on antenna (np.int64(173), 'Jee'). A total of 361 integrations were phase-flipped relative to the 0th integration between 2461046.461278954 and 2461046.5043404787.

2 phase flips detected on antenna (np.int64(231), 'Jnn'). A total of 360 integrations were phase-flipped relative to the 0th integration between 2461046.4656410306 and 2461046.505794504.

24 phase flips detected on antenna (np.int64(112), 'Jee'). A total of 254 integrations were phase-flipped relative to the 0th integration between 2461046.4692201703 and 2461046.499866554.

14 phase flips detected on antenna (np.int64(112), 'Jnn'). A total of 233 integrations were phase-flipped relative to the 0th integration between 2461046.473805943 and 2461046.5010968833.

6 phase flips detected on antenna (np.int64(246), 'Jnn'). A total of 230 integrations were phase-flipped relative to the 0th integration between 2461046.476713994 and 2461046.502662757.

18 phase flips detected on antenna (np.int64(111), 'Jnn'). A total of 41 integrations were phase-flipped relative to the 0th integration between 2461046.4823064 and 2461046.489017287.

18 phase flips detected on antenna (np.int64(319), 'Jee'). A total of 175 integrations were phase-flipped relative to the 0th integration between 2461046.4692201703 and 2461046.499866554.

18 phase flips detected on antenna (np.int64(262), 'Jnn'). A total of 149 integrations were phase-flipped relative to the 0th integration between 2461046.4809642225 and 2461046.499642858.

20 phase flips detected on antenna (np.int64(317), 'Jnn'). A total of 45 integrations were phase-flipped relative to the 0th integration between 2461046.4821945517 and 2461046.4972940474.

4 phase flips detected on antenna (np.int64(293), 'Jnn'). A total of 230 integrations were phase-flipped relative to the 0th integration between 2461046.4771613865 and 2461046.502886453.

10 phase flips detected on antenna (np.int64(190), 'Jee'). A total of 14 integrations were phase-flipped relative to the 0th integration between 2461046.485549995 and 2461046.489017287.

16 phase flips detected on antenna (np.int64(170), 'Jnn'). A total of 159 integrations were phase-flipped relative to the 0th integration between 2461046.4781680196 and 2461046.499866554.

6 phase flips detected on antenna (np.int64(232), 'Jee'). A total of 395 integrations were phase-flipped relative to the 0th integration between 2461046.461167106 and 2461046.505570808.

4 phase flips detected on antenna (np.int64(175), 'Jee'). A total of 397 integrations were phase-flipped relative to the 0th integration between 2461046.461278954 and 2461046.5061300485.

2 phase flips detected on antenna (np.int64(96), 'Jnn'). A total of 349 integrations were phase-flipped relative to the 0th integration between 2461046.4657528787 and 2461046.504676023.

8 phase flips detected on antenna (np.int64(278), 'Jnn'). A total of 193 integrations were phase-flipped relative to the 0th integration between 2461046.479622045 and 2461046.5014324277.

6 phase flips detected on antenna (np.int64(292), 'Jnn'). A total of 181 integrations were phase-flipped relative to the 0th integration between 2461046.48051683 and 2461046.5013205796.

6 phase flips detected on antenna (np.int64(228), 'Jnn'). A total of 134 integrations were phase-flipped relative to the 0th integration between 2461046.4808523743 and 2461046.498636225.

4 phase flips detected on antenna (np.int64(96), 'Jee'). A total of 407 integrations were phase-flipped relative to the 0th integration between 2461046.4588182955 and 2461046.5043404787.

18 phase flips detected on antenna (np.int64(292), 'Jee'). A total of 85 integrations were phase-flipped relative to the 0th integration between 2461046.482418248 and 2461046.4953926294.

26 phase flips detected on antenna (np.int64(212), 'Jee'). A total of 237 integrations were phase-flipped relative to the 0th integration between 2461046.4692201703 and 2461046.500985035.

4 phase flips detected on antenna (np.int64(193), 'Jee'). A total of 409 integrations were phase-flipped relative to the 0th integration between 2461046.4589301436 and 2461046.504676023.

4 phase flips detected on antenna (np.int64(191), 'Jnn'). A total of 233 integrations were phase-flipped relative to the 0th integration between 2461046.4761547535 and 2461046.5022153645.

18 phase flips detected on antenna (np.int64(229), 'Jee'). A total of 190 integrations were phase-flipped relative to the 0th integration between 2461046.4692201703 and 2461046.499866554.

8 phase flips detected on antenna (np.int64(294), 'Jnn'). A total of 336 integrations were phase-flipped relative to the 0th integration between 2461046.466983208 and 2461046.5050115674.

14 phase flips detected on antenna (np.int64(191), 'Jee'). A total of 184 integrations were phase-flipped relative to the 0th integration between 2461046.4692201703 and 2461046.498859921.

8 phase flips detected on antenna (np.int64(79), 'Jnn'). A total of 330 integrations were phase-flipped relative to the 0th integration between 2461046.4666476636 and 2461046.503893086.

6 phase flips detected on antenna (np.int64(113), 'Jnn'). A total of 330 integrations were phase-flipped relative to the 0th integration between 2461046.466983208 and 2461046.5040049343.

6 phase flips detected on antenna (np.int64(113), 'Jee'). A total of 368 integrations were phase-flipped relative to the 0th integration between 2461046.461278954 and 2461046.5032219975.

4 phase flips detected on antenna (np.int64(79), 'Jee'). A total of 376 integrations were phase-flipped relative to the 0th integration between 2461046.461167106 and 2461046.5032219975.

6 phase flips detected on antenna (np.int64(229), 'Jnn'). A total of 236 integrations were phase-flipped relative to the 0th integration between 2461046.4763784497 and 2461046.502886453.

10 phase flips detected on antenna (np.int64(211), 'Jee'). A total of 260 integrations were phase-flipped relative to the 0th integration between 2461046.4692201703 and 2461046.5010968833.

2 phase flips detected on antenna (np.int64(153), 'Jnn'). A total of 422 integrations were phase-flipped relative to the 0th integration between 2461046.4596012323 and 2461046.506689289.

4 phase flips detected on antenna (np.int64(195), 'Jnn'). A total of 442 integrations were phase-flipped relative to the 0th integration between 2461046.458147207 and 2461046.507584074.

4 phase flips detected on antenna (np.int64(115), 'Jnn'). A total of 435 integrations were phase-flipped relative to the 0th integration between 2461046.458370903 and 2461046.5070248335.

2 phase flips detected on antenna (np.int64(134), 'Jnn'). A total of 434 integrations were phase-flipped relative to the 0th integration between 2461046.4589301436 and 2461046.507360378.

2 phase flips detected on antenna (np.int64(9), 'Jnn'). A total of 1 integrations were phase-flipped relative to the 0th integration between 2461046.4821945517 and 2461046.4821945517.

6 phase flips detected on antenna (np.int64(154), 'Jnn'). A total of 449 integrations were phase-flipped relative to the 0th integration between 2461046.4576998143 and 2461046.5080314665.

6 phase flips detected on antenna (np.int64(174), 'Jnn'). A total of 448 integrations were phase-flipped relative to the 0th integration between 2461046.4576998143 and 2461046.5079196184.

10 phase flips detected on antenna (np.int64(175), 'Jnn'). A total of 390 integrations were phase-flipped relative to the 0th integration between 2461046.461278954 and 2461046.506577441.
In [17]:
# calculate average chi^2 per antenna before additional flagging
avg_cspa_vs_time = {ant: np.nanmean(np.where(cs.flag_grids[ant], np.nan, cs.cspa_grids[ant]), axis=1) for ant in cs.ants}
avg_cspa_vs_freq = {ant: np.nanmean(np.where(cs.flag_grids[ant], np.nan, cs.cspa_grids[ant]), axis=0) for ant in cs.ants}
avg_cspa = {ant: np.nanmean(np.where(cs.flag_grids[ant], np.nan, cs.cspa_grids[ant])) for ant in cs.ants}

Mean of empty slice

Mean of empty slice

Mean of empty slice
In [18]:
# Pick out antennas with too high relative differences before and after smoothing and flag them.
avg_relative_diffs = {ant: np.nanmean(rel_diff) for ant, rel_diff in meta['freq_avg_rel_diff'].items()}
to_cut = sorted([ant for ant, diff in avg_relative_diffs.items() if ant[0] >= 0 and diff > SC_RELATIVE_DIFF_CUTOFF])
if len(to_cut) > 0:
    for ant in to_cut:
        print(f'Flagging antenna {ant[0]}{ant[1][-1]} with a relative difference before and after smoothing of {avg_relative_diffs[ant]:.2%} '
              f'(compared to the {SC_RELATIVE_DIFF_CUTOFF:.2%} cutoff).')
        cs.flag_grids[ant] |= True
else:
    print(f'No antennas have a relative difference above the {SC_RELATIVE_DIFF_CUTOFF:.2%} cutoff.')

Flagging antenna 4e with a relative difference before and after smoothing of 28.53% (compared to the 20.00% cutoff).
Flagging antenna 5e with a relative difference before and after smoothing of 31.05% (compared to the 20.00% cutoff).
Flagging antenna 5n with a relative difference before and after smoothing of 26.85% (compared to the 20.00% cutoff).
Flagging antenna 7e with a relative difference before and after smoothing of 38.89% (compared to the 20.00% cutoff).
Flagging antenna 7n with a relative difference before and after smoothing of 32.31% (compared to the 20.00% cutoff).
Flagging antenna 8e with a relative difference before and after smoothing of 41.00% (compared to the 20.00% cutoff).
Flagging antenna 9e with a relative difference before and after smoothing of 44.86% (compared to the 20.00% cutoff).
Flagging antenna 9n with a relative difference before and after smoothing of 38.62% (compared to the 20.00% cutoff).
Flagging antenna 15e with a relative difference before and after smoothing of 27.09% (compared to the 20.00% cutoff).
Flagging antenna 15n with a relative difference before and after smoothing of 23.29% (compared to the 20.00% cutoff).
Flagging antenna 16n with a relative difference before and after smoothing of 24.62% (compared to the 20.00% cutoff).
Flagging antenna 17e with a relative difference before and after smoothing of 30.44% (compared to the 20.00% cutoff).
Flagging antenna 17n with a relative difference before and after smoothing of 27.04% (compared to the 20.00% cutoff).
Flagging antenna 18e with a relative difference before and after smoothing of 34.34% (compared to the 20.00% cutoff).
Flagging antenna 19e with a relative difference before and after smoothing of 38.49% (compared to the 20.00% cutoff).
Flagging antenna 19n with a relative difference before and after smoothing of 34.60% (compared to the 20.00% cutoff).
Flagging antenna 20e with a relative difference before and after smoothing of 42.38% (compared to the 20.00% cutoff).
Flagging antenna 20n with a relative difference before and after smoothing of 37.79% (compared to the 20.00% cutoff).
Flagging antenna 29e with a relative difference before and after smoothing of 28.43% (compared to the 20.00% cutoff).
Flagging antenna 30e with a relative difference before and after smoothing of 30.77% (compared to the 20.00% cutoff).
Flagging antenna 31e with a relative difference before and after smoothing of 35.29% (compared to the 20.00% cutoff).
Flagging antenna 31n with a relative difference before and after smoothing of 30.98% (compared to the 20.00% cutoff).
Flagging antenna 32e with a relative difference before and after smoothing of 39.27% (compared to the 20.00% cutoff).
Flagging antenna 33e with a relative difference before and after smoothing of 44.83% (compared to the 20.00% cutoff).
Flagging antenna 36e with a relative difference before and after smoothing of 23.37% (compared to the 20.00% cutoff).
Flagging antenna 36n with a relative difference before and after smoothing of 24.71% (compared to the 20.00% cutoff).
Flagging antenna 38e with a relative difference before and after smoothing of 20.45% (compared to the 20.00% cutoff).
Flagging antenna 41e with a relative difference before and after smoothing of 22.84% (compared to the 20.00% cutoff).
Flagging antenna 42e with a relative difference before and after smoothing of 24.00% (compared to the 20.00% cutoff).
Flagging antenna 43e with a relative difference before and after smoothing of 27.40% (compared to the 20.00% cutoff).
Flagging antenna 43n with a relative difference before and after smoothing of 24.47% (compared to the 20.00% cutoff).
Flagging antenna 45e with a relative difference before and after smoothing of 35.49% (compared to the 20.00% cutoff).
Flagging antenna 45n with a relative difference before and after smoothing of 31.81% (compared to the 20.00% cutoff).
Flagging antenna 46n with a relative difference before and after smoothing of 35.28% (compared to the 20.00% cutoff).
Flagging antenna 50e with a relative difference before and after smoothing of 21.52% (compared to the 20.00% cutoff).
Flagging antenna 50n with a relative difference before and after smoothing of 21.49% (compared to the 20.00% cutoff).
Flagging antenna 57e with a relative difference before and after smoothing of 24.62% (compared to the 20.00% cutoff).
Flagging antenna 57n with a relative difference before and after smoothing of 21.58% (compared to the 20.00% cutoff).
Flagging antenna 58e with a relative difference before and after smoothing of 27.83% (compared to the 20.00% cutoff).
Flagging antenna 58n with a relative difference before and after smoothing of 25.15% (compared to the 20.00% cutoff).
Flagging antenna 59e with a relative difference before and after smoothing of 31.28% (compared to the 20.00% cutoff).
Flagging antenna 59n with a relative difference before and after smoothing of 28.52% (compared to the 20.00% cutoff).
Flagging antenna 60e with a relative difference before and after smoothing of 36.09% (compared to the 20.00% cutoff).
Flagging antenna 65e with a relative difference before and after smoothing of 21.39% (compared to the 20.00% cutoff).
Flagging antenna 65n with a relative difference before and after smoothing of 20.49% (compared to the 20.00% cutoff).
Flagging antenna 73e with a relative difference before and after smoothing of 25.31% (compared to the 20.00% cutoff).
Flagging antenna 73n with a relative difference before and after smoothing of 22.62% (compared to the 20.00% cutoff).
Flagging antenna 74e with a relative difference before and after smoothing of 28.48% (compared to the 20.00% cutoff).
Flagging antenna 75e with a relative difference before and after smoothing of 32.72% (compared to the 20.00% cutoff).
Flagging antenna 75n with a relative difference before and after smoothing of 29.32% (compared to the 20.00% cutoff).
Flagging antenna 79e with a relative difference before and after smoothing of 56.47% (compared to the 20.00% cutoff).
Flagging antenna 79n with a relative difference before and after smoothing of 52.53% (compared to the 20.00% cutoff).
Flagging antenna 91e with a relative difference before and after smoothing of 29.52% (compared to the 20.00% cutoff).
Flagging antenna 91n with a relative difference before and after smoothing of 26.99% (compared to the 20.00% cutoff).
Flagging antenna 92n with a relative difference before and after smoothing of 33.00% (compared to the 20.00% cutoff).
Flagging antenna 93n with a relative difference before and after smoothing of 38.30% (compared to the 20.00% cutoff).
Flagging antenna 94e with a relative difference before and after smoothing of 44.90% (compared to the 20.00% cutoff).

Flagging antenna 94n with a relative difference before and after smoothing of 42.61% (compared to the 20.00% cutoff).
Flagging antenna 96e with a relative difference before and after smoothing of 58.27% (compared to the 20.00% cutoff).
Flagging antenna 96n with a relative difference before and after smoothing of 53.39% (compared to the 20.00% cutoff).
Flagging antenna 97e with a relative difference before and after smoothing of 60.19% (compared to the 20.00% cutoff).
Flagging antenna 98e with a relative difference before and after smoothing of 24.41% (compared to the 20.00% cutoff).
Flagging antenna 108e with a relative difference before and after smoothing of 25.90% (compared to the 20.00% cutoff).
Flagging antenna 108n with a relative difference before and after smoothing of 23.32% (compared to the 20.00% cutoff).
Flagging antenna 110e with a relative difference before and after smoothing of 39.09% (compared to the 20.00% cutoff).
Flagging antenna 110n with a relative difference before and after smoothing of 34.34% (compared to the 20.00% cutoff).
Flagging antenna 111n with a relative difference before and after smoothing of 37.96% (compared to the 20.00% cutoff).
Flagging antenna 112e with a relative difference before and after smoothing of 48.49% (compared to the 20.00% cutoff).
Flagging antenna 112n with a relative difference before and after smoothing of 44.95% (compared to the 20.00% cutoff).
Flagging antenna 113e with a relative difference before and after smoothing of 55.38% (compared to the 20.00% cutoff).
Flagging antenna 113n with a relative difference before and after smoothing of 50.99% (compared to the 20.00% cutoff).
Flagging antenna 115e with a relative difference before and after smoothing of 61.13% (compared to the 20.00% cutoff).
Flagging antenna 115n with a relative difference before and after smoothing of 60.28% (compared to the 20.00% cutoff).
Flagging antenna 116e with a relative difference before and after smoothing of 25.45% (compared to the 20.00% cutoff).
Flagging antenna 116n with a relative difference before and after smoothing of 23.45% (compared to the 20.00% cutoff).
Flagging antenna 126e with a relative difference before and after smoothing of 22.05% (compared to the 20.00% cutoff).
Flagging antenna 126n with a relative difference before and after smoothing of 20.12% (compared to the 20.00% cutoff).
Flagging antenna 128e with a relative difference before and after smoothing of 34.85% (compared to the 20.00% cutoff).
Flagging antenna 128n with a relative difference before and after smoothing of 30.76% (compared to the 20.00% cutoff).
Flagging antenna 129e with a relative difference before and after smoothing of 40.94% (compared to the 20.00% cutoff).
Flagging antenna 129n with a relative difference before and after smoothing of 36.51% (compared to the 20.00% cutoff).
Flagging antenna 130e with a relative difference before and after smoothing of 44.09% (compared to the 20.00% cutoff).
Flagging antenna 130n with a relative difference before and after smoothing of 39.91% (compared to the 20.00% cutoff).
Flagging antenna 131e with a relative difference before and after smoothing of 51.45% (compared to the 20.00% cutoff).
Flagging antenna 131n with a relative difference before and after smoothing of 46.96% (compared to the 20.00% cutoff).
Flagging antenna 132e with a relative difference before and after smoothing of 55.77% (compared to the 20.00% cutoff).
Flagging antenna 132n with a relative difference before and after smoothing of 52.83% (compared to the 20.00% cutoff).
Flagging antenna 133e with a relative difference before and after smoothing of 59.09% (compared to the 20.00% cutoff).
Flagging antenna 133n with a relative difference before and after smoothing of 54.44% (compared to the 20.00% cutoff).
Flagging antenna 134e with a relative difference before and after smoothing of 60.87% (compared to the 20.00% cutoff).
Flagging antenna 134n with a relative difference before and after smoothing of 60.54% (compared to the 20.00% cutoff).
Flagging antenna 136e with a relative difference before and after smoothing of 25.46% (compared to the 20.00% cutoff).
Flagging antenna 145e with a relative difference before and after smoothing of 21.07% (compared to the 20.00% cutoff).
Flagging antenna 147e with a relative difference before and after smoothing of 30.98% (compared to the 20.00% cutoff).
Flagging antenna 147n with a relative difference before and after smoothing of 27.73% (compared to the 20.00% cutoff).
Flagging antenna 148n with a relative difference before and after smoothing of 32.70% (compared to the 20.00% cutoff).
Flagging antenna 149e with a relative difference before and after smoothing of 42.14% (compared to the 20.00% cutoff).
Flagging antenna 149n with a relative difference before and after smoothing of 38.34% (compared to the 20.00% cutoff).
Flagging antenna 150e with a relative difference before and after smoothing of 46.15% (compared to the 20.00% cutoff).
Flagging antenna 150n with a relative difference before and after smoothing of 42.82% (compared to the 20.00% cutoff).
Flagging antenna 152e with a relative difference before and after smoothing of 57.75% (compared to the 20.00% cutoff).
Flagging antenna 152n with a relative difference before and after smoothing of 52.06% (compared to the 20.00% cutoff).
Flagging antenna 153e with a relative difference before and after smoothing of 60.11% (compared to the 20.00% cutoff).
Flagging antenna 153n with a relative difference before and after smoothing of 58.49% (compared to the 20.00% cutoff).
Flagging antenna 154n with a relative difference before and after smoothing of 62.26% (compared to the 20.00% cutoff).
Flagging antenna 155e with a relative difference before and after smoothing of 29.97% (compared to the 20.00% cutoff).
Flagging antenna 155n with a relative difference before and after smoothing of 29.89% (compared to the 20.00% cutoff).
Flagging antenna 156e with a relative difference before and after smoothing of 23.12% (compared to the 20.00% cutoff).
Flagging antenna 156n with a relative difference before and after smoothing of 20.63% (compared to the 20.00% cutoff).
Flagging antenna 166e with a relative difference before and after smoothing of 23.78% (compared to the 20.00% cutoff).
Flagging antenna 166n with a relative difference before and after smoothing of 21.82% (compared to the 20.00% cutoff).
Flagging antenna 167e with a relative difference before and after smoothing of 27.52% (compared to the 20.00% cutoff).
Flagging antenna 167n with a relative difference before and after smoothing of 24.72% (compared to the 20.00% cutoff).
Flagging antenna 168e with a relative difference before and after smoothing of 32.35% (compared to the 20.00% cutoff).

Flagging antenna 168n with a relative difference before and after smoothing of 29.22% (compared to the 20.00% cutoff).
Flagging antenna 169n with a relative difference before and after smoothing of 33.50% (compared to the 20.00% cutoff).
Flagging antenna 170n with a relative difference before and after smoothing of 38.93% (compared to the 20.00% cutoff).
Flagging antenna 173e with a relative difference before and after smoothing of 56.86% (compared to the 20.00% cutoff).
Flagging antenna 173n with a relative difference before and after smoothing of 53.38% (compared to the 20.00% cutoff).
Flagging antenna 174e with a relative difference before and after smoothing of 60.37% (compared to the 20.00% cutoff).
Flagging antenna 174n with a relative difference before and after smoothing of 60.15% (compared to the 20.00% cutoff).
Flagging antenna 175e with a relative difference before and after smoothing of 60.74% (compared to the 20.00% cutoff).
Flagging antenna 175n with a relative difference before and after smoothing of 56.77% (compared to the 20.00% cutoff).
Flagging antenna 176n with a relative difference before and after smoothing of 22.30% (compared to the 20.00% cutoff).
Flagging antenna 186e with a relative difference before and after smoothing of 21.15% (compared to the 20.00% cutoff).
Flagging antenna 187e with a relative difference before and after smoothing of 24.22% (compared to the 20.00% cutoff).
Flagging antenna 188e with a relative difference before and after smoothing of 29.32% (compared to the 20.00% cutoff).
Flagging antenna 188n with a relative difference before and after smoothing of 26.83% (compared to the 20.00% cutoff).
Flagging antenna 189e with a relative difference before and after smoothing of 33.74% (compared to the 20.00% cutoff).
Flagging antenna 189n with a relative difference before and after smoothing of 30.34% (compared to the 20.00% cutoff).
Flagging antenna 190e with a relative difference before and after smoothing of 38.54% (compared to the 20.00% cutoff).
Flagging antenna 190n with a relative difference before and after smoothing of 35.80% (compared to the 20.00% cutoff).
Flagging antenna 191e with a relative difference before and after smoothing of 43.82% (compared to the 20.00% cutoff).
Flagging antenna 191n with a relative difference before and after smoothing of 41.28% (compared to the 20.00% cutoff).
Flagging antenna 192e with a relative difference before and after smoothing of 51.17% (compared to the 20.00% cutoff).
Flagging antenna 192n with a relative difference before and after smoothing of 47.81% (compared to the 20.00% cutoff).
Flagging antenna 193e with a relative difference before and after smoothing of 57.42% (compared to the 20.00% cutoff).
Flagging antenna 193n with a relative difference before and after smoothing of 50.99% (compared to the 20.00% cutoff).
Flagging antenna 194n with a relative difference before and after smoothing of 54.31% (compared to the 20.00% cutoff).
Flagging antenna 195e with a relative difference before and after smoothing of 61.90% (compared to the 20.00% cutoff).
Flagging antenna 195n with a relative difference before and after smoothing of 58.39% (compared to the 20.00% cutoff).
Flagging antenna 196e with a relative difference before and after smoothing of 21.90% (compared to the 20.00% cutoff).
Flagging antenna 196n with a relative difference before and after smoothing of 20.62% (compared to the 20.00% cutoff).
Flagging antenna 206e with a relative difference before and after smoothing of 22.74% (compared to the 20.00% cutoff).
Flagging antenna 207e with a relative difference before and after smoothing of 26.53% (compared to the 20.00% cutoff).
Flagging antenna 208e with a relative difference before and after smoothing of 30.74% (compared to the 20.00% cutoff).
Flagging antenna 208n with a relative difference before and after smoothing of 28.74% (compared to the 20.00% cutoff).
Flagging antenna 209e with a relative difference before and after smoothing of 36.46% (compared to the 20.00% cutoff).
Flagging antenna 211e with a relative difference before and after smoothing of 47.13% (compared to the 20.00% cutoff).
Flagging antenna 212e with a relative difference before and after smoothing of 49.50% (compared to the 20.00% cutoff).
Flagging antenna 214e with a relative difference before and after smoothing of 60.81% (compared to the 20.00% cutoff).
Flagging antenna 214n with a relative difference before and after smoothing of 57.92% (compared to the 20.00% cutoff).
Flagging antenna 225e with a relative difference before and after smoothing of 24.21% (compared to the 20.00% cutoff).
Flagging antenna 225n with a relative difference before and after smoothing of 22.62% (compared to the 20.00% cutoff).
Flagging antenna 226e with a relative difference before and after smoothing of 28.85% (compared to the 20.00% cutoff).
Flagging antenna 228e with a relative difference before and after smoothing of 37.77% (compared to the 20.00% cutoff).
Flagging antenna 228n with a relative difference before and after smoothing of 36.26% (compared to the 20.00% cutoff).
Flagging antenna 229e with a relative difference before and after smoothing of 43.21% (compared to the 20.00% cutoff).
Flagging antenna 229n with a relative difference before and after smoothing of 40.58% (compared to the 20.00% cutoff).
Flagging antenna 231e with a relative difference before and after smoothing of 53.93% (compared to the 20.00% cutoff).
Flagging antenna 231n with a relative difference before and after smoothing of 50.42% (compared to the 20.00% cutoff).
Flagging antenna 232e with a relative difference before and after smoothing of 56.51% (compared to the 20.00% cutoff).
Flagging antenna 232n with a relative difference before and after smoothing of 53.88% (compared to the 20.00% cutoff).
Flagging antenna 242e with a relative difference before and after smoothing of 21.24% (compared to the 20.00% cutoff).
Flagging antenna 242n with a relative difference before and after smoothing of 21.15% (compared to the 20.00% cutoff).
Flagging antenna 243e with a relative difference before and after smoothing of 25.56% (compared to the 20.00% cutoff).
Flagging antenna 243n with a relative difference before and after smoothing of 24.36% (compared to the 20.00% cutoff).
Flagging antenna 244e with a relative difference before and after smoothing of 29.80% (compared to the 20.00% cutoff).
Flagging antenna 244n with a relative difference before and after smoothing of 27.66% (compared to the 20.00% cutoff).
Flagging antenna 245e with a relative difference before and after smoothing of 35.04% (compared to the 20.00% cutoff).
Flagging antenna 246n with a relative difference before and after smoothing of 39.62% (compared to the 20.00% cutoff).

Flagging antenna 262n with a relative difference before and after smoothing of 36.22% (compared to the 20.00% cutoff).
Flagging antenna 277n with a relative difference before and after smoothing of 34.34% (compared to the 20.00% cutoff).
Flagging antenna 278n with a relative difference before and after smoothing of 40.84% (compared to the 20.00% cutoff).
Flagging antenna 292e with a relative difference before and after smoothing of 37.54% (compared to the 20.00% cutoff).
Flagging antenna 292n with a relative difference before and after smoothing of 38.40% (compared to the 20.00% cutoff).
Flagging antenna 293n with a relative difference before and after smoothing of 42.28% (compared to the 20.00% cutoff).
Flagging antenna 294n with a relative difference before and after smoothing of 49.06% (compared to the 20.00% cutoff).
Flagging antenna 301n with a relative difference before and after smoothing of 21.48% (compared to the 20.00% cutoff).
Flagging antenna 302n with a relative difference before and after smoothing of 23.92% (compared to the 20.00% cutoff).
Flagging antenna 306e with a relative difference before and after smoothing of 40.03% (compared to the 20.00% cutoff).
Flagging antenna 307n with a relative difference before and after smoothing of 44.92% (compared to the 20.00% cutoff).
Flagging antenna 315e with a relative difference before and after smoothing of 24.38% (compared to the 20.00% cutoff).
Flagging antenna 315n with a relative difference before and after smoothing of 26.32% (compared to the 20.00% cutoff).
Flagging antenna 316e with a relative difference before and after smoothing of 29.39% (compared to the 20.00% cutoff).
Flagging antenna 316n with a relative difference before and after smoothing of 30.29% (compared to the 20.00% cutoff).
Flagging antenna 317e with a relative difference before and after smoothing of 33.84% (compared to the 20.00% cutoff).
Flagging antenna 317n with a relative difference before and after smoothing of 34.01% (compared to the 20.00% cutoff).
Flagging antenna 319e with a relative difference before and after smoothing of 43.35% (compared to the 20.00% cutoff).
Flagging antenna 319n with a relative difference before and after smoothing of 43.02% (compared to the 20.00% cutoff).
In [19]:
if not PER_POL_REFANT:
    # put back in the smoothed phasor, ensuring the amplitude is 1 and that data are flagged anywhere either polarization's refant is flagged
    smoothed_relative_pol_phasor = cs.gain_grids[(-1, other_refant[-1])] / np.abs(cs.gain_grids[(-1, other_refant[-1])])
    for ant in cs.gain_grids:
        if ant[0] >= 0 and ant[1] == other_refant[1]:
            cs.gain_grids[ant] /= smoothed_relative_pol_phasor
        cs.flag_grids[ant] |= (cs.flag_grids[(-1, other_refant[1])])
    cs.refant = overall_refant
In [20]:
def phase_flip_diagnostic_plot():
    '''Shows time-smoothed antenna avg phases after taking out a delay and filtering in time.'''
    if not np.any([np.any(meta['phase_flipped'][ant]) for ant in meta['phase_flipped']]):
        print("No antennas have phase flips identified. Nothing to plot.")
        return
    
    plt.figure(figsize=(14,4))
    for ant in meta['phase_flipped']:
        if np.any(meta['phase_flipped'][ant]):
            to_plot = np.angle(np.exp(1.0j * (meta['phases'][ant] - meta['time_smoothed_phases'][ant])))
            to_plot[to_plot < -np.pi / 2] += 2 * np.pi
            plt.plot(cs.time_grid - int(cs.time_grid[0]), to_plot, label=f'{ant[0]}{ant[1][-1]}')
    plt.legend(title='Antennas with Identified Phase Flips', ncol=4)
    plt.xlabel(f'JD - {int(cs.time_grid[0])}')
    plt.ylabel('Average Phase After Filtering (radians)')
    plt.tight_layout()

Figure 2: Antenna Phases with Identified Phase Flips¶

In [21]:
phase_flip_diagnostic_plot()
No description has been provided for this image

Plot results¶

In [22]:
def amplitude_plot(ant_to_plot):
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")
        # Pick vmax to not saturate 90% of the abscal gains
        vmax = np.max([np.percentile(np.abs(cs.gain_grids[ant_to_plot, pol][~cs.flag_grids[ant_to_plot, pol]]), 99) for pol in ['Jee', 'Jnn']])

        display(HTML(f'<h2>Antenna {ant_to_plot} Amplitude Waterfalls</h2>'))    

        # Plot abscal gain amplitude waterfalls for a single antenna
        fig, axes = plt.subplots(4, 2, figsize=(14,14), gridspec_kw={'height_ratios': [1, 1, .4, .4]})
        for ax, pol in zip(axes[0], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            extent=[cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
            im = ax.imshow(np.where(cs.flag_grids[ant], np.nan, np.abs(cs.gain_grids[ant])), aspect='auto', cmap='inferno', 
                           interpolation='nearest', vmin=0, vmax=vmax, extent=extent)
            ax.set_title(f'Smoothcal Gain Amplitude of Antenna {ant[0]}: {pol[-1]}-polarized' )
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('LST (Hours)')
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])
            ax.set_yticklabels(ax.get_yticks() % 24)
            plt.colorbar(im, ax=ax,  orientation='horizontal', pad=.15)

        # Now flagged plot abscal waterfall    
        for ax, pol in zip(axes[1], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            extent=[cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
            im = ax.imshow(np.where(cs.flag_grids[ant], np.nan, np.abs(abscal_gains[ant])), aspect='auto', cmap='inferno', 
                           interpolation='nearest', vmin=0, vmax=vmax, extent=extent)
            ax.set_title(f'Abscal Gain Amplitude of Antenna {ant[0]}: {pol[-1]}-polarized' )
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('LST (Hours)')
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])
            ax.set_yticklabels(ax.get_yticks() % 24)
            plt.colorbar(im, ax=ax,  orientation='horizontal', pad=.15)
            
        # Now plot mean gain spectra 
        for ax, pol in zip(axes[2], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)   
            nflags_spectrum = np.sum(cs.flag_grids[ant], axis=0)
            to_plot = nflags_spectrum <= np.percentile(nflags_spectrum, 75)
            ax.plot(cs.freqs[to_plot] / 1e6, np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.abs(abscal_gains[ant])), axis=0)[to_plot], 'r.', label='Abscal')        
            ax.plot(cs.freqs[to_plot] / 1e6, np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.abs(cs.gain_grids[ant])), axis=0)[to_plot], 'k.', ms=2, label='Smoothed')        
            ax.set_ylim([0, vmax])
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])    
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('|g| (unitless)')
            ax.set_title(f'Mean Infrequently-Flagged Gain Amplitude of Antenna {ant[0]}: {pol[-1]}-polarized')
            ax.legend(loc='upper left')

        # Now plot mean gain time series
        for ax, pol in zip(axes[3], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            nflags_series = np.sum(cs.flag_grids[ant], axis=1)
            to_plot = nflags_series <= np.percentile(nflags_series, 75)
            ax.plot(lst_grid[to_plot], np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.abs(abscal_gains[ant])), axis=1)[to_plot], 'r.', label='Abscal')        
            ax.plot(lst_grid[to_plot], np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.abs(cs.gain_grids[ant])), axis=1)[to_plot], 'k.', ms=2, label='Smoothed')        
            ax.set_ylim([0, vmax])
            ax.set_xlabel('LST (hours)')
            ax.set_ylabel('|g| (unitless)')
            ax.set_title(f'Mean Infrequently-Flagged Gain Amplitude of Antenna {ant[0]}: {pol[-1]}-polarized')
            ax.set_xticklabels(ax.get_xticks() % 24)
            ax.legend(loc='upper left')

        plt.tight_layout()
        plt.show()
In [23]:
def phase_plot(ant_to_plot):
    with warnings.catch_warnings():
        warnings.simplefilter("ignore")    
        display(HTML(f'<h2>Antenna {ant_to_plot} Phase Waterfalls</h2>'))
        fig, axes = plt.subplots(4, 2, figsize=(14,14), gridspec_kw={'height_ratios': [1, 1, .4, .4]})
        
        # Plot phase waterfalls for a single antenna    
        for ax, pol in zip(axes[0], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            extent=[cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
            im = ax.imshow(np.where(cs.flag_grids[ant], np.nan, np.angle(cs.gain_grids[ant])), aspect='auto', cmap='inferno', 
                           interpolation='nearest', vmin=-np.pi, vmax=np.pi, extent=extent)

            refant = (cs.refant[pol] if isinstance(cs.refant, dict) else cs.refant)
            ax.set_title(f'Smoothcal Gain Phase of Ant {ant[0]}{pol[-1]} / Ant {refant[0]}{refant[1][-1]}')
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('LST (Hours)')
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])
            ax.set_yticklabels(ax.get_yticks() % 24)
            plt.colorbar(im, ax=ax,  orientation='horizontal', pad=.15)

        # Now plot abscal phase waterfall    
        for ax, pol in zip(axes[1], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            extent=[cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
            im = ax.imshow(np.where(cs.flag_grids[ant], np.nan, np.angle(abscal_gains[ant])), aspect='auto', cmap='inferno', 
                           interpolation='nearest', vmin=-np.pi, vmax=np.pi, extent=extent)
            refant = (cs.refant[pol] if isinstance(cs.refant, dict) else cs.refant)
            ax.set_title(f'Abscal Gain Phase of Ant {ant[0]}{pol[-1]} / Ant {refant[0]}{refant[1][-1]}')
            ax.set_xlabel('Frequency (MHz)')
            ax.set_ylabel('LST (Hours)')
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])
            ax.set_yticklabels(ax.get_yticks() % 24)
            plt.colorbar(im, ax=ax,  orientation='horizontal', pad=.15)
            
        # Now plot median gain spectra 
        for ax, pol in zip(axes[2], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)   
            nflags_spectrum = np.sum(cs.flag_grids[ant], axis=0)
            to_plot = nflags_spectrum <= np.percentile(nflags_spectrum, 75)
            ax.plot(cs.freqs[to_plot] / 1e6, np.nanmedian(np.where(cs.flag_grids[ant], np.nan, np.angle(abscal_gains[ant])), axis=0)[to_plot], 'r.', label='Abscal')        
            ax.plot(cs.freqs[to_plot] / 1e6, np.nanmedian(np.where(cs.flag_grids[ant], np.nan, np.angle(cs.gain_grids[ant])), axis=0)[to_plot], 'k.', ms=2, label='Smoothed')        
            ax.set_ylim([-np.pi, np.pi])
            ax.set_xlim([cs.freqs[0]/1e6, cs.freqs[-1]/1e6])    
            ax.set_xlabel('Frequency (MHz)')
            refant = (cs.refant[pol] if isinstance(cs.refant, dict) else cs.refant)
            ax.set_ylabel(f'Phase of g$_{{{ant[0]}{pol[-1]}}}$ / g$_{{{refant[0]}{refant[1][-1]}}}$')
            ax.set_title(f'Median Infrequently-Flagged Gain Phase of Ant {ant[0]}{pol[-1]} / Ant {refant[0]}{refant[1][-1]}')
            ax.legend(loc='upper left')

        # # Now plot median gain time series
        for ax, pol in zip(axes[3], ['Jee', 'Jnn']):
            ant = (ant_to_plot, pol)
            nflags_series = np.sum(cs.flag_grids[ant], axis=1)
            to_plot = nflags_series <= np.percentile(nflags_series, 75)
            ax.plot(lst_grid[to_plot], np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.angle(abscal_gains[ant])), axis=1)[to_plot], 'r.', label='Abscal')        
            ax.plot(lst_grid[to_plot], np.nanmean(np.where(cs.flag_grids[ant], np.nan, np.angle(cs.gain_grids[ant])), axis=1)[to_plot], 'k.', ms=2, label='Smoothed')        
            ax.set_ylim([-np.pi, np.pi])    
            ax.set_xlabel('LST (hours)')
            refant = (cs.refant[pol] if isinstance(cs.refant, dict) else cs.refant)
            ax.set_ylabel(f'Phase of g$_{{{ant[0]}{pol[-1]}}}$ / g$_{{{refant[0]}{refant[1][-1]}}}$')
            ax.set_title(f'Mean Infrequently-Flagged Gain Phase of Ant {ant[0]}{pol[-1]} / Ant {refant[0]}{refant[1][-1]}')
            ax.set_xticklabels(ax.get_xticks() % 24)    
            ax.legend(loc='upper left')

        plt.tight_layout()
        plt.show()
In [24]:
# Select first 2 unflagged antennas from candidates for amplitude plotting
ants_to_plot = []
for ant_candidate in ants_to_plot_candidates:
    if not (np.all(cs.flag_grids[ant_candidate, 'Jee']) and np.all(cs.flag_grids[ant_candidate, 'Jnn'])):
        ants_to_plot.append(ant_candidate)
        if len(ants_to_plot) >= 2:
            break

Figure 3: Full-Day Gain Amplitudes Before and After smooth_cal¶

Here we plot abscal and smooth_cal gain amplitudes for both of the sample antennas. We also show means across time/frequency, excluding frequencies/times that are frequently flagged.

In [25]:
if len(ants_to_plot) == 0:
    print("Warning: No unflagged antennas available for plotting.")
else:
    for ant_to_plot in ants_to_plot:
        amplitude_plot(ant_to_plot)

Antenna 101 Amplitude Waterfalls

No description has been provided for this image

Antenna 223 Amplitude Waterfalls

No description has been provided for this image

Figure 4: Full-Day Gain Phases Before and After smooth_cal¶

Here we plot abscal and smooth_cal phases relative to each polarization's reference antenna for both of the sample antennas. We also show medians across time/frequency, excluding frequencies/times that are frequently flagged.

In [26]:
# Use the same selected unflagged antennas for phase plotting
if len(ants_to_plot) == 0:
    print("Warning: No unflagged antennas available for plotting.")
else:
    for ant_to_plot in ants_to_plot:
        phase_plot(ant_to_plot)

Antenna 101 Phase Waterfalls

No description has been provided for this image

Antenna 223 Phase Waterfalls

No description has been provided for this image

Examine $\chi^2$¶

In [27]:
def chisq_plot():
    fig, axes = plt.subplots(1, 2, figsize=(14, 10), sharex=True, sharey=True)
    extent = [cs.freqs[0]/1e6, cs.freqs[-1]/1e6, lst_grid[-1], lst_grid[0]]
    for ax, pol in zip(axes, ['Jee', 'Jnn']):
        refant = (cs.refant[pol] if isinstance(cs.refant, dict) else cs.refant)
        im = ax.imshow(np.where(cs.flag_grids[refant], np.nan, cs.chisq_grids[pol]), vmin=1, vmax=5, 
                       aspect='auto', cmap='turbo', interpolation='none', extent=extent)
        ax.set_yticklabels(ax.get_yticks() % 24)
        ax.set_title(f'{pol[1:]}-Polarized $\\chi^2$ / DoF')
        ax.set_xlabel('Frequency (MHz)')

    axes[0].set_ylabel('LST (hours)')
    plt.tight_layout()
    fig.colorbar(im, ax=axes, pad=.07, label='$\\chi^2$ / DoF', orientation='horizontal', extend='both', aspect=50)

Figure 5: Full-Day $\chi^2$ / DoF Waterfall from Redundant-Baseline Calibration¶

Here we plot $\chi^2$ per degree of freedom from redundant-baseline calibration for both polarizations separately. While this plot is a little out of place, as it was not produced by this notebook, it is a convenient place where all the necessary components are readily available. If the array were perfectly redundant and any non-redundancies in the calibrated visibilities were explicable by thermal noise alone, this waterfall should be all 1.

In [28]:
chisq_plot()

set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
No description has been provided for this image
In [29]:
def cspa_vs_time_plot():
    fig, axes = plt.subplots(2, 1, figsize=(14, 6), sharex=True, sharey=True, gridspec_kw={'hspace': 0})
    for ax, pol in zip(axes, ['Jee', 'Jnn']):
        detail_cutoff = np.percentile([np.nanmean(m) for ant, m in avg_cspa_vs_time.items() 
                                       if ant[1] == pol and np.isfinite(np.nanmean(m))], 95)
        for ant in avg_cspa_vs_time:
            if ant[1] == pol and not np.all(cs.flag_grids[ant]):
                if np.nanmean(avg_cspa_vs_time[ant]) > detail_cutoff:
                    ax.plot(lst_grid, avg_cspa_vs_time[ant], label=str((int(ant[0]), ant[1])), zorder=100)
                else:
                    ax.plot(lst_grid, avg_cspa_vs_time[ant], c='grey', alpha=.2, lw=.5)
        ax.legend(title=f'{pol[1:]}-Polarized', ncol=2)
        ax.set_ylabel('Mean Unflagged $\\chi^2$ per Antenna')
        ax.set_xlabel('LST (hours)')
        ax.set_xticklabels(ax.get_xticks() % 24)

    plt.ylim([1, 5.4])
    plt.tight_layout()
In [30]:
def cspa_vs_freq_plot():
    fig, axes = plt.subplots(2, 1, figsize=(14, 6), sharex=True, sharey=True, gridspec_kw={'hspace': 0})
    for ax, pol in zip(axes, ['Jee', 'Jnn']):
        detail_cutoff = np.percentile([np.nanmean(m) for ant, m in avg_cspa_vs_freq.items() 
                                       if ant[1] == pol and np.isfinite(np.nanmean(m))], 95)
        for ant in avg_cspa_vs_freq:
            if ant[1] == pol and not np.all(cs.flag_grids[ant]):
                if np.nanmean(avg_cspa_vs_freq[ant]) > detail_cutoff:
                    ax.plot(cs.freqs / 1e6, avg_cspa_vs_freq[ant], label=str((int(ant[0]), ant[1])), zorder=100)
                else:
                    ax.plot(cs.freqs / 1e6, avg_cspa_vs_freq[ant], c='grey', alpha=.2, lw=.5)
        ax.legend(title=f'{pol[1:]}-Polarized', ncol=2)
        ax.set_ylabel('Mean Unflagged $\\chi^2$ per Antenna')
        ax.set_xlabel('Frequency (MHz)')

    plt.ylim([1, 5.4])
    plt.tight_layout()
In [31]:
def avg_cspa_array_plot():
    hd = io.HERAData(SUM_FILE)
    
    fig, axes = plt.subplots(1, 2, figsize=(14, 8), sharex=True, sharey=True, gridspec_kw={'wspace': 0})
    for pol, ax in zip(['Jee', 'Jnn'], axes):

        ants_here = [ant for ant in avg_cspa if np.isfinite(avg_cspa[ant]) and ant[1] == pol if ant[0] in hd.antpos]
        avg_chisqs = [avg_cspa[ant] for ant in ants_here]
        xs = [hd.antpos[ant[0]][0] for ant in ants_here]
        ys = [hd.antpos[ant[0]][1] for ant in ants_here]
        names = [ant[0] for ant in ants_here]
        
        im = ax.scatter(x=xs, y=ys, c=avg_chisqs, s=200, vmin=1, vmax=3, cmap='turbo')
        ax.set_aspect('equal')
        for x,y,n in zip(xs, ys, names):
            ax.text(x, y, str(n), va='center', ha='center', fontsize=8)
        ax.set_title(pol)
        ax.set_xlabel('East-West Antenna Position (m)')
    
    axes[0].set_ylabel('North-South Antenna Position (m)')

    plt.tight_layout()
    plt.colorbar(im, ax=axes, location='top', aspect=60, pad=.04, label='Mean Unflagged $\\chi^2$ per Antenna', extend='both')

Figure 6: Average $\chi^2$ per Antenna¶

Here we plot $\chi^2$ per antenna from redundant-baseline calibration, separating polarizations and averaging the unflagged pixels in the waterfalls over frequency or time. The worst 5% of antennas are shown in color and highlighted in the legends, the rest are shown in grey. We also show time- and frequency-averaged $\chi^2$ for each antennas as a scatter plot with array position.

In [32]:
cspa_vs_freq_plot()
cspa_vs_time_plot()
avg_cspa_array_plot()

Mean of empty slice

set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
Mean of empty slice
set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image

Examine relative differences before and after smoothing¶

In [33]:
def time_avg_diff_plot():
    fig, axes = plt.subplots(2, 1, figsize=(14, 6), sharex=True, sharey=True, gridspec_kw={'hspace': 0})
    for ax, pol in zip(axes, ['Jee', 'Jnn']):
        detail_cutoff = np.percentile([np.nanmean(diff) for ant, diff in meta['time_avg_rel_diff'].items() 
                                       if ant[1] == pol and np.isfinite(np.nanmean(diff))], 95)    
        for ant, rel_diff in meta['time_avg_rel_diff'].items():
            if ant[0] >= 0 and ant[1] == pol and np.any(np.isfinite(rel_diff)):
                if np.nanmean(rel_diff) > detail_cutoff:
                    if np.all(cs.flag_grids[ant]):
                        ax.plot(cs.freqs / 1e6, rel_diff, label=str((int(ant[0]), ant[1])), zorder=99, ls='--', c='r', lw=.5)    
                    else:
                        ax.plot(cs.freqs / 1e6, rel_diff, label=str((int(ant[0]), ant[1])), zorder=100)
                else:
                    ax.plot(cs.freqs / 1e6, rel_diff, c='grey', alpha=.2, lw=.5)
        med_rel_diff = np.nanmedian([diff for ant, diff in meta['time_avg_rel_diff'].items() if ant[1] == pol], axis=0)
        ax.plot(cs.freqs / 1e6, med_rel_diff, 'k--', label='Median')
        ax.set_ylim([0, 1.05])
        ax.legend(title=f'{pol[1:]}-Polarized', ncol=2)
        ax.set_ylabel('Time-Averaged Relative Difference\nBefore and After Smoothing')
        ax.set_xlabel('Frequency (MHz)')
    plt.tight_layout()
In [34]:
def freq_avg_diff_plot():
    fig, axes = plt.subplots(2, 1, figsize=(14, 6), sharex=True, sharey=True, gridspec_kw={'hspace': 0})
    for ax, pol in zip(axes, ['Jee', 'Jnn']):
        detail_cutoff = np.percentile([np.nanmean(m) for ant, m in meta['freq_avg_rel_diff'].items() 
                                       if ant[1] == pol and np.isfinite(np.nanmean(m))], 95)    
        for ant, rel_diff in meta['freq_avg_rel_diff'].items():
            if ant[0] >= 0 and ant[1] == pol and np.any(np.isfinite(rel_diff)):
                if np.nanmean(rel_diff) > detail_cutoff:
                    if np.all(cs.flag_grids[ant]):
                        ax.plot(lst_grid, rel_diff, label=str((int(ant[0]), ant[1])), zorder=99, ls='--', c='r', lw=.5)    
                    else:
                        ax.plot(lst_grid, rel_diff, label=str((int(ant[0]), ant[1])), zorder=100)
                else:
                    ax.plot(lst_grid, rel_diff, c='grey', alpha=.2, lw=.5)
        
        med_rel_diff = np.nanmedian([diff for ant, diff in meta['freq_avg_rel_diff'].items() if ant[1] == pol], axis=0)
        ax.plot(lst_grid, med_rel_diff, 'k--', label='Median', zorder=101)
        ax.set_ylim([0, 1.05])
        ax.legend(title=f'{pol[1:]}-Polarized', ncol=2)
        ax.set_ylabel('Frequency-Averaged Relative Difference\nBefore and After Smoothing')
        ax.set_xlabel('LST (hours)')
        ax.set_xticklabels(ax.get_xticks() % 24)
    plt.tight_layout()
In [35]:
def avg_difference_array_plot():
    hd = io.HERAData(SUM_FILE)
    
    fig, axes = plt.subplots(1, 2, figsize=(14, 8), sharex=True, sharey=True, gridspec_kw={'wspace': 0})
    for pol, ax in zip(['Jee', 'Jnn'], axes):
    
        avg_diffs = [np.nanmean(meta['time_avg_rel_diff'][ant]) for ant in meta['time_avg_rel_diff'] if ant[1] == pol if ant[0] in hd.antpos]
        xs = [hd.antpos[ant[0]][0] for ant in meta['time_avg_rel_diff'] if ant[1] == pol if ant[0] in hd.antpos]
        ys = [hd.antpos[ant[0]][1] for ant in meta['time_avg_rel_diff'] if ant[1] == pol if ant[0] in hd.antpos]
        names = [ant[0] for ant in meta['time_avg_rel_diff'] if ant[1] == pol if ant[0] in hd.antpos]
        
        im = ax.scatter(x=xs, y=ys, c=avg_diffs, s=200, vmin=0, vmax=.25, cmap='turbo')
        ax.set_aspect('equal')
        for x,y,n in zip(xs, ys, names):
            color = ('w' if np.all(cs.flag_grids[n, pol]) else 'k')
            ax.text(x, y, str(n), va='center', ha='center', fontsize=8, c=color)
        ax.set_title(pol)
        ax.set_xlabel('East-West Antenna Position (m)')
    
    axes[0].set_ylabel('North-South Antenna Position (m)')

    plt.tight_layout()
    plt.colorbar(im, ax=axes, location='top', aspect=60, pad=.04, label='Average Relative Difference Before and After Smoothing', extend='max')

Figure 7: Relative Difference Before and After Smoothing¶

Similar to the above plots, here we show the relative difference before and after smoothing, compared to the magnitude of the smoothed calibration solution. Totally flagged antennas (because they are above the SC_RELATIVE_DIFF_CUTOFF) are red in the first two plots, and their numbers are white in the last plot.

In [36]:
time_avg_diff_plot()
freq_avg_diff_plot()
avg_difference_array_plot()

All-NaN slice encountered

All-NaN slice encountered
set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
All-NaN slice encountered
set_ticklabels() should only be used with a fixed number of ticks, i.e. after set_ticks() or using a FixedLocator.
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image

Save Results¶

In [37]:
add_to_history = 'Produced by calibration_smoothing notebook with the following environment:\n' + '=' * 65 + '\n' + os.popen('conda env export').read() + '=' * 65
In [38]:
cs.write_smoothed_cal(output_replace=(CAL_SUFFIX, SMOOTH_CAL_SUFFIX), add_to_history=add_to_history, clobber=True)

Mean of empty slice

invalid value encountered in multiply
invalid value encountered in divide

Metadata¶

In [39]:
for repo in ['hera_cal', 'hera_qm', 'hera_filters', 'hera_notebook_templates', 'pyuvdata']:
    exec(f'from {repo} import __version__')
    print(f'{repo}: {__version__}')

hera_cal: 3.7.7.dev97+gc2668d3f7
hera_qm: 2.2.1.dev4+gf6d02113b
hera_filters: 0.1.7

hera_notebook_templates: 0.0.1.dev1313+g92178a09c
pyuvdata: 3.2.5.dev1+g5a985ae31
In [40]:
print(f'Finished execution in {(time.time() - tstart) / 60:.2f} minutes.')

Finished execution in 52.80 minutes.