Second Round of Full Day RFI Flagging¶
by Josh Dillon, last updated July 31, 2023
This notebook is synthesizes information from individual delay_filtered_average_zscore notebooks to find low-level RFI and flag it. That notebook takes smooth_calibrated data, redundantly averages it, performs a high-pass delay filter, and then incoherently averages across baselines, creating a per-polarization z-score. This notebook then takes that whole night of z-scores and finds a new set of flags to both add to the smooth_cal files, which are updated in place, and to write down as new UVFlag waterfall-type .h5 files.
Here's a set of links to skip to particular figures and tables:
• Figure 1: Waterfall of Maximum z-Score of Either Polarization Before Round 2 Flagging¶
• Figure 2: Histogram of z-scores¶
• Figure 3: Waterfall of Maximum z-Score of Either Polarization After Round 2 Flagging¶
• Figure 4: Spectra of Time-Averaged z-Scores¶
• Figure 5: Summary of Flags Before and After Round 2 Flagging¶
In [1]:
import time
tstart = time.time()
In [2]:
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin # REQUIRED to have the compression plugins available
import numpy as np
import glob
import matplotlib.pyplot as plt
import matplotlib
import copy
import warnings
from pyuvdata import UVFlag, UVCal
from hera_cal import utils
from hera_qm import xrfi
from hera_qm.time_series_metrics import true_stretches
from IPython.display import display, HTML
%matplotlib inline
display(HTML("<style>.container { width:100% !important; }</style>"))
_ = np.seterr(all='ignore') # get rid of red warnings
%config InlineBackend.figure_format = 'retina'
In [3]:
# get input data file names
SUM_FILE = os.environ.get("SUM_FILE", None)
# SUM_FILE = '/lustre/aoc/projects/hera/h6c-analysis/IDR2/2459861/zen.2459861.25297.sum.uvh5'
SUM_SUFFIX = os.environ.get("SUM_SUFFIX", 'sum.uvh5')
# get input and output suffixes
SMOOTH_CAL_SUFFIX = os.environ.get("SMOOTH_CAL_SUFFIX", 'sum.smooth.calfits')
ZSCORE_SUFFIX = os.environ.get("ZSCORE_SUFFIX", 'sum.red_avg_zscore.h5')
FLAG_WATERFALL2_SUFFIX = os.environ.get("FLAG_WATERFALL2_SUFFIX", 'sum.flag_waterfall_round_2.h5')
OUT_YAML_SUFFIX = os.environ.get("OUT_YAML_SUFFIX", '_aposteriori_flags.yaml')
OUT_YAML_DIR = os.environ.get("OUT_YAML_DIR", None)
# build globs
sum_glob = '.'.join(SUM_FILE.split('.')[:-3]) + '.*.' + SUM_SUFFIX
cal_files_glob = sum_glob.replace(SUM_SUFFIX, SMOOTH_CAL_SUFFIX)
zscore_glob = sum_glob.replace(SUM_SUFFIX, ZSCORE_SUFFIX)
# build out yaml file
if OUT_YAML_DIR is None:
OUT_YAML_DIR = os.path.dirname(SUM_FILE)
out_yaml_file = os.path.join(OUT_YAML_DIR, SUM_FILE.split('.')[-4] + OUT_YAML_SUFFIX)
# get flagging parameters
Z_THRESH = float(os.environ.get("Z_THRESH", 5))
WS_Z_THRESH = float(os.environ.get("WS_Z_THRESH", 4))
AVG_Z_THRESH = float(os.environ.get("AVG_Z_THRESH", 1))
MAX_FREQ_FLAG_FRAC = float(os.environ.get("MAX_FREQ_FLAG_FRAC", .25))
MAX_TIME_FLAG_FRAC = float(os.environ.get("MAX_TIME_FLAG_FRAC", .1))
for setting in ['Z_THRESH', 'WS_Z_THRESH', 'AVG_Z_THRESH', 'MAX_FREQ_FLAG_FRAC', 'MAX_TIME_FLAG_FRAC']:
print(f'{setting} = {eval(setting)}')
Z_THRESH = 5.0 WS_Z_THRESH = 4.0 AVG_Z_THRESH = 1.0 MAX_FREQ_FLAG_FRAC = 0.25 MAX_TIME_FLAG_FRAC = 0.1
Load z-scores¶
In [4]:
# load z-scores
zscore_files = sorted(glob.glob(zscore_glob))
print(f'Found {len(zscore_files)} *.{ZSCORE_SUFFIX} files starting with {zscore_files[0]}.')
uvf = UVFlag(zscore_files, use_future_array_shapes=True)
Found 1757 *.sum.red_avg_zscore.h5 files starting with /mnt/sn1/2460362/zen.2460362.25448.sum.red_avg_zscore.h5.
In [5]:
# get calibration solution files
cal_files = sorted(glob.glob(cal_files_glob))
print(f'Found {len(cal_files)} *.{SMOOTH_CAL_SUFFIX} files starting with {cal_files[0]}.')
Found 1757 *.sum.smooth.calfits files starting with /mnt/sn1/2460362/zen.2460362.25448.sum.smooth.calfits.
In [6]:
assert len(zscore_files) == len(cal_files)
In [7]:
# extract z-scores and correct by a single number per polarization to account for biases created by filtering
zscore = {pol: uvf.metric_array[:, :, np.argwhere(uvf.polarization_array == utils.polstr2num(pol, x_orientation=uvf.x_orientation))[0][0]] for pol in ['ee', 'nn']}
zscore = {pol: zscore[pol] - np.nanmedian(zscore[pol]) for pol in zscore}
In [8]:
freqs = uvf.freq_array
times = uvf.time_array
In [9]:
extent = [freqs[0] / 1e6, freqs[-1] / 1e6, times[-1] - int(times[0]), times[0] - int(times[0])]
In [10]:
def plot_max_z_score(zscore, flags=None):
if flags is None:
flags = np.any(~np.isfinite(list(zscore.values())), axis=0)
plt.figure(figsize=(14,10), dpi=100)
plt.imshow(np.where(flags, np.nan, np.nanmax([zscore['ee'], zscore['nn']], axis=0)), aspect='auto',
cmap='coolwarm', interpolation='none', vmin=-10, vmax=10, extent=extent)
plt.colorbar(location='top', label='Max z-score of either polarization', extend='both', aspect=40, pad=.02)
plt.xlabel('Frequency (MHz)')
plt.ylabel(f'JD - {int(times[0])}')
plt.tight_layout()
Figure 1: Waterfall of Maximum z-Score of Either Polarization Before Round 2 Flagging¶
Shows the worse of the two results from delay_filtered_average_zscore from either polarization. Dips near flagged channels are expected, due to overfitting of noise. Positive-going excursions are problematic and likely evidence of RFI.
In [11]:
plot_max_z_score(zscore)
All-NaN axis encountered
In [12]:
def plot_histogram():
plt.figure(figsize=(14,4), dpi=100)
bins = np.arange(-50, 100, .1)
hist_ee = plt.hist(np.ravel(zscore['ee']), bins=bins, density=True, label='ee-polarized z-scores', alpha=.5)
hist_nn = plt.hist(np.ravel(zscore['nn']), bins=bins, density=True, label='nn-polarized z-scores', alpha=.5)
plt.plot(bins, (2*np.pi)**-.5 * np.exp(-bins**2 / 2), 'k:', label='Gaussian approximate\nnoise-only distribution')
plt.axvline(WS_Z_THRESH, c='r', ls='--', label='Watershed z-score')
plt.axvline(Z_THRESH, c='r', ls='-', label='Threshold z-score')
plt.yscale('log')
all_densities = np.concatenate([hist_ee[0][hist_ee[0] > 0], hist_nn[0][hist_nn[0] > 0]])
plt.ylim(np.min(all_densities) / 2, np.max(all_densities) * 2)
plt.xlim([-50, 100])
plt.legend()
plt.xlabel('z-score')
plt.ylabel('Density')
plt.tight_layout()
Figure 2: Histogram of z-scores¶
Shows a comparison of the histogram of z-scores in this file (one per polarization) to a Gaussian approximation of what one might expect from thermal noise. Without filtering, the actual distribution is a weighted sum of Rayleigh distributions. Filtering further complicates this. To make the z-scores more reliable, a single per-polarization median is subtracted from each waterfall, which allows us to flag low-level outliers with more confidence. Any points beyond the solid red line are flagged. Any points neighboring a flag beyond the dashed red line are also flagged. Finally, flagging is performed for low-level outliers in whole times or channels.
In [13]:
plot_histogram()
Perform flagging¶
In [14]:
def iteratively_flag_on_averaged_zscore(flags, zscore, avg_z_thresh=1.5, verbose=True):
'''Flag whole integrations or channels based on average z-score. This is done
iteratively to prevent bad times affecting channel averages or vice versa.'''
flagged_chan_count = 0
flagged_int_count = 0
while True:
zspec = np.nanmean(np.where(flags, np.nan, zscore), axis=0)
ztseries = np.nanmean(np.where(flags, np.nan, zscore), axis=1)
if (np.nanmax(zspec) < avg_z_thresh) and (np.nanmax(ztseries) < avg_z_thresh):
break
if np.nanmax(zspec) >= np.nanmax(ztseries):
flagged_chan_count += np.sum((zspec >= np.nanmax(ztseries)) & (zspec >= avg_z_thresh))
flags[:, (zspec >= np.nanmax(ztseries)) & (zspec >= avg_z_thresh)] = True
else:
flagged_int_count += np.sum((ztseries >= np.nanmax(zspec)) & (ztseries >= avg_z_thresh))
flags[(ztseries >= np.nanmax(zspec)) & (ztseries >= avg_z_thresh), :] = True
if verbose:
print(f'\tFlagging an additional {flagged_int_count} integrations and {flagged_chan_count} channels.')
def impose_max_chan_flag_frac(flags, max_flag_frac=.25, verbose=True):
'''Flag channels already flagged more than max_flag_frac (excluding completely flagged times).'''
unflagged_times = ~np.all(flags, axis=1)
frequently_flagged_chans = np.mean(flags[unflagged_times, :], axis=0) >= max_flag_frac
if verbose:
print(f'\tFlagging {np.sum(frequently_flagged_chans) - np.sum(np.all(flags, axis=0))} channels previously flagged {max_flag_frac:.2%} or more.')
flags[:, frequently_flagged_chans] = True
def impose_max_time_flag_frac(flags, max_flag_frac=.25, verbose=True):
'''Flag times already flagged more than max_flag_frac (excluding completely flagged channels).'''
unflagged_chans = ~np.all(flags, axis=0)
frequently_flagged_times = np.mean(flags[:, unflagged_chans], axis=1) >= max_flag_frac
if verbose:
print(f'\tFlagging {np.sum(frequently_flagged_times) - np.sum(np.all(flags, axis=1))} times previously flagged {max_flag_frac:.2%} or more.')
flags[frequently_flagged_times, :] = True
In [15]:
flags = np.any(~np.isfinite(list(zscore.values())), axis=0)
print(f'{np.mean(flags):.3%} of waterfall flagged to start.')
# flag largest outliers
for pol in ['ee', 'nn']:
flags |= (zscore[pol] > Z_THRESH)
print(f'{np.mean(flags):.3%} of waterfall flagged after flagging z > {Z_THRESH} outliers.')
# watershed flagging
while True:
nflags = np.sum(flags)
for pol in ['ee', 'nn']:
flags |= xrfi._ws_flag_waterfall(zscore[pol], flags, WS_Z_THRESH)
if np.sum(flags) == nflags:
break
print(f'{np.mean(flags):.3%} of waterfall flagged after watershed flagging on z > {WS_Z_THRESH} neightbors of prior flags.')
# flag whole integrations or channels
while True:
nflags = np.sum(flags)
for pol in ['ee', 'nn']:
iteratively_flag_on_averaged_zscore(flags, zscore[pol], avg_z_thresh=AVG_Z_THRESH, verbose=True)
impose_max_chan_flag_frac(flags, max_flag_frac=MAX_FREQ_FLAG_FRAC, verbose=True)
impose_max_time_flag_frac(flags, max_flag_frac=MAX_TIME_FLAG_FRAC, verbose=True)
if np.sum(flags) == nflags:
break
print(f'{np.mean(flags):.3%} of waterfall flagged after flagging whole times and channels with average z > {AVG_Z_THRESH}.')
20.589% of waterfall flagged to start. 59.032% of waterfall flagged after flagging z > 5.0 outliers. 61.524% of waterfall flagged after watershed flagging on z > 4.0 neightbors of prior flags.
Mean of empty slice Mean of empty slice
Flagging an additional 120 integrations and 81 channels. Flagging 1192 channels previously flagged 25.00% or more. Flagging 1796 times previously flagged 10.00% or more. Flagging an additional 0 integrations and 0 channels. Flagging 0 channels previously flagged 25.00% or more. Flagging 0 times previously flagged 10.00% or more. Flagging an additional 0 integrations and 0 channels. Flagging 0 channels previously flagged 25.00% or more. Flagging 0 times previously flagged 10.00% or more. Flagging an additional 0 integrations and 0 channels. Flagging 0 channels previously flagged 25.00% or more. Flagging 0 times previously flagged 10.00% or more. 99.809% of waterfall flagged after flagging whole times and channels with average z > 1.0.
Show results of flagging¶
Figure 3: Waterfall of Maximum z-Score of Either Polarization After Round 2 Flagging¶
The same as Figure 1, but after the flagging performed in this notebook.
In [16]:
plot_max_z_score(zscore, flags=flags)
All-NaN axis encountered
In [17]:
def zscore_spectra():
fig, axes = plt.subplots(2, 1, figsize=(14,6), dpi=100, sharex=True, sharey=True, gridspec_kw={'hspace': 0})
for ax, pol in zip(axes, ['ee', 'nn']):
ax.plot(freqs / 1e6, np.nanmean(zscore[pol], axis=0),'r', label=f'{pol}-Polarization Before Round 2 Flagging', lw=.5)
ax.plot(freqs / 1e6, np.nanmean(np.where(flags, np.nan, zscore[pol]), axis=0), label=f'{pol}-Polarization After Round 2 Flagging')
ax.legend(loc='lower right')
ax.set_ylabel('Time-Averged Z-Score\n(Excluding Flags)')
ax.set_ylim(-11, 11)
axes[1].set_xlabel('Frequency (MHz)')
plt.tight_layout()
Figure 4: Spectra of Time-Averaged z-Scores¶
The average along the time axis of Figures 1 and 3 (though now separated per-polarization). This plot is useful for showing channels with repeated low-level RFI.
In [18]:
zscore_spectra()
Mean of empty slice Mean of empty slice
In [19]:
def summarize_flagging():
plt.figure(figsize=(14,10), dpi=100)
cmap = matplotlib.colors.ListedColormap(((0, 0, 0),) + matplotlib.cm.get_cmap("Set2").colors[0:2])
plt.imshow(np.where(np.any(~np.isfinite(list(zscore.values())), axis=0), 1, np.where(flags, 2, 0)),
aspect='auto', cmap=cmap, interpolation='none', extent=extent)
plt.clim([-.5, 2.5])
cbar = plt.colorbar(location='top', aspect=40, pad=.02)
cbar.set_ticks([0, 1, 2])
cbar.set_ticklabels(['Unflagged', 'Previously Flagged', 'Flagged Here Using Delayed Filtered z-Scores'])
plt.xlabel('Frequency (MHz)')
plt.ylabel(f'JD - {int(times[0])}')
plt.tight_layout()
Figure 5: Summary of Flags Before and After Round 2 Flagging¶
This plot shows which times and frequencies were flagged before and after this notebook. It is directly comparable to Figure 5 of the first round full_day_rfi notebook.
In [20]:
summarize_flagging()
Save results¶
In [21]:
add_to_history = 'by full_day_rfi_round_2 notebook with the following environment:\n' + '=' * 65 + '\n' + os.popen('conda env export').read() + '=' * 65
In [22]:
tind = 0
always_flagged_ants = set()
ever_unflagged_ants = set()
for cal_file in cal_files:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# update cal_file
uvc = UVCal()
uvc.read(cal_file, use_future_array_shapes=True)
uvc.flag_array |= (flags[tind:tind + len(uvc.time_array), :].T)[None, :, :, None]
uvc.history += 'Modified ' + add_to_history
uvc.write_calfits(cal_file, clobber=True)
# keep track of flagged antennas
for antnum in uvc.ant_array:
for antpol in ['Jee', 'Jnn']:
if np.all(uvc.get_flags(antnum, antpol)):
if (antnum, antpol) not in ever_unflagged_ants:
always_flagged_ants.add((antnum, antpol))
else:
ever_unflagged_ants.add((antnum, antpol))
always_flagged_ants.discard((antnum, antpol))
# Create new flag object
uvf_out = UVFlag(uvc, waterfall=True, mode='flag')
uvf_out.flag_array |= flags[tind:tind + len(uvc.time_array), :, None]
uvf_out.history += 'Produced ' + add_to_history
uvf_out.write(cal_file.replace(SMOOTH_CAL_SUFFIX, FLAG_WATERFALL2_SUFFIX), clobber=True)
# increment time index
tind += len(uvc.time_array)
print(f'Saved {len(cal_files)} *.{FLAG_WATERFALL2_SUFFIX} files starting with {cal_files[0].replace(SMOOTH_CAL_SUFFIX, FLAG_WATERFALL2_SUFFIX)}.')
Saved 1757 *.sum.flag_waterfall_round_2.h5 files starting with /mnt/sn1/2460362/zen.2460362.25448.sum.flag_waterfall_round_2.h5.
In [23]:
# write summary of entirely flagged times/freqs/ants to yaml
all_flagged_times = np.all(flags, axis=1)
all_flagged_freqs = np.all(flags, axis=0)
all_flagged_ants = sorted(always_flagged_ants)
dt = np.median(np.diff(times))
out_yml_str = 'JD_flags: ' + str([[times[flag_stretch][0] - dt / 2, times[flag_stretch][-1] + dt / 2]
for flag_stretch in true_stretches(all_flagged_times)])
df = np.median(np.diff(freqs))
out_yml_str += '\n\nfreq_flags: ' + str([[freqs[flag_stretch][0] - df / 2, freqs[flag_stretch][-1] + df / 2]
for flag_stretch in true_stretches(all_flagged_freqs)])
out_yml_str += '\n\nex_ants: ' + str(all_flagged_ants).replace("'", "").replace('(', '[').replace(')', ']')
print(f'Writing the following to {out_yaml_file}\n' + '-' * (25 + len(out_yaml_file)))
print(out_yml_str)
with open(out_yaml_file, 'w') as outfile:
outfile.writelines(out_yml_str)
Writing the following to /mnt/sn1/2460362/2460362_aposteriori_flags.yaml ------------------------------------------------------------------------ JD_flags: [[2460362.2543714694, 2460362.2660036725], [2460362.2661155206, 2460362.2674576975], [2460362.2675695457, 2460362.270030204], [2460362.2702539004, 2460362.273385647], [2460362.2734974953, 2460362.2743922803], [2460362.2745041284, 2460362.2753989133], [2460362.2755107614, 2460362.2757344577], [2460362.275846306, 2460362.276070002], [2460362.27618185, 2460362.2777477237], [2460362.27797142, 2460362.2809913186], [2460362.2811031668, 2460362.2816624073], [2460362.2817742554, 2460362.2821098], [2460362.282221648, 2460362.284123066], [2460362.2842349135, 2460362.2844586098], [2460362.284570458, 2460362.2878140532], [2460362.2879259014, 2460362.2882614457], [2460362.288373294, 2460362.2889325344], [2460362.2890443825, 2460362.2903865594], [2460362.291057648, 2460362.291728737], [2460362.292064281, 2460362.2921761293], [2460362.2922879774, 2460362.2923998255], [2460362.2925116736, 2460362.29273537], [2460362.292959066, 2460362.293070914], [2460362.2931827623, 2460362.2932946105], [2460362.2934064586, 2460362.2935183067], [2460362.293630155, 2460362.2941893954], [2460362.2944130916, 2460362.2945249397], [2460362.294636788, 2460362.294860484], [2460362.2950841803, 2460362.2951960284], [2460362.2954197247, 2460362.295531573], [2460362.2956434204, 2460362.2958671167], [2460362.296090813, 2460362.297209294], [2460362.2975448384, 2460362.2977685346], [2460362.297992231, 2460362.2984396233], [2460362.298887016, 2460362.298998864], [2460362.29922256, 2460362.2993344082], [2460362.2994462564, 2460362.2997818007], [2460362.300005497, 2460362.3005647375], [2460362.3007884338, 2460362.3011239776], [2460362.3012358258, 2460362.301347674], [2460362.301683218, 2460362.3017950663], [2460362.3019069145, 2460362.3020187626], [2460362.3021306107, 2460362.302578003], [2460362.3030253956, 2460362.3031372437], [2460362.3036964843, 2460362.304143877], [2460362.304255725, 2460362.304367573], [2460362.304479421, 2460362.304591269], [2460362.3047031173, 2460362.3048149655], [2460362.305374206, 2460362.3055979023], [2460362.3057097504, 2460362.3058215985], [2460362.3059334466, 2460362.306492687], [2460362.306828231, 2460362.306940079], [2460362.30749932, 2460362.307611168], [2460362.307834864, 2460362.3079467122], [2460362.3081704085, 2460362.308505953], [2460362.308841497, 2460362.3089533453], [2460362.3102955227, 2460362.310631067], [2460362.310742915, 2460362.3110784595], [2460362.3116377, 2460362.311749548], [2460362.3123087883, 2460362.3124206364], [2460362.3126443326, 2460362.3127561808], [2460362.313091725, 2460362.313203573], [2460362.3134272695, 2460362.3135391176], [2460362.31398651, 2460362.314098358], [2460362.3144339025, 2460362.3145457506], [2460362.3154405355, 2460362.3155523837], [2460362.315664232, 2460362.315887928], [2460362.315999776, 2460362.3162234724], [2460362.3163353205, 2460362.3164471686], [2460362.316670865, 2460362.316782713], [2460362.316894561, 2460362.3171182573], [2460362.3173419535, 2460362.3174538016], [2460362.317789346, 2460362.317901194], [2460362.318460434, 2460362.3185722823], [2460362.319131523, 2460362.319355219], [2460362.3198026116, 2460362.320026308], [2460362.3204737003, 2460362.3206973965], [2460362.321032941, 2460362.321144789], [2460362.321256637, 2460362.321368485], [2460362.3214803333, 2460362.3215921815], [2460362.321927726, 2460362.322151422], [2460362.3227106626, 2460362.3228225107], [2460362.322934359, 2460362.323046207], [2460362.3233817513, 2460362.3234935994], [2460362.3240528395, 2460362.3241646877], [2460362.324276536, 2460362.3249476245], [2460362.325506865, 2460362.325618713], [2460362.3257305613, 2460362.3258424094], [2460362.3261779537, 2460362.326289802], [2460362.3268490424, 2460362.3269608906], [2460362.327408283, 2460362.3276319792], [2460362.32819122, 2460362.328526764], [2460362.3287504604, 2460362.3288623085], [2460362.3289741566, 2460362.3290860048], [2460362.329645245, 2460362.329868941], [2460362.3300926373, 2460362.3302044854], [2460362.3303163336, 2460362.33054003], [2460362.330763726, 2460362.3313229666], [2460362.331770359, 2460362.331882207], [2460362.3322177515, 2460362.3323295997], [2460362.332441448, 2460362.332665144], [2460362.3332243846, 2460362.3333362327], [2460362.333671777, 2460362.333783625], [2460362.3338954733, 2460362.3340073214], [2460362.3342310176, 2460362.334454714], [2460362.334566562, 2460362.335013954], [2460362.33523765, 2460362.3353494983], [2460362.3354613464, 2460362.3355731945], [2460362.336020587, 2460362.3363561314], [2460362.3366916757, 2460362.336803524], [2460362.336915372, 2460362.33702722], [2460362.3372509163, 2460362.3374746125], [2460362.3375864606, 2460362.3376983088], [2460362.338033853, 2460362.338145701], [2460362.3382575493, 2460362.33881679], [2460362.338928638, 2460362.339040486], [2460362.3393760305, 2460362.3395997267], [2460362.339711575, 2460362.339823423], [2460362.340047119, 2460362.3406063593], [2460362.3407182074, 2460362.3409419036], [2460362.3410537518, 2460362.341277448], [2460362.342060385, 2460362.342172233], [2460362.342284081, 2460362.343402562], [2460362.3438499547, 2460362.343961803], [2460362.3444091952, 2460362.3445210434], [2460362.34530398, 2460362.3454158283], [2460362.3455276764, 2460362.3456395245], [2460362.3461987646, 2460362.3463106127], [2460362.346534309, 2460362.3469817014], [2460362.347317246, 2460362.347429094], [2460362.3482120307, 2460362.348323879], [2460362.3491068156, 2460362.3492186638], [2460362.34944236, 2460362.349666056], [2460362.3497779043, 2460362.350225297], [2460362.350560841, 2460362.3506726893], [2460362.35123193, 2460362.351343778], [2460362.351903018, 2460362.352014866], [2460362.3524622587, 2460362.352574107], [2460362.352685955, 2460362.3533570436], [2460362.35358074, 2460362.3541399804], [2460362.3542518285, 2460362.3543636766], [2460362.354699221, 2460362.354811069], [2460362.356824335, 2460362.3569361833], [2460362.3574954234, 2460362.3576072715], [2460362.358166512, 2460362.35827836], [2460362.358837601, 2460362.358949449], [2460362.3596205376, 2460362.3597323857], [2460362.3602916263, 2460362.3604034744], [2460362.360962715, 2460362.361186411], [2460362.3614101075, 2460362.361745652], [2460362.361969348, 2460362.362081196], [2460362.3621930443, 2460362.3624167405], [2460362.3625285886, 2460362.362752285], [2460362.3630878287, 2460362.363311525], [2460362.363535221, 2460362.3636470693], [2460362.3637589174, 2460362.3639826137], [2460362.364430006, 2460362.3645418542], [2460362.365101095, 2460362.365212943], [2460362.365324791, 2460362.3656603354], [2460362.3657721835, 2460362.3658840316], [2460362.3665551203, 2460362.3666669684], [2460362.367226209, 2460362.367338057], [2460362.3678972977, 2460362.368009146], [2460362.368568386, 2460362.368680234], [2460362.368792082, 2460362.3689039303], [2460362.3690157784, 2460362.3692394746], [2460362.3693513228, 2460362.369463171], [2460362.369686867, 2460362.369798715], [2460362.3700224115, 2460362.3701342596], [2460362.3709171964, 2460362.3711408926], [2460362.3742726394, 2460362.3743844875], [2460362.3751674243, 2460362.3753911206], [2460362.3760622093, 2460362.3761740574], [2460362.3763977536, 2460362.3765096017], [2460362.376733298, 2460362.3771806904], [2460362.377739931, 2460362.3784110197], [2460362.379529501, 2460362.379641349], [2460362.3798650447, 2460362.380312437], [2460362.3805361334, 2460362.3806479815], [2460362.382996792, 2460362.383220488], [2460362.3833323363, 2460362.3834441844], [2460362.384115273, 2460362.3842271212], [2460362.384786362, 2460362.385010058], [2460362.3854574505, 2460362.3855692986], [2460362.386240387, 2460362.386352235], [2460362.3869114756, 2460362.3870233237], [2460362.387358868, 2460362.3876944124], [2460362.388253653, 2460362.388365501], [2460362.3889247417, 2460362.38903659], [2460362.390155071, 2460362.3904906153], [2460362.3910498554, 2460362.3911617035], [2460362.391497248, 2460362.391609096], [2460362.392503881, 2460362.392615729], [2460362.3931749696, 2460362.3932868177], [2460362.3938460583, 2460362.3939579064], [2460362.394517147, 2460362.394628995], [2460362.3948526913, 2460362.3949645394], [2460362.3951882357, 2460362.395300084], [2460362.39552378, 2460362.395635628], [2460362.3959711725, 2460362.3960830206], [2460362.3961948687, 2460362.396306717], [2460362.396642261, 2460362.396977805], [2460362.397089653, 2460362.3972015013], [2460362.3974251975, 2460362.3975370456], [2460362.3976488938, 2460362.397760742], [2460362.398096286, 2460362.3982081343], [2460362.398767375, 2460362.398879223], [2460362.398991071, 2460362.3991029193], [2460362.3993266155, 2460362.3994384636], [2460362.39966216, 2460362.399885856], [2460362.4003332485, 2460362.4004450967], [2460362.400668793, 2460362.400780641], [2460362.4011161854, 2460362.4012280335], [2460362.401787274, 2460362.401899122], [2460362.4024583623, 2460362.4025702104], [2460362.403129451, 2460362.4034649953], [2460362.4036886916, 2460362.4038005397], [2460362.404024236, 2460362.404247932], [2460362.4048071727, 2460362.404919021], [2460362.405254565, 2460362.4053664133], [2460362.4054782614, 2460362.4055901095], [2460362.405925654, 2460362.406037502], [2460362.406261198, 2460362.4063730463], [2460362.4073796794, 2460362.4074915275], [2460362.4076033756, 2460362.4077152237], [2460362.4081626157, 2460362.408274464], [2460362.4088337044, 2460362.4089455525], [2460362.4090574007, 2460362.409169249], [2460362.410399578, 2460362.410511426], [2460362.4110706667, 2460362.411182515], [2460362.4119654517, 2460362.4120773], [2460362.4126365404, 2460362.4127483885], [2460362.413307629, 2460362.413419477], [2460362.4139787173, 2460362.4143142616], [2460362.414649806, 2460362.414761654], [2460362.4149853503, 2460362.4152090466], [2460362.4153208947, 2460362.415432743], [2460362.415768287, 2460362.4158801353], [2460362.4161038315, 2460362.4162156796], [2460362.41677492, 2460362.4168867683], [2460362.417446009, 2460362.417557857], [2460362.4181170976, 2460362.4182289457], [2460362.4187881863, 2460362.4190118825], [2460362.4193474264, 2460362.4194592745], [2460362.4195711226, 2460362.4196829707], [2460362.419906667, 2460362.420018515], [2460362.420130363, 2460362.4203540594], [2460362.4209133, 2460362.421025148], [2460362.4215843887, 2460362.421696237], [2460362.423038414, 2460362.4231502623], [2460362.4234858067, 2460362.424604288], [2460362.42505168, 2460362.425163528], [2460362.4258346166, 2460362.4259464648], [2460362.426282009, 2460362.426393857], [2460362.4265057053, 2460362.4266175535], [2460362.426953098, 2460362.427064946], [2460362.4275123384, 2460362.4277360346], [2460362.428295275, 2460362.4284071233], [2460362.4286308195, 2460362.4287426677], [2460362.428966364, 2460362.42919006], [2460362.4297493007, 2460362.430084845], [2460362.4304203894, 2460362.4305322375], [2460362.4307559333, 2460362.4308677814], [2460362.4310914776, 2460362.4312033257], [2460362.4317625663, 2460362.4318744144], [2460362.4319862626, 2460362.4320981107], [2460362.432545503, 2460362.4326573513], [2460362.4329928956, 2460362.4331047437], [2460362.433216592, 2460362.43332844], [2460362.4338876805, 2460362.4339995286], [2460362.434558769, 2460362.4348943136], [2460362.435229858, 2460362.435341706], [2460362.4360127947, 2460362.436124643], [2460362.4363483386, 2460362.436572035], [2460362.436683883, 2460362.436907579], [2460362.4380260604, 2460362.4381379085], [2460362.438697149, 2460362.4390326934], [2460362.439480086, 2460362.439703782], [2460362.4400393264, 2460362.440486719], [2460362.4408222632, 2460362.4410459595], [2460362.441381504, 2460362.4416052], [2460362.441717048, 2460362.4418288963], [2460362.442052592, 2460362.4423881364], [2460362.4426118326, 2460362.4427236808], [2460362.442947377, 2460362.4433947694], [2460362.4435066176, 2460362.4436184657], [2460362.443730314, 2460362.443842162], [2460362.44395401, 2460362.444065858], [2460362.4441777063, 2460362.4445132506], [2460362.4446250987, 2460362.445072491], [2460362.4451843393, 2460362.445631732], [2460362.44574358, 2460362.4463028205], [2460362.4464146686, 2460362.446973909], [2460362.4470857573, 2460362.447421301], [2460362.4478686936, 2460362.4479805417], [2460362.44809239, 2460362.448316086], [2460362.448427934, 2460362.4485397823], [2460362.4486516304, 2460362.4487634785], [2460362.4488753267, 2460362.4495464154], [2460362.4496582635, 2460362.450105656], [2460362.450217504, 2460362.4504412003], [2460362.4505530484, 2460362.4507767446], [2460362.451335985, 2460362.4516715296], [2460362.452007074, 2460362.452118922], [2460362.4527900107, 2460362.453684795], [2460362.4541321876, 2460362.4542440358], [2460362.454355884, 2460362.454467732], [2460362.4548032763, 2460362.4550269726], [2460362.455474365, 2460362.4558099094], [2460362.4560336056, 2460362.4561454537], [2460362.456257302, 2460362.45636915], [2460362.4567046943, 2460362.4568165424], [2460362.4569283905, 2460362.4570402387], [2460362.457375783, 2460362.457487631], [2460362.4575994792, 2460362.458382416], [2460362.458829808, 2460362.4591653524], [2460362.459724593, 2460362.459948289], [2460362.4602838336, 2460362.460619378], [2460362.4610667704, 2460362.4612904666], [2460362.4622970996, 2460362.462520796], [2460362.4638629733, 2460362.4640866695], [2460362.4646459096, 2460362.4647577577], [2460362.4653169983, 2460362.4654288464], [2460362.465764391, 2460362.465876239], [2460362.466099935, 2460362.4663236313], [2460362.4665473276, 2460362.4666591757], [2460362.466771024, 2460362.46699472], [2460362.4674421125, 2460362.4676658087], [2460362.467889505, 2460362.4683368974], [2460362.46878429, 2460362.468896138], [2460362.469007986, 2460362.4691198342], [2460362.4694553786, 2460362.469790923], [2460362.4699027706, 2460362.4700146187], [2460362.470126467, 2460362.4707975555], [2460362.4709094036, 2460362.4711331], [2460362.4715804923, 2460362.4716923404], [2460362.4718041886, 2460362.4719160367], [2460362.472139733, 2460362.472251581], [2460362.472363429, 2460362.4725871254], [2460362.4726989735, 2460362.4729226697], [2460362.473034518, 2460362.473258214], [2460362.4735937584, 2460362.474264847], [2460362.474376695, 2460362.4744885433], [2460362.4748240877, 2460362.474935936], [2460362.475047784, 2460362.475159632], [2460362.47527148, 2460362.4754951764], [2460362.475607024, 2460362.4759425684], [2460362.4760544165, 2460362.476389961], [2460362.476501809, 2460362.4824297586], [2460362.4825416068, 2460362.482653455], [2460362.4832126955, 2460362.4833245436], [2460362.4834363917, 2460362.48354824], [2460362.483771936, 2460362.483883784], [2460362.4839956323, 2460362.4851141134], [2460362.4852259615, 2460362.4858970502], [2460362.4860088984, 2460362.4862325946], [2460362.4863444427, 2460362.487015531], [2460362.487127379, 2460362.487239227], [2460362.4873510753, 2460362.48824586], [2460362.4883577083, 2460362.4885814046], [2460362.4886932527, 2460362.490930215], [2460362.491153911, 2460362.4928316325], [2460362.4930553287, 2460362.494061962], [2460362.49417381, 2460362.4948448986], [2460362.495404139, 2460362.4955159873], [2460362.4956278354, 2460362.4958515316], [2460362.4959633797, 2460362.497193709], [2460362.497305557, 2460362.4975292534], [2460362.4976411015, 2460362.4977529496], [2460362.497976646, 2460362.5008846964], [2460362.5009965445, 2460362.5011083926], [2460362.501443937, 2460362.502338722], [2460362.502562418, 2460362.504687532], [2460362.50479938, 2460362.5054704687], [2460362.505582317, 2460362.505806013], [2460362.5062534055, 2460362.507707431], [2460362.507931127, 2460362.508490368], [2460362.508602216, 2460362.5090496084], [2460362.5093851523, 2460362.5094970004], [2460362.5096088485, 2460362.5097206966], [2460362.5098325447, 2460362.509944393], [2460362.510056241, 2460362.5105036334], [2460362.5106154815, 2460362.51128657], [2460362.5115102665, 2460362.5116221146], [2460362.511957659, 2460362.5122932033], [2460362.5124050514, 2460362.512852444], [2460362.51307614, 2460362.5146420137], [2460362.514753862, 2460362.51542495], [2460362.515536798, 2460362.5157604944], [2460362.5158723425, 2460362.5159841906], [2460362.5160960387, 2460362.517326368], [2460362.5175500643, 2460362.5176619124], [2460362.5178856086, 2460362.5179974567], [2460362.518333001, 2460362.5195633303], [2460362.5197870266, 2460362.5198988747], [2460362.520010723, 2460362.520234419], [2460362.520681811, 2460362.5209055073], [2460362.5213528997, 2460362.521464748], [2460362.521800292, 2460362.5219121403], [2460362.522471381, 2460362.5229187733], [2460362.5230306215, 2460362.52370171], [2460362.5238135583, 2460362.5241491026], [2460362.524484647, 2460362.524708343], [2460362.5248201913, 2460362.5249320394], [2460362.5250438876, 2460362.5260505206], [2460362.5262742164, 2460362.528063786], [2460362.5281756343, 2460362.528623027], [2460362.528734875, 2460362.52962966], [2460362.529741508, 2460362.5305244448], [2460362.530748141, 2460362.530971837], [2460362.5310836853, 2460362.531531078], [2460362.531642926, 2460362.531754774], [2460362.5318666217, 2460362.5327614066], [2460362.5328732547, 2460362.533096951], [2460362.533991736, 2460362.5345509765], [2460362.5347746727, 2460362.535110217], [2460362.536005002, 2460362.536228698], [2460362.538241964, 2460362.538353812], [2460362.53846566, 2460362.538577508], [2460362.5386893563, 2460362.5388012044], [2460362.5389130525, 2460362.5391367488], [2460362.539248597, 2460362.539584141], [2460362.5398078375, 2460362.5399196856], [2460362.54025523, 2460362.540367078], [2460362.5407026224, 2460362.5409263186], [2460362.5439462173, 2460362.5440580654], [2460362.5443936097, 2460362.544505458], [2460362.545400243, 2460362.545512091], [2460362.5460713315, 2460362.5461831796], [2460362.54674242, 2460362.5468542683], [2460362.547413509, 2460362.547525357], [2460362.5480845976, 2460362.548755686], [2460362.548867534, 2460362.548979382], [2460362.5493149264, 2460362.5494267745], [2460362.5495386226, 2460362.5496504707], [2460362.550097863, 2460362.5505452557], [2460362.5508808, 2460362.5512163443], [2460362.5514400406, 2460362.5521111293], [2460362.5523348255, 2460362.552894066], [2460362.553005914, 2460362.5544599392], [2460362.5546836355, 2460362.55501918], [2460362.555131028, 2460362.555242876], [2460362.5554665723, 2460362.5555784204], [2460362.5558021166, 2460362.556025813], [2460362.556361357, 2460362.5565850534], [2460362.556920598, 2460362.557032446], [2460362.557144294, 2460362.5577035346], [2460362.5578153827, 2460362.5588220158], [2460362.558933864, 2460362.5597168007], [2460362.5599404965, 2460362.5601641927], [2460362.560387889, 2460362.560499737], [2460362.5608352814, 2460362.562401155], [2460362.5628485475, 2460362.563519636], [2460362.5636314843, 2460362.5639670286], [2460362.564190725, 2460362.565756598], [2460362.565868446, 2460362.566651383], [2460362.5670987754, 2460362.567658016], [2460362.567769864, 2460362.568664649], [2460362.568776497, 2460362.5690001934], [2460362.5691120415, 2460362.570677915], [2460362.5707897632, 2460362.5709016114], [2460362.5711253076, 2460362.5751518393], [2460362.5752636874, 2460362.5754873836], [2460362.57571108, 2460362.578171738], [2460362.578283586, 2460362.5786191304], [2460362.5787309785, 2460362.5788428267], [2460362.578954675, 2460362.579066523], [2460362.579178371, 2460362.5798494597], [2460362.579961308, 2460362.5806323965], [2460362.5807442446, 2460362.5824219664], [2460362.582533814, 2460362.583428599], [2460362.583540447, 2460362.583652295], [2460362.5838759914, 2460362.584323384], [2460362.584435232, 2460362.584658928], [2460362.5849944726, 2460362.5851063207], [2460362.585218169, 2460362.586560346], [2460362.5866721943, 2460362.5868958905], [2460362.5870077386, 2460362.587343283], [2460362.587455131, 2460362.587566979], [2460362.5876788273, 2460362.58812622], [2460362.5882380675, 2460362.5883499156], [2460362.588573612, 2460362.58868546], [2460362.5891328524, 2460362.5893565486], [2460362.5894683967, 2460362.592600144], [2460362.5931593846, 2460362.5956200426], [2460362.5957318908, 2460362.595955587], [2460362.597074068, 2460362.5971859163], [2460362.5972977644, 2460362.5999821187], [2460362.600205815, 2460362.600317663], [2460362.6007650555, 2460362.6008769036], [2460362.6009887517, 2460362.6011006], [2460362.601324296, 2460362.6024427772], [2460362.6027783216, 2460362.603113866], [2460362.603225714, 2460362.6034494103], [2460362.6035612584, 2460362.6038968028], [2460362.604008651, 2460362.604232347], [2460362.604344195, 2460362.6044560433], [2460362.6057982203, 2460362.6060219165], [2460362.6062456127, 2460362.606357461], [2460362.606469309, 2460362.6068048533], [2460362.6070285495, 2460362.607364094], [2460362.607699638, 2460362.608482575], [2460362.608594423, 2460362.609601056], [2460362.6097129043, 2460362.6100484487], [2460362.610160297, 2460362.610383993], [2460362.610495841, 2460362.6149697653], [2460362.6150816134, 2460362.6151934615], [2460362.615529006, 2460362.6160882465], [2460362.616200094, 2460362.616311942], [2460362.6164237903, 2460362.6193318414], [2460362.6194436895, 2460362.6273849052], [2460362.627608601, 2460362.627720449], [2460362.6278322972, 2460362.629621867], [2460362.629733715, 2460362.6320825256], [2460362.6321943738, 2460362.635549817], [2460362.635661665, 2460362.637786779], [2460362.637898627, 2460362.638681564], [2460362.6387934117, 2460362.6398000447], [2460362.639911893, 2460362.6403592853], [2460362.6405829815, 2460362.6414777664], [2460362.641925159, 2460362.642148855], [2460362.6423725514, 2460362.6476294124]] freq_flags: [[46981811.5234375, 87265014.6484375], [87509155.2734375, 108139038.0859375], [108383178.7109375, 234115600.5859375]] ex_ants: [[3, Jnn], [10, Jee], [18, Jee], [18, Jnn], [27, Jee], [27, Jnn], [28, Jee], [28, Jnn], [31, Jnn], [32, Jnn], [34, Jee], [34, Jnn], [35, Jee], [40, Jnn], [45, Jee], [45, Jnn], [46, Jee], [47, Jee], [47, Jnn], [51, Jee], [54, Jee], [55, Jee], [56, Jee], [56, Jnn], [61, Jee], [61, Jnn], [63, Jee], [63, Jnn], [64, Jee], [64, Jnn], [68, Jnn], [73, Jee], [73, Jnn], [77, Jee], [77, Jnn], [78, Jee], [78, Jnn], [86, Jee], [86, Jnn], [87, Jee], [88, Jee], [88, Jnn], [89, Jee], [89, Jnn], [90, Jee], [90, Jnn], [92, Jee], [93, Jee], [93, Jnn], [95, Jee], [97, Jnn], [98, Jnn], [99, Jee], [99, Jnn], [104, Jee], [104, Jnn], [107, Jee], [107, Jnn], [108, Jnn], [109, Jnn], [111, Jee], [112, Jee], [114, Jnn], [115, Jee], [115, Jnn], [119, Jnn], [127, Jee], [127, Jnn], [130, Jnn], [136, Jnn], [142, Jnn], [161, Jnn], [165, Jee], [165, Jnn], [166, Jee], [166, Jnn], [170, Jee], [171, Jnn], [174, Jee], [174, Jnn], [176, Jee], [177, Jee], [180, Jnn], [183, Jee], [183, Jnn], [184, Jee], [184, Jnn], [188, Jnn], [194, Jee], [194, Jnn], [196, Jee], [196, Jnn], [197, Jnn], [199, Jnn], [200, Jee], [202, Jnn], [205, Jee], [208, Jee], [212, Jnn], [213, Jee], [217, Jee], [218, Jnn], [229, Jee], [232, Jee], [232, Jnn], [243, Jee], [245, Jnn], [251, Jee], [253, Jee], [253, Jnn], [255, Jnn], [256, Jee], [256, Jnn], [266, Jee], [266, Jnn], [269, Jnn], [270, Jee], [270, Jnn], [272, Jee], [272, Jnn], [283, Jee], [283, Jnn], [321, Jnn], [323, Jee], [323, Jnn], [324, Jee], [324, Jnn], [325, Jnn], [327, Jee], [327, Jnn], [328, Jee], [328, Jnn], [329, Jee], [329, Jnn], [331, Jee], [331, Jnn], [332, Jee], [332, Jnn], [333, Jee], [336, Jnn]]
Metadata¶
In [24]:
for repo in ['hera_cal', 'hera_qm', 'hera_filters', 'hera_notebook_templates', 'pyuvdata']:
exec(f'from {repo} import __version__')
print(f'{repo}: {__version__}')
hera_cal: 3.4.2.dev11+ga9e1297 hera_qm: 2.1.3.dev5+g3e71720 hera_filters: 0.1.0 hera_notebook_templates: 0.1.dev486+gfb8560a pyuvdata: 2.4.0
In [25]:
print(f'Finished execution in {(time.time() - tstart) / 60:.2f} minutes.')
Finished execution in 28.11 minutes.