Second Round of Full Day RFI Flagging¶
by Josh Dillon, last updated July 31, 2023
This notebook is synthesizes information from individual delay_filtered_average_zscore notebooks to find low-level RFI and flag it. That notebook takes smooth_calibrated data, redundantly averages it, performs a high-pass delay filter, and then incoherently averages across baselines, creating a per-polarization z-score. This notebook then takes that whole night of z-scores and finds a new set of flags to both add to the smooth_cal files, which are updated in place, and to write down as new UVFlag waterfall-type .h5 files.
Here's a set of links to skip to particular figures and tables:
• Figure 1: Waterfall of Maximum z-Score of Either Polarization Before Round 2 Flagging¶
• Figure 2: Histogram of z-scores¶
• Figure 3: Waterfall of Maximum z-Score of Either Polarization After Round 2 Flagging¶
• Figure 4: Spectra of Time-Averaged z-Scores¶
• Figure 5: Summary of Flags Before and After Round 2 Flagging¶
In [1]:
import time
tstart = time.time()
In [2]:
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin # REQUIRED to have the compression plugins available
import numpy as np
import glob
import matplotlib.pyplot as plt
import matplotlib
import copy
import warnings
from pyuvdata import UVFlag, UVCal
from hera_cal import utils
from hera_qm import xrfi
from hera_qm.time_series_metrics import true_stretches
from IPython.display import display, HTML
%matplotlib inline
display(HTML("<style>.container { width:100% !important; }</style>"))
_ = np.seterr(all='ignore') # get rid of red warnings
%config InlineBackend.figure_format = 'retina'
In [3]:
# get input data file names
SUM_FILE = os.environ.get("SUM_FILE", None)
# SUM_FILE = '/lustre/aoc/projects/hera/h6c-analysis/IDR2/2459861/zen.2459861.25297.sum.uvh5'
SUM_SUFFIX = os.environ.get("SUM_SUFFIX", 'sum.uvh5')
# get input and output suffixes
SMOOTH_CAL_SUFFIX = os.environ.get("SMOOTH_CAL_SUFFIX", 'sum.smooth.calfits')
ZSCORE_SUFFIX = os.environ.get("ZSCORE_SUFFIX", 'sum.red_avg_zscore.h5')
FLAG_WATERFALL2_SUFFIX = os.environ.get("FLAG_WATERFALL2_SUFFIX", 'sum.flag_waterfall_round_2.h5')
OUT_YAML_SUFFIX = os.environ.get("OUT_YAML_SUFFIX", '_aposteriori_flags.yaml')
OUT_YAML_DIR = os.environ.get("OUT_YAML_DIR", None)
# build globs
sum_glob = '.'.join(SUM_FILE.split('.')[:-3]) + '.*.' + SUM_SUFFIX
cal_files_glob = sum_glob.replace(SUM_SUFFIX, SMOOTH_CAL_SUFFIX)
zscore_glob = sum_glob.replace(SUM_SUFFIX, ZSCORE_SUFFIX)
# build out yaml file
if OUT_YAML_DIR is None:
OUT_YAML_DIR = os.path.dirname(SUM_FILE)
out_yaml_file = os.path.join(OUT_YAML_DIR, SUM_FILE.split('.')[-4] + OUT_YAML_SUFFIX)
# get flagging parameters
Z_THRESH = float(os.environ.get("Z_THRESH", 5))
WS_Z_THRESH = float(os.environ.get("WS_Z_THRESH", 4))
AVG_Z_THRESH = float(os.environ.get("AVG_Z_THRESH", 1))
MAX_FREQ_FLAG_FRAC = float(os.environ.get("MAX_FREQ_FLAG_FRAC", .25))
MAX_TIME_FLAG_FRAC = float(os.environ.get("MAX_TIME_FLAG_FRAC", .1))
for setting in ['Z_THRESH', 'WS_Z_THRESH', 'AVG_Z_THRESH', 'MAX_FREQ_FLAG_FRAC', 'MAX_TIME_FLAG_FRAC']:
print(f'{setting} = {eval(setting)}')
Z_THRESH = 5.0 WS_Z_THRESH = 4.0 AVG_Z_THRESH = 1.0 MAX_FREQ_FLAG_FRAC = 0.25 MAX_TIME_FLAG_FRAC = 0.1
Load z-scores¶
In [4]:
# load z-scores
zscore_files = sorted(glob.glob(zscore_glob))
print(f'Found {len(zscore_files)} *.{ZSCORE_SUFFIX} files starting with {zscore_files[0]}.')
uvf = UVFlag(zscore_files, use_future_array_shapes=True)
Found 1936 *.sum.red_avg_zscore.h5 files starting with /mnt/sn1/data2/2460592/zen.2460592.25243.sum.red_avg_zscore.h5.
In [5]:
# get calibration solution files
cal_files = sorted(glob.glob(cal_files_glob))
print(f'Found {len(cal_files)} *.{SMOOTH_CAL_SUFFIX} files starting with {cal_files[0]}.')
Found 1936 *.sum.smooth.calfits files starting with /mnt/sn1/data2/2460592/zen.2460592.25243.sum.smooth.calfits.
In [6]:
assert len(zscore_files) == len(cal_files)
In [7]:
# extract z-scores and correct by a single number per polarization to account for biases created by filtering
zscore = {pol: uvf.metric_array[:, :, np.argwhere(uvf.polarization_array == utils.polstr2num(pol, x_orientation=uvf.x_orientation))[0][0]] for pol in ['ee', 'nn']}
zscore = {pol: zscore[pol] - np.nanmedian(zscore[pol]) for pol in zscore}
In [8]:
freqs = uvf.freq_array
times = uvf.time_array
In [9]:
extent = [freqs[0] / 1e6, freqs[-1] / 1e6, times[-1] - int(times[0]), times[0] - int(times[0])]
In [10]:
def plot_max_z_score(zscore, flags=None):
if flags is None:
flags = np.any(~np.isfinite(list(zscore.values())), axis=0)
plt.figure(figsize=(14,10), dpi=100)
plt.imshow(np.where(flags, np.nan, np.nanmax([zscore['ee'], zscore['nn']], axis=0)), aspect='auto',
cmap='coolwarm', interpolation='none', vmin=-10, vmax=10, extent=extent)
plt.colorbar(location='top', label='Max z-score of either polarization', extend='both', aspect=40, pad=.02)
plt.xlabel('Frequency (MHz)')
plt.ylabel(f'JD - {int(times[0])}')
plt.tight_layout()
Figure 1: Waterfall of Maximum z-Score of Either Polarization Before Round 2 Flagging¶
Shows the worse of the two results from delay_filtered_average_zscore from either polarization. Dips near flagged channels are expected, due to overfitting of noise. Positive-going excursions are problematic and likely evidence of RFI.
In [11]:
plot_max_z_score(zscore)
All-NaN axis encountered
In [12]:
def plot_histogram():
plt.figure(figsize=(14,4), dpi=100)
bins = np.arange(-50, 100, .1)
hist_ee = plt.hist(np.ravel(zscore['ee']), bins=bins, density=True, label='ee-polarized z-scores', alpha=.5)
hist_nn = plt.hist(np.ravel(zscore['nn']), bins=bins, density=True, label='nn-polarized z-scores', alpha=.5)
plt.plot(bins, (2*np.pi)**-.5 * np.exp(-bins**2 / 2), 'k:', label='Gaussian approximate\nnoise-only distribution')
plt.axvline(WS_Z_THRESH, c='r', ls='--', label='Watershed z-score')
plt.axvline(Z_THRESH, c='r', ls='-', label='Threshold z-score')
plt.yscale('log')
all_densities = np.concatenate([hist_ee[0][hist_ee[0] > 0], hist_nn[0][hist_nn[0] > 0]])
plt.ylim(np.min(all_densities) / 2, np.max(all_densities) * 2)
plt.xlim([-50, 100])
plt.legend()
plt.xlabel('z-score')
plt.ylabel('Density')
plt.tight_layout()
Figure 2: Histogram of z-scores¶
Shows a comparison of the histogram of z-scores in this file (one per polarization) to a Gaussian approximation of what one might expect from thermal noise. Without filtering, the actual distribution is a weighted sum of Rayleigh distributions. Filtering further complicates this. To make the z-scores more reliable, a single per-polarization median is subtracted from each waterfall, which allows us to flag low-level outliers with more confidence. Any points beyond the solid red line are flagged. Any points neighboring a flag beyond the dashed red line are also flagged. Finally, flagging is performed for low-level outliers in whole times or channels.
In [13]:
plot_histogram()
Perform flagging¶
In [14]:
def iteratively_flag_on_averaged_zscore(flags, zscore, avg_z_thresh=1.5, verbose=True):
'''Flag whole integrations or channels based on average z-score. This is done
iteratively to prevent bad times affecting channel averages or vice versa.'''
flagged_chan_count = 0
flagged_int_count = 0
while True:
zspec = np.nanmean(np.where(flags, np.nan, zscore), axis=0)
ztseries = np.nanmean(np.where(flags, np.nan, zscore), axis=1)
if (np.nanmax(zspec) < avg_z_thresh) and (np.nanmax(ztseries) < avg_z_thresh):
break
if np.nanmax(zspec) >= np.nanmax(ztseries):
flagged_chan_count += np.sum((zspec >= np.nanmax(ztseries)) & (zspec >= avg_z_thresh))
flags[:, (zspec >= np.nanmax(ztseries)) & (zspec >= avg_z_thresh)] = True
else:
flagged_int_count += np.sum((ztseries >= np.nanmax(zspec)) & (ztseries >= avg_z_thresh))
flags[(ztseries >= np.nanmax(zspec)) & (ztseries >= avg_z_thresh), :] = True
if verbose:
print(f'\tFlagging an additional {flagged_int_count} integrations and {flagged_chan_count} channels.')
def impose_max_chan_flag_frac(flags, max_flag_frac=.25, verbose=True):
'''Flag channels already flagged more than max_flag_frac (excluding completely flagged times).'''
unflagged_times = ~np.all(flags, axis=1)
frequently_flagged_chans = np.mean(flags[unflagged_times, :], axis=0) >= max_flag_frac
if verbose:
print(f'\tFlagging {np.sum(frequently_flagged_chans) - np.sum(np.all(flags, axis=0))} channels previously flagged {max_flag_frac:.2%} or more.')
flags[:, frequently_flagged_chans] = True
def impose_max_time_flag_frac(flags, max_flag_frac=.25, verbose=True):
'''Flag times already flagged more than max_flag_frac (excluding completely flagged channels).'''
unflagged_chans = ~np.all(flags, axis=0)
frequently_flagged_times = np.mean(flags[:, unflagged_chans], axis=1) >= max_flag_frac
if verbose:
print(f'\tFlagging {np.sum(frequently_flagged_times) - np.sum(np.all(flags, axis=1))} times previously flagged {max_flag_frac:.2%} or more.')
flags[frequently_flagged_times, :] = True
In [15]:
flags = np.any(~np.isfinite(list(zscore.values())), axis=0)
print(f'{np.mean(flags):.3%} of waterfall flagged to start.')
# flag largest outliers
for pol in ['ee', 'nn']:
flags |= (zscore[pol] > Z_THRESH)
print(f'{np.mean(flags):.3%} of waterfall flagged after flagging z > {Z_THRESH} outliers.')
# watershed flagging
while True:
nflags = np.sum(flags)
for pol in ['ee', 'nn']:
flags |= xrfi._ws_flag_waterfall(zscore[pol], flags, WS_Z_THRESH)
if np.sum(flags) == nflags:
break
print(f'{np.mean(flags):.3%} of waterfall flagged after watershed flagging on z > {WS_Z_THRESH} neightbors of prior flags.')
# flag whole integrations or channels
while True:
nflags = np.sum(flags)
for pol in ['ee', 'nn']:
iteratively_flag_on_averaged_zscore(flags, zscore[pol], avg_z_thresh=AVG_Z_THRESH, verbose=True)
impose_max_chan_flag_frac(flags, max_flag_frac=MAX_FREQ_FLAG_FRAC, verbose=True)
impose_max_time_flag_frac(flags, max_flag_frac=MAX_TIME_FLAG_FRAC, verbose=True)
if np.sum(flags) == nflags:
break
print(f'{np.mean(flags):.3%} of waterfall flagged after flagging whole times and channels with average z > {AVG_Z_THRESH}.')
30.484% of waterfall flagged to start. 32.449% of waterfall flagged after flagging z > 5.0 outliers.
32.941% of waterfall flagged after watershed flagging on z > 4.0 neightbors of prior flags.
Mean of empty slice
Mean of empty slice
Flagging an additional 0 integrations and 7 channels. Flagging 17 channels previously flagged 25.00% or more. Flagging 284 times previously flagged 10.00% or more.
Flagging an additional 0 integrations and 6 channels. Flagging 0 channels previously flagged 25.00% or more. Flagging 0 times previously flagged 10.00% or more.
Flagging an additional 0 integrations and 0 channels. Flagging 0 channels previously flagged 25.00% or more. Flagging 0 times previously flagged 10.00% or more.
Flagging an additional 0 integrations and 0 channels. Flagging 0 channels previously flagged 25.00% or more. Flagging 0 times previously flagged 10.00% or more. 39.221% of waterfall flagged after flagging whole times and channels with average z > 1.0.
Show results of flagging¶
Figure 3: Waterfall of Maximum z-Score of Either Polarization After Round 2 Flagging¶
The same as Figure 1, but after the flagging performed in this notebook.
In [16]:
plot_max_z_score(zscore, flags=flags)
All-NaN axis encountered
In [17]:
def zscore_spectra():
fig, axes = plt.subplots(2, 1, figsize=(14,6), dpi=100, sharex=True, sharey=True, gridspec_kw={'hspace': 0})
for ax, pol in zip(axes, ['ee', 'nn']):
ax.plot(freqs / 1e6, np.nanmean(zscore[pol], axis=0),'r', label=f'{pol}-Polarization Before Round 2 Flagging', lw=.5)
ax.plot(freqs / 1e6, np.nanmean(np.where(flags, np.nan, zscore[pol]), axis=0), label=f'{pol}-Polarization After Round 2 Flagging')
ax.legend(loc='lower right')
ax.set_ylabel('Time-Averged Z-Score\n(Excluding Flags)')
ax.set_ylim(-11, 11)
axes[1].set_xlabel('Frequency (MHz)')
plt.tight_layout()
Figure 4: Spectra of Time-Averaged z-Scores¶
The average along the time axis of Figures 1 and 3 (though now separated per-polarization). This plot is useful for showing channels with repeated low-level RFI.
In [18]:
zscore_spectra()
Mean of empty slice
Mean of empty slice
In [19]:
def summarize_flagging():
plt.figure(figsize=(14,10), dpi=100)
cmap = matplotlib.colors.ListedColormap(((0, 0, 0),) + matplotlib.cm.get_cmap("Set2").colors[0:2])
plt.imshow(np.where(np.any(~np.isfinite(list(zscore.values())), axis=0), 1, np.where(flags, 2, 0)),
aspect='auto', cmap=cmap, interpolation='none', extent=extent)
plt.clim([-.5, 2.5])
cbar = plt.colorbar(location='top', aspect=40, pad=.02)
cbar.set_ticks([0, 1, 2])
cbar.set_ticklabels(['Unflagged', 'Previously Flagged', 'Flagged Here Using Delayed Filtered z-Scores'])
plt.xlabel('Frequency (MHz)')
plt.ylabel(f'JD - {int(times[0])}')
plt.tight_layout()
Figure 5: Summary of Flags Before and After Round 2 Flagging¶
This plot shows which times and frequencies were flagged before and after this notebook. It is directly comparable to Figure 5 of the first round full_day_rfi notebook.
In [20]:
summarize_flagging()
The get_cmap function was deprecated in Matplotlib 3.7 and will be removed two minor releases later. Use ``matplotlib.colormaps[name]`` or ``matplotlib.colormaps.get_cmap(obj)`` instead.
Save results¶
In [21]:
add_to_history = 'by full_day_rfi_round_2 notebook with the following environment:\n' + '=' * 65 + '\n' + os.popen('conda env export').read() + '=' * 65
In [22]:
tind = 0
always_flagged_ants = set()
ever_unflagged_ants = set()
for cal_file in cal_files:
with warnings.catch_warnings():
warnings.simplefilter("ignore")
# update cal_file
uvc = UVCal()
uvc.read(cal_file, use_future_array_shapes=True)
uvc.flag_array |= (flags[tind:tind + len(uvc.time_array), :].T)[None, :, :, None]
uvc.history += 'Modified ' + add_to_history
uvc.write_calfits(cal_file, clobber=True)
# keep track of flagged antennas
for antnum in uvc.ant_array:
for antpol in ['Jee', 'Jnn']:
if np.all(uvc.get_flags(antnum, antpol)):
if (antnum, antpol) not in ever_unflagged_ants:
always_flagged_ants.add((antnum, antpol))
else:
ever_unflagged_ants.add((antnum, antpol))
always_flagged_ants.discard((antnum, antpol))
# Create new flag object
uvf_out = UVFlag(uvc, waterfall=True, mode='flag')
uvf_out.flag_array |= flags[tind:tind + len(uvc.time_array), :, None]
uvf_out.history += 'Produced ' + add_to_history
uvf_out.write(cal_file.replace(SMOOTH_CAL_SUFFIX, FLAG_WATERFALL2_SUFFIX), clobber=True)
# increment time index
tind += len(uvc.time_array)
print(f'Saved {len(cal_files)} *.{FLAG_WATERFALL2_SUFFIX} files starting with {cal_files[0].replace(SMOOTH_CAL_SUFFIX, FLAG_WATERFALL2_SUFFIX)}.')
Saved 1936 *.sum.flag_waterfall_round_2.h5 files starting with /mnt/sn1/data2/2460592/zen.2460592.25243.sum.flag_waterfall_round_2.h5.
In [23]:
# write summary of entirely flagged times/freqs/ants to yaml
all_flagged_times = np.all(flags, axis=1)
all_flagged_freqs = np.all(flags, axis=0)
all_flagged_ants = sorted(always_flagged_ants)
dt = np.median(np.diff(times))
out_yml_str = 'JD_flags: ' + str([[times[flag_stretch][0] - dt / 2, times[flag_stretch][-1] + dt / 2]
for flag_stretch in true_stretches(all_flagged_times)])
df = np.median(np.diff(freqs))
out_yml_str += '\n\nfreq_flags: ' + str([[freqs[flag_stretch][0] - df / 2, freqs[flag_stretch][-1] + df / 2]
for flag_stretch in true_stretches(all_flagged_freqs)])
out_yml_str += '\n\nex_ants: ' + str(all_flagged_ants).replace("'", "").replace('(', '[').replace(')', ']')
print(f'Writing the following to {out_yaml_file}\n' + '-' * (25 + len(out_yaml_file)))
print(out_yml_str)
with open(out_yaml_file, 'w') as outfile:
outfile.writelines(out_yml_str)
Writing the following to /mnt/sn1/data2/2460592/2460592_aposteriori_flags.yaml ------------------------------------------------------------------------------ JD_flags: [[2460592.2523226016, 2460592.253664779], [2460592.253776627, 2460592.2545595635], [2460592.2546714116, 2460592.2565728296], [2460592.256796526, 2460592.2573557664], [2460592.2574676145, 2460592.258809792], [2460592.25892164, 2460592.2593690325], [2460592.2594808806, 2460592.260151969], [2460592.260263817, 2460592.260487513], [2460592.2605993613, 2460592.2607112094], [2460592.2608230575, 2460592.2610467537], [2460592.261382298, 2460592.2618296905], [2460592.262053387, 2460592.262388931], [2460592.2627244755, 2460592.2628363236], [2460592.2629481717, 2460592.26306002], [2460592.2637311085, 2460592.2639548047], [2460592.264402197, 2460592.2645140453], [2460592.2646258934, 2460592.2647377416], [2460592.264961438, 2460592.265073286], [2460592.265744374, 2460592.2659680704], [2460592.2661917666, 2460592.266415463], [2460592.2669747034, 2460592.2673102478], [2460592.267645792, 2460592.26775764], [2460592.2678694883, 2460592.2680931846], [2460592.2688761214, 2460592.2689879695], [2460592.2697709063, 2460592.2698827544], [2460592.270441995, 2460592.270553843], [2460592.2712249313, 2460592.2713367795], [2460592.2725671087, 2460592.272678957], [2460592.2733500456, 2460592.2734618937], [2460592.273797438, 2460592.273909286], [2460592.2741329824, 2460592.2742448305], [2460592.274915919, 2460592.2750277673], [2460592.275698856, 2460592.2763699447], [2460592.2786069065, 2460592.2787187546], [2460592.278942451, 2460592.279166147], [2460592.279277995, 2460592.2795016915], [2460592.28017278, 2460592.2803964764], [2460592.2812912613, 2460592.282074198], [2460592.282409742, 2460592.2827452864], [2460592.2828571345, 2460592.283304527], [2460592.2836400713, 2460592.284423008], [2460592.284534856, 2460592.2846467043], [2460592.2857651855, 2460592.28610073], [2460592.286212578, 2460592.2865481223], [2460592.288785084, 2460592.2888969323], [2460592.289791717, 2460592.2901272615], [2460592.2911338946, 2460592.2912457427], [2460592.291357591, 2460592.291469439], [2460592.2941537932, 2460592.2942656414], [2460592.2956078188, 2460592.295719667], [2460592.2963907556, 2460592.2965026037], [2460592.2970618443, 2460592.2971736924], [2460592.297844781, 2460592.297956629], [2460592.2997461986, 2460592.2998580467], [2460592.3043319713, 2460592.3044438194], [2460592.304667515, 2460592.3048912114], [2460592.306233389, 2460592.306345237], [2460592.310036225, 2460592.310148073], [2460592.310707313, 2460592.310819161], [2460592.3131679716, 2460592.3132798197], [2460592.315404934, 2460592.315516782], [2460592.3171945033, 2460592.3175300476], [2460592.318089288, 2460592.318872225], [2460592.3195433137, 2460592.3202144024], [2460592.320997339, 2460592.3211091873], [2460592.3213328836, 2460592.3214447317], [2460592.3217802756, 2460592.3218921237], [2460592.32211582, 2460592.322227668], [2460592.3227869086, 2460592.323122453], [2460592.3234579973, 2460592.3235698454], [2460592.3239053898, 2460592.327596377], [2460592.327708225, 2460592.3280437696], [2460592.3281556177, 2460592.328379314], [2460592.328491162, 2460592.3308399725], [2460592.3309518206, 2460592.3316229093], [2460592.3317347574, 2460592.3319584536], [2460592.3320703018, 2460592.332405846], [2460592.3333006306, 2460592.3334124787], [2460592.333748023, 2460592.3340835674], [2460592.3341954155, 2460592.3343072636], [2460592.3344191117, 2460592.33453096], [2460592.3363205297, 2460592.3371034665], [2460592.3373271627, 2460592.337550859], [2460592.337662707, 2460592.3385574915], [2460592.3386693397, 2460592.3392285802], [2460592.3393404284, 2460592.3396759727], [2460592.340123365, 2460592.3403470614], [2460592.3405707576, 2460592.3406826057], [2460592.340794454, 2460592.340906302], [2460592.341129998, 2460592.3412418463], [2460592.3415773907, 2460592.341912935], [2460592.3421366313, 2460592.342695872], [2460592.34280772, 2460592.3431432643], [2460592.3432551124, 2460592.3433669605], [2460592.3434788086, 2460592.343814353], [2460592.344149897, 2460592.3447091375], [2460592.345044682, 2460592.34515653], [2460592.3457157705, 2460592.3458276186], [2460592.3459394667, 2460592.346163163], [2460592.346386859, 2460592.3464987073], [2460592.3468342517, 2460592.347281644], [2460592.3486238215, 2460592.3487356696], [2460592.349183062, 2460592.34929491], [2460592.3511963277, 2460592.351420024], [2460592.3517555683, 2460592.3518674164], [2460592.3539925306, 2460592.3541043787], [2460592.3552228594, 2460592.3554465557], [2460592.3573479736, 2460592.3574598217], [2460592.3580190623, 2460592.3581309104], [2460592.358578303, 2460592.358690151], [2460592.359696784, 2460592.359808632], [2460592.360479721, 2460592.360591569], [2460592.3626048346, 2460592.3627166827], [2460592.3676379994, 2460592.3677498475], [2460592.3687564805, 2460592.3688683286], [2460592.372671164, 2460592.372783012], [2460592.3728948603, 2460592.3730067085], [2460592.3742370377, 2460592.374460734], [2460592.3760266076, 2460592.3761384557], [2460592.377256937, 2460592.377368785], [2460592.377704329, 2460592.377816177], [2460592.3784872657, 2460592.37882281], [2460592.379605747, 2460592.379717595], [2460592.3812834686, 2460592.3813953167], [2460592.3836322785, 2460592.3837441267], [2460592.3844152153, 2460592.3845270635], [2460592.3855336965, 2460592.3856455446], [2460592.3863166333, 2460592.3865403295], [2460592.386875874, 2460592.386987722], [2460592.3876588107, 2460592.387770659], [2460592.388106203, 2460592.3882180513], [2460592.3885535956, 2460592.388777292], [2460592.389224684, 2460592.389336532], [2460592.3896720763, 2460592.3897839244], [2460592.390343165, 2460592.390455013], [2460592.391461646, 2460592.3915734943], [2460592.3920208868, 2460592.392132735], [2460592.392356431, 2460592.3925801273], [2460592.393139368, 2460592.393251216], [2460592.3935867604, 2460592.3936986085], [2460592.3939223047, 2460592.394146001], [2460592.394257849, 2460592.394369697], [2460592.3944815453, 2460592.3945933934], [2460592.3949289373, 2460592.3950407854], [2460592.3952644817, 2460592.395488178], [2460592.395600026, 2460592.395711874], [2460592.3960474185, 2460592.3961592666], [2460592.3971658996, 2460592.397389596], [2460592.397501444, 2460592.397613292], [2460592.39772514, 2460592.3978369883], [2460592.3979488364, 2460592.3980606846], [2460592.39951471, 2460592.399626558], [2460592.3998502544, 2460592.3999621025], [2460592.4001857983, 2460592.4002976464], [2460592.4004094945, 2460592.4005213426], [2460592.4006331908, 2460592.400745039], [2460592.400856887, 2460592.4014161276], [2460592.4015279757, 2460592.401751672], [2460592.40186352, 2460592.401975368], [2460592.4021990644, 2460592.4024227606], [2460592.402758305, 2460592.402870153], [2460592.4033175455, 2460592.4034293937], [2460592.403541242, 2460592.40365309], [2460592.4039886342, 2460592.4041004824], [2460592.4044360267, 2460592.404547875], [2460592.4051071154, 2460592.4052189635], [2460592.405554508, 2460592.405778204], [2460592.406113748, 2460592.406337444], [2460592.4065611404, 2460592.4067848367], [2460592.407120381, 2460592.407232229], [2460592.4073440772, 2460592.4074559254], [2460592.4076796216, 2460592.4077914697], [2460592.4085744065, 2460592.4086862546], [2460592.408909951, 2460592.409021799], [2460592.409133647, 2460592.409245495], [2460592.4094691915, 2460592.4095810396], [2460592.41014028, 2460592.4102521283], [2460592.4105876726, 2460592.410923217], [2460592.4113706094, 2460592.4114824575], [2460592.4118180014, 2460592.4120416977], [2460592.412265394, 2460592.41248909], [2460592.412600938, 2460592.4127127863], [2460592.413383875, 2460592.413495723], [2460592.4138312675, 2460592.4139431156], [2460592.414166812, 2460592.41427866], [2460592.4146142043, 2460592.4147260524], [2460592.4149497487, 2460592.415173445], [2460592.4165156223, 2460592.4166274704], [2460592.4167393185, 2460592.4168511666], [2460592.417074863, 2460592.417186711], [2460592.417522255, 2460592.417634103], [2460592.417745951, 2460592.4180814954], [2460592.418752584, 2460592.4189762804], [2460592.4190881285, 2460592.4193118247], [2460592.419535521, 2460592.4199829134], [2460592.4200947615, 2460592.4202066096], [2460592.420430306, 2460592.4208776983], [2460592.4211013946, 2460592.421325091], [2460592.421548787, 2460592.421660635], [2460592.4217724833, 2460592.4218843314], [2460592.4219961795, 2460592.4222198757], [2460592.422443572, 2460592.422779116], [2460592.4232265083, 2460592.4234502045], [2460592.423785749, 2460592.423897597], [2460592.4243449895, 2460592.4244568376], [2460592.4245686857, 2460592.4251279263], [2460592.4253516225, 2460592.425687167], [2460592.425910863, 2460592.4262464074], [2460592.4263582556, 2460592.4264701037], [2460592.426581952, 2460592.4266938], [2460592.426805648, 2460592.4270293443], [2460592.4272530405, 2460592.4273648886], [2460592.4274767367, 2460592.427588585], [2460592.427924129, 2460592.4281478254], [2460592.428371521, 2460592.4285952174], [2460592.42904261, 2460592.429266306], [2460592.4294900023, 2460592.4296018505], [2460592.430161091, 2460592.4303847873], [2460592.4304966354, 2460592.4306084835], [2460592.4308321797, 2460592.431055876], [2460592.431167724, 2460592.431279572], [2460592.4313914203, 2460592.4315032684], [2460592.4316151165, 2460592.4317269647], [2460592.432174357, 2460592.4322862052], [2460592.4325099015, 2460592.432845446], [2460592.433069142, 2460592.43318099], [2460592.4335165345, 2460592.4336283826], [2460592.433852079, 2460592.433963927], [2460592.434411319, 2460592.434635015], [2460592.4347468633, 2460592.4349705596], [2460592.4350824077, 2460592.435194256], [2460592.4355298, 2460592.4356416482], [2460592.4358653445, 2460592.436200889], [2460592.4367601294, 2460592.4368719775], [2460592.4369838256, 2460592.4370956738], [2460592.43731937, 2460592.437431218], [2460592.4378786106, 2460592.438102307], [2460592.438214155, 2460592.438437851], [2460592.4388852436, 2460592.43910894], [2460592.439332636, 2460592.4395563323], [2460592.439780028, 2460592.439891876], [2460592.4400037243, 2460592.4402274205], [2460592.4403392687, 2460592.440562965], [2460592.440898509, 2460592.4411222055], [2460592.441681446, 2460592.4420169904], [2460592.4422406866, 2460592.442464383], [2460592.4429117753, 2460592.443471016], [2460592.443582864, 2460592.4438065602], [2460592.4440302565, 2460592.4441421046], [2460592.4442539527, 2460592.444365801], [2460592.4448131933, 2460592.4453724334], [2460592.4455961296, 2460592.4457079777], [2460592.4462672183, 2460592.4463790664], [2460592.4464909146, 2460592.446714611], [2460592.446938307, 2460592.447050155], [2460592.4471620033, 2460592.4473856995], [2460592.447833092, 2460592.448056788], [2460592.4482804844, 2460592.4483923325], [2460592.4485041806, 2460592.448727877], [2460592.449063421, 2460592.4491752693], [2460592.4492871175, 2460592.4495108137], [2460592.449846358, 2460592.449958206], [2460592.4500700543, 2460592.4502937505], [2460592.450741143, 2460592.4509648387], [2460592.451188535, 2460592.451300383], [2460592.451412231, 2460592.4516359274], [2460592.452195168, 2460592.452307016], [2460592.4526425605, 2460592.4527544086], [2460592.4528662567, 2460592.452978105], [2460592.453201801, 2460592.4534254973], [2460592.4538728897, 2460592.453984738], [2460592.4545439784, 2460592.454991371], [2460592.455103219, 2460592.455215067], [2460592.4557743077, 2460592.455886156], [2460592.4563335483, 2460592.456557244], [2460592.4567809403, 2460592.4568927884], [2460592.4571164846, 2460592.4572283328], [2460592.457675725, 2460592.4577875733], [2460592.4578994215, 2460592.4581231177], [2460592.459353447, 2460592.459465295], [2460592.4599126875, 2460592.460136384], [2460592.4605837762, 2460592.4606956244], [2460592.461366713, 2460592.461478561], [2460592.4615904093, 2460592.4618141055], [2460592.462932586, 2460592.4630444343], [2460592.463491827, 2460592.4639392192], [2460592.4668472703, 2460592.4669591184], [2460592.4715448907, 2460592.471768587], [2460592.4721041312, 2460592.4735581563], [2460592.476018815, 2460592.4764662073], [2460592.4765780554, 2460592.4766899035], [2460592.477025448, 2460592.477137296], [2460592.479486106, 2460592.4800453465], [2460592.4801571947, 2460592.4809401315], [2460592.484295575, 2460592.484407423], [2460592.485973296, 2460592.4860851443], [2460592.486532537, 2460592.486756233], [2460592.4916775497, 2460592.491789398], [2460592.4952566894, 2460592.4953685375], [2460592.4988358286, 2460592.499059525], [2460592.501296487, 2460592.5014083353], [2460592.501632031, 2460592.501743879], [2460592.5019675754, 2460592.5020794235], [2460592.5023031197, 2460592.502526816], [2460592.5029742084, 2460592.5030860566], [2460592.506329652, 2460592.5064415], [2460592.5075599807, 2460592.507671829], [2460592.5199751207, 2460592.520086969], [2460592.521093602, 2460592.521317298], [2460592.5219883868, 2460592.522100235], [2460592.5281400327, 2460592.528587425], [2460592.530824387, 2460592.531048083], [2460592.531607324, 2460592.531719172], [2460592.5342916786, 2460592.534515375], [2460592.535298311, 2460592.5354101593], [2460592.539772236, 2460592.539884084], [2460592.545364641, 2460592.5454764892], [2460592.5457001855, 2460592.5459238817], [2460592.5524110715, 2460592.5525229196], [2460592.5536414008, 2460592.553753249], [2460592.556549452, 2460592.556773148], [2460592.5578916287, 2460592.558003477], [2460592.5638195784, 2460592.5639314265], [2460592.5668394775, 2460592.5669513256], [2460592.5703067686, 2460592.570530465], [2460592.574556997, 2460592.574668845], [2460592.576011022, 2460592.57612287], [2460592.576793959, 2460592.576905807], [2460592.5783598325, 2460592.5784716806], [2460592.5785835288, 2460592.578695377], [2460592.579590162, 2460592.57970201], [2460592.5819389718, 2460592.58205082], [2460592.5899920356, 2460592.590215732], [2460592.5924526937, 2460592.592788238], [2460592.593683023, 2460592.5941304155], [2460592.594801504, 2460592.595472593], [2460592.5983806434, 2460592.5987161878], [2460592.6012886944, 2460592.6015123907], [2460592.6036375044, 2460592.6037493525], [2460592.604420441, 2460592.6045322893], [2460592.6064337073, 2460592.6065455554], [2460592.608111429, 2460592.6088943654], [2460592.6099009984, 2460592.610348391], [2460592.6109076315, 2460592.611355024], [2460592.6126972013, 2460592.613144594], [2460592.6135919862, 2460592.6138156825], [2460592.6141512264, 2460592.614598619], [2460592.6153815556, 2460592.6154934037], [2460592.615828948, 2460592.6160526443], [2460592.6161644924, 2460592.616611885], [2460592.6169474293, 2460592.61750667], [2460592.618736999, 2460592.618848847], [2460592.620079176, 2460592.6204147204], [2460592.6206384166, 2460592.6207502647], [2460592.621197657, 2460592.6213095053], [2460592.6215332015, 2460592.621756898], [2460592.622987227, 2460592.6232109233], [2460592.623770164, 2460592.623882012], [2460592.6242175563, 2460592.6243294044], [2460592.624888645, 2460592.6255597332], [2460592.6257834295, 2460592.6260071257], [2460592.627349303, 2460592.627461151], [2460592.629138873, 2460592.629250721], [2460592.6295862654, 2460592.6298099617], [2460592.6311521386, 2460592.6312639867], [2460592.6321587716, 2460592.632382468], [2460592.6347312783, 2460592.6350668226], [2460592.6358497594, 2460592.636185304], [2460592.6373037845, 2460592.6375274807], [2460592.637751177, 2460592.6381985694], [2460592.6393170506, 2460592.639540747], [2460592.641665861, 2460592.641777709], [2460592.644238367, 2460592.6443502153], [2460592.646027937, 2460592.6462516333], [2460592.6465871776, 2460592.6466990258], [2460592.6490478357, 2460592.6497189244], [2460592.6516203424, 2460592.6519558867], [2460592.65352176, 2460592.6538573042], [2460592.6543046967, 2460592.654416545], [2460592.6560942666, 2460592.656317963], [2460592.656541659, 2460592.656653507], [2460592.6581075327, 2460592.658331229], [2460592.658443077, 2460592.658554925], [2460592.6590023176, 2460592.6591141657], [2460592.660680039, 2460592.660903735], [2460592.661015583, 2460592.6611274313], [2460592.662581457, 2460592.662917001], [2460592.6632525455, 2460592.6634762418], [2460592.664930267, 2460592.665042115], [2460592.666272444, 2460592.6664961404], [2460592.6687331027, 2460592.668844951], [2460592.669739736, 2460592.669851584], [2460592.67007528, 2460592.6862932555]] freq_flags: [[47714233.3984375, 48080444.3359375], [49911499.0234375, 50033569.3359375], [54428100.5859375, 54672241.2109375], [62240600.5859375, 62728881.8359375], [69931030.2734375, 70053100.5859375], [87509155.2734375, 108016967.7734375], [109970092.7734375, 110092163.0859375], [112655639.6484375, 112777709.9609375], [113632202.1484375, 113754272.4609375], [116439819.3359375, 116561889.6484375], [124618530.2734375, 125350952.1484375], [129989624.0234375, 130111694.3359375], [136215209.9609375, 136459350.5859375], [136825561.5234375, 138046264.6484375], [138656616.2109375, 138778686.5234375], [140365600.5859375, 140487670.8984375], [140975952.1484375, 142562866.2109375], [143051147.4609375, 143173217.7734375], [143783569.3359375, 144027709.9609375], [145736694.3359375, 145980834.9609375], [147445678.7109375, 147567749.0234375], [149154663.0859375, 149276733.3984375], [149887084.9609375, 150009155.2734375], [154159545.8984375, 154403686.5234375], [169906616.2109375, 170150756.8359375], [175155639.6484375, 175277709.9609375], [181137084.9609375, 181259155.2734375], [187362670.8984375, 187606811.5234375], [189926147.4609375, 190048217.7734375], [191146850.5859375, 191513061.5234375], [197128295.8984375, 197372436.5234375], [198104858.3984375, 198348999.0234375], [199203491.2109375, 199325561.5234375], [201766967.7734375, 201889038.0859375], [204940795.8984375, 205062866.2109375], [208480834.9609375, 208724975.5859375], [209945678.7109375, 210067749.0234375], [212142944.3359375, 212265014.6484375], [220687866.2109375, 220809936.5234375], [223129272.4609375, 223373413.0859375], [227401733.3984375, 227523803.7109375], [229110717.7734375, 229354858.3984375], [229965209.9609375, 230087280.2734375], [231063842.7734375, 231185913.0859375]] ex_ants: [[7, Jee], [8, Jee], [9, Jee], [15, Jnn], [16, Jee], [17, Jnn], [18, Jee], [18, Jnn], [21, Jee], [22, Jee], [22, Jnn], [27, Jee], [27, Jnn], [28, Jee], [28, Jnn], [31, Jnn], [33, Jnn], [34, Jee], [34, Jnn], [35, Jee], [35, Jnn], [36, Jee], [36, Jnn], [37, Jee], [37, Jnn], [40, Jnn], [42, Jnn], [45, Jee], [46, Jee], [47, Jee], [47, Jnn], [48, Jee], [48, Jnn], [49, Jee], [49, Jnn], [50, Jnn], [51, Jee], [54, Jnn], [55, Jee], [61, Jee], [61, Jnn], [62, Jee], [62, Jnn], [63, Jee], [63, Jnn], [64, Jee], [64, Jnn], [69, Jee], [72, Jnn], [73, Jee], [77, Jee], [77, Jnn], [78, Jee], [78, Jnn], [80, Jnn], [81, Jee], [81, Jnn], [82, Jnn], [83, Jee], [84, Jnn], [85, Jnn], [86, Jee], [86, Jnn], [88, Jee], [88, Jnn], [90, Jee], [90, Jnn], [92, Jee], [93, Jee], [95, Jee], [96, Jee], [97, Jnn], [98, Jee], [98, Jnn], [99, Jnn], [100, Jee], [100, Jnn], [101, Jnn], [102, Jnn], [103, Jnn], [104, Jnn], [107, Jee], [107, Jnn], [109, Jnn], [116, Jee], [116, Jnn], [119, Jee], [119, Jnn], [120, Jee], [120, Jnn], [121, Jee], [121, Jnn], [122, Jnn], [123, Jnn], [130, Jee], [130, Jnn], [131, Jnn], [134, Jee], [136, Jee], [136, Jnn], [137, Jee], [137, Jnn], [142, Jnn], [144, Jee], [144, Jnn], [155, Jee], [161, Jnn], [170, Jee], [171, Jnn], [176, Jee], [176, Jnn], [177, Jee], [177, Jnn], [178, Jee], [178, Jnn], [179, Jee], [179, Jnn], [180, Jnn], [181, Jnn], [182, Jee], [184, Jee], [188, Jnn], [189, Jee], [189, Jnn], [193, Jee], [194, Jee], [198, Jnn], [199, Jnn], [200, Jee], [200, Jnn], [201, Jnn], [202, Jnn], [204, Jee], [204, Jnn], [206, Jee], [208, Jee], [208, Jnn], [209, Jee], [209, Jnn], [210, Jee], [210, Jnn], [212, Jnn], [213, Jee], [215, Jee], [215, Jnn], [216, Jee], [218, Jnn], [221, Jee], [226, Jnn], [227, Jee], [232, Jee], [235, Jee], [235, Jnn], [240, Jee], [240, Jnn], [241, Jee], [241, Jnn], [242, Jee], [242, Jnn], [243, Jee], [243, Jnn], [245, Jnn], [246, Jee], [246, Jnn], [250, Jee], [251, Jee], [251, Jnn], [253, Jnn], [255, Jnn], [256, Jee], [256, Jnn], [261, Jee], [261, Jnn], [262, Jee], [262, Jnn], [266, Jnn], [268, Jnn], [270, Jee], [270, Jnn], [272, Jee], [281, Jee], [281, Jnn], [295, Jee], [295, Jnn], [320, Jee], [320, Jnn], [321, Jee], [321, Jnn], [323, Jee], [323, Jnn], [324, Jee], [324, Jnn], [325, Jee], [325, Jnn], [326, Jee], [326, Jnn], [327, Jee], [327, Jnn], [328, Jee], [328, Jnn], [329, Jee], [329, Jnn], [331, Jee], [331, Jnn], [332, Jee], [332, Jnn], [333, Jee], [333, Jnn], [336, Jee], [336, Jnn], [340, Jee], [340, Jnn]]
Metadata¶
In [24]:
for repo in ['hera_cal', 'hera_qm', 'hera_filters', 'hera_notebook_templates', 'pyuvdata']:
exec(f'from {repo} import __version__')
print(f'{repo}: {__version__}')
hera_cal: 3.6.2.dev110+g0529798 hera_qm: 2.2.0 hera_filters: 0.1.6.dev1+g297dcce
hera_notebook_templates: 0.1.dev936+gdc93cad pyuvdata: 3.0.1.dev70+g283dda3
In [25]:
print(f'Finished execution in {(time.time() - tstart) / 60:.2f} minutes.')
Finished execution in 137.04 minutes.