
import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import matplotlib.pyplot as plt
import matplotlib
import glob
import os
import operator
from hera_cal.io import HERAData
from hera_cal.utils import split_pol
from hera_qm.metrics_io import load_metric_file
from hera_notebook_templates.utils import status_colors
from IPython.display import display, HTML

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the first few lines of this cell.
# JD = '2459122'
# data_path = '/lustre/aoc/projects/hera/H4C/2459122'
# os.environ["JULIANDATE"] = JD
# os.environ["DATA_PATH"] = data_path


# Use environment variables to figure out path to data
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
print(f'JD = "{JD}"')
print(f'data_path = "{data_path}"')

JD = "2459746"
data_path = "/mnt/sn1/2459746"


from astropy.time import Time
utc = Time(JD, format='jd').datetime
print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 6-15-2022


print(f'Looking for data in {data_path} on JD {JD}...')

auto_metrics_file = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}*.auto_metrics.h5')))
if len(auto_metrics_file) > 0:
    auto_metrics_file = auto_metrics_file[0]
    print(f'Found auto_metrics results file at {auto_metrics_file}.')
else:
    raise OSError(f'{auto_metrics_file} not found.')

raw_auto_files = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}.?????.sum.autos.uvh5')))
if len(raw_auto_files) > 0:
    print(f'Found {len(raw_auto_files)} extracted autocorrelation files.')
else: 
    raise OSError(f'No files of the form zen.{JD}.?????.sum.autos.uvh5 found in {data_path}.')

Looking for data in /mnt/sn1/2459746 on JD 2459746...
Found auto_metrics results file at /mnt/sn1/2459746/zen.2459746.25308.sum.auto_metrics.h5.
Found 372 extracted autocorrelation files.


# load auto_metrics and define use someful quantities
am = load_metric_file(auto_metrics_file)
mean_round_modz_cut = am['parameters']['mean_round_modz_cut']
ex_ants = am['ex_ants']['r2_ex_ants']
ants = sorted(set(bl[0] for bl in am['modzs']['r2_shape_modzs']))

# load raw autocorrelation waterfalls and define some useful quantities
hd = HERAData(raw_auto_files)
autos, _, _ = hd.read(axis='blt')
wf_shape = next(iter(autos.values())).shape
freqs = autos.freqs / 1e6
times = autos.times
lsts = autos.lsts * 12 / np.pi


# try to load a priori antenna statusesm but fail gracefully if this doesn't work.
a_priori_statuses = {ant: 'Not Found' for ant in ants}
nodes = {ant: np.nan for ant in ants}
try:
    from hera_mc import cm_hookup

    # get node numbers
    hookup = cm_hookup.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in nodes:
            nodes[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])            
            
    # get apriori antenna status
    for ant_name, data in hookup.items():
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in a_priori_statuses:
            a_priori_statuses[ant] = data.apriori
    
except Exception as err:
    print(f'Could not load node numbers and a priori antenna statuses.\nEncountered {type(err)} with message: {err}')

Could not load node numbers and a priori antenna statuses.
Encountered <class 'TypeError'> with message: 'NoneType' object is not subscriptable


# print ex_ants for easy copy-pasting to YAML file
print('ex_ants: [' + ", ".join(str(ant) for ant in ex_ants) + ']')

ex_ants: [0, 1, 2, 3, 4, 5, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 45, 46, 50, 51, 52, 53, 57, 65, 66, 67, 68, 69, 70, 71, 72, 73, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 116, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 138, 140, 141, 142, 143, 144, 150, 155, 161, 162, 163, 165, 166, 167, 168, 169, 170, 177, 178, 179, 180, 181, 182, 184, 185, 187, 189, 191, 220, 221, 222, 320, 321, 323, 324, 325, 329, 333]


def Array_Plot():
    plt.figure(figsize=(16,16))
    plt.scatter(np.array([autos.antpos[ant][0] for ant in ants]), 
                np.array([autos.antpos[ant][1] for ant in ants]), c='w', s=0)    
    for ant in ants:
        pos = autos.antpos[ant]
        bad = ant in ex_ants
        plt.gca().add_artist(plt.Circle(tuple(pos[0:2]), radius=7, 
                                        fill=(~bad), color=['grey','r'][bad]))
        plt.text(pos[0],pos[1],str(ant), va='center', ha='center', color='w')
    plt.xlabel("Antenna East-West Position (meters)")
    plt.ylabel("Antenna North-South Position (meters)")
    plt.title(f'Antenna Positions and Auto_Metrics Flags on {JD}\n(Maximum Modified Z-Score > {mean_round_modz_cut} in Red)');
    plt.axis('equal')
    plt.tight_layout()


Array_Plot()


# Parse modzs for Table 1 and other figures
modzs_to_check = {'Shape': 'r2_shape_modzs', 'Power': 'r2_power_modzs', 
                  'Temporal Variability': 'r2_temp_var_modzs', 'Temporal Discontinuties': 'r2_temp_diff_modzs'}
worst_metrics = []
worst_zs = []
all_modzs = {}
binary_flags = {rationale: [] for rationale in modzs_to_check}

for ant in ants:
    # parse modzs and figure out flag counts
    modzs = {f'{pol} {rationale}': am['modzs'][dict_name][(ant, ant, pol)] 
             for rationale, dict_name in modzs_to_check.items() for pol in autos.pols()}
    for pol in autos.pols():
        for rationale, dict_name in modzs_to_check.items():
            binary_flags[rationale].append(am['modzs'][dict_name][(ant, ant, pol)] > mean_round_modz_cut)
    
    # figure out which metric is the largest outlier
    worst_metric, worst_z = max(modzs.items(), key=operator.itemgetter(1))
    worst_metrics.append(worst_metric)
    worst_zs.append(worst_z)
    
    # parse out all metrics for dataframe
    for k in modzs:
        col_label = k + ' Modified Z-Score'        
        if col_label in all_modzs:
            all_modzs[col_label].append(modzs[k])
        else:
            all_modzs[col_label] = [modzs[k]]


# build dataframe
to_show = {'Ant': ants, 
           'Node': [f'N{nodes[ant]:02}' for ant in ants],
           'A Priori Status': [a_priori_statuses[ant] for ant in ants],
           'Worst Metric': worst_metrics, 'Worst Modified Z-Score': worst_zs}
to_show.update(all_modzs)
df = pd.DataFrame(to_show).sort_values('Worst Modified Z-Score', ascending=False)

# style dataframe
z_score_cols = [col for col in df.columns if col not in ['Ant', 'Node', 'A Priori Status', 'Worst Metric']]
table1 = df.style.hide_index()\
           .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
           .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=z_score_cols) \
           .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])


HTML(table1.render())


def Flag_Bar_Chart():
    plt.figure(figsize=(8, 4), dpi=100)

    # count 
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    antpol_flags = [np.sum(f) for f in flags_list]
    ant_flags = [np.sum(np.array(f)[0::2] | np.array(f)[1::2]) for f in flags_list]

    # make bar chart
    plt.bar(np.arange(len(rationales)), antpol_flags, width=.7, color='yellow', ec='k', tick_label=rationales)
    for x, (nflags, nants) in enumerate(zip(antpol_flags, ant_flags)):
        plt.text(x, nflags/2, f'{nflags} Feeds on \n{nants} Antennas\nFlagged', va='center', ha='center')

    # set labels
    plt.ylabel('Antenna-Polarizations Flagged')
    plt.xlabel('Reason for Flagging')
    plt.tight_layout()


Flag_Bar_Chart()


def Rationale_Corr_Plot():
    plt.figure(figsize=(6,6), dpi=100)
    
    # compute correlation matrix
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    corrs = np.corrcoef(flags_list)
    
    # plot and label correlation matrix
    plt.imshow(corrs, cmap='viridis', interpolation='nearest', origin='upper')
    for i in range(corrs.shape[0]):
        for j in range(corrs.shape[1]):
            plt.text(i, j, np.round(corrs[i, j], 3), va='center', ha='center',
                     bbox={'facecolor': 'w', 'ec': 'w', 'alpha': .75})

    # colorbar, labels, and style
    plt.yticks(range(len(rationales)), rationales)
    plt.xticks(range(len(rationales)), rationales, rotation=-45, ha='right')
    plt.gca().xaxis.tick_top()
    plt.clim([0, 1])
    plt.colorbar(fraction=0.046, pad=0.04)
    plt.tight_layout()


Rationale_Corr_Plot()


def plot_all_spectra(spectra, mod_zs, modz_cut, overall, freqs, reason, ex_ants=[], 
                     xlabel='Frequency (MHz)', ylabel='', yscale='linear', ylim_factor=None):
    '''Helper function for plotting all spectra and showing which ones were flagged and why.''' 
    fig, axes = plt.subplots(1,2, figsize=(14,5), dpi=100)
    pols = sorted(set([bl[2] for bl in spectra]))
    for ax, pol in zip(axes, pols):
        # sort antennas into good, bad, and bad but not for this reason
        bad_here = [bl for bl in spectra if (bl[2] == pol) and mod_zs[bl] > modz_cut]
        other_bad = [bl for bl in spectra if (bl[2] == pol) and (bl[0] in ex_ants) and (bl not in bad_here)]
        good = [bl for bl in spectra if (bl[2] == pol) and (bl[0] not in ex_ants)]
        
        # plot all spectra
        l1, l2, l3 = None, None, None
        for bl in other_bad:
            l2, = ax.plot(freqs, spectra[bl], 'darkviolet', lw=.5)
        for bl in bad_here:
            l1, = ax.plot(freqs, spectra[bl], 'r', lw=.5)
        for bl in good:
            l3, = ax.plot(freqs, spectra[bl], 'grey', alpha=.5)
        l4, = ax.plot(freqs, overall[bl[2]], 'k--')
        
        ax.set_xlabel(xlabel)
        ax.set_ylabel(ylabel)
        ax.set_yscale(yscale)
        if ylim_factor is not None:
            ax.set_ylim([np.nanmin([spectra[bl] for bl in good]) / ylim_factor, 
                         np.nanmax([spectra[bl] for bl in good]) * ylim_factor])
        ax.set_title(f'Outliers in Autocorrelation {reason}: {pol}')
        ax.legend([l1, l2, l3, l4], [f'Flagged for {reason}', 'Flagged for Another Reason', 'Unflagged', 'Average Unflagged'], loc='lower right')
    plt.tight_layout()


overall_shape = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra_normed'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_power = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_var = {pol: np.nanmean([spec for bl, spec in am['spectra']['std_spectra_normed'].items() if (bl[2] == pol) 
                                     and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_diff = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_abs_diff_spectra_normed'].items() if (bl[2] == pol) 
                                      and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}


plot_all_spectra(am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], mean_round_modz_cut, 
                 overall_shape, freqs, 'Shape', ex_ants=ex_ants, yscale='linear', ylim_factor=1.2)


plot_all_spectra(am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], mean_round_modz_cut, 
                 overall_power, freqs, 'Power', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'], mean_round_modz_cut,
                 overall_temp_var, freqs, 'Temporal Variability', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'], mean_round_modz_cut, 
                 overall_temp_diff, freqs, 'Temporal Discontinutities', ex_ants=ex_ants, yscale='log')


# compute average good autocorrelations for each polarization
avg_good_autos = {pol: np.zeros(wf_shape, dtype=float) for pol in autos.pols()}
for pol in autos.pols():
    for i in range(wf_shape[0]):
        avg_good_autos[pol][i] = np.mean([np.abs(autos[bl][i, :]) for bl in autos 
                                          if (bl[0] not in ex_ants) and (bl[2] == pol)], axis=0)
    avg_good_autos[pol][am['flags']] = np.nan


def Avg_Auto_Plot():
    fig, axes = plt.subplots(1, 2, figsize=(14,5), dpi=100)
    for ax, pol in zip(axes, sorted(autos.pols())):
        im = ax.imshow(avg_good_autos[pol], aspect='auto', interpolation='nearest',
                       extent=[freqs[0], freqs[-1], times[-1], times[0]])
        ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))    
        plt.colorbar(im, ax=ax)
        ax.set_title(f'Average Good Raw {pol} Autocorrelation After Flagging')
        ax.set_ylabel('LST (hours)')
        ax.set_xlabel('Frequency (MHz)')
    plt.tight_layout()


Avg_Auto_Plot()


def plot_spectra(axes, ant, spec, modzs, modz_cut, overall, reason, yscale='linear'):
    '''Helper function for plotting both antennas of a given polarization.'''
    for pol, ax in zip(sorted(autos.pols()), axes):
        bl = (ant, ant, pol)

        # plot good antennas
        for bl2 in modzs:
            if (bl2[0] not in ex_ants) and (bl2[2] == pol):
                ax.plot(freqs, spec[bl2], 'grey', lw=.5, alpha=.5)
        ax.plot(freqs, overall[pol], 'k--')

        # plot this anetnna
        color = 'r'
        alpha = .75
        if modzs[bl] >= modz_cut:
            alpha = 1
        elif bl[0] in ex_ants:
            color = 'darkviolet'
        else:
            color = 'darkgreen'
        ax.plot(freqs, spec[bl], color, alpha=alpha, label=f'{ant} {pol}\n(z = {np.round(modzs[bl],1)})')

        # decorate axis
        ax.set_yscale(yscale)
        ax.legend(loc=2)
        ax.set_title(f'{pol} {reason}')


len(ants)

150


def Plot_All_Auto_Metrics():
    for row, ant in enumerate(df['Ant']): 
        # print title of section
        display(HTML(f'<h2>Antenna {ant}: {JD}</h2>'))

        # print metrics
        df_row = df.loc[df['Ant'] == ant]
        df_row = df_row.style.hide_index()\
                       .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])\
                       .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
                       .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
                       .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols)
        display(HTML(df_row.render()))

        # plot spectra and waterfalls
        fig, axes = plt.subplots(2, 6, figsize=(18, 5.5), dpi=(50 * (104 / len(ants))**.5)) # this should help manage filesize

        # plot individual spectra compared to all good antennas
        plot_spectra(axes[:, 0], ant, am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], 
                     mean_round_modz_cut, overall_shape, 'Shape', yscale='linear')
        plot_spectra(axes[:, 1], ant, am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], 
                     mean_round_modz_cut, overall_power, 'Power', yscale='log')
        plot_spectra(axes[:, 2], ant, am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'],
                     mean_round_modz_cut, overall_temp_var, 'Temporal Variability', yscale='linear')
        plot_spectra(axes[:, 3], ant, am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'],
                     mean_round_modz_cut, overall_temp_diff, 'Temporal Discontinutities', yscale='linear')
        plt.tight_layout()

        # plot linear-scale waterfalls 
        for pol, ax in zip(sorted(autos.pols()), axes[:, 4]):
            bl = (ant, ant, pol)        
            im = ax.imshow(np.where(am['flags'], np.nan, autos[bl].real), 
                           aspect='auto', interpolation='nearest', cmap='inferno',
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Waterfall (Linear Scale)')

        # plot log-scale mean-divided waterfalls
        for pol, ax in zip(sorted(autos.pols()), axes[:, 5]):
            bl = (ant, ant, pol)
            to_plot = autos[bl].real / avg_good_autos[pol]
            to_plot[am['flags']] = np.nan
            to_plot /= np.nanmean(to_plot)
            im = ax.imshow(np.log10(to_plot), aspect='auto', cmap='seismic', interpolation='nearest', vmin=-.07, vmax=.07,
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Log(Normalized Waterfall)')

        plt.show()

        # print some whitespace
        display(HTML('<hr style="height:3px">'))


Plot_All_Auto_Metrics()


from hera_qm import __version__
print(__version__)

2.0.2

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
0	N00	Not Found	ee Temporal Variability	21892.706336	36.525032	41.345784	44.754964	48.104708	20710.778660	21892.706336	20114.451540	21500.407441
94	N10	Not Found	ee Temporal Discontinuties	1953.584148	62.055803	61.186984	68.806734	93.174571	154.381801	433.713941	929.244806	1953.584148
93	N10	Not Found	nn Temporal Discontinuties	1507.975832	61.099270	49.671796	77.454717	58.759956	273.916777	145.536546	1507.975832	701.990416
130	N10	Not Found	nn Temporal Discontinuties	1380.503838	76.213222	52.837047	93.710381	70.058702	231.878922	289.194246	1380.503838	1182.506806
109	N10	Not Found	ee Temporal Discontinuties	1207.513926	68.050291	51.664629	72.447604	67.898200	215.534007	264.355880	1118.288974	1207.513926
112	N10	Not Found	nn Temporal Discontinuties	1118.755399	67.993894	49.779659	69.764331	57.859589	155.723014	135.788477	1118.755399	798.342326
111	N10	Not Found	nn Temporal Discontinuties	1005.287569	59.482655	48.843972	60.024682	53.492717	176.935889	90.041246	1005.287569	685.940751
92	N10	Not Found	ee Temporal Discontinuties	981.908466	61.799192	53.770455	60.960875	67.481856	174.238273	244.033039	757.472520	981.908466
110	N10	Not Found	nn Temporal Discontinuties	948.356755	53.735097	46.923465	61.184466	55.263904	142.853901	155.472587	948.356755	860.051535
129	N10	Not Found	ee Temporal Discontinuties	947.173064	60.046364	57.277768	66.110060	71.074424	192.491905	152.495453	862.132454	947.173064
127	N10	Not Found	nn Temporal Discontinuties	782.472930	65.164942	48.229581	61.072594	52.507382	135.570937	116.104720	782.472930	687.730742
128	N10	Not Found	nn Temporal Discontinuties	746.932490	55.940158	49.475830	60.208571	52.305941	127.438275	82.667653	746.932490	461.825931
45	N05	Not Found	nn Temporal Discontinuties	346.543529	24.991617	0.058798	9.308129	-0.586096	344.437850	131.294977	346.543529	132.818344
178	N12	Not Found	nn Temporal Discontinuties	105.273152	4.898733	1.849997	8.775533	3.815275	30.137931	28.090238	105.273152	94.566819
187	N14	Not Found	nn Temporal Discontinuties	80.607031	0.986380	-0.735169	-0.606125	-0.809711	12.280913	4.584586	80.607031	5.376402
167	N15	Not Found	nn Temporal Discontinuties	69.083301	28.762721	21.314056	46.844982	43.302694	39.132005	21.526885	69.083301	21.177647
19	N02	Not Found	nn Temporal Variability	59.887842	5.601697	1.654517	4.214630	0.613134	59.887842	4.542487	3.528303	46.351419
140	Nnan	Not Found	ee Temporal Discontinuties	58.461960	6.807728	6.333876	10.735524	10.133928	34.315800	34.589191	57.837465	58.461960
10	N02	Not Found	ee Temporal Discontinuties	55.249030	4.039460	7.615179	2.484446	3.883493	10.394830	7.813547	43.359976	55.249030
46	N05	Not Found	nn Temporal Discontinuties	54.762029	3.200280	1.345027	5.544337	4.130811	14.338113	15.504367	54.762029	53.570608
70	N04	Not Found	nn Temporal Variability	52.595135	43.912466	51.198849	27.122327	24.904878	52.595135	31.246720	29.783692	22.953595
168	N15	Not Found	nn Power	50.423638	41.215259	41.787222	50.423638	45.993401	4.577842	14.233198	-3.351424	-1.643897
169	N15	Not Found	nn Power	49.050676	40.755390	41.809908	49.050676	48.077677	5.626995	7.643226	-0.914413	-2.198022
170	N15	Not Found	ee Power	48.404721	40.661855	41.293869	48.126571	48.404721	10.274216	6.647718	-0.422762	-2.147188
8	N02	Not Found	nn Power	48.085394	41.535494	42.154990	48.085394	45.053884	34.255254	37.437809	32.822084	37.383308
321	N02	Not Found	nn Power	46.695584	42.488621	42.238778	46.695584	44.531039	8.733516	15.190551	-1.581514	-1.019194
99	Nnan	Not Found	nn Shape	45.484775	45.484775	43.984869	36.053812	35.528263	24.878823	34.641544	11.300993	8.714461
33	N02	Not Found	nn Temporal Discontinuties	44.774326	11.025531	-1.288653	20.568632	-0.317171	35.045052	32.860159	44.774326	33.754054
40	N04	Not Found	ee Shape	43.489070	43.073992	43.489070	39.562626	37.049773	20.467790	32.919443	7.670940	7.852256
72	N04	Not Found	nn Shape	43.297980	43.297980	40.148188	41.754399	37.810182	16.966201	36.639681	4.330620	8.056191
143	N14	Not Found	nn Shape	42.921690	42.921690	41.347828	41.700199	40.182563	23.778199	28.761101	4.927765	4.358336
189	N15	Not Found	nn Temporal Discontinuties	39.339712	0.341406	0.375968	0.923356	-0.683131	11.654852	7.837626	39.339712	34.147601
324	N04	Not Found	nn Power	29.251898	6.499091	6.001289	29.251898	26.844552	-0.182324	2.608165	-1.567596	-0.849198
155	N12	Not Found	ee Shape	28.677857	26.916737	28.677857	9.794424	10.135665	0.502007	1.538585	6.388996	6.350068
98	N07	Not Found	ee Shape	28.174691	26.411200	28.174691	4.444449	5.056616	-0.412229	1.334970	4.333735	5.682760
185	N14	Not Found	ee Temporal Discontinuties	28.122381	4.724977	5.168426	16.131167	15.222772	7.788671	8.199090	26.596077	28.122381
73	N05	Not Found	ee Temporal Discontinuties	27.140453	9.020873	2.029330	0.718700	2.763089	3.708286	10.585550	16.714025	27.140453
150	N15	Not Found	ee Shape	26.928043	23.883690	26.928043	11.350864	10.882652	0.561738	0.724051	7.664473	6.516569
116	N07	Not Found	ee Shape	26.720926	24.930502	26.720926	5.268077	5.794869	1.909554	2.531496	3.895009	3.893506
333	N12	Not Found	ee Shape	26.507819	1.403690	26.507819	15.928319	7.691798	5.355143	6.599295	16.480384	10.853004
180	N13	Not Found	nn Shape	25.871477	25.871477	0.698458	10.464930	-0.622876	-0.319580	0.842396	-2.936506	4.333078
142	N13	Not Found	nn Shape	24.089519	24.089519	3.297896	10.134711	2.524768	-1.116681	-0.593268	-3.427764	-2.901228
329	N12	Not Found	ee Temporal Variability	24.074032	3.707486	4.824394	20.161590	14.072311	0.972166	24.074032	7.478994	5.825182
323	Nnan	Not Found	nn Power	22.387920	3.038255	11.125594	22.387920	11.105578	2.532199	0.803939	1.976210	4.289141
177	N12	Not Found	nn Temporal Variability	21.722346	-0.355686	0.693692	0.375709	0.439394	21.722346	0.346734	9.167953	1.273089
57	Nnan	Not Found	ee Shape	18.465929	3.647799	18.465929	4.217900	5.494840	2.345522	2.253847	0.347932	0.505564
141	N13	Not Found	nn Shape	16.480113	16.480113	0.039384	7.575059	0.569131	1.494794	1.030394	4.271095	-2.870985
7	N02	Not Found	nn Temporal Discontinuties	12.918006	4.851490	4.789689	3.095385	2.313796	2.535376	0.354801	12.918006	2.517802
181	N13	Not Found	ee Shape	12.212047	3.471249	12.212047	1.803318	5.634055	-0.291769	-0.430276	-3.479802	-3.218516
32	N02	Not Found	ee Temporal Discontinuties	11.713885	1.865535	1.534440	1.148265	1.050638	3.306547	4.202138	10.995910	11.713885
144	N14	Not Found	ee Shape	10.538114	4.294225	10.538114	1.458103	4.420098	-0.465080	4.585061	1.946467	3.684977
179	N12	Not Found	ee Temporal Variability	10.432848	1.256460	1.707061	-0.608400	0.364572	-1.384717	10.432848	-0.840306	3.181738
69	N04	Not Found	nn Temporal Variability	10.093021	2.290080	-0.094438	0.542475	-0.233921	10.093021	4.511429	5.978153	1.727914
31	N02	Not Found	nn Temporal Variability	9.270623	1.075230	6.802100	-0.483955	3.432188	9.270623	5.018718	1.867854	7.677767
21	N02	Not Found	nn Shape	8.111459	8.111459	-0.517851	4.842172	-0.203274	1.659918	2.235144	1.574385	3.754250
163	N14	Not Found	nn Temporal Discontinuties	7.784646	-1.295757	0.785075	-0.989642	0.710979	2.498433	1.220450	7.784646	6.689472
71	N04	Not Found	nn Power	7.199521	5.826307	4.362021	7.199521	5.570037	2.665469	1.140254	0.985717	5.894966
20	N02	Not Found	nn Temporal Variability	6.899997	-0.347097	1.500978	0.491914	2.875411	6.899997	1.584019	-0.570585	2.990038
191	N15	Not Found	ee Temporal Variability	4.849220	-1.230133	1.693035	-0.446192	0.045664	-0.400715	4.849220	0.234540	1.213655
145	N14	Not Found	ee Shape	3.987762	-1.281245	3.987762	-0.919179	2.172143	-0.172126	-0.877444	0.579194	1.608276
186	N14	Not Found	ee Temporal Discontinuties	2.898563	0.612938	0.105270	-0.670268	-0.678714	0.795934	0.768681	0.511209	2.898563
158	N12	Not Found	nn Temporal Variability	2.856304	-0.629568	1.300403	0.516966	0.821074	2.856304	2.067291	0.501471	-0.272256
55	N04	Not Found	nn Temporal Variability	2.851433	-0.465714	-1.348088	0.102896	0.087679	2.851433	-1.293176	2.260242	-0.024113
160	Nnan	Not Found	ee Temporal Variability	2.538252	2.500469	1.979623	1.172644	1.327376	-0.556098	2.538252	-1.386947	-4.421135
157	N12	Not Found	ee Power	2.460169	0.818391	1.866749	-0.732006	2.460169	0.036912	0.001524	-1.400872	0.419614
156	N12	Not Found	nn Temporal Discontinuties	2.174272	-0.601878	-0.520805	0.147811	-0.873955	0.244873	0.045522	2.174272	-0.575036
56	N04	Not Found	nn Shape	2.129194	2.129194	1.341388	1.614038	0.609487	-0.290721	0.661521	0.337546	1.614404
42	N04	Not Found	ee Shape	1.897924	1.809081	1.897924	-0.316391	-0.354536	0.152923	-0.435400	-1.452785	-0.131853
176	N12	Not Found	ee Shape	1.896764	0.083848	1.896764	-0.309268	1.553923	-1.381567	-0.748174	-1.870048	-0.772092
135	N12	Not Found	ee Temporal Discontinuties	1.889935	-0.508424	-0.600633	-0.837339	-0.019130	-0.687461	-0.001524	1.083890	1.889935
164	N14	Not Found	nn Temporal Variability	1.820929	1.761655	0.169125	0.277343	-0.448963	1.820929	-0.636810	-0.566973	-0.115684
9	N02	Not Found	ee Temporal Variability	1.636619	-0.630640	-0.718342	-0.956892	-0.047200	0.328271	1.636619	-0.590869	-0.125866
183	N13	Not Found	nn Shape	1.207799	1.207799	0.406224	0.480655	0.242031	0.858289	-0.309950	-1.560630	-4.408410
190	N15	Not Found	nn Power	1.058829	-0.083848	-0.510945	1.058829	0.005410	-0.951241	-1.078287	-0.180439	0.024113
136	Nnan	Not Found	ee Temporal Variability	1.014372	-0.301223	-0.819558	0.814951	-0.203636	-0.630467	1.014372	0.701878	0.843184
54	N04	Not Found	nn Temporal Discontinuties	0.647104	-0.311366	-0.532589	-0.005410	-0.724358	-0.144844	-0.259560	0.647104	0.222458
41	N04	Not Found	ee Temporal Variability	0.302980	-0.245951	-1.165451	-0.865087	-1.057552	-1.734907	0.302980	-1.043085	0.070057
1	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
2	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
3	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
4	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
5	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
11	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
12	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
13	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
14	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
15	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
16	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
17	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
18	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
23	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
24	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
25	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
26	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
27	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
28	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
29	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
30	N01	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
36	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
37	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
38	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
39	N00	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
50	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
51	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
52	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
53	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
65	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
66	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
67	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
68	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
81	N07	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
82	N07	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
83	N07	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
84	N08	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
85	N08	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
86	N08	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
87	N08	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
88	N09	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
89	N09	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
90	N09	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
91	N09	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
100	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
101	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
102	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
103	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
104	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
105	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
106	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
107	N09	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
108	N09	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
119	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
120	N08	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
121	N08	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
122	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
123	N08	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
124	N09	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
125	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
126	N09	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
138	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
161	N13	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
162	N13	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
165	N14	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
166	N14	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
182	N13	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
184	N14	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
220	N18	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
221	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
222	N18	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
320	N03	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
325	Nnan	Not Found	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan

auto_metrics Nightly Notebook¶

Statistics computed (after removing the worst offenders with a median-based metrics and then RFI flagging):¶

Parse Inputs and Load Data¶

Summary Plots and Tables¶

Figure 1: Antenna Positions with auto_metrics flags.¶

Table 1: Modified Z-Score Summary¶

Figure 2: Flagging Rationale Summary¶

Figure 3: Flagging Rationale Correlations¶

Figure 4: Outliers in Autocorrelation Shape¶

Figure 5: Outliers in Autocorrelation Power¶

Figure 6: Outliers in Autocorrelation Temporal Variability¶

Figure 7: Outliers in Autocorrelation Temporal Discontinuities¶

Figure 8: Average Good Autocorrelations and Flags¶

Per-Antenna Plots¶

Figure 9: Per-Antenna Statistics, Spectra, and Waterfalls¶

Antenna 0: 2459746

Antenna 94: 2459746

Antenna 93: 2459746

Antenna 130: 2459746

Antenna 109: 2459746

Antenna 112: 2459746

Antenna 111: 2459746

Antenna 92: 2459746

Antenna 110: 2459746

Antenna 129: 2459746

Antenna 127: 2459746

Antenna 128: 2459746

Antenna 45: 2459746

Antenna 178: 2459746

Antenna 187: 2459746

Antenna 167: 2459746

Antenna 19: 2459746

Antenna 140: 2459746

Antenna 10: 2459746

Antenna 46: 2459746

Antenna 70: 2459746

Antenna 168: 2459746

Antenna 169: 2459746

Antenna 170: 2459746

Antenna 8: 2459746

Antenna 321: 2459746

Antenna 99: 2459746

Antenna 33: 2459746

Antenna 40: 2459746

Antenna 72: 2459746

Antenna 143: 2459746

Antenna 189: 2459746

Antenna 324: 2459746

Antenna 155: 2459746

Antenna 98: 2459746

Antenna 185: 2459746

Antenna 73: 2459746

Antenna 150: 2459746

Antenna 116: 2459746

Antenna 333: 2459746

Antenna 180: 2459746

Antenna 142: 2459746

Antenna 329: 2459746

Antenna 323: 2459746

Antenna 177: 2459746

Antenna 57: 2459746

Antenna 141: 2459746

Antenna 7: 2459746

Antenna 181: 2459746

Antenna 32: 2459746

Antenna 144: 2459746

Antenna 179: 2459746

Antenna 69: 2459746

Antenna 31: 2459746

Antenna 21: 2459746

Antenna 163: 2459746

Antenna 71: 2459746

Antenna 20: 2459746

Antenna 191: 2459746

Antenna 145: 2459746

Antenna 186: 2459746

Antenna 158: 2459746

Antenna 55: 2459746

Antenna 160: 2459746

Antenna 157: 2459746

`auto_metrics` Nightly Notebook¶

Figure 1: Antenna Positions with `auto_metrics` flags.¶