
import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import matplotlib.pyplot as plt
import matplotlib
import glob
import os
import operator
from hera_cal.io import HERAData
from hera_cal.utils import split_pol
from hera_qm.metrics_io import load_metric_file
from hera_notebook_templates.utils import status_colors
from IPython.display import display, HTML

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the first few lines of this cell.
# JD = '2459122'
# data_path = '/lustre/aoc/projects/hera/H4C/2459122'
# os.environ["JULIANDATE"] = JD
# os.environ["DATA_PATH"] = data_path


# Use environment variables to figure out path to data
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
print(f'JD = "{JD}"')
print(f'data_path = "{data_path}"')

JD = "2460101"
data_path = "/mnt/sn1/2460101"


from astropy.time import Time
utc = Time(JD, format='jd').datetime
print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 6-5-2023


print(f'Looking for data in {data_path} on JD {JD}...')

auto_metrics_file = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}*.auto_metrics.h5')))
if len(auto_metrics_file) > 0:
    auto_metrics_file = auto_metrics_file[0]
    print(f'Found auto_metrics results file at {auto_metrics_file}.')
else:
    raise OSError(f'{auto_metrics_file} not found.')

raw_auto_files = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}.?????.sum.autos.uvh5')))
if len(raw_auto_files) > 0:
    print(f'Found {len(raw_auto_files)} extracted autocorrelation files.')
else: 
    raise OSError(f'No files of the form zen.{JD}.?????.sum.autos.uvh5 found in {data_path}.')

Looking for data in /mnt/sn1/2460101 on JD 2460101...
Found auto_metrics results file at /mnt/sn1/2460101/zen.2460101.42115.sum.auto_metrics.h5.
Found 361 extracted autocorrelation files.


# load auto_metrics and define use someful quantities
am = load_metric_file(auto_metrics_file)
mean_round_modz_cut = am['parameters']['mean_round_modz_cut']
ex_ants = am['ex_ants']['r2_ex_ants']
ants = sorted(set(bl[0] for bl in am['modzs']['r2_shape_modzs']))

# load raw autocorrelation waterfalls and define some useful quantities
hd = HERAData(raw_auto_files)
autos, _, _ = hd.read(axis='blt')
wf_shape = next(iter(autos.values())).shape
freqs = autos.freqs / 1e6
times = autos.times
lsts = autos.lsts * 12 / np.pi


# try to load a priori antenna statusesm but fail gracefully if this doesn't work.
a_priori_statuses = {ant: 'Not Found' for ant in ants}
nodes = {ant: np.nan for ant in ants}
try:
    from hera_mc import cm_hookup

    # get node numbers
    hookup = cm_hookup.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in nodes:
            nodes[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])            
            
    # get apriori antenna status
    for ant_name, data in hookup.items():
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in a_priori_statuses:
            a_priori_statuses[ant] = data.apriori
    
except Exception as err:
    print(f'Could not load node numbers and a priori antenna statuses.\nEncountered {type(err)} with message: {err}')


# print ex_ants for easy copy-pasting to YAML file
print('ex_ants: [' + ", ".join(str(ant) for ant in ex_ants) + ']')

ex_ants: [4, 5, 9, 15, 17, 18, 27, 28, 29, 32, 34, 40, 41, 47, 51, 54, 55, 58, 59, 61, 66, 68, 329, 78, 81, 82, 83, 86, 87, 90, 92, 93, 94, 96, 104, 109, 110, 111, 112, 117, 118, 121, 124, 127, 131, 136, 137, 140, 143, 147, 148, 149, 150, 151, 153, 155, 156, 159, 160, 161, 164, 165, 166, 167, 168, 169, 170, 180, 182, 184, 189, 190, 191, 200, 205, 207, 208, 209, 210, 211, 225, 227, 246]


def Array_Plot():
    plt.figure(figsize=(16,16))
    plt.scatter(np.array([autos.antpos[ant][0] for ant in ants]), 
                np.array([autos.antpos[ant][1] for ant in ants]), c='w', s=0)    
    for ant in ants:
        pos = autos.antpos[ant]
        bad = ant in ex_ants
        plt.gca().add_artist(plt.Circle(tuple(pos[0:2]), radius=7, 
                                        fill=(~bad), color=['grey','r'][bad]))
        plt.text(pos[0],pos[1],str(ant), va='center', ha='center', color='w')
    plt.xlabel("Antenna East-West Position (meters)")
    plt.ylabel("Antenna North-South Position (meters)")
    plt.title(f'Antenna Positions and Auto_Metrics Flags on {JD}\n(Maximum Modified Z-Score > {mean_round_modz_cut} in Red)');
    plt.axis('equal')
    plt.tight_layout()


Array_Plot()


# Parse modzs for Table 1 and other figures
modzs_to_check = {'Shape': 'r2_shape_modzs', 'Power': 'r2_power_modzs', 
                  'Temporal Variability': 'r2_temp_var_modzs', 'Temporal Discontinuties': 'r2_temp_diff_modzs'}
worst_metrics = []
worst_zs = []
all_modzs = {}
binary_flags = {rationale: [] for rationale in modzs_to_check}

for ant in ants:
    # parse modzs and figure out flag counts
    modzs = {f'{pol} {rationale}': am['modzs'][dict_name][(ant, ant, pol)] 
             for rationale, dict_name in modzs_to_check.items() for pol in autos.pols()}
    for pol in autos.pols():
        for rationale, dict_name in modzs_to_check.items():
            binary_flags[rationale].append(am['modzs'][dict_name][(ant, ant, pol)] > mean_round_modz_cut)
    
    # figure out which metric is the largest outlier
    worst_metric, worst_z = max(modzs.items(), key=operator.itemgetter(1))
    worst_metrics.append(worst_metric)
    worst_zs.append(worst_z)
    
    # parse out all metrics for dataframe
    for k in modzs:
        col_label = k + ' Modified Z-Score'        
        if col_label in all_modzs:
            all_modzs[col_label].append(modzs[k])
        else:
            all_modzs[col_label] = [modzs[k]]


# build dataframe
to_show = {'Ant': ants, 
           'Node': [f'N{nodes[ant]:02}' for ant in ants],
           'A Priori Status': [a_priori_statuses[ant] for ant in ants],
           'Worst Metric': worst_metrics, 'Worst Modified Z-Score': worst_zs}
to_show.update(all_modzs)
df = pd.DataFrame(to_show).sort_values('Worst Modified Z-Score', ascending=False)

# style dataframe
z_score_cols = [col for col in df.columns if col not in ['Ant', 'Node', 'A Priori Status', 'Worst Metric']]
table1 = df.style.hide_index()\
           .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
           .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=z_score_cols) \
           .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])


HTML(table1.render())


def Flag_Bar_Chart():
    plt.figure(figsize=(8, 4), dpi=100)

    # count 
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    antpol_flags = [np.sum(f) for f in flags_list]
    ant_flags = [np.sum(np.array(f)[0::2] | np.array(f)[1::2]) for f in flags_list]

    # make bar chart
    plt.bar(np.arange(len(rationales)), antpol_flags, width=.7, color='yellow', ec='k', tick_label=rationales)
    for x, (nflags, nants) in enumerate(zip(antpol_flags, ant_flags)):
        plt.text(x, nflags/2, f'{nflags} Feeds on \n{nants} Antennas\nFlagged', va='center', ha='center')

    # set labels
    plt.ylabel('Antenna-Polarizations Flagged')
    plt.xlabel('Reason for Flagging')
    plt.tight_layout()


Flag_Bar_Chart()


def Rationale_Corr_Plot():
    plt.figure(figsize=(6,6), dpi=100)
    
    # compute correlation matrix
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    corrs = np.corrcoef(flags_list)
    
    # plot and label correlation matrix
    plt.imshow(corrs, cmap='viridis', interpolation='nearest', origin='upper')
    for i in range(corrs.shape[0]):
        for j in range(corrs.shape[1]):
            plt.text(i, j, np.round(corrs[i, j], 3), va='center', ha='center',
                     bbox={'facecolor': 'w', 'ec': 'w', 'alpha': .75})

    # colorbar, labels, and style
    plt.yticks(range(len(rationales)), rationales)
    plt.xticks(range(len(rationales)), rationales, rotation=-45, ha='right')
    plt.gca().xaxis.tick_top()
    plt.clim([0, 1])
    plt.colorbar(fraction=0.046, pad=0.04)
    plt.tight_layout()


Rationale_Corr_Plot()


def plot_all_spectra(spectra, mod_zs, modz_cut, overall, freqs, reason, ex_ants=[], 
                     xlabel='Frequency (MHz)', ylabel='', yscale='linear', ylim_factor=None):
    '''Helper function for plotting all spectra and showing which ones were flagged and why.''' 
    fig, axes = plt.subplots(1,2, figsize=(14,5), dpi=100)
    pols = sorted(set([bl[2] for bl in spectra]))
    for ax, pol in zip(axes, pols):
        # sort antennas into good, bad, and bad but not for this reason
        bad_here = [bl for bl in spectra if (bl[2] == pol) and mod_zs[bl] > modz_cut]
        other_bad = [bl for bl in spectra if (bl[2] == pol) and (bl[0] in ex_ants) and (bl not in bad_here)]
        good = [bl for bl in spectra if (bl[2] == pol) and (bl[0] not in ex_ants)]
        
        # plot all spectra
        l1, l2, l3 = None, None, None
        for bl in other_bad:
            l2, = ax.plot(freqs, spectra[bl], 'darkviolet', lw=.5)
        for bl in bad_here:
            l1, = ax.plot(freqs, spectra[bl], 'r', lw=.5)
        for bl in good:
            l3, = ax.plot(freqs, spectra[bl], 'grey', alpha=.5)
        l4, = ax.plot(freqs, overall[bl[2]], 'k--')
        
        ax.set_xlabel(xlabel)
        ax.set_ylabel(ylabel)
        ax.set_yscale(yscale)
        if ylim_factor is not None:
            ax.set_ylim([np.nanmin([spectra[bl] for bl in good]) / ylim_factor, 
                         np.nanmax([spectra[bl] for bl in good]) * ylim_factor])
        ax.set_title(f'Outliers in Autocorrelation {reason}: {pol}')
        ax.legend([l1, l2, l3, l4], [f'Flagged for {reason}', 'Flagged for Another Reason', 'Unflagged', 'Average Unflagged'], loc='lower right')
    plt.tight_layout()


overall_shape = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra_normed'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_power = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_var = {pol: np.nanmean([spec for bl, spec in am['spectra']['std_spectra_normed'].items() if (bl[2] == pol) 
                                     and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_diff = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_abs_diff_spectra_normed'].items() if (bl[2] == pol) 
                                      and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}


plot_all_spectra(am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], mean_round_modz_cut, 
                 overall_shape, freqs, 'Shape', ex_ants=ex_ants, yscale='linear', ylim_factor=1.2)


plot_all_spectra(am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], mean_round_modz_cut, 
                 overall_power, freqs, 'Power', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'], mean_round_modz_cut,
                 overall_temp_var, freqs, 'Temporal Variability', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'], mean_round_modz_cut, 
                 overall_temp_diff, freqs, 'Temporal Discontinutities', ex_ants=ex_ants, yscale='log')


# compute average good autocorrelations for each polarization
avg_good_autos = {pol: np.zeros(wf_shape, dtype=float) for pol in autos.pols()}
for pol in autos.pols():
    for i in range(wf_shape[0]):
        avg_good_autos[pol][i] = np.mean([np.abs(autos[bl][i, :]) for bl in autos 
                                          if (bl[0] not in ex_ants) and (bl[2] == pol)], axis=0)
    avg_good_autos[pol][am['flags']] = np.nan


def Avg_Auto_Plot():
    fig, axes = plt.subplots(1, 2, figsize=(14,5), dpi=100)
    for ax, pol in zip(axes, sorted(autos.pols())):
        im = ax.imshow(avg_good_autos[pol], aspect='auto', interpolation='nearest',
                       extent=[freqs[0], freqs[-1], times[-1], times[0]])
        ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))    
        plt.colorbar(im, ax=ax)
        ax.set_title(f'Average Good Raw {pol} Autocorrelation After Flagging')
        ax.set_ylabel('LST (hours)')
        ax.set_xlabel('Frequency (MHz)')
    plt.tight_layout()


Avg_Auto_Plot()


def plot_spectra(axes, ant, spec, modzs, modz_cut, overall, reason, yscale='linear'):
    '''Helper function for plotting both antennas of a given polarization.'''
    for pol, ax in zip(sorted(autos.pols()), axes):
        bl = (ant, ant, pol)

        # plot good antennas
        for bl2 in modzs:
            if (bl2[0] not in ex_ants) and (bl2[2] == pol):
                ax.plot(freqs, spec[bl2], 'grey', lw=.5, alpha=.5)
        ax.plot(freqs, overall[pol], 'k--')

        # plot this anetnna
        color = 'r'
        alpha = .75
        if modzs[bl] >= modz_cut:
            alpha = 1
        elif bl[0] in ex_ants:
            color = 'darkviolet'
        else:
            color = 'darkgreen'
        ax.plot(freqs, spec[bl], color, alpha=alpha, label=f'{ant} {pol}\n(z = {np.round(modzs[bl],1)})')

        # decorate axis
        ax.set_yscale(yscale)
        ax.legend(loc=2)
        ax.set_title(f'{pol} {reason}')


len(ants)

202


def Plot_All_Auto_Metrics():
    for row, ant in enumerate(df['Ant']): 
        # print title of section
        display(HTML(f'<h2>Antenna {ant}: {JD}</h2>'))

        # print metrics
        df_row = df.loc[df['Ant'] == ant]
        df_row = df_row.style.hide_index()\
                       .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])\
                       .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
                       .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
                       .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols)
        display(HTML(df_row.render()))

        # plot spectra and waterfalls
        fig, axes = plt.subplots(2, 6, figsize=(18, 5.5), dpi=(40 * (104 / len(ants))**.5)) # this should help manage filesize

        # plot individual spectra compared to all good antennas
        plot_spectra(axes[:, 0], ant, am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], 
                     mean_round_modz_cut, overall_shape, 'Shape', yscale='linear')
        plot_spectra(axes[:, 1], ant, am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], 
                     mean_round_modz_cut, overall_power, 'Power', yscale='log')
        plot_spectra(axes[:, 2], ant, am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'],
                     mean_round_modz_cut, overall_temp_var, 'Temporal Variability', yscale='linear')
        plot_spectra(axes[:, 3], ant, am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'],
                     mean_round_modz_cut, overall_temp_diff, 'Temporal Discontinutities', yscale='linear')
        plt.tight_layout()

        # plot linear-scale waterfalls 
        for pol, ax in zip(sorted(autos.pols()), axes[:, 4]):
            bl = (ant, ant, pol)        
            im = ax.imshow(np.where(am['flags'], np.nan, autos[bl].real), 
                           aspect='auto', interpolation='nearest', cmap='inferno',
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Waterfall (Linear Scale)')

        # plot log-scale mean-divided waterfalls
        for pol, ax in zip(sorted(autos.pols()), axes[:, 5]):
            bl = (ant, ant, pol)
            to_plot = autos[bl].real / avg_good_autos[pol]
            to_plot[am['flags']] = np.nan
            to_plot /= np.nanmean(to_plot)
            im = ax.imshow(np.log10(to_plot), aspect='auto', cmap='seismic', interpolation='nearest', vmin=-.07, vmax=.07,
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Log(Normalized Waterfall)')

        plt.show()

        # print some whitespace
        display(HTML('<hr style="height:3px">'))


Plot_All_Auto_Metrics()


from hera_qm import __version__
print(__version__)

2.1.1.dev3+gb291d34

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
83	N07	RF_maintenance	ee Power	inf	46.904418	47.020159	inf	inf	1.169278	1.557951	6.000932	7.448181
170	N15	digital_ok	ee Power	inf	80.172235	80.197817	inf	inf	631.987558	630.409273	5276.477694	5298.611916
117	N07	RF_maintenance	ee Power	inf	46.900926	47.017195	inf	inf	2.340728	2.337733	5.759450	7.496054
118	N07	RF_maintenance	ee Power	inf	46.899941	47.016657	inf	inf	1.576598	1.464266	6.129831	6.221548
82	N07	RF_maintenance	ee Power	inf	46.906447	47.017421	inf	inf	1.727866	1.676207	6.775951	8.715330
137	N07	RF_maintenance	ee Power	inf	46.908821	47.019820	inf	inf	2.569549	3.159368	12.308362	16.481273
81	N07	RF_maintenance	ee Power	inf	46.908656	47.313603	inf	inf	2.190151	2.519344	9.378201	34.940709
27	N01	RF_ok	nn Temporal Discontinuties	56.919476	5.061153	10.092797	19.015120	11.578539	2.232198	0.732464	4.258164	56.919476
104	N08	dish_maintenance	nn Shape	30.084350	4.512615	30.084350	4.360020	8.533792	1.858905	3.602328	2.151100	3.222259
18	N01	RF_ok	ee Temporal Discontinuties	29.017939	4.797904	0.435651	5.232672	0.836442	2.658323	0.320743	29.017939	7.555853
55	N04	digital_ok	nn Shape	26.096966	0.948790	26.096966	1.577200	13.867813	1.953221	2.595719	2.940670	2.725923
40	N04	digital_ok	ee Temporal Discontinuties	23.364590	1.350655	0.152930	1.003223	-0.348676	1.039510	-0.230989	23.364590	0.223829
66	N03	digital_ok	ee Power	22.863776	12.779963	1.927804	22.863776	12.799844	2.254126	1.127651	8.249384	3.093231
93	N10	RF_maintenance	nn Power	19.612301	5.196913	6.320288	18.920162	19.612301	2.241520	2.224681	4.300892	4.740540
143	N14	RF_maintenance	nn Power	19.538046	10.547819	6.412997	8.561861	19.538046	1.957391	2.232720	3.267252	5.618904
58	N05	RF_maintenance	nn Power	19.491258	6.592993	7.999898	18.889230	19.491258	2.230530	2.218868	3.674980	3.655411
59	N05	RF_ok	nn Power	19.459261	-0.573888	8.248621	-0.296677	19.459261	-0.233525	2.214645	0.101797	3.261397
92	N10	RF_maintenance	nn Power	19.380944	4.844199	5.891706	19.012842	19.380944	2.235777	2.217917	4.566262	4.573063
124	N09	RF_maintenance	ee Power	19.248548	5.077124	0.278141	19.248548	0.745350	2.229259	0.832842	3.749394	1.230351
94	N10	RF_maintenance	ee Power	19.197144	5.606045	3.409350	19.197144	9.555273	2.256562	2.437343	4.438065	2.742915
109	N10	RF_maintenance	nn Power	19.190818	3.733114	6.193284	18.242783	19.190818	1.097454	2.223061	2.844574	4.379156
166	N14	RF_maintenance	ee Power	19.181460	7.253974	-0.388291	19.181460	-0.299732	2.238439	0.072506	3.620484	0.008761
182	N13	RF_maintenance	nn Power	19.168507	-0.853170	6.132526	-0.294192	19.168507	-1.087977	2.173983	-0.676758	4.509422
127	N10	RF_maintenance	ee Power	18.988367	4.986684	17.974986	18.988367	4.895677	2.232752	3.301821	3.413178	2.517595
160	N13	RF_maintenance	ee Power	18.910796	5.537217	0.191286	18.910796	0.403852	2.244893	0.594782	4.498442	0.501656
29	N01	RF_maintenance	nn Power	18.800111	5.273460	6.203964	18.473173	18.800111	2.230593	2.206461	4.196929	3.186912
155	N12	RF_maintenance	ee Power	18.744925	5.030676	-0.348526	18.744925	0.288613	2.220155	0.507124	4.328533	0.598196
136	N12	digital_ok	ee Power	18.609900	4.628050	1.069284	18.609900	0.927086	2.205318	0.887489	4.620735	1.280115
87	N08	RF_maintenance	ee Shape	18.188526	18.188526	0.384777	3.931167	-0.826579	1.079948	-0.534950	3.955566	-0.143329
180	N13	RF_ok	nn Power	16.227957	-0.345929	12.699473	-0.172372	16.227957	-0.107593	-0.036856	1.000101	2.706496
184	N14	dish_maintenance	ee Power	16.087229	7.026080	0.594701	16.087229	1.082504	-0.179743	0.843290	2.924987	0.844135
32	N02	RF_maintenance	nn Temporal Discontinuties	13.969818	10.442809	-0.698375	0.746811	-1.119736	0.117572	-0.241129	3.043544	13.969818
112	N10	digital_ok	nn Power	13.425576	0.254119	2.410948	0.496563	13.425576	0.763670	0.796646	0.770595	2.039561
17	N01	digital_ok	nn Power	13.367517	0.158268	1.992889	0.619299	13.367517	0.857372	0.916011	0.771930	3.229184
161	N13	digital_ok	nn Shape	13.324999	0.494567	13.324999	0.585223	0.462013	0.532502	-0.621550	0.634458	0.143244
110	N10	RF_maintenance	ee Shape	12.706220	12.706220	-0.953478	0.534520	-0.182325	-0.278355	-0.325157	0.802214	0.121977
329	N12	dish_maintenance	nn Power	12.209103	5.730560	6.122783	11.436352	12.209103	2.247067	2.217463	2.974870	3.757626
211	N20	RF_maintenance	nn Power	12.064714	-0.558879	6.203261	-0.177657	12.064714	-0.274821	2.212444	0.193112	3.308702
34	N06	not_connected	ee Power	11.863108	5.763069	0.163950	11.863108	-0.154079	2.245202	-0.887834	3.472467	-0.888650
200	N18	RF_maintenance	ee Power	11.676635	6.075018	-0.180655	11.676635	-1.159747	2.245563	-1.100516	3.306184	-0.377428

auto_metrics Nightly Notebook¶

Statistics computed (after removing the worst offenders with a median-based metrics and then RFI flagging):¶

Parse Inputs and Load Data¶

Summary Plots and Tables¶

Figure 1: Antenna Positions with auto_metrics flags.¶

Table 1: Modified Z-Score Summary¶

Figure 2: Flagging Rationale Summary¶

Figure 3: Flagging Rationale Correlations¶

Figure 4: Outliers in Autocorrelation Shape¶

Figure 5: Outliers in Autocorrelation Power¶

Figure 6: Outliers in Autocorrelation Temporal Variability¶

Figure 7: Outliers in Autocorrelation Temporal Discontinuities¶

Figure 8: Average Good Autocorrelations and Flags¶

Per-Antenna Plots¶

Figure 9: Per-Antenna Statistics, Spectra, and Waterfalls¶

Antenna 83: 2460101

Antenna 170: 2460101

Antenna 117: 2460101

Antenna 118: 2460101

Antenna 82: 2460101

Antenna 137: 2460101

Antenna 81: 2460101

Antenna 27: 2460101

Antenna 104: 2460101

Antenna 18: 2460101

Antenna 55: 2460101

Antenna 40: 2460101

Antenna 66: 2460101

Antenna 93: 2460101

Antenna 143: 2460101

Antenna 58: 2460101

Antenna 59: 2460101

Antenna 92: 2460101

Antenna 124: 2460101

Antenna 94: 2460101

Antenna 109: 2460101

Antenna 166: 2460101

Antenna 182: 2460101

Antenna 127: 2460101

Antenna 160: 2460101

Antenna 29: 2460101

Antenna 155: 2460101

Antenna 136: 2460101

Antenna 87: 2460101

Antenna 180: 2460101

Antenna 184: 2460101

Antenna 32: 2460101

Antenna 112: 2460101

Antenna 17: 2460101

Antenna 161: 2460101

Antenna 110: 2460101

Antenna 329: 2460101

Antenna 211: 2460101

Antenna 34: 2460101

Antenna 200: 2460101

Antenna 131: 2460101

Antenna 153: 2460101

Antenna 47: 2460101

Antenna 246: 2460101

Antenna 225: 2460101

Antenna 61: 2460101

Antenna 207: 2460101

Antenna 51: 2460101

Antenna 90: 2460101

Antenna 96: 2460101

Antenna 28: 2460101

Antenna 68: 2460101

Antenna 121: 2460101

Antenna 78: 2460101

Antenna 205: 2460101

Antenna 4: 2460101

Antenna 54: 2460101

Antenna 111: 2460101

Antenna 159: 2460101

Antenna 156: 2460101

Antenna 86: 2460101

Antenna 165: 2460101

Antenna 227: 2460101

Antenna 15: 2460101

Antenna 140: 2460101

`auto_metrics` Nightly Notebook¶

Figure 1: Antenna Positions with `auto_metrics` flags.¶