
import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import matplotlib.pyplot as plt
import matplotlib
import glob
import os
import operator
from hera_cal.io import HERAData
from hera_cal.utils import split_pol
from hera_qm.metrics_io import load_metric_file
from hera_notebook_templates.utils import status_colors
from IPython.display import display, HTML

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the first few lines of this cell.
# JD = '2459122'
# data_path = '/lustre/aoc/projects/hera/H4C/2459122'
# os.environ["JULIANDATE"] = JD
# os.environ["DATA_PATH"] = data_path


# Use environment variables to figure out path to data
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
print(f'JD = "{JD}"')
print(f'data_path = "{data_path}"')

JD = "2459925"
data_path = "/mnt/sn1/2459925"


from astropy.time import Time
utc = Time(JD, format='jd').datetime
print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 12-11-2022


print(f'Looking for data in {data_path} on JD {JD}...')

auto_metrics_file = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}*.auto_metrics.h5')))
if len(auto_metrics_file) > 0:
    auto_metrics_file = auto_metrics_file[0]
    print(f'Found auto_metrics results file at {auto_metrics_file}.')
else:
    raise OSError(f'{auto_metrics_file} not found.')

raw_auto_files = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}.?????.sum.autos.uvh5')))
if len(raw_auto_files) > 0:
    print(f'Found {len(raw_auto_files)} extracted autocorrelation files.')
else: 
    raise OSError(f'No files of the form zen.{JD}.?????.sum.autos.uvh5 found in {data_path}.')

Looking for data in /mnt/sn1/2459925 on JD 2459925...
Found auto_metrics results file at /mnt/sn1/2459925/zen.2459925.21296.sum.auto_metrics.h5.
Found 1850 extracted autocorrelation files.


# load auto_metrics and define use someful quantities
am = load_metric_file(auto_metrics_file)
mean_round_modz_cut = am['parameters']['mean_round_modz_cut']
ex_ants = am['ex_ants']['r2_ex_ants']
ants = sorted(set(bl[0] for bl in am['modzs']['r2_shape_modzs']))

# load raw autocorrelation waterfalls and define some useful quantities
hd = HERAData(raw_auto_files)
autos, _, _ = hd.read(axis='blt')
wf_shape = next(iter(autos.values())).shape
freqs = autos.freqs / 1e6
times = autos.times
lsts = autos.lsts * 12 / np.pi


# try to load a priori antenna statusesm but fail gracefully if this doesn't work.
a_priori_statuses = {ant: 'Not Found' for ant in ants}
nodes = {ant: np.nan for ant in ants}
try:
    from hera_mc import cm_hookup

    # get node numbers
    hookup = cm_hookup.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in nodes:
            nodes[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])            
            
    # get apriori antenna status
    for ant_name, data in hookup.items():
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in a_priori_statuses:
            a_priori_statuses[ant] = data.apriori
    
except Exception as err:
    print(f'Could not load node numbers and a priori antenna statuses.\nEncountered {type(err)} with message: {err}')


# print ex_ants for easy copy-pasting to YAML file
print('ex_ants: [' + ", ".join(str(ant) for ant in ex_ants) + ']')

ex_ants: [3, 4, 9, 10, 15, 16, 18, 19, 21, 27, 28, 29, 30, 32, 34, 35, 36, 42, 47, 49, 50, 51, 54, 55, 56, 57, 58, 59, 60, 63, 68, 72, 78, 80, 81, 84, 86, 87, 90, 92, 93, 94, 96, 97, 99, 100, 101, 103, 104, 106, 108, 109, 110, 111, 113, 117, 119, 121, 122, 123, 126, 128, 131, 133, 136, 138, 142, 143, 145, 146, 147, 148, 149, 155, 156, 158, 159, 161, 164, 165, 170, 180, 182, 183, 185, 187, 189, 200, 201, 203, 207, 208, 209, 210, 219, 222, 223, 224, 225, 228, 237, 239, 240, 241, 242, 243, 244, 246, 261, 262, 320, 329]


def Array_Plot():
    plt.figure(figsize=(16,16))
    plt.scatter(np.array([autos.antpos[ant][0] for ant in ants]), 
                np.array([autos.antpos[ant][1] for ant in ants]), c='w', s=0)    
    for ant in ants:
        pos = autos.antpos[ant]
        bad = ant in ex_ants
        plt.gca().add_artist(plt.Circle(tuple(pos[0:2]), radius=7, 
                                        fill=(~bad), color=['grey','r'][bad]))
        plt.text(pos[0],pos[1],str(ant), va='center', ha='center', color='w')
    plt.xlabel("Antenna East-West Position (meters)")
    plt.ylabel("Antenna North-South Position (meters)")
    plt.title(f'Antenna Positions and Auto_Metrics Flags on {JD}\n(Maximum Modified Z-Score > {mean_round_modz_cut} in Red)');
    plt.axis('equal')
    plt.tight_layout()


Array_Plot()


# Parse modzs for Table 1 and other figures
modzs_to_check = {'Shape': 'r2_shape_modzs', 'Power': 'r2_power_modzs', 
                  'Temporal Variability': 'r2_temp_var_modzs', 'Temporal Discontinuties': 'r2_temp_diff_modzs'}
worst_metrics = []
worst_zs = []
all_modzs = {}
binary_flags = {rationale: [] for rationale in modzs_to_check}

for ant in ants:
    # parse modzs and figure out flag counts
    modzs = {f'{pol} {rationale}': am['modzs'][dict_name][(ant, ant, pol)] 
             for rationale, dict_name in modzs_to_check.items() for pol in autos.pols()}
    for pol in autos.pols():
        for rationale, dict_name in modzs_to_check.items():
            binary_flags[rationale].append(am['modzs'][dict_name][(ant, ant, pol)] > mean_round_modz_cut)
    
    # figure out which metric is the largest outlier
    worst_metric, worst_z = max(modzs.items(), key=operator.itemgetter(1))
    worst_metrics.append(worst_metric)
    worst_zs.append(worst_z)
    
    # parse out all metrics for dataframe
    for k in modzs:
        col_label = k + ' Modified Z-Score'        
        if col_label in all_modzs:
            all_modzs[col_label].append(modzs[k])
        else:
            all_modzs[col_label] = [modzs[k]]


# build dataframe
to_show = {'Ant': ants, 
           'Node': [f'N{nodes[ant]:02}' for ant in ants],
           'A Priori Status': [a_priori_statuses[ant] for ant in ants],
           'Worst Metric': worst_metrics, 'Worst Modified Z-Score': worst_zs}
to_show.update(all_modzs)
df = pd.DataFrame(to_show).sort_values('Worst Modified Z-Score', ascending=False)

# style dataframe
z_score_cols = [col for col in df.columns if col not in ['Ant', 'Node', 'A Priori Status', 'Worst Metric']]
table1 = df.style.hide_index()\
           .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
           .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=z_score_cols) \
           .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])


HTML(table1.render())


def Flag_Bar_Chart():
    plt.figure(figsize=(8, 4), dpi=100)

    # count 
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    antpol_flags = [np.sum(f) for f in flags_list]
    ant_flags = [np.sum(np.array(f)[0::2] | np.array(f)[1::2]) for f in flags_list]

    # make bar chart
    plt.bar(np.arange(len(rationales)), antpol_flags, width=.7, color='yellow', ec='k', tick_label=rationales)
    for x, (nflags, nants) in enumerate(zip(antpol_flags, ant_flags)):
        plt.text(x, nflags/2, f'{nflags} Feeds on \n{nants} Antennas\nFlagged', va='center', ha='center')

    # set labels
    plt.ylabel('Antenna-Polarizations Flagged')
    plt.xlabel('Reason for Flagging')
    plt.tight_layout()


Flag_Bar_Chart()


def Rationale_Corr_Plot():
    plt.figure(figsize=(6,6), dpi=100)
    
    # compute correlation matrix
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    corrs = np.corrcoef(flags_list)
    
    # plot and label correlation matrix
    plt.imshow(corrs, cmap='viridis', interpolation='nearest', origin='upper')
    for i in range(corrs.shape[0]):
        for j in range(corrs.shape[1]):
            plt.text(i, j, np.round(corrs[i, j], 3), va='center', ha='center',
                     bbox={'facecolor': 'w', 'ec': 'w', 'alpha': .75})

    # colorbar, labels, and style
    plt.yticks(range(len(rationales)), rationales)
    plt.xticks(range(len(rationales)), rationales, rotation=-45, ha='right')
    plt.gca().xaxis.tick_top()
    plt.clim([0, 1])
    plt.colorbar(fraction=0.046, pad=0.04)
    plt.tight_layout()


Rationale_Corr_Plot()


def plot_all_spectra(spectra, mod_zs, modz_cut, overall, freqs, reason, ex_ants=[], 
                     xlabel='Frequency (MHz)', ylabel='', yscale='linear', ylim_factor=None):
    '''Helper function for plotting all spectra and showing which ones were flagged and why.''' 
    fig, axes = plt.subplots(1,2, figsize=(14,5), dpi=100)
    pols = sorted(set([bl[2] for bl in spectra]))
    for ax, pol in zip(axes, pols):
        # sort antennas into good, bad, and bad but not for this reason
        bad_here = [bl for bl in spectra if (bl[2] == pol) and mod_zs[bl] > modz_cut]
        other_bad = [bl for bl in spectra if (bl[2] == pol) and (bl[0] in ex_ants) and (bl not in bad_here)]
        good = [bl for bl in spectra if (bl[2] == pol) and (bl[0] not in ex_ants)]
        
        # plot all spectra
        l1, l2, l3 = None, None, None
        for bl in other_bad:
            l2, = ax.plot(freqs, spectra[bl], 'darkviolet', lw=.5)
        for bl in bad_here:
            l1, = ax.plot(freqs, spectra[bl], 'r', lw=.5)
        for bl in good:
            l3, = ax.plot(freqs, spectra[bl], 'grey', alpha=.5)
        l4, = ax.plot(freqs, overall[bl[2]], 'k--')
        
        ax.set_xlabel(xlabel)
        ax.set_ylabel(ylabel)
        ax.set_yscale(yscale)
        if ylim_factor is not None:
            ax.set_ylim([np.nanmin([spectra[bl] for bl in good]) / ylim_factor, 
                         np.nanmax([spectra[bl] for bl in good]) * ylim_factor])
        ax.set_title(f'Outliers in Autocorrelation {reason}: {pol}')
        ax.legend([l1, l2, l3, l4], [f'Flagged for {reason}', 'Flagged for Another Reason', 'Unflagged', 'Average Unflagged'], loc='lower right')
    plt.tight_layout()


overall_shape = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra_normed'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_power = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_var = {pol: np.nanmean([spec for bl, spec in am['spectra']['std_spectra_normed'].items() if (bl[2] == pol) 
                                     and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_diff = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_abs_diff_spectra_normed'].items() if (bl[2] == pol) 
                                      and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}


plot_all_spectra(am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], mean_round_modz_cut, 
                 overall_shape, freqs, 'Shape', ex_ants=ex_ants, yscale='linear', ylim_factor=1.2)


plot_all_spectra(am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], mean_round_modz_cut, 
                 overall_power, freqs, 'Power', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'], mean_round_modz_cut,
                 overall_temp_var, freqs, 'Temporal Variability', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'], mean_round_modz_cut, 
                 overall_temp_diff, freqs, 'Temporal Discontinutities', ex_ants=ex_ants, yscale='log')


# compute average good autocorrelations for each polarization
avg_good_autos = {pol: np.zeros(wf_shape, dtype=float) for pol in autos.pols()}
for pol in autos.pols():
    for i in range(wf_shape[0]):
        avg_good_autos[pol][i] = np.mean([np.abs(autos[bl][i, :]) for bl in autos 
                                          if (bl[0] not in ex_ants) and (bl[2] == pol)], axis=0)
    avg_good_autos[pol][am['flags']] = np.nan


def Avg_Auto_Plot():
    fig, axes = plt.subplots(1, 2, figsize=(14,5), dpi=100)
    for ax, pol in zip(axes, sorted(autos.pols())):
        im = ax.imshow(avg_good_autos[pol], aspect='auto', interpolation='nearest',
                       extent=[freqs[0], freqs[-1], times[-1], times[0]])
        ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))    
        plt.colorbar(im, ax=ax)
        ax.set_title(f'Average Good Raw {pol} Autocorrelation After Flagging')
        ax.set_ylabel('LST (hours)')
        ax.set_xlabel('Frequency (MHz)')
    plt.tight_layout()


Avg_Auto_Plot()


def plot_spectra(axes, ant, spec, modzs, modz_cut, overall, reason, yscale='linear'):
    '''Helper function for plotting both antennas of a given polarization.'''
    for pol, ax in zip(sorted(autos.pols()), axes):
        bl = (ant, ant, pol)

        # plot good antennas
        for bl2 in modzs:
            if (bl2[0] not in ex_ants) and (bl2[2] == pol):
                ax.plot(freqs, spec[bl2], 'grey', lw=.5, alpha=.5)
        ax.plot(freqs, overall[pol], 'k--')

        # plot this anetnna
        color = 'r'
        alpha = .75
        if modzs[bl] >= modz_cut:
            alpha = 1
        elif bl[0] in ex_ants:
            color = 'darkviolet'
        else:
            color = 'darkgreen'
        ax.plot(freqs, spec[bl], color, alpha=alpha, label=f'{ant} {pol}\n(z = {np.round(modzs[bl],1)})')

        # decorate axis
        ax.set_yscale(yscale)
        ax.legend(loc=2)
        ax.set_title(f'{pol} {reason}')


len(ants)

201


def Plot_All_Auto_Metrics():
    for row, ant in enumerate(df['Ant']): 
        # print title of section
        display(HTML(f'<h2>Antenna {ant}: {JD}</h2>'))

        # print metrics
        df_row = df.loc[df['Ant'] == ant]
        df_row = df_row.style.hide_index()\
                       .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])\
                       .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
                       .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
                       .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols)
        display(HTML(df_row.render()))

        # plot spectra and waterfalls
        fig, axes = plt.subplots(2, 6, figsize=(18, 5.5), dpi=(50 * (104 / len(ants))**.5)) # this should help manage filesize

        # plot individual spectra compared to all good antennas
        plot_spectra(axes[:, 0], ant, am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], 
                     mean_round_modz_cut, overall_shape, 'Shape', yscale='linear')
        plot_spectra(axes[:, 1], ant, am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], 
                     mean_round_modz_cut, overall_power, 'Power', yscale='log')
        plot_spectra(axes[:, 2], ant, am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'],
                     mean_round_modz_cut, overall_temp_var, 'Temporal Variability', yscale='linear')
        plot_spectra(axes[:, 3], ant, am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'],
                     mean_round_modz_cut, overall_temp_diff, 'Temporal Discontinutities', yscale='linear')
        plt.tight_layout()

        # plot linear-scale waterfalls 
        for pol, ax in zip(sorted(autos.pols()), axes[:, 4]):
            bl = (ant, ant, pol)        
            im = ax.imshow(np.where(am['flags'], np.nan, autos[bl].real), 
                           aspect='auto', interpolation='nearest', cmap='inferno',
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Waterfall (Linear Scale)')

        # plot log-scale mean-divided waterfalls
        for pol, ax in zip(sorted(autos.pols()), axes[:, 5]):
            bl = (ant, ant, pol)
            to_plot = autos[bl].real / avg_good_autos[pol]
            to_plot[am['flags']] = np.nan
            to_plot /= np.nanmean(to_plot)
            im = ax.imshow(np.log10(to_plot), aspect='auto', cmap='seismic', interpolation='nearest', vmin=-.07, vmax=.07,
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Log(Normalized Waterfall)')

        plt.show()

        # print some whitespace
        display(HTML('<hr style="height:3px">'))


Plot_All_Auto_Metrics()


from hera_qm import __version__
print(__version__)

2.0.5.dev11+g87299d5

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
138	N07	RF_maintenance	ee Power	inf	120.537560	115.473460	inf	inf	3195.597897	3392.333729	261.812911	311.123040
149	N15	digital_ok	ee Power	inf	174.648418	174.216670	inf	inf	4229.892561	4365.473679	214.075849	246.113052
148	N15	digital_ok	ee Power	inf	104.465129	104.851383	inf	inf	3761.078669	3931.481162	309.581504	310.189234
147	N15	digital_ok	ee Power	inf	133.805114	111.207095	inf	inf	4691.479732	4566.041017	332.904250	346.387259
187	N14	digital_ok	ee Temporal Variability	55.806312	-0.484941	-1.096666	-0.146077	2.291159	55.806312	1.502197	-0.932306	2.067614
19	N02	digital_ok	nn Temporal Variability	51.104373	-0.384329	-1.048363	-0.237381	2.577726	10.208402	51.104373	-0.200392	0.761341
103	N08	digital_ok	ee Temporal Variability	49.967059	2.208015	2.860871	1.261585	-0.655075	49.967059	1.016946	0.193568	-0.273091
122	N08	digital_ok	ee Temporal Variability	47.716477	4.126810	3.967867	0.180076	0.707254	47.716477	0.112050	-0.631292	0.568821
243	N19	RF_ok	ee Shape	45.206616	45.206616	2.922119	0.128548	-1.991612	4.058255	0.192931	1.485927	-1.331549
18	N01	RF_maintenance	nn Shape	35.489159	7.643992	35.489159	11.228925	-0.247208	4.750467	14.099806	3.024642	1.103490
28	N01	RF_maintenance	nn Shape	35.418022	13.784687	35.418022	-0.672480	1.097332	6.798901	14.425912	1.291203	2.257688
136	N12	digital_ok	ee Temporal Variability	34.183165	0.297308	-0.875853	5.759523	2.275130	34.183165	26.016038	5.888725	2.688364
200	N18	RF_maintenance	nn Shape	30.284076	7.686571	30.284076	5.080603	-1.137534	4.811337	6.738676	3.087347	0.568427
242	N19	RF_ok	ee Temporal Variability	30.221869	26.651056	1.463661	-0.669083	1.085230	30.221869	-1.250149	2.183816	-0.704513
240	N19	RF_maintenance	nn Shape	30.000193	1.080697	30.000193	2.209944	-0.050945	2.614456	6.134338	1.166301	2.178077
156	N12	RF_maintenance	ee Temporal Variability	29.477088	0.039626	7.561497	5.306245	11.458880	29.477088	5.817615	6.612628	1.886316
223	N19	RF_ok	nn Temporal Variability	29.395718	2.806583	2.037839	-1.974812	-0.406854	-1.492844	29.395718	-1.904622	-1.194962
4	N01	RF_maintenance	nn Temporal Variability	28.291408	0.047895	-0.264888	0.037517	-0.234807	14.590640	28.291408	-0.932510	-0.254701
92	N10	RF_maintenance	nn Shape	28.229748	26.816226	28.229748	1.811615	2.309763	5.644268	6.549813	1.381630	0.687728
97	N11	not_connected	ee Temporal Variability	23.972507	-0.646031	2.255673	-1.930643	1.991096	23.972507	9.777942	-1.440112	-0.285966
329	N12	dish_maintenance	ee Temporal Variability	21.015595	2.675649	0.104797	-1.830725	-1.588210	21.015595	-0.141678	1.293418	-0.026578
10	N02	digital_ok	ee Temporal Variability	20.560620	0.304896	-1.115834	0.828110	-0.986808	20.560620	0.118971	1.051421	-0.282555
207	N19	RF_ok	nn Temporal Variability	19.353597	4.045079	2.295920	0.609401	-1.235571	-1.008113	19.353597	-0.541829	-1.508382
32	N02	RF_maintenance	ee Temporal Variability	17.592685	12.696815	-0.073825	1.140640	1.575293	17.592685	9.108353	1.304300	1.459199
30	N01	digital_ok	ee Temporal Variability	17.068934	-0.681692	-0.849537	-0.861258	0.299029	17.068934	-0.070903	-0.824759	-0.171758
145	N14	RF_maintenance	nn Temporal Variability	16.638998	-0.779796	0.198070	-0.437012	5.293670	-0.587728	16.638998	0.305031	2.862877
239	N18	RF_ok	nn Temporal Variability	16.460276	0.441505	0.721196	-0.408125	0.356746	0.076184	16.460276	-1.559099	-1.306429
182	N13	digital_ok	ee Temporal Variability	16.380769	0.411065	1.601570	-0.464310	2.755114	16.380769	1.280026	-1.892286	0.544726
68	N03	digital_ok	nn Power	15.803283	-0.140626	14.217768	0.463454	15.803283	0.681397	5.962051	0.981251	1.984846
110	N10	RF_maintenance	nn Power	15.465179	14.354531	14.442270	15.181028	15.465179	4.740524	5.831378	3.053155	1.883039
104	N08	RF_maintenance	nn Shape	15.368675	0.823957	15.368675	7.110714	8.495960	6.623081	5.321574	3.659496	3.326061
161	N13	digital_ok	nn Shape	15.348508	-0.391458	15.348508	-0.276056	-0.246664	-0.531677	1.636883	0.345342	-0.366746
84	N08	RF_maintenance	nn Power	14.962031	10.026507	9.579431	14.280645	14.962031	3.342634	5.901357	0.786645	1.961470
51	N03	digital_ok	ee Power	14.493640	12.560732	1.358298	14.493640	0.229254	5.085972	1.903120	3.333543	0.589782
126	N09	RF_maintenance	ee Temporal Variability	14.384189	5.965164	1.395367	-0.022982	1.044468	14.384189	0.267138	-0.103642	0.502621
108	N09	RF_maintenance	nn Shape	14.132654	7.305850	14.132654	11.194290	0.205870	4.755216	5.949403	3.056585	-0.549880
117	N07	RF_maintenance	nn Power	12.211426	7.339565	8.750341	11.354068	12.211426	4.717637	5.880445	3.048932	1.950786
72	N04	digital_ok	nn Power	12.210780	2.601773	9.211028	0.770826	12.210780	0.523046	5.798237	0.794552	1.918881
42	N04	digital_ok	nn Power	12.175305	7.653693	8.935616	11.539188	12.175305	4.665510	5.736313	3.004852	1.871643
56	N04	digital_ok	nn Power	12.040403	0.172937	9.322631	0.248008	12.040403	-0.031221	5.825108	0.459604	1.913290

auto_metrics Nightly Notebook¶

Statistics computed (after removing the worst offenders with a median-based metrics and then RFI flagging):¶

Parse Inputs and Load Data¶

Summary Plots and Tables¶

Figure 1: Antenna Positions with auto_metrics flags.¶

Table 1: Modified Z-Score Summary¶

Figure 2: Flagging Rationale Summary¶

Figure 3: Flagging Rationale Correlations¶

Figure 4: Outliers in Autocorrelation Shape¶

Figure 5: Outliers in Autocorrelation Power¶

Figure 6: Outliers in Autocorrelation Temporal Variability¶

Figure 7: Outliers in Autocorrelation Temporal Discontinuities¶

Figure 8: Average Good Autocorrelations and Flags¶

Per-Antenna Plots¶

Figure 9: Per-Antenna Statistics, Spectra, and Waterfalls¶

Antenna 138: 2459925

Antenna 149: 2459925

Antenna 148: 2459925

Antenna 147: 2459925

Antenna 187: 2459925

Antenna 19: 2459925

Antenna 103: 2459925

Antenna 122: 2459925

Antenna 243: 2459925

Antenna 18: 2459925

Antenna 28: 2459925

Antenna 136: 2459925

Antenna 200: 2459925

Antenna 242: 2459925

Antenna 240: 2459925

Antenna 156: 2459925

Antenna 223: 2459925

Antenna 4: 2459925

Antenna 92: 2459925

Antenna 97: 2459925

Antenna 329: 2459925

Antenna 10: 2459925

Antenna 207: 2459925

Antenna 32: 2459925

Antenna 30: 2459925

Antenna 145: 2459925

Antenna 239: 2459925

Antenna 182: 2459925

Antenna 68: 2459925

Antenna 110: 2459925

Antenna 104: 2459925

Antenna 161: 2459925

Antenna 84: 2459925

Antenna 51: 2459925

Antenna 126: 2459925

Antenna 108: 2459925

Antenna 117: 2459925

Antenna 72: 2459925

Antenna 42: 2459925

Antenna 56: 2459925

Antenna 180: 2459925

Antenna 54: 2459925

Antenna 208: 2459925

Antenna 55: 2459925

Antenna 142: 2459925

Antenna 60: 2459925

Antenna 58: 2459925

Antenna 128: 2459925

Antenna 94: 2459925

Antenna 111: 2459925

Antenna 237: 2459925

Antenna 27: 2459925

Antenna 109: 2459925

Antenna 170: 2459925

Antenna 222: 2459925

Antenna 3: 2459925

Antenna 59: 2459925

Antenna 29: 2459925

Antenna 16: 2459925

Antenna 164: 2459925

Antenna 155: 2459925

Antenna 262: 2459925

Antenna 189: 2459925

Antenna 15: 2459925

Antenna 81: 2459925

`auto_metrics` Nightly Notebook¶

Figure 1: Antenna Positions with `auto_metrics` flags.¶