
import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import matplotlib.pyplot as plt
import matplotlib
import glob
import os
import operator
from hera_cal.io import HERAData
from hera_cal.utils import split_pol
from hera_qm.metrics_io import load_metric_file
from hera_notebook_templates.utils import status_colors
from IPython.display import display, HTML

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the first few lines of this cell.
# JD = '2459122'
# data_path = '/lustre/aoc/projects/hera/H4C/2459122'
# os.environ["JULIANDATE"] = JD
# os.environ["DATA_PATH"] = data_path


# Use environment variables to figure out path to data
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
print(f'JD = "{JD}"')
print(f'data_path = "{data_path}"')

JD = "2459741"
data_path = "/mnt/sn1/2459741"


from astropy.time import Time
utc = Time(JD, format='jd').datetime
print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 6-10-2022


print(f'Looking for data in {data_path} on JD {JD}...')

auto_metrics_file = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}*.auto_metrics.h5')))
if len(auto_metrics_file) > 0:
    auto_metrics_file = auto_metrics_file[0]
    print(f'Found auto_metrics results file at {auto_metrics_file}.')
else:
    raise OSError(f'{auto_metrics_file} not found.')

raw_auto_files = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}.?????.sum.autos.uvh5')))
if len(raw_auto_files) > 0:
    print(f'Found {len(raw_auto_files)} extracted autocorrelation files.')
else: 
    raise OSError(f'No files of the form zen.{JD}.?????.sum.autos.uvh5 found in {data_path}.')

Looking for data in /mnt/sn1/2459741 on JD 2459741...
Found auto_metrics results file at /mnt/sn1/2459741/zen.2459741.25313.sum.auto_metrics.h5.
Found 372 extracted autocorrelation files.


# load auto_metrics and define use someful quantities
am = load_metric_file(auto_metrics_file)
mean_round_modz_cut = am['parameters']['mean_round_modz_cut']
ex_ants = am['ex_ants']['r2_ex_ants']
ants = sorted(set(bl[0] for bl in am['modzs']['r2_shape_modzs']))

# load raw autocorrelation waterfalls and define some useful quantities
hd = HERAData(raw_auto_files)
autos, _, _ = hd.read(axis='blt')
wf_shape = next(iter(autos.values())).shape
freqs = autos.freqs / 1e6
times = autos.times
lsts = autos.lsts * 12 / np.pi


# try to load a priori antenna statusesm but fail gracefully if this doesn't work.
a_priori_statuses = {ant: 'Not Found' for ant in ants}
nodes = {ant: np.nan for ant in ants}
try:
    from hera_mc import cm_hookup

    # get node numbers
    hookup = cm_hookup.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in nodes:
            nodes[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])            
            
    # get apriori antenna status
    for ant_name, data in hookup.items():
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in a_priori_statuses:
            a_priori_statuses[ant] = data.apriori
    
except Exception as err:
    print(f'Could not load node numbers and a priori antenna statuses.\nEncountered {type(err)} with message: {err}')


# print ex_ants for easy copy-pasting to YAML file
print('ex_ants: [' + ", ".join(str(ant) for ant in ex_ants) + ']')

ex_ants: [0, 1, 2, 3, 4, 5, 7, 8, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 21, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 36, 37, 38, 39, 40, 45, 46, 50, 51, 52, 53, 55, 57, 65, 66, 67, 68, 69, 70, 71, 72, 73, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 116, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 136, 138, 140, 141, 142, 143, 144, 145, 150, 155, 157, 160, 163, 164, 166, 167, 168, 169, 170, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 189, 191, 220, 221, 222, 320, 321, 323, 324, 325, 329, 333]


def Array_Plot():
    plt.figure(figsize=(16,16))
    plt.scatter(np.array([autos.antpos[ant][0] for ant in ants]), 
                np.array([autos.antpos[ant][1] for ant in ants]), c='w', s=0)    
    for ant in ants:
        pos = autos.antpos[ant]
        bad = ant in ex_ants
        plt.gca().add_artist(plt.Circle(tuple(pos[0:2]), radius=7, 
                                        fill=(~bad), color=['grey','r'][bad]))
        plt.text(pos[0],pos[1],str(ant), va='center', ha='center', color='w')
    plt.xlabel("Antenna East-West Position (meters)")
    plt.ylabel("Antenna North-South Position (meters)")
    plt.title(f'Antenna Positions and Auto_Metrics Flags on {JD}\n(Maximum Modified Z-Score > {mean_round_modz_cut} in Red)');
    plt.axis('equal')
    plt.tight_layout()


Array_Plot()


# Parse modzs for Table 1 and other figures
modzs_to_check = {'Shape': 'r2_shape_modzs', 'Power': 'r2_power_modzs', 
                  'Temporal Variability': 'r2_temp_var_modzs', 'Temporal Discontinuties': 'r2_temp_diff_modzs'}
worst_metrics = []
worst_zs = []
all_modzs = {}
binary_flags = {rationale: [] for rationale in modzs_to_check}

for ant in ants:
    # parse modzs and figure out flag counts
    modzs = {f'{pol} {rationale}': am['modzs'][dict_name][(ant, ant, pol)] 
             for rationale, dict_name in modzs_to_check.items() for pol in autos.pols()}
    for pol in autos.pols():
        for rationale, dict_name in modzs_to_check.items():
            binary_flags[rationale].append(am['modzs'][dict_name][(ant, ant, pol)] > mean_round_modz_cut)
    
    # figure out which metric is the largest outlier
    worst_metric, worst_z = max(modzs.items(), key=operator.itemgetter(1))
    worst_metrics.append(worst_metric)
    worst_zs.append(worst_z)
    
    # parse out all metrics for dataframe
    for k in modzs:
        col_label = k + ' Modified Z-Score'        
        if col_label in all_modzs:
            all_modzs[col_label].append(modzs[k])
        else:
            all_modzs[col_label] = [modzs[k]]


# build dataframe
to_show = {'Ant': ants, 
           'Node': [f'N{nodes[ant]:02}' for ant in ants],
           'A Priori Status': [a_priori_statuses[ant] for ant in ants],
           'Worst Metric': worst_metrics, 'Worst Modified Z-Score': worst_zs}
to_show.update(all_modzs)
df = pd.DataFrame(to_show).sort_values('Worst Modified Z-Score', ascending=False)

# style dataframe
z_score_cols = [col for col in df.columns if col not in ['Ant', 'Node', 'A Priori Status', 'Worst Metric']]
table1 = df.style.hide_index()\
           .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
           .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=z_score_cols) \
           .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])


HTML(table1.render())


def Flag_Bar_Chart():
    plt.figure(figsize=(8, 4), dpi=100)

    # count 
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    antpol_flags = [np.sum(f) for f in flags_list]
    ant_flags = [np.sum(np.array(f)[0::2] | np.array(f)[1::2]) for f in flags_list]

    # make bar chart
    plt.bar(np.arange(len(rationales)), antpol_flags, width=.7, color='yellow', ec='k', tick_label=rationales)
    for x, (nflags, nants) in enumerate(zip(antpol_flags, ant_flags)):
        plt.text(x, nflags/2, f'{nflags} Feeds on \n{nants} Antennas\nFlagged', va='center', ha='center')

    # set labels
    plt.ylabel('Antenna-Polarizations Flagged')
    plt.xlabel('Reason for Flagging')
    plt.tight_layout()


Flag_Bar_Chart()


def Rationale_Corr_Plot():
    plt.figure(figsize=(6,6), dpi=100)
    
    # compute correlation matrix
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    corrs = np.corrcoef(flags_list)
    
    # plot and label correlation matrix
    plt.imshow(corrs, cmap='viridis', interpolation='nearest', origin='upper')
    for i in range(corrs.shape[0]):
        for j in range(corrs.shape[1]):
            plt.text(i, j, np.round(corrs[i, j], 3), va='center', ha='center',
                     bbox={'facecolor': 'w', 'ec': 'w', 'alpha': .75})

    # colorbar, labels, and style
    plt.yticks(range(len(rationales)), rationales)
    plt.xticks(range(len(rationales)), rationales, rotation=-45, ha='right')
    plt.gca().xaxis.tick_top()
    plt.clim([0, 1])
    plt.colorbar(fraction=0.046, pad=0.04)
    plt.tight_layout()


Rationale_Corr_Plot()


def plot_all_spectra(spectra, mod_zs, modz_cut, overall, freqs, reason, ex_ants=[], 
                     xlabel='Frequency (MHz)', ylabel='', yscale='linear', ylim_factor=None):
    '''Helper function for plotting all spectra and showing which ones were flagged and why.''' 
    fig, axes = plt.subplots(1,2, figsize=(14,5), dpi=100)
    pols = sorted(set([bl[2] for bl in spectra]))
    for ax, pol in zip(axes, pols):
        # sort antennas into good, bad, and bad but not for this reason
        bad_here = [bl for bl in spectra if (bl[2] == pol) and mod_zs[bl] > modz_cut]
        other_bad = [bl for bl in spectra if (bl[2] == pol) and (bl[0] in ex_ants) and (bl not in bad_here)]
        good = [bl for bl in spectra if (bl[2] == pol) and (bl[0] not in ex_ants)]
        
        # plot all spectra
        l1, l2, l3 = None, None, None
        for bl in other_bad:
            l2, = ax.plot(freqs, spectra[bl], 'darkviolet', lw=.5)
        for bl in bad_here:
            l1, = ax.plot(freqs, spectra[bl], 'r', lw=.5)
        for bl in good:
            l3, = ax.plot(freqs, spectra[bl], 'grey', alpha=.5)
        l4, = ax.plot(freqs, overall[bl[2]], 'k--')
        
        ax.set_xlabel(xlabel)
        ax.set_ylabel(ylabel)
        ax.set_yscale(yscale)
        if ylim_factor is not None:
            ax.set_ylim([np.nanmin([spectra[bl] for bl in good]) / ylim_factor, 
                         np.nanmax([spectra[bl] for bl in good]) * ylim_factor])
        ax.set_title(f'Outliers in Autocorrelation {reason}: {pol}')
        ax.legend([l1, l2, l3, l4], [f'Flagged for {reason}', 'Flagged for Another Reason', 'Unflagged', 'Average Unflagged'], loc='lower right')
    plt.tight_layout()


overall_shape = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra_normed'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_power = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_var = {pol: np.nanmean([spec for bl, spec in am['spectra']['std_spectra_normed'].items() if (bl[2] == pol) 
                                     and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_diff = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_abs_diff_spectra_normed'].items() if (bl[2] == pol) 
                                      and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}


plot_all_spectra(am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], mean_round_modz_cut, 
                 overall_shape, freqs, 'Shape', ex_ants=ex_ants, yscale='linear', ylim_factor=1.2)


plot_all_spectra(am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], mean_round_modz_cut, 
                 overall_power, freqs, 'Power', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'], mean_round_modz_cut,
                 overall_temp_var, freqs, 'Temporal Variability', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'], mean_round_modz_cut, 
                 overall_temp_diff, freqs, 'Temporal Discontinutities', ex_ants=ex_ants, yscale='log')


# compute average good autocorrelations for each polarization
avg_good_autos = {pol: np.zeros(wf_shape, dtype=float) for pol in autos.pols()}
for pol in autos.pols():
    for i in range(wf_shape[0]):
        avg_good_autos[pol][i] = np.mean([np.abs(autos[bl][i, :]) for bl in autos 
                                          if (bl[0] not in ex_ants) and (bl[2] == pol)], axis=0)
    avg_good_autos[pol][am['flags']] = np.nan


def Avg_Auto_Plot():
    fig, axes = plt.subplots(1, 2, figsize=(14,5), dpi=100)
    for ax, pol in zip(axes, sorted(autos.pols())):
        im = ax.imshow(avg_good_autos[pol], aspect='auto', interpolation='nearest',
                       extent=[freqs[0], freqs[-1], times[-1], times[0]])
        ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))    
        plt.colorbar(im, ax=ax)
        ax.set_title(f'Average Good Raw {pol} Autocorrelation After Flagging')
        ax.set_ylabel('LST (hours)')
        ax.set_xlabel('Frequency (MHz)')
    plt.tight_layout()


Avg_Auto_Plot()


def plot_spectra(axes, ant, spec, modzs, modz_cut, overall, reason, yscale='linear'):
    '''Helper function for plotting both antennas of a given polarization.'''
    for pol, ax in zip(sorted(autos.pols()), axes):
        bl = (ant, ant, pol)

        # plot good antennas
        for bl2 in modzs:
            if (bl2[0] not in ex_ants) and (bl2[2] == pol):
                ax.plot(freqs, spec[bl2], 'grey', lw=.5, alpha=.5)
        ax.plot(freqs, overall[pol], 'k--')

        # plot this anetnna
        color = 'r'
        alpha = .75
        if modzs[bl] >= modz_cut:
            alpha = 1
        elif bl[0] in ex_ants:
            color = 'darkviolet'
        else:
            color = 'darkgreen'
        ax.plot(freqs, spec[bl], color, alpha=alpha, label=f'{ant} {pol}\n(z = {np.round(modzs[bl],1)})')

        # decorate axis
        ax.set_yscale(yscale)
        ax.legend(loc=2)
        ax.set_title(f'{pol} {reason}')


len(ants)

150


def Plot_All_Auto_Metrics():
    for row, ant in enumerate(df['Ant']): 
        # print title of section
        display(HTML(f'<h2>Antenna {ant}: {JD}</h2>'))

        # print metrics
        df_row = df.loc[df['Ant'] == ant]
        df_row = df_row.style.hide_index()\
                       .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])\
                       .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
                       .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
                       .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols)
        display(HTML(df_row.render()))

        # plot spectra and waterfalls
        fig, axes = plt.subplots(2, 6, figsize=(18, 5.5), dpi=(50 * (104 / len(ants))**.5)) # this should help manage filesize

        # plot individual spectra compared to all good antennas
        plot_spectra(axes[:, 0], ant, am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], 
                     mean_round_modz_cut, overall_shape, 'Shape', yscale='linear')
        plot_spectra(axes[:, 1], ant, am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], 
                     mean_round_modz_cut, overall_power, 'Power', yscale='log')
        plot_spectra(axes[:, 2], ant, am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'],
                     mean_round_modz_cut, overall_temp_var, 'Temporal Variability', yscale='linear')
        plot_spectra(axes[:, 3], ant, am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'],
                     mean_round_modz_cut, overall_temp_diff, 'Temporal Discontinutities', yscale='linear')
        plt.tight_layout()

        # plot linear-scale waterfalls 
        for pol, ax in zip(sorted(autos.pols()), axes[:, 4]):
            bl = (ant, ant, pol)        
            im = ax.imshow(np.where(am['flags'], np.nan, autos[bl].real), 
                           aspect='auto', interpolation='nearest', cmap='inferno',
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Waterfall (Linear Scale)')

        # plot log-scale mean-divided waterfalls
        for pol, ax in zip(sorted(autos.pols()), axes[:, 5]):
            bl = (ant, ant, pol)
            to_plot = autos[bl].real / avg_good_autos[pol]
            to_plot[am['flags']] = np.nan
            to_plot /= np.nanmean(to_plot)
            im = ax.imshow(np.log10(to_plot), aspect='auto', cmap='seismic', interpolation='nearest', vmin=-.07, vmax=.07,
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Log(Normalized Waterfall)')

        plt.show()

        # print some whitespace
        display(HTML('<hr style="height:3px">'))


Plot_All_Auto_Metrics()


from hera_qm import __version__
print(__version__)

2.0.2

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	nn Shape Modified Z-Score	ee Shape Modified Z-Score	nn Power Modified Z-Score	ee Power Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Discontinuties Modified Z-Score	ee Temporal Discontinuties Modified Z-Score
0	N00	RF_maintenance	ee Temporal Variability	54105.480332	45.844133	50.157071	56.821716	60.040943	51482.209662	54105.480332	20233.983656	21477.953226
130	N10	digital_maintenance	ee Temporal Discontinuties	4568.018280	82.322324	99.315968	91.693968	169.033259	530.856570	2039.501451	1569.879013	4568.018280
92	N10	digital_ok	ee Temporal Discontinuties	1933.347995	72.497430	75.349908	73.270553	102.983147	427.863464	657.347849	1182.078647	1933.347995
93	N10	RF_maintenance	nn Temporal Discontinuties	1873.992497	80.067438	62.338084	91.015863	79.822438	737.553088	604.693518	1873.992497	1255.310128
128	N10	digital_ok	nn Temporal Discontinuties	1407.292782	79.310168	65.317179	91.098608	80.851332	533.584058	676.131345	1407.292782	1315.329210
32	N02	RF_maintenance	nn Temporal Variability	1406.358999	0.876825	1.378419	1.541844	2.334874	1406.358999	1167.259779	565.510766	475.244565
112	N10	RF_maintenance	ee Temporal Discontinuties	1386.567877	75.555445	62.820810	61.448831	77.649334	330.327057	510.242302	851.725922	1386.567877
94	N10	RF_maintenance	nn Temporal Discontinuties	1274.986797	67.849638	60.804466	75.104547	71.425574	435.822006	413.610155	1274.986797	1156.039816
109	N10	digital_ok	ee Temporal Discontinuties	1273.490878	81.050474	65.923976	78.435469	81.447549	425.446326	629.879909	1258.223383	1273.490878
127	N10	RF_maintenance	nn Temporal Discontinuties	1124.058382	92.899191	60.560218	77.537533	70.466959	451.210385	430.939136	1124.058382	1081.763245
129	N10	digital_ok	ee Temporal Discontinuties	1085.850115	65.288645	73.427111	59.873262	84.973500	274.499693	455.476092	793.579064	1085.850115
110	N10	RF_maintenance	nn Temporal Discontinuties	1081.711477	70.419105	56.560616	83.579099	57.298283	454.728638	215.995965	1081.711477	605.588063
111	N10	digital_ok	nn Temporal Discontinuties	861.675811	68.271561	62.682593	63.427930	72.228370	306.216962	208.122596	861.675811	663.456901
45	N05	RF_maintenance	nn Temporal Variability	808.020830	31.258896	0.198577	11.633352	-0.446851	808.020830	314.295675	329.485608	128.394069
182	N13	RF_maintenance	nn Temporal Discontinuties	152.797250	49.024704	45.794403	6.723858	57.114418	147.003500	45.325080	152.797250	3.802329
70	N04	RF_maintenance	ee Temporal Variability	131.869057	54.689055	61.253376	34.543453	36.128948	110.116135	131.869057	50.618294	38.547722
99	N07	digital_ok	ee Temporal Variability	109.669224	55.447452	55.892539	45.892892	44.331214	96.369711	109.669224	21.953819	25.858725
8	N02	RF_maintenance	ee Temporal Variability	98.540930	50.935234	50.613007	60.257228	57.342401	90.080209	98.540930	37.446653	47.813820
40	N04	digital_ok	ee Temporal Variability	97.043167	52.180172	52.342524	48.955201	46.837988	74.904004	97.043167	17.871775	18.400954
72	N04	digital_ok	ee Temporal Variability	95.180542	53.141587	47.714932	52.184309	47.496827	62.974690	95.180542	12.109532	19.778221
33	N02	RF_maintenance	nn Temporal Variability	84.598258	13.529366	-0.919696	25.273221	-0.880587	84.598258	81.872135	44.036415	32.824722
178	N12	digital_ok	nn Temporal Discontinuties	77.096493	6.885148	4.053926	11.889936	6.728045	37.685452	34.646011	77.096493	68.890674
143	N14	digital_ok	ee Temporal Variability	73.413141	53.004338	49.753619	51.039119	50.041955	69.633887	73.413141	14.564991	13.867191
184	N14	digital_ok	ee Temporal Discontinuties	73.072276	2.229896	1.717808	3.518770	3.253482	48.705713	45.358302	72.868282	73.072276
168	N15	RF_maintenance	nn Power	63.234267	50.375908	50.363457	63.234267	58.284551	20.600095	42.768170	2.477981	2.392186
170	N15	digital_ok	ee Power	61.693552	49.747414	49.903270	60.090713	61.693552	33.207849	28.248428	7.610808	0.079520
169	N15	digital_ok	nn Power	61.554417	49.820173	50.390926	61.554417	61.350610	24.166654	27.396214	3.561712	-0.330512
321	N02	not_connected	nn Power	58.254490	52.372864	50.921051	58.254490	56.330176	42.103149	54.775597	5.055436	7.075528
46	N05	digital_ok	ee Temporal Discontinuties	52.238522	4.817751	3.909691	7.174790	8.047859	32.608120	35.048381	51.035711	52.238522
140	N13	digital_ok	ee Temporal Discontinuties	42.132499	7.367058	6.644568	13.662540	13.672075	23.217071	28.764642	40.684086	42.132499
10	N02	digital_ok	nn Temporal Discontinuties	40.824030	5.083621	0.316851	3.216476	-0.667280	20.792315	14.166374	40.824030	37.552437
324	N04	not_connected	nn Power	36.588874	8.515883	8.393946	36.588874	34.678591	-0.265726	2.619056	-0.430291	-0.315569
155	N12	RF_maintenance	nn Shape	35.407002	35.407002	35.393616	12.289020	11.662486	1.758516	0.644905	7.250459	6.118075
98	N07	digital_maintenance	ee Shape	33.697911	32.192990	33.697911	5.764781	5.781508	1.454361	3.784528	4.757359	6.525749
100	N07	RF_maintenance	ee Shape	33.657050	31.548466	33.657050	7.721475	5.924100	3.503943	3.801044	8.181902	7.786537
7	N02	digital_ok	nn Temporal Discontinuties	33.453736	5.543721	5.838953	3.794960	3.848354	23.362650	5.412616	33.453736	4.305766
119	N07	RF_maintenance	ee Shape	33.214196	2.441309	33.214196	2.060872	6.137524	-0.660912	1.672916	0.383121	4.707774
83	N07	digital_ok	ee Shape	32.956981	31.696265	32.956981	6.739609	6.772358	32.183687	13.362610	4.754761	6.264234
179	N12	digital_ok	ee Temporal Variability	32.905626	1.367393	2.923474	-0.519355	1.883259	-3.330156	32.905626	-0.221412	9.331883
19	N02	digital_ok	nn Temporal Variability	31.839115	6.256809	4.875705	5.368769	1.482289	31.839115	4.048309	4.636922	19.174858
116	N07	RF_maintenance	ee Shape	31.468441	29.867480	31.468441	6.874316	6.693457	1.403967	2.312611	4.853195	4.456453
180	N13	RF_maintenance	nn Shape	31.211457	31.211457	1.063384	13.217923	-0.036866	0.573171	7.085164	3.798143	21.804234
333	N12	dish_maintenance	ee Shape	30.284291	0.986233	30.284291	20.188726	11.390981	17.483114	4.146619	14.711596	5.637264
142	N13	digital_ok	nn Shape	29.375644	29.375644	5.588417	12.753037	3.107399	-0.582470	-2.105193	2.616101	2.655199
323	N02	not_connected	nn Power	27.653819	3.222339	13.998939	27.653819	15.475598	6.695107	3.132792	2.213191	4.948862
329	N12	dish_maintenance	nn Power	24.708403	3.858006	19.750369	24.708403	14.314808	7.164488	0.060882	6.676482	3.461043
69	N04	digital_ok	nn Temporal Variability	24.080601	2.091372	1.336311	0.607901	1.603856	24.080601	3.668777	2.703096	2.000658
73	N05	RF_maintenance	ee Temporal Variability	23.478576	11.371368	2.514627	1.480622	4.373987	8.287189	23.478576	13.159233	22.179671
185	N14	digital_ok	ee Temporal Discontinuties	21.533039	7.442392	7.115683	19.479888	16.605614	12.646502	17.081553	18.607563	21.533039
189	N15	digital_ok	ee Temporal Discontinuties	20.937306	1.738061	1.552607	0.145585	-0.368243	6.439429	10.647854	7.922528	20.937306
141	N13	digital_ok	nn Shape	20.548295	20.548295	1.652016	9.733703	0.898710	-0.058187	3.634706	4.542222	2.370822
177	N12	digital_ok	nn Temporal Variability	17.893115	5.248994	1.129135	5.058036	0.085050	17.893115	-0.147617	7.080277	1.456751
144	N14	digital_ok	nn Shape	15.369164	15.369164	12.071871	6.408662	4.318027	-0.707937	-0.444249	4.621341	3.366959
191	N15	digital_ok	nn Temporal Discontinuties	15.227962	6.664337	2.048566	4.140557	0.783164	1.092856	2.072331	15.227962	1.390691
181	N13	digital_ok	ee Shape	15.226441	4.360642	15.226441	2.515978	6.515161	-2.161302	-1.852567	2.587395	1.234950
21	N02	digital_ok	nn Temporal Discontinuties	14.317295	0.074012	0.374944	-0.662595	-0.696832	2.284473	2.619376	14.317295	3.318410
57	N04	digital_ok	ee Temporal Variability	14.265250	5.571751	7.118239	6.140383	1.212282	5.019840	14.265250	0.735769	0.491401
82	N07	RF_maintenance	ee Temporal Discontinuties	12.500411	8.442200	9.902920	1.162649	1.928400	6.369537	3.908758	11.610276	12.500411
31	N02	digital_ok	ee Temporal Variability	11.626577	-0.067374	9.259503	1.122528	3.759476	9.550677	11.626577	2.508968	5.292228
71	N04	digital_ok	nn Power	10.232537	8.157496	5.112964	10.232537	7.112463	1.933223	4.492425	1.932998	5.709867
166	N14	RF_maintenance	nn Shape	10.123281	10.123281	6.905745	3.236105	2.009820	-1.902664	-0.042375	0.320053	0.829775
160	N13	digital_ok	nn Temporal Discontinuties	9.119094	2.503447	3.796788	1.284343	3.568989	1.017043	6.509571	9.119094	0.809829
164	N14	digital_ok	nn Temporal Variability	8.834918	2.370829	1.083597	0.298234	0.617433	8.834918	2.799334	1.074938	1.355607
163	N14	digital_ok	nn Temporal Discontinuties	8.548622	-0.624014	1.715066	-0.726590	0.524168	5.676393	3.436934	8.548622	5.857395
138	N07	digital_ok	ee Shape	8.535862	5.861856	8.535862	0.930582	1.443879	0.315137	0.168483	0.567149	1.012917
145	N14	digital_ok	ee Shape	8.334283	-0.942097	8.334283	-0.673832	3.678231	-0.245369	-0.644130	2.233228	2.948286
183	N13	digital_ok	nn Temporal Discontinuties	7.876964	1.393173	1.465522	0.677006	0.160722	-1.087873	-2.250958	7.876964	0.526656
81	N07	digital_ok	ee Shape	6.599382	1.952441	6.599382	3.729742	2.538515	-0.389860	-1.888546	1.200175	0.295310
186	N14	digital_ok	ee Temporal Variability	6.228308	2.237898	1.669220	-0.087449	-0.371497	-1.668131	6.228308	1.879677	5.195632
55	N04	digital_ok	nn Temporal Discontinuties	6.197277	1.922036	-0.938260	-0.319985	-0.171962	5.670172	0.020647	6.197277	0.743392
187	N14	digital_ok	nn Temporal Discontinuties	5.998170	2.296801	0.442187	2.422568	0.024809	2.057163	0.286057	5.998170	3.640184
136	N12	RF_maintenance	ee Temporal Variability	4.798903	-0.540211	-0.683756	0.804127	0.957349	0.110916	4.798903	1.834405	2.002773
157	N12	RF_maintenance	ee Power	4.592813	1.875783	3.259687	-0.395567	4.592813	1.332395	-0.639558	-0.057153	2.482098
158	N12	RF_maintenance	ee Shape	3.542999	1.407624	3.542999	1.923599	1.214161	0.188040	0.424419	3.167578	-0.293192
54	N04	digital_ok	nn Temporal Discontinuties	3.529523	0.077282	0.005561	0.465841	-0.697420	0.628863	0.176445	3.529523	0.693352
42	N04	digital_ok	nn Temporal Variability	3.273585	2.853945	2.137027	-0.118050	0.132427	3.273585	0.720119	-0.960515	-0.056594
20	N02	digital_ok	ee Power	3.131218	1.486254	2.333507	2.313388	3.131218	-0.020222	1.122168	-1.488216	1.910688
176	N12	digital_ok	ee Power	3.009313	0.017608	2.574765	-0.143092	3.009313	-1.589515	-1.914572	-1.522127	-0.931078
56	N04	digital_ok	ee Temporal Variability	2.706072	2.346759	1.607046	2.077160	0.202401	1.309198	2.706072	0.853872	1.852233
165	N14	digital_ok	nn Power	1.493852	-0.257743	-1.386162	1.493852	0.820246	-0.247394	0.348938	-0.769921	-0.475398
190	N15	digital_ok	nn Power	1.384187	-0.005561	-0.028557	1.384187	-0.425860	0.819420	0.020222	0.224774	-0.096952
135	N12	digital_ok	ee Temporal Discontinuties	1.256344	-0.550252	-0.652859	-0.831872	-0.469071	-0.067904	-0.314653	0.771355	1.256344
156	N12	RF_maintenance	nn Temporal Discontinuties	1.206610	-0.923886	-0.696122	0.146785	0.118050	-0.348309	-2.087588	1.206610	-0.472457
162	N13	digital_ok	nn Temporal Variability	0.522148	-0.508092	-1.373662	-0.838990	-0.326401	0.522148	-1.505112	0.020925	0.225686
161	N13	digital_ok	ee Shape	0.502005	-0.112356	0.502005	-0.357925	-0.351773	-1.146610	0.288116	-0.020925	0.464599
41	N04	digital_ok	nn Shape	0.415383	0.415383	-0.925821	-0.596005	-0.778929	-2.998692	-0.175214	-0.655629	0.056463
9	N02	digital_ok	ee Temporal Discontinuties	-0.190118	-0.544953	-0.463501	-0.651561	-0.816381	-1.625908	-1.112658	-0.413725	-0.190118
1	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
2	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
3	N01	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
4	N01	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
5	N01	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
11	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
12	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
13	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
14	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
15	N01	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
16	N01	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
17	N01	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
18	N01	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
23	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
24	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
25	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
26	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
27	N01	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
28	N01	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
29	N01	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
30	N01	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
36	N03	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
37	N03	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
38	N03	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
39	N00	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
50	N03	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
51	N03	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
52	N03	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
53	N03	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
65	N03	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
66	N03	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
67	N03	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
68	N03	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
84	N08	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
85	N08	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
86	N08	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
87	N08	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
88	N09	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
89	N09	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
90	N09	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
91	N09	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
101	N08	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
102	N08	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
103	N08	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
104	N08	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
105	N09	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
106	N09	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
107	N09	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
108	N09	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
120	N08	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
121	N08	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
122	N08	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
123	N08	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
124	N09	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
125	N09	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
126	N09	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
150	N15	RF_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
167	N15	digital_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
220	N18	RF_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
221	N18	RF_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
222	N18	RF_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
320	N03	dish_maintenance	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan
325	N09	dish_ok	nn Shape	nan	nan	nan	inf	inf	nan	nan	nan	nan

auto_metrics Nightly Notebook¶

Statistics computed (after removing the worst offenders with a median-based metrics and then RFI flagging):¶

Parse Inputs and Load Data¶

Summary Plots and Tables¶

Figure 1: Antenna Positions with auto_metrics flags.¶

Table 1: Modified Z-Score Summary¶

Figure 2: Flagging Rationale Summary¶

Figure 3: Flagging Rationale Correlations¶

Figure 4: Outliers in Autocorrelation Shape¶

Figure 5: Outliers in Autocorrelation Power¶

Figure 6: Outliers in Autocorrelation Temporal Variability¶

Figure 7: Outliers in Autocorrelation Temporal Discontinuities¶

Figure 8: Average Good Autocorrelations and Flags¶

Per-Antenna Plots¶

Figure 9: Per-Antenna Statistics, Spectra, and Waterfalls¶

Antenna 0: 2459741

Antenna 130: 2459741

Antenna 92: 2459741

Antenna 93: 2459741

Antenna 128: 2459741

Antenna 32: 2459741

Antenna 112: 2459741

Antenna 94: 2459741

Antenna 109: 2459741

Antenna 127: 2459741

Antenna 129: 2459741

Antenna 110: 2459741

Antenna 111: 2459741

Antenna 45: 2459741

Antenna 182: 2459741

Antenna 70: 2459741

Antenna 99: 2459741

Antenna 8: 2459741

Antenna 40: 2459741

Antenna 72: 2459741

Antenna 33: 2459741

Antenna 178: 2459741

Antenna 143: 2459741

Antenna 184: 2459741

Antenna 168: 2459741

Antenna 170: 2459741

Antenna 169: 2459741

Antenna 321: 2459741

Antenna 46: 2459741

Antenna 140: 2459741

Antenna 10: 2459741

Antenna 324: 2459741

Antenna 155: 2459741

Antenna 98: 2459741

Antenna 100: 2459741

Antenna 7: 2459741

Antenna 119: 2459741

Antenna 83: 2459741

Antenna 179: 2459741

Antenna 19: 2459741

Antenna 116: 2459741

Antenna 180: 2459741

Antenna 333: 2459741

Antenna 142: 2459741

Antenna 323: 2459741

Antenna 329: 2459741

Antenna 69: 2459741

Antenna 73: 2459741

Antenna 185: 2459741

Antenna 189: 2459741

Antenna 141: 2459741

Antenna 177: 2459741

Antenna 144: 2459741

Antenna 191: 2459741

Antenna 181: 2459741

Antenna 21: 2459741

Antenna 57: 2459741

Antenna 82: 2459741

Antenna 31: 2459741

Antenna 71: 2459741

Antenna 166: 2459741

Antenna 160: 2459741

Antenna 164: 2459741

Antenna 163: 2459741

Antenna 138: 2459741

`auto_metrics` Nightly Notebook¶

Figure 1: Antenna Positions with `auto_metrics` flags.¶