
import numpy as np
import pandas as pd
pd.set_option('display.max_rows', 1000)
import matplotlib.pyplot as plt
import matplotlib
import glob
import os
import operator
from hera_cal.io import HERAData
from hera_cal.utils import split_pol
from hera_qm.metrics_io import load_metric_file
from hera_notebook_templates.utils import status_colors
from IPython.display import display, HTML

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))


# If you want to run this notebook locally, copy the output of the next cell into the first few lines of this cell.
# JD = '2459122'
# data_path = '/lustre/aoc/projects/hera/H4C/2459122'
# os.environ["JULIANDATE"] = JD
# os.environ["DATA_PATH"] = data_path


# Use environment variables to figure out path to data
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
print(f'JD = "{JD}"')
print(f'data_path = "{data_path}"')

JD = "2460106"
data_path = "/mnt/sn1/2460106"


from astropy.time import Time
utc = Time(JD, format='jd').datetime
print(f'Date: {utc.month}-{utc.day}-{utc.year}')

Date: 6-10-2023


print(f'Looking for data in {data_path} on JD {JD}...')

auto_metrics_file = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}*.auto_metrics.h5')))
if len(auto_metrics_file) > 0:
    auto_metrics_file = auto_metrics_file[0]
    print(f'Found auto_metrics results file at {auto_metrics_file}.')
else:
    raise OSError(f'{auto_metrics_file} not found.')

raw_auto_files = sorted(glob.glob(os.path.join(data_path, f'zen.{JD}.?????.sum.autos.uvh5')))
if len(raw_auto_files) > 0:
    print(f'Found {len(raw_auto_files)} extracted autocorrelation files.')
else: 
    raise OSError(f'No files of the form zen.{JD}.?????.sum.autos.uvh5 found in {data_path}.')

Looking for data in /mnt/sn1/2460106 on JD 2460106...
Found auto_metrics results file at /mnt/sn1/2460106/zen.2460106.42132.sum.auto_metrics.h5.
Found 343 extracted autocorrelation files.


# load auto_metrics and define use someful quantities
am = load_metric_file(auto_metrics_file)
mean_round_modz_cut = am['parameters']['mean_round_modz_cut']
ex_ants = am['ex_ants']['r2_ex_ants']
ants = sorted(set(bl[0] for bl in am['modzs']['r2_shape_modzs']))

# load raw autocorrelation waterfalls and define some useful quantities
hd = HERAData(raw_auto_files)
autos, _, _ = hd.read(axis='blt')
wf_shape = next(iter(autos.values())).shape
freqs = autos.freqs / 1e6
times = autos.times
lsts = autos.lsts * 12 / np.pi


# try to load a priori antenna statusesm but fail gracefully if this doesn't work.
a_priori_statuses = {ant: 'Not Found' for ant in ants}
nodes = {ant: np.nan for ant in ants}
try:
    from hera_mc import cm_hookup

    # get node numbers
    hookup = cm_hookup.get_hookup('default')
    for ant_name in hookup:
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in nodes:
            nodes[ant] = int(hookup[ant_name].get_part_from_type('node')['E<ground'][1:])            
            
    # get apriori antenna status
    for ant_name, data in hookup.items():
        ant = int("".join(filter(str.isdigit, ant_name)))
        if ant in a_priori_statuses:
            a_priori_statuses[ant] = data.apriori
    
except Exception as err:
    print(f'Could not load node numbers and a priori antenna statuses.\nEncountered {type(err)} with message: {err}')


# print ex_ants for easy copy-pasting to YAML file
print('ex_ants: [' + ", ".join(str(ant) for ant in ex_ants) + ']')

ex_ants: [4, 7, 9, 15, 16, 17, 18, 19, 27, 28, 29, 32, 34, 36, 37, 38, 40, 41, 42, 47, 51, 54, 55, 56, 58, 59, 61, 77, 78, 81, 82, 83, 86, 87, 90, 92, 93, 94, 96, 101, 104, 109, 110, 111, 112, 117, 118, 121, 124, 127, 131, 134, 136, 137, 140, 142, 143, 151, 153, 155, 156, 159, 160, 161, 164, 165, 166, 169, 180, 182, 184, 185, 200, 201, 202, 204, 205, 207, 208, 209, 210, 211, 225, 226, 227, 242, 243, 246, 262, 329]


def Array_Plot():
    plt.figure(figsize=(16,16))
    plt.scatter(np.array([autos.antpos[ant][0] for ant in ants]), 
                np.array([autos.antpos[ant][1] for ant in ants]), c='w', s=0)    
    for ant in ants:
        pos = autos.antpos[ant]
        bad = ant in ex_ants
        plt.gca().add_artist(plt.Circle(tuple(pos[0:2]), radius=7, 
                                        fill=(~bad), color=['grey','r'][bad]))
        plt.text(pos[0],pos[1],str(ant), va='center', ha='center', color='w')
    plt.xlabel("Antenna East-West Position (meters)")
    plt.ylabel("Antenna North-South Position (meters)")
    plt.title(f'Antenna Positions and Auto_Metrics Flags on {JD}\n(Maximum Modified Z-Score > {mean_round_modz_cut} in Red)');
    plt.axis('equal')
    plt.tight_layout()


Array_Plot()


# Parse modzs for Table 1 and other figures
modzs_to_check = {'Shape': 'r2_shape_modzs', 'Power': 'r2_power_modzs', 
                  'Temporal Variability': 'r2_temp_var_modzs', 'Temporal Discontinuties': 'r2_temp_diff_modzs'}
worst_metrics = []
worst_zs = []
all_modzs = {}
binary_flags = {rationale: [] for rationale in modzs_to_check}

for ant in ants:
    # parse modzs and figure out flag counts
    modzs = {f'{pol} {rationale}': am['modzs'][dict_name][(ant, ant, pol)] 
             for rationale, dict_name in modzs_to_check.items() for pol in autos.pols()}
    for pol in autos.pols():
        for rationale, dict_name in modzs_to_check.items():
            binary_flags[rationale].append(am['modzs'][dict_name][(ant, ant, pol)] > mean_round_modz_cut)
    
    # figure out which metric is the largest outlier
    worst_metric, worst_z = max(modzs.items(), key=operator.itemgetter(1))
    worst_metrics.append(worst_metric)
    worst_zs.append(worst_z)
    
    # parse out all metrics for dataframe
    for k in modzs:
        col_label = k + ' Modified Z-Score'        
        if col_label in all_modzs:
            all_modzs[col_label].append(modzs[k])
        else:
            all_modzs[col_label] = [modzs[k]]


# build dataframe
to_show = {'Ant': ants, 
           'Node': [f'N{nodes[ant]:02}' for ant in ants],
           'A Priori Status': [a_priori_statuses[ant] for ant in ants],
           'Worst Metric': worst_metrics, 'Worst Modified Z-Score': worst_zs}
to_show.update(all_modzs)
df = pd.DataFrame(to_show).sort_values('Worst Modified Z-Score', ascending=False)

# style dataframe
z_score_cols = [col for col in df.columns if col not in ['Ant', 'Node', 'A Priori Status', 'Worst Metric']]
table1 = df.style.hide_index()\
           .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
           .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
           .background_gradient(cmap='viridis', vmax=mean_round_modz_cut * 3, vmin=0, axis=None, subset=z_score_cols) \
           .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
           .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])


HTML(table1.render())


def Flag_Bar_Chart():
    plt.figure(figsize=(8, 4), dpi=100)

    # count 
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    antpol_flags = [np.sum(f) for f in flags_list]
    ant_flags = [np.sum(np.array(f)[0::2] | np.array(f)[1::2]) for f in flags_list]

    # make bar chart
    plt.bar(np.arange(len(rationales)), antpol_flags, width=.7, color='yellow', ec='k', tick_label=rationales)
    for x, (nflags, nants) in enumerate(zip(antpol_flags, ant_flags)):
        plt.text(x, nflags/2, f'{nflags} Feeds on \n{nants} Antennas\nFlagged', va='center', ha='center')

    # set labels
    plt.ylabel('Antenna-Polarizations Flagged')
    plt.xlabel('Reason for Flagging')
    plt.tight_layout()


Flag_Bar_Chart()


def Rationale_Corr_Plot():
    plt.figure(figsize=(6,6), dpi=100)
    
    # compute correlation matrix
    rationales = list(binary_flags.keys())
    flags_list = np.array([binary_flags[rationale] for rationale in rationales])
    corrs = np.corrcoef(flags_list)
    
    # plot and label correlation matrix
    plt.imshow(corrs, cmap='viridis', interpolation='nearest', origin='upper')
    for i in range(corrs.shape[0]):
        for j in range(corrs.shape[1]):
            plt.text(i, j, np.round(corrs[i, j], 3), va='center', ha='center',
                     bbox={'facecolor': 'w', 'ec': 'w', 'alpha': .75})

    # colorbar, labels, and style
    plt.yticks(range(len(rationales)), rationales)
    plt.xticks(range(len(rationales)), rationales, rotation=-45, ha='right')
    plt.gca().xaxis.tick_top()
    plt.clim([0, 1])
    plt.colorbar(fraction=0.046, pad=0.04)
    plt.tight_layout()


Rationale_Corr_Plot()


def plot_all_spectra(spectra, mod_zs, modz_cut, overall, freqs, reason, ex_ants=[], 
                     xlabel='Frequency (MHz)', ylabel='', yscale='linear', ylim_factor=None):
    '''Helper function for plotting all spectra and showing which ones were flagged and why.''' 
    fig, axes = plt.subplots(1,2, figsize=(14,5), dpi=100)
    pols = sorted(set([bl[2] for bl in spectra]))
    for ax, pol in zip(axes, pols):
        # sort antennas into good, bad, and bad but not for this reason
        bad_here = [bl for bl in spectra if (bl[2] == pol) and mod_zs[bl] > modz_cut]
        other_bad = [bl for bl in spectra if (bl[2] == pol) and (bl[0] in ex_ants) and (bl not in bad_here)]
        good = [bl for bl in spectra if (bl[2] == pol) and (bl[0] not in ex_ants)]
        
        # plot all spectra
        l1, l2, l3 = None, None, None
        for bl in other_bad:
            l2, = ax.plot(freqs, spectra[bl], 'darkviolet', lw=.5)
        for bl in bad_here:
            l1, = ax.plot(freqs, spectra[bl], 'r', lw=.5)
        for bl in good:
            l3, = ax.plot(freqs, spectra[bl], 'grey', alpha=.5)
        l4, = ax.plot(freqs, overall[bl[2]], 'k--')
        
        ax.set_xlabel(xlabel)
        ax.set_ylabel(ylabel)
        ax.set_yscale(yscale)
        if ylim_factor is not None:
            ax.set_ylim([np.nanmin([spectra[bl] for bl in good]) / ylim_factor, 
                         np.nanmax([spectra[bl] for bl in good]) * ylim_factor])
        ax.set_title(f'Outliers in Autocorrelation {reason}: {pol}')
        ax.legend([l1, l2, l3, l4], [f'Flagged for {reason}', 'Flagged for Another Reason', 'Unflagged', 'Average Unflagged'], loc='lower right')
    plt.tight_layout()


overall_shape = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra_normed'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_power = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_spectra'].items() if (bl[2] == pol) 
                                  and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_var = {pol: np.nanmean([spec for bl, spec in am['spectra']['std_spectra_normed'].items() if (bl[2] == pol) 
                                     and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}
overall_temp_diff = {pol: np.nanmean([spec for bl, spec in am['spectra']['mean_abs_diff_spectra_normed'].items() if (bl[2] == pol) 
                                      and (bl[0] not in ex_ants)], axis=0) for pol in autos.pols()}


plot_all_spectra(am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], mean_round_modz_cut, 
                 overall_shape, freqs, 'Shape', ex_ants=ex_ants, yscale='linear', ylim_factor=1.2)


plot_all_spectra(am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], mean_round_modz_cut, 
                 overall_power, freqs, 'Power', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'], mean_round_modz_cut,
                 overall_temp_var, freqs, 'Temporal Variability', ex_ants=ex_ants, yscale='log')


plot_all_spectra(am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'], mean_round_modz_cut, 
                 overall_temp_diff, freqs, 'Temporal Discontinutities', ex_ants=ex_ants, yscale='log')


# compute average good autocorrelations for each polarization
avg_good_autos = {pol: np.zeros(wf_shape, dtype=float) for pol in autos.pols()}
for pol in autos.pols():
    for i in range(wf_shape[0]):
        avg_good_autos[pol][i] = np.mean([np.abs(autos[bl][i, :]) for bl in autos 
                                          if (bl[0] not in ex_ants) and (bl[2] == pol)], axis=0)
    avg_good_autos[pol][am['flags']] = np.nan


def Avg_Auto_Plot():
    fig, axes = plt.subplots(1, 2, figsize=(14,5), dpi=100)
    for ax, pol in zip(axes, sorted(autos.pols())):
        im = ax.imshow(avg_good_autos[pol], aspect='auto', interpolation='nearest',
                       extent=[freqs[0], freqs[-1], times[-1], times[0]])
        ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))    
        plt.colorbar(im, ax=ax)
        ax.set_title(f'Average Good Raw {pol} Autocorrelation After Flagging')
        ax.set_ylabel('LST (hours)')
        ax.set_xlabel('Frequency (MHz)')
    plt.tight_layout()


Avg_Auto_Plot()


def plot_spectra(axes, ant, spec, modzs, modz_cut, overall, reason, yscale='linear'):
    '''Helper function for plotting both antennas of a given polarization.'''
    for pol, ax in zip(sorted(autos.pols()), axes):
        bl = (ant, ant, pol)

        # plot good antennas
        for bl2 in modzs:
            if (bl2[0] not in ex_ants) and (bl2[2] == pol):
                ax.plot(freqs, spec[bl2], 'grey', lw=.5, alpha=.5)
        ax.plot(freqs, overall[pol], 'k--')

        # plot this anetnna
        color = 'r'
        alpha = .75
        if modzs[bl] >= modz_cut:
            alpha = 1
        elif bl[0] in ex_ants:
            color = 'darkviolet'
        else:
            color = 'darkgreen'
        ax.plot(freqs, spec[bl], color, alpha=alpha, label=f'{ant} {pol}\n(z = {np.round(modzs[bl],1)})')

        # decorate axis
        ax.set_yscale(yscale)
        ax.legend(loc=2)
        ax.set_title(f'{pol} {reason}')


len(ants)

202


def Plot_All_Auto_Metrics():
    for row, ant in enumerate(df['Ant']): 
        # print title of section
        display(HTML(f'<h2>Antenna {ant}: {JD}</h2>'))

        # print metrics
        df_row = df.loc[df['Ant'] == ant]
        df_row = df_row.style.hide_index()\
                       .set_table_styles([dict(selector="th",props=[('max-width', '70pt')])])\
                       .applymap(lambda val: f'background-color: {status_colors[val]}' if val in status_colors else '', subset=['A Priori Status']) \
                       .applymap(lambda val: 'font-weight: bold' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'color: red' if val in ex_ants else '', subset=['Ant']) \
                       .applymap(lambda val: 'font-weight: bold' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols) \
                       .applymap(lambda val: 'color: red' if val > am['parameters']['mean_round_modz_cut'] else '', subset=z_score_cols)
        display(HTML(df_row.render()))

        # plot spectra and waterfalls
        fig, axes = plt.subplots(2, 6, figsize=(18, 5.5), dpi=(40 * (104 / len(ants))**.5)) # this should help manage filesize

        # plot individual spectra compared to all good antennas
        plot_spectra(axes[:, 0], ant, am['spectra']['mean_spectra_normed'], am['modzs']['r2_shape_modzs'], 
                     mean_round_modz_cut, overall_shape, 'Shape', yscale='linear')
        plot_spectra(axes[:, 1], ant, am['spectra']['mean_spectra'], am['modzs']['r2_power_modzs'], 
                     mean_round_modz_cut, overall_power, 'Power', yscale='log')
        plot_spectra(axes[:, 2], ant, am['spectra']['std_spectra_normed'], am['modzs']['r2_temp_var_modzs'],
                     mean_round_modz_cut, overall_temp_var, 'Temporal Variability', yscale='linear')
        plot_spectra(axes[:, 3], ant, am['spectra']['mean_abs_diff_spectra_normed'], am['modzs']['r2_temp_diff_modzs'],
                     mean_round_modz_cut, overall_temp_diff, 'Temporal Discontinutities', yscale='linear')
        plt.tight_layout()

        # plot linear-scale waterfalls 
        for pol, ax in zip(sorted(autos.pols()), axes[:, 4]):
            bl = (ant, ant, pol)        
            im = ax.imshow(np.where(am['flags'], np.nan, autos[bl].real), 
                           aspect='auto', interpolation='nearest', cmap='inferno',
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Waterfall (Linear Scale)')

        # plot log-scale mean-divided waterfalls
        for pol, ax in zip(sorted(autos.pols()), axes[:, 5]):
            bl = (ant, ant, pol)
            to_plot = autos[bl].real / avg_good_autos[pol]
            to_plot[am['flags']] = np.nan
            to_plot /= np.nanmean(to_plot)
            im = ax.imshow(np.log10(to_plot), aspect='auto', cmap='seismic', interpolation='nearest', vmin=-.07, vmax=.07,
                           extent=[freqs[0], freqs[-1], times[-1], times[0]])
            ax.set_yticklabels(np.around(lsts[[min(max(np.searchsorted(times, t), 0), len(times) - 1) for t in ax.get_yticks()]], 2))
            ax.set_title(f'{pol} Log(Normalized Waterfall)')

        plt.show()

        # print some whitespace
        display(HTML('<hr style="height:3px">'))


Plot_All_Auto_Metrics()


from hera_qm import __version__
print(__version__)

2.1.1.dev3+gb291d34

Ant	Node	A Priori Status	Worst Metric	Worst Modified Z-Score	ee Shape Modified Z-Score	nn Shape Modified Z-Score	ee Power Modified Z-Score	nn Power Modified Z-Score	ee Temporal Variability Modified Z-Score	nn Temporal Variability Modified Z-Score	ee Temporal Discontinuties Modified Z-Score	nn Temporal Discontinuties Modified Z-Score
82	N07	RF_maintenance	ee Power	inf	94.542745	95.007337	inf	inf	-1.614950	-1.631070	-4.035313	-4.150820
137	N07	RF_maintenance	ee Power	inf	94.547442	95.012097	inf	inf	-1.614950	-1.631070	-4.035313	-4.150820
117	N07	RF_maintenance	ee Power	inf	94.498900	94.932024	inf	inf	-1.614950	-1.631070	-4.035313	-4.150820
118	N07	digital_ok	ee Power	inf	94.529527	94.984651	inf	inf	-1.614950	-1.631070	-4.035313	-4.150820
81	N07	RF_maintenance	ee Power	inf	94.534657	94.967755	inf	inf	-1.614950	-1.631070	-4.035313	-4.150820
83	N07	RF_maintenance	ee Power	inf	94.544115	95.001922	inf	inf	-1.614950	-1.631070	-4.035313	-4.150820
208	N20	dish_maintenance	nn Temporal Discontinuties	44.086273	4.012583	5.406392	10.361757	21.047241	1.882164	2.243667	3.696122	44.086273
18	N01	RF_maintenance	nn Temporal Discontinuties	38.818783	-0.238435	4.435318	0.582124	3.114879	0.483856	0.580723	7.665063	38.818783
55	N04	RF_maintenance	ee Shape	35.320023	35.320023	-0.623039	14.070148	-0.537218	2.598584	-0.862565	3.141198	-0.255704
104	N08	dish_maintenance	nn Shape	35.137157	5.019570	35.137157	4.124639	7.390471	2.702897	3.411440	4.324425	5.036466
51	N03	digital_ok	ee Temporal Discontinuties	34.901406	0.802849	1.136543	-0.594775	1.109269	-0.398146	1.395449	34.901406	1.249573
87	N08	RF_maintenance	ee Shape	30.847250	30.847250	1.584614	3.971725	-1.007143	2.035694	-0.524396	10.352451	-0.212273
166	N14	RF_maintenance	nn Shape	29.140055	17.429033	29.140055	21.472430	6.444783	2.454047	2.984746	4.014284	2.982550
110	N10	RF_maintenance	ee Shape	28.820437	28.820437	-0.167639	3.069468	0.281737	0.603843	0.596766	1.671670	0.281713
27	N01	RF_ok	nn Temporal Discontinuties	27.406961	14.434412	16.317396	19.012614	8.210835	2.397681	2.696926	5.403567	27.406961
40	N04	digital_ok	nn Temporal Discontinuties	24.656713	0.106653	0.825914	0.687489	0.613923	0.874593	0.743553	1.260889	24.656713
161	N13	digital_ok	nn Shape	22.462855	0.086828	22.462855	0.405256	0.087039	0.412425	-0.371847	0.595786	-0.067741
209	N20	dish_maintenance	nn Temporal Discontinuties	22.444574	4.006920	5.055875	19.906073	20.648911	2.296091	2.736432	14.928767	22.444574
124	N09	RF_maintenance	nn Power	21.540374	8.835151	10.591252	21.091588	21.540374	2.371472	2.714210	3.657402	7.093173
93	N10	RF_maintenance	nn Power	21.419268	9.264049	10.396515	20.736571	21.419268	2.374906	2.711336	4.174383	4.766256
143	N14	RF_maintenance	nn Power	21.331848	10.140125	10.143067	-0.518426	21.331848	2.559943	2.708464	1.320466	5.115129
58	N05	RF_maintenance	nn Power	21.316036	8.647284	9.990512	20.730947	21.316036	2.365459	2.701766	4.518327	4.456107
59	N05	RF_maintenance	nn Power	21.277546	-0.351611	10.590264	0.213754	21.277546	0.193116	2.706556	0.334626	3.530834
16	N01	RF_maintenance	nn Power	21.184173	9.371415	10.387470	20.673281	21.184173	2.372551	2.585604	3.407940	3.743398
92	N10	RF_maintenance	nn Power	21.171377	8.507081	9.680939	20.841165	21.171377	2.356139	2.692855	4.451459	4.831008
109	N10	RF_maintenance	nn Power	20.971346	5.383747	10.196242	2.625795	20.971346	2.038898	2.697057	2.595110	3.687711
182	N13	RF_maintenance	nn Power	20.955039	-0.423917	10.392352	-0.185866	20.955039	-1.087807	2.652836	-0.470955	4.376560
127	N10	RF_maintenance	ee Power	20.814649	9.075923	4.391741	20.814649	3.287192	2.362107	2.398214	3.187734	3.210088
160	N13	RF_maintenance	ee Power	20.718756	9.573722	10.693400	20.718756	3.485711	2.373146	2.374025	4.141797	2.394292
29	N01	RF_maintenance	nn Power	20.663551	9.010117	10.445382	20.377026	20.663551	2.365059	2.697061	5.301236	4.661400
155	N12	RF_maintenance	ee Power	20.555551	8.570057	10.998164	20.555551	4.306911	2.355543	2.398458	4.314801	2.637253
32	N02	RF_maintenance	ee Shape	20.491376	20.491376	-0.312129	0.398416	-1.193836	0.317157	-0.029069	2.546697	7.959364
136	N12	RF_maintenance	ee Power	20.424691	8.236527	15.301576	20.424691	3.636237	2.352663	2.477636	4.231969	3.825709
86	N08	digital_ok	ee Power	18.517519	9.429992	0.126715	18.517519	-0.392187	2.371804	0.116660	2.693890	4.549413
184	N14	dish_maintenance	ee Shape	17.953671	17.953671	10.229031	15.647958	7.749105	1.516988	2.486053	3.147208	2.840863
112	N10	digital_ok	ee Power	17.066292	3.091442	0.539008	17.066292	0.301507	-0.753997	-0.191397	1.953293	-1.246230
165	N14	digital_ok	ee Shape	16.471171	16.471171	11.406956	1.533749	0.233443	-0.318490	-0.175477	2.645672	0.766719
207	N19	RF_maintenance	nn Shape	14.206490	10.001925	14.206490	12.442072	5.465350	2.376727	2.660078	3.366249	3.888962
329	N12	dish_maintenance	nn Power	13.368897	9.780241	10.330340	12.674985	13.368897	2.381840	2.709769	3.598450	3.994468
211	N20	RF_maintenance	nn Power	13.297162	-1.109052	10.256009	-0.434271	13.297162	-0.753395	2.703503	-0.005456	4.306198

auto_metrics Nightly Notebook¶

Statistics computed (after removing the worst offenders with a median-based metrics and then RFI flagging):¶

Parse Inputs and Load Data¶

Summary Plots and Tables¶

Figure 1: Antenna Positions with auto_metrics flags.¶

Table 1: Modified Z-Score Summary¶

Figure 2: Flagging Rationale Summary¶

Figure 3: Flagging Rationale Correlations¶

Figure 4: Outliers in Autocorrelation Shape¶

Figure 5: Outliers in Autocorrelation Power¶

Figure 6: Outliers in Autocorrelation Temporal Variability¶

Figure 7: Outliers in Autocorrelation Temporal Discontinuities¶

Figure 8: Average Good Autocorrelations and Flags¶

Per-Antenna Plots¶

Figure 9: Per-Antenna Statistics, Spectra, and Waterfalls¶

Antenna 82: 2460106

Antenna 137: 2460106

Antenna 117: 2460106

Antenna 118: 2460106

Antenna 81: 2460106

Antenna 83: 2460106

Antenna 208: 2460106

Antenna 18: 2460106

Antenna 55: 2460106

Antenna 104: 2460106

Antenna 51: 2460106

Antenna 87: 2460106

Antenna 166: 2460106

Antenna 110: 2460106

Antenna 27: 2460106

Antenna 40: 2460106

Antenna 161: 2460106

Antenna 209: 2460106

Antenna 124: 2460106

Antenna 93: 2460106

Antenna 143: 2460106

Antenna 58: 2460106

Antenna 59: 2460106

Antenna 16: 2460106

Antenna 92: 2460106

Antenna 109: 2460106

Antenna 182: 2460106

Antenna 127: 2460106

Antenna 160: 2460106

Antenna 29: 2460106

Antenna 155: 2460106

Antenna 32: 2460106

Antenna 136: 2460106

Antenna 86: 2460106

Antenna 184: 2460106

Antenna 112: 2460106

Antenna 165: 2460106

Antenna 207: 2460106

Antenna 329: 2460106

Antenna 211: 2460106

Antenna 34: 2460106

Antenna 153: 2460106

Antenna 246: 2460106

Antenna 131: 2460106

Antenna 225: 2460106

Antenna 185: 2460106

Antenna 54: 2460106

Antenna 96: 2460106

Antenna 227: 2460106

Antenna 111: 2460106

Antenna 37: 2460106

Antenna 90: 2460106

Antenna 15: 2460106

Antenna 28: 2460106

Antenna 180: 2460106

Antenna 121: 2460106

Antenna 205: 2460106

Antenna 42: 2460106

Antenna 134: 2460106

Antenna 226: 2460106

Antenna 151: 2460106

Antenna 142: 2460106

Antenna 159: 2460106

Antenna 4: 2460106

Antenna 169: 2460106

`auto_metrics` Nightly Notebook¶

Figure 1: Antenna Positions with `auto_metrics` flags.¶