In [1]:
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData, UVFlag
import pyuvdata
import os
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, SkyCoord, AltAz, Angle
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import h5py
import importlib
from scipy import stats
import scipy
import pandas as pd
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
In [2]:
#get data location
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
nb_outdir = os.environ['NB_OUTDIR']
utc = Time(JD, format='jd').datetime
print(f'JD = {JD}')
print(f'Date = {utc.month}-{utc.day}-{utc.year}')
print(f'data_path = "{data_path}"')

JD = 2459772
Date = 7-11-2022
data_path = "/mnt/sn1/2459772"
In [3]:
# Load in data
HHfiles, difffiles, uvdx, uvdy = utils.load_data_ds(data_path,JD)
    
uvd = UVData()
uvd_diff = UVData()
uvd.read(HHfiles[0])
use_ants = [int(ant) for ant in uvd.get_ants()]
bls = [(ant, ant) for ant in use_ants]
uvd.read(HHfiles[::10], skip_bad_files=True, bls=bls)
uvd_diff.read(difffiles[::10], skip_bad_files=True, bls=bls)
lsts = uvd.lst_array

flagfile = glob.glob(os.path.join(HHfiles[0].split('zen')[0],'zen.{}*total_stage_1_threshold_flags.h5'.format(JD)))
uvf = UVFlag()
uvf.read(flagfile)
bls = [(ant, ant) for ant in uvd.get_ants()]
times_uvf = np.unique(uvf.time_array)
times_uvd = np.unique(uvd.time_array)
idx_times = [np.where(time_uvd == times_uvf)[0][0] for time_uvd in times_uvd]
uvd.flag_array[:,0,:,:] = np.repeat(uvf.flag_array[idx_times], len(bls), axis=0)

372 sum files found between JDs 2459772.25304 and 2459772.33603
372 diff files found between JDs 2459772.25304 and 2459772.33603

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [4]:
utils.plot_lst_coverage(uvd)

Delay spectrum¶

Delay spectrum CLEANed using uvtools.dspec.high_pass_fourier_filter with 7th-order Blackman-Harris window function. Odd/even visibilities are used to remove noise bias.

In [5]:
_data_cleaned_sq, d_even, d_odd = utils.clean_ds(bls, uvd, uvd_diff, N_threads=14)

Waterfalls of delay spectra for autocorrelation¶

These plots show autocorrelation delay spectrum waterfalls of each antenna that is active and whose status qualifies for this notebook. For nn/ee polarization, the autocorrelation delay spectrum is normalized by the max of the delay spectrum. For ne polarization, the autocorrelation delay spectrum is normalized by max(sqrt(|nn| |ee|)). ne and en are the same for autocorrelations, and thus only ne is shown here. The delay spectra are presented in dB with 10log10($|\tilde{V}|$).

For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

nn polarization¶

In [6]:
utils.plot_wfds(uvd, _data_cleaned_sq, 0)

ee polarization¶

In [7]:
utils.plot_wfds(uvd, _data_cleaned_sq, 1)

ne polarization¶

In [8]:
utils.plot_wfds(uvd, _data_cleaned_sq, 2)

Analysis of 2700ns features in delay spectra¶

This plot shows the relative amplitude at 2700 ns feature. The relative amplitude is calculated in dB with the mean amplitude at 2500-3000 ns compared to the mean amplitude at 2000-2500 ns. Larger values of relative feature amplitude indicate higher probability of detecting the peak at 2700 ns. Antennas in the same node are grouped by the shaded region.

In [9]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='nn')
In [10]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='ee')

This plot shows a matrix representing the 2700ns feature correlation of each baseline. The color bar indicates the amplitude of 2700ns (mean amplitude of 2500-3000ns delay spectrum) in dB which is the same as that in the above plot.

In [11]:
# utils.CorrMatrix_2700ns(uvd, HHfiles, difffiles, flagfile, JD, N_threads=14)

Analysis of noise floor in delay spectra¶

This plot shows the ratio of delay spectrum to noise floor (averaged over 1000-4000ns). Near 1 indicates the delay spectrum reaches to the noise floor, which may mean good.

In [12]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='nn')
In [13]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='ee')
In [14]:
# get the ratio of delay spectum to noise for different freqeuncy bands and pols
ds_noise_ratio = utils.get_ds_noise_ratio(uvd, uvd_diff, bls)

nodes, antDict, inclNodes = utils.generate_nodeDict(uvd)
ants = uvd.get_ants()
# build dataframe
to_show = {'Ant': ants, 'Node': [int(antDict[ant]['node']) for ant in ants], 'Snap': [int(antDict[ant]['snapLocs'][0]) for ant in ants]}
df = pd.DataFrame(to_show)
 
cols_ratio = []
for key in ds_noise_ratio.keys():
    if(key[0] == 40):
        col = r'Full '
    else:
        col = r'{}-{} MHz '.format(key[0], key[1])
    col += key[2]
    df[col] = ds_noise_ratio[key]
    cols_ratio.append(col)
    

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

ratio_cut = 3
# style dataframe
table = df.style.hide_index() \
          .applymap(lambda val: 'color: red' if val > ratio_cut else '', subset=cols_ratio) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

This table shows the ratio of the delay spectrum to the noise level from diff files for different frequency bands and pols. The ratio > 3 is colored in red

In [15]:
HTML(table.render())
Out[15]:
Ant Node Snap Full nn Full ee 50-85 MHz nn 50-85 MHz ee 120-155 MHz nn 120-155 MHz ee 155-190 MHz nn 155-190 MHz ee 190-225 MHz nn 190-225 MHz ee
3 1 2 1.776925 1.021344 1.800655 1.942551 4.758544 2.847029 1.882691 2.233895 1.996835 2.928661
4 1 2 1.499746 0.965182 5.849640 4.546507 4.097689 2.303265 2.029057 2.635552 2.976926 3.188565
5 1 2 1.523808 1.123910 1.786519 1.904537 3.794258 2.067439 2.166283 2.497781 7.669860 8.913386
7 2 0 1.370457 0.951727 2.215662 1.788024 3.387402 2.108981 1.985535 1.807480 2.825246 2.462627
8 2 0 1.363637 1.157441 2.518534 2.058138 3.692010 2.169355 2.228558 1.921146 3.027236 4.568059
9 2 0 1.537259 1.092147 1.875003 2.045800 4.696351 2.533248 2.669709 2.796145 2.777004 3.499523
10 2 1 1.589761 1.124851 2.218455 2.094441 4.093687 2.141037 2.116903 3.346921 2.907241 3.906343
15 1 3 1.741792 1.108802 2.361484 1.976702 4.287440 2.695770 2.364852 2.391113 5.345985 7.355501
16 1 3 1.849185 1.369834 1.973503 1.959558 3.855956 3.239754 2.138582 2.526801 19.987859 22.627484
17 1 3 1.901154 1.213145 1.891174 1.767265 4.684492 2.531353 2.221805 1.982492 2.120244 4.281029
18 1 0 58.670982 47.537114 1.237269 1.727421 280.150714 251.325510 1.560813 1.923361 2.063249 2.156722
19 2 1 1.498134 1.061621 2.036324 2.125624 3.312046 2.544405 2.464810 2.145470 4.312478 6.891231
20 2 1 1.346628 1.084644 1.898040 1.629052 3.957930 2.071564 1.848054 1.846090 2.363637 2.645335
21 2 2 2.854380 1.586276 1.721929 6.923610 12.182017 2.357390 2.094285 1.649843 1.967420 2.302819
27 1 0 3.385001 3.634098 9.501373 11.056331 7.968469 7.777579 4.547786 2.438050 3.523404 2.071951
28 1 0 163.293089 1.932243 1.528212 3.237308 814.057862 5.071144 1.892514 5.429037 2.710078 6.097623
29 1 1 nan nan nan nan nan nan nan nan nan nan
30 1 1 nan nan nan nan nan nan nan nan nan nan
31 2 2 1.597160 1.520169 1.883622 1.869240 4.469185 2.360974 3.412773 9.967740 5.342193 19.225057
32 2 2 2.557898 3.339233 2.123795 1.871115 4.600726 3.339126 6.579617 8.796128 7.170089 9.920727
33 2 3 32.346772 0.927658 1.512444 1.818352 157.701874 1.914733 1.866306 2.019760 2.465878 2.630314
36 3 3 1.957883 1.419840 2.094724 1.906868 3.412701 2.047307 2.105250 1.840114 2.510736 3.046547
37 3 1 1.428337 1.059728 3.116931 2.242203 4.183902 2.780009 1.974732 2.211106 4.553396 4.424138
38 3 1 1.474682 1.147054 2.049502 1.920883 3.602873 2.644221 2.921560 2.953652 2.315316 3.239207
40 4 1 1.727410 1.185526 1.944684 2.111801 4.825170 2.652613 2.356394 2.711770 2.802603 5.015959
41 4 3 1.670136 1.184998 1.965577 2.270727 4.714304 2.737948 2.289730 2.145071 2.204995 2.705263
42 4 0 2.026971 1.152123 1.954182 1.968584 5.230097 3.073407 2.363771 2.670425 5.414564 11.410586
45 5 0 1.712235 1.122601 4.599123 1.709446 5.036536 2.289433 3.053753 2.297452 3.536373 2.183361
46 5 0 2.486792 1.383660 1.993724 1.763863 6.738856 2.724595 2.123157 1.937252 3.018500 3.162729
50 3 3 1.494656 2.315747 2.032572 1.630889 3.717310 3.371278 2.496471 2.557402 3.147844 2.593649
51 3 2 5.055471 1.109277 14.052546 3.201411 5.427605 2.185326 5.118968 1.830932 6.706385 3.565262
52 3 3 6.394381 1.385599 17.371056 2.053820 3.276715 2.491216 3.610887 2.209973 4.674133 4.425802
53 3 2 1.677534 0.930542 2.030786 1.849533 4.497298 2.174543 2.076066 2.090630 5.960002 4.769100
54 4 0 1.815724 1.123280 1.765939 2.118975 5.558367 2.768201 2.177194 2.258936 5.291135 4.556077
55 4 2 1.205831 0.880168 2.264629 1.451374 2.885115 2.267991 2.505011 1.813565 3.649090 2.079389
56 4 1 1.617131 1.220254 1.783160 1.977487 5.250753 2.516710 2.383631 2.149555 3.077040 3.339421
57 4 3 1.663754 1.276444 3.477503 1.672524 4.870193 2.328948 2.390037 1.798207 3.149662 2.255002
65 3 0 1.294109 0.870080 2.614456 1.997891 3.014905 2.007969 2.082220 1.992143 2.568326 2.507890
66 3 0 1.736278 1.041304 3.178632 1.828422 4.158897 2.117017 2.067945 1.928993 2.590551 3.287311
67 3 0 1.878485 1.032957 1.807756 1.754683 4.685617 2.242766 1.677055 1.779276 1.976471 2.193464
68 3 1 1.403038 0.913517 2.057331 1.909600 4.707885 2.091402 1.994711 2.070654 2.852764 3.110435
69 4 1 1.485464 1.006528 2.015892 2.379297 4.193054 2.252887 2.052277 2.263728 2.900179 3.267181
70 4 2 3.046106 1.803374 2.665073 2.093009 7.986006 3.253045 4.597381 2.742000 3.414087 3.470915
71 4 2 1.177211 1.083154 1.910999 2.061995 2.361427 2.243473 2.045762 2.109202 2.163966 2.339984
72 4 0 1.620311 1.269412 2.121685 2.386880 4.797541 3.916752 2.116883 2.550315 3.456714 5.164845
73 5 0 1.612398 1.325375 1.832496 1.647800 3.793206 2.804019 1.842638 2.346269 3.138838 3.125601
81 7 0 1.997070 1.509158 2.207539 1.752749 3.371080 3.068424 3.538033 2.355191 14.300864 7.911206
82 7 0 1.507365 1.245276 1.857330 2.392331 3.818069 2.840797 2.178597 2.360639 3.226364 4.483227
83 7 0 1.298634 1.034190 1.987778 3.257100 3.247868 2.157528 2.470004 1.934071 2.986745 3.752301
88 9 0 5.929546 18.799135 23.974321 37.521920 9.480293 20.341951 5.091408 4.378511 7.265547 3.765059
89 9 2 nan nan nan nan nan nan nan nan nan nan
90 9 0 7.223443 2.901727 3.234424 2.571075 12.504768 5.413609 32.838694 13.315302 40.029193 21.168020
91 9 1 4.174468 5.605944 13.179975 11.882482 4.673070 14.440906 6.074596 5.424731 7.549285 3.085927
92 10 0 2.929430 1.889958 2.011828 2.107063 6.847056 4.070345 1.972903 1.953188 1.871004 1.916094
93 10 0 4.367619 1.940242 13.556916 2.534308 6.641361 4.535897 3.335625 2.099291 3.199624 2.542304
94 10 0 1.529925 1.401307 1.653822 2.792388 4.331770 3.060892 2.438580 2.981131 3.056266 15.195090
98 7 2 2.839208 0.876696 2.384674 2.313862 7.568167 2.039127 8.793869 2.440506 30.644896 3.709928
99 7 2 1.377909 1.058660 1.985268 1.960668 3.748187 2.730650 1.961661 1.897903 4.367772 4.310410
100 7 1 1.394816 0.963018 2.113948 2.559197 3.225881 1.869005 2.050719 1.919291 2.715179 2.647511
105 9 1 2.715448 2.395137 18.425132 7.219683 3.190932 5.347145 5.269926 2.239110 9.920869 2.630120
106 9 3 2.051462 2.004664 1.894778 2.018818 6.448464 4.869568 3.575115 5.605751 4.834166 3.950443
107 9 0 6.811204 4.910598 28.629814 35.040841 10.120080 2.852544 5.543155 6.989880 4.703502 1.982495
108 9 1 1.823071 0.977927 1.792548 1.742574 4.116889 1.941379 1.956347 1.978145 2.822736 2.529967
109 10 1 2.515608 1.586161 2.409372 1.896679 6.314452 2.933904 1.852522 1.968938 2.631822 2.556347
110 10 1 1.280681 2.266072 2.132099 2.048311 3.558705 4.455581 2.115513 3.137187 2.398735 2.406011
111 10 1 1.676346 1.090041 2.791926 1.760492 5.108382 2.523654 2.157772 1.967541 2.578296 1.885325
112 10 2 1.349625 1.425539 10.664573 2.273703 2.123300 4.082277 2.112102 2.013039 2.323326 2.511445
116 7 2 1.178287 0.948327 1.798255 2.037475 3.081693 2.189185 1.723973 1.952427 3.154174 2.838011
117 7 3 nan nan nan nan nan nan nan nan nan nan
118 7 3 nan nan nan nan nan nan nan nan nan nan
119 7 1 1.432800 1.027573 2.005743 1.839162 3.794307 2.253983 1.967370 1.956212 4.983853 2.636839
124 9 2 nan nan nan nan nan nan nan nan nan nan
125 9 3 4.096608 2.800088 36.401829 12.509595 9.052294 5.188782 3.384212 2.373103 4.814973 2.355617
126 9 3 4.012524 2.715678 29.086608 12.652601 6.373913 8.594158 4.351884 4.841631 5.492819 4.514091
127 10 2 3.176937 2.725711 4.207215 2.203926 8.990052 7.803941 7.900714 7.376896 11.602877 12.544472
128 10 2 1.353484 1.086899 2.001294 2.518817 4.538218 2.544077 2.186520 2.014307 5.616742 8.358638
129 10 3 1.735927 1.051811 2.006277 1.569231 3.830773 2.259201 1.952276 1.685992 2.153064 2.422411
130 10 3 1.822824 1.758361 3.304263 13.278050 4.176662 1.892800 2.487538 2.185240 3.773325 2.002690
135 12 2 nan nan nan nan nan nan nan nan nan nan
136 12 2 nan nan nan nan nan nan nan nan nan nan
137 7 3 nan nan nan nan nan nan nan nan nan nan
138 7 1 1.968920 3.309349 1.705418 3.928737 5.567831 2.214408 2.061539 2.333520 11.919387 2.326643
140 13 2 1.474601 1.113217 1.893938 1.804422 3.821183 2.579230 2.044116 2.165754 2.487856 2.340320
141 13 2 6.284487 1.137079 16.839419 1.976992 21.613307 2.888873 15.197582 2.161089 15.130179 4.145110
142 13 2 3.000785 3.462853 6.119211 3.205413 5.488177 8.253354 2.555204 1.682793 4.355584 3.062816
143 14 2 2.083385 1.105404 2.676857 1.670286 4.243220 2.426165 4.248318 2.472137 2.307600 2.809861
144 14 3 4.299608 1.197087 1.919599 1.389066 7.314518 2.495459 3.030506 1.641978 2.880647 2.685784
145 14 3 3.738331 2.944686 14.250438 18.559245 9.509747 10.437471 3.105009 1.830164 3.102740 1.959993
150 15 1 3.966250 5.340166 21.754186 36.413709 6.658204 8.700509 3.500224 2.227553 3.000441 1.838597
155 12 2 nan nan nan nan nan nan nan nan nan nan
156 12 3 2.124383 1.671654 1.809551 2.034402 4.921833 4.361902 2.104358 2.877540 1.939613 3.802624
157 12 3 1.804066 1.261535 1.771020 2.034777 3.794179 2.496989 1.871659 2.170040 2.186495 2.681541
158 12 3 1.908756 1.125317 4.804003 1.523787 5.655630 2.334968 3.266640 2.271852 4.544159 7.255538
160 13 3 2.764800 3.143299 5.871831 22.133799 4.275127 4.001055 3.541112 3.629692 3.733634 2.546813
161 13 0 2.702610 1.133377 1.706764 1.912307 8.391760 2.429432 3.143084 2.196486 4.565981 5.139809
162 13 0 1.554761 1.751558 2.083461 2.114113 3.022152 3.419855 2.546922 2.860763 3.669909 15.253824
163 14 2 1.548680 1.155682 1.992939 1.794431 3.397232 1.982072 2.341727 1.929144 3.065240 2.960407
164 14 2 1.391107 1.138736 1.881893 1.831472 3.035889 2.311823 2.194943 3.092783 3.089420 3.241315
165 14 0 7.292417 5.855369 12.970217 5.101910 23.041003 16.235678 5.774004 7.189790 14.397000 7.116650
166 14 0 4.264114 2.325019 2.011374 1.559393 7.993534 4.214364 9.530528 4.357261 6.745893 6.965281
167 15 1 2.328361 2.472623 2.438846 1.852000 4.408479 2.923729 2.572523 3.098519 3.055421 2.451423
168 15 2 1.874406 1.508671 3.347180 2.319931 3.600960 2.701049 6.085450 8.239439 10.848399 11.257657
169 15 2 1.465344 1.404243 2.167695 2.549586 2.772037 2.846882 3.989936 5.882490 4.514108 6.928105
170 15 2 1.864383 1.420782 3.202470 2.588291 5.014900 3.104686 3.992702 6.247593 5.391067 5.438786
176 12 0 1.304665 0.988545 1.863503 1.827107 3.029030 2.190419 2.245575 2.075516 3.124192 2.537437
177 12 0 1.211388 1.085420 1.883923 1.799363 2.798711 2.499616 2.095978 1.851664 2.803976 2.478082
178 12 0 1.596600 1.034083 2.002625 1.633757 4.835239 2.237855 2.180205 2.258892 2.525089 2.659746
179 12 1 2.318625 1.043037 1.528492 1.803611 4.827841 1.858238 5.723660 2.624824 5.183085 3.092511
180 13 3 3.410044 13.173238 14.589585 0.794757 2.805411 46.837233 2.523599 12.352419 2.168640 17.321513
181 13 3 1.802318 1.166993 1.914610 1.755383 4.407509 3.334572 1.900524 2.055063 2.387747 2.177125
182 13 0 1.773487 1.409994 2.153765 2.138173 4.404911 4.427916 2.379155 2.045984 2.775254 2.686158
183 13 1 1.815372 1.142712 1.816935 2.103941 6.000745 2.516595 1.836023 1.981576 2.528174 2.736485
184 14 0 1.680721 0.955047 1.830391 1.928245 4.707967 2.102332 2.432323 1.982699 3.855524 6.456574
185 14 1 2.027297 1.586554 1.818468 2.258545 3.617379 2.190924 3.163136 4.890889 4.331699 16.666592
186 14 1 1.900906 1.105978 1.748065 1.738961 4.846709 2.178622 2.027109 2.181490 2.817038 2.967722
187 14 1 2.002852 1.693392 2.578896 4.596682 4.740603 4.062896 4.081876 6.039837 13.368512 30.952258
189 15 3 nan nan nan nan nan nan nan nan nan nan
190 15 3 nan nan nan nan nan nan nan nan nan nan
191 15 3 nan nan nan nan nan nan nan nan nan nan
220 18 2 nan nan nan nan nan nan nan nan nan nan
221 18 2 nan nan nan nan nan nan nan nan nan nan
222 18 2 nan nan nan nan nan nan nan nan nan nan
320 3 2 4.594307 5.601729 6.106976 24.360745 4.431552 5.486331 6.802948 4.989952 6.759638 2.125285
321 2 3 1.769396 0.945145 1.956908 2.116255 3.213974 2.105392 1.890651 1.874387 2.138997 2.035221
323 2 3 1.183365 3.923881 5.073515 7.252838 2.391877 4.000968 2.078208 2.216622 2.310362 4.179041
324 4 3 0.865872 0.915453 1.932848 2.134839 1.920210 2.009938 1.827430 2.046766 2.043054 2.214247
329 12 1 1.097406 1.366253 3.035705 12.814205 2.888848 2.265645 1.755282 1.861524 2.286621 2.201841
333 12 1 1.314818 1.174891 2.171472 1.932948 4.466300 2.714827 2.118882 1.849366 2.042794 2.175246
In [16]:
csv_file = os.path.join(nb_outdir, 'ds_noise_ratio_{}.csv'.format(JD))
df.to_csv(csv_file, index=False)

Delay spectrum and autocorrelation plot per baseline per polarization for a given frequency (sub-)band¶

Left panel: time averaged delay spectum of autocorrelation in dB with 10*log10($|\tilde{V}|$) (blue) and noise from diff file representing the expected variance of the delay spectrum (red). The time-averaging is performed by 1. binning three time integrations of each even and odd visibility, 2. Fouier transform the binned even and odd visibilities, and 3. multiply the even and odd delay spectra at alternating time bin and average the squared delay spectrum along the time axis. This helps to reduce the noise bias. Both autocorrelation delay spectrum and diff delay spectrum are averaged in the same way

Right panel: time averaged autocorrelations w/o (orange) and w/ xRFI flags (blue). Flagged one is shifted from the unflagged one for clarity

In [17]:
utils.interactive_plots_dspec(bls, uvd, uvd_diff, JD)
In [ ]: