In [1]:
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData, UVFlag
import pyuvdata
import os
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, SkyCoord, AltAz, Angle
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import h5py
import importlib
from scipy import stats
import scipy
import pandas as pd
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
In [2]:
#get data location
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
nb_outdir = os.environ['NB_OUTDIR']
utc = Time(JD, format='jd').datetime
print(f'JD = {JD}')
print(f'Date = {utc.month}-{utc.day}-{utc.year}')
print(f'data_path = "{data_path}"')

JD = 2459767
Date = 7-6-2022
data_path = "/mnt/sn1/2459767"
In [3]:
# Load in data
HHfiles, difffiles, uvdx, uvdy = utils.load_data_ds(data_path,JD)
    
uvd = UVData()
uvd_diff = UVData()
uvd.read(HHfiles[0])
use_ants = [int(ant) for ant in uvd.get_ants()]
bls = [(ant, ant) for ant in use_ants]
uvd.read(HHfiles[::10], skip_bad_files=True, bls=bls)
uvd_diff.read(difffiles[::10], skip_bad_files=True, bls=bls)
lsts = uvd.lst_array

flagfile = glob.glob(os.path.join(HHfiles[0].split('zen')[0],'zen.{}*total_stage_1_threshold_flags.h5'.format(JD)))
uvf = UVFlag()
uvf.read(flagfile)
bls = [(ant, ant) for ant in uvd.get_ants()]
times_uvf = np.unique(uvf.time_array)
times_uvd = np.unique(uvd.time_array)
idx_times = [np.where(time_uvd == times_uvf)[0][0] for time_uvd in times_uvd]
uvd.flag_array[:,0,:,:] = np.repeat(uvf.flag_array[idx_times], len(bls), axis=0)

372 sum files found between JDs 2459767.25308 and 2459767.33607
372 diff files found between JDs 2459767.25308 and 2459767.33607

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [4]:
utils.plot_lst_coverage(uvd)

Delay spectrum¶

Delay spectrum CLEANed using uvtools.dspec.high_pass_fourier_filter with 7th-order Blackman-Harris window function. Odd/even visibilities are used to remove noise bias.

In [5]:
_data_cleaned_sq, d_even, d_odd = utils.clean_ds(bls, uvd, uvd_diff, N_threads=14)

Waterfalls of delay spectra for autocorrelation¶

These plots show autocorrelation delay spectrum waterfalls of each antenna that is active and whose status qualifies for this notebook. For nn/ee polarization, the autocorrelation delay spectrum is normalized by the max of the delay spectrum. For ne polarization, the autocorrelation delay spectrum is normalized by max(sqrt(|nn| |ee|)). ne and en are the same for autocorrelations, and thus only ne is shown here. The delay spectra are presented in dB with 10log10($|\tilde{V}|$).

For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

nn polarization¶

In [6]:
utils.plot_wfds(uvd, _data_cleaned_sq, 0)

ee polarization¶

In [7]:
utils.plot_wfds(uvd, _data_cleaned_sq, 1)

ne polarization¶

In [8]:
utils.plot_wfds(uvd, _data_cleaned_sq, 2)

Analysis of 2700ns features in delay spectra¶

This plot shows the relative amplitude at 2700 ns feature. The relative amplitude is calculated in dB with the mean amplitude at 2500-3000 ns compared to the mean amplitude at 2000-2500 ns. Larger values of relative feature amplitude indicate higher probability of detecting the peak at 2700 ns. Antennas in the same node are grouped by the shaded region.

In [9]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='nn')
In [10]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='ee')

This plot shows a matrix representing the 2700ns feature correlation of each baseline. The color bar indicates the amplitude of 2700ns (mean amplitude of 2500-3000ns delay spectrum) in dB which is the same as that in the above plot.

In [11]:
# utils.CorrMatrix_2700ns(uvd, HHfiles, difffiles, flagfile, JD, N_threads=14)

Analysis of noise floor in delay spectra¶

This plot shows the ratio of delay spectrum to noise floor (averaged over 1000-4000ns). Near 1 indicates the delay spectrum reaches to the noise floor, which may mean good.

In [12]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='nn')
In [13]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='ee')
In [14]:
# get the ratio of delay spectum to noise for different freqeuncy bands and pols
ds_noise_ratio = utils.get_ds_noise_ratio(uvd, uvd_diff, bls)

nodes, antDict, inclNodes = utils.generate_nodeDict(uvd)
ants = uvd.get_ants()
# build dataframe
to_show = {'Ant': ants, 'Node': [int(antDict[ant]['node']) for ant in ants], 'Snap': [int(antDict[ant]['snapLocs'][0]) for ant in ants]}
df = pd.DataFrame(to_show)
 
cols_ratio = []
for key in ds_noise_ratio.keys():
    if(key[0] == 40):
        col = r'Full '
    else:
        col = r'{}-{} MHz '.format(key[0], key[1])
    col += key[2]
    df[col] = ds_noise_ratio[key]
    cols_ratio.append(col)
    

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

ratio_cut = 3
# style dataframe
table = df.style.hide_index() \
          .applymap(lambda val: 'color: red' if val > ratio_cut else '', subset=cols_ratio) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

This table shows the ratio of the delay spectrum to the noise level from diff files for different frequency bands and pols. The ratio > 3 is colored in red

In [15]:
HTML(table.render())
Out[15]:
Ant Node Snap Full nn Full ee 50-85 MHz nn 50-85 MHz ee 120-155 MHz nn 120-155 MHz ee 155-190 MHz nn 155-190 MHz ee 190-225 MHz nn 190-225 MHz ee
3 1 2 nan nan nan nan nan nan nan nan nan nan
4 1 2 nan nan nan nan nan nan nan nan nan nan
5 1 2 nan nan nan nan nan nan nan nan nan nan
7 2 0 1.623112 0.966473 2.045093 1.697040 7.570341 2.289896 1.963144 1.723875 3.063299 3.153608
8 2 0 1.364716 1.257296 2.106939 2.977484 5.168973 2.243712 1.860879 1.860369 3.136679 4.519368
9 2 0 1.844852 1.058538 1.933657 1.434877 8.909412 2.023954 2.384110 1.923616 2.892691 3.475544
10 2 1 1.664116 1.151198 1.903889 3.885965 5.820116 2.168493 1.621128 3.012695 3.626182 5.731768
15 1 3 1.832524 1.305314 1.999368 1.752754 5.194221 3.012942 2.339550 2.248670 7.086611 9.912723
16 1 3 2.033738 1.315457 1.692128 1.978631 6.645805 2.504164 2.138216 2.465414 24.201373 26.043632
17 1 3 1.913711 1.263277 1.560507 1.717477 5.823792 2.495732 1.735947 1.907588 1.857731 4.212135
18 1 0 55.352890 32.552368 5.712596 7.023946 263.339384 173.177195 4.900427 3.356428 2.612546 2.598301
19 2 1 1.532326 1.119108 1.724827 1.616598 3.921215 2.081585 1.855323 1.584929 2.889150 3.129956
20 2 1 1.476975 1.167566 1.772427 1.422715 5.522896 1.533056 1.613910 1.646234 2.090623 3.576409
21 2 2 1.907992 1.008157 1.556113 1.781023 7.909396 2.204161 2.098758 2.165300 2.337852 4.228172
27 1 0 4.404352 6.679620 7.628627 11.432904 12.930511 7.252147 3.636083 2.430910 2.676506 1.659197
28 1 0 81.261723 1.242502 7.602250 3.226638 399.756018 4.655556 2.122368 2.724054 3.124764 3.658377
29 1 1 1.822659 1.390532 1.849458 1.873252 6.314641 2.291398 2.389499 3.199696 5.477092 6.707191
30 1 1 2.173221 1.237087 1.712590 1.649887 7.014640 1.932229 4.682641 5.488998 3.355173 4.743529
31 2 2 1.966493 1.694047 1.883158 2.109726 7.386498 2.715458 3.491278 10.951128 5.945860 23.441354
32 2 2 2.475223 2.342876 4.694925 2.516847 6.250345 2.264538 9.043520 9.313532 11.413963 13.312869
33 2 3 33.404742 1.128501 6.150598 1.575394 164.921482 1.988360 4.818135 1.875209 3.298284 3.299315
36 3 3 2.447518 2.102834 2.017767 1.781870 3.545919 2.564120 2.029832 2.014928 3.195112 4.018794
37 3 1 1.635266 1.052116 2.589289 1.685972 5.707727 2.283216 1.956314 1.832420 5.683978 4.258461
38 3 1 1.557190 1.166363 1.957529 1.543528 5.344699 2.001312 2.306864 1.987560 2.784534 3.270400
40 4 1 1.787491 1.210127 1.563038 1.739031 6.308552 2.070808 1.706020 2.104463 2.693128 4.798169
41 4 3 1.903889 1.671464 1.884816 2.278277 7.228384 3.148661 1.786749 1.947780 2.244545 2.874187
42 4 0 1.803672 1.303562 1.551785 1.691276 5.700552 2.523179 1.787045 1.843076 4.896468 9.681032
45 5 0 1.206463 1.257623 1.867427 1.507168 4.260652 1.760919 2.359225 1.922199 3.449920 3.509259
46 5 0 3.109813 1.886903 2.660454 1.898487 10.130675 3.760584 2.027934 2.381600 4.146912 6.550758
50 3 3 1.886514 2.099142 2.066598 3.262230 4.066479 3.289165 2.415580 2.330100 3.995749 2.544515
51 3 2 5.649400 1.149744 14.628026 2.692911 11.665249 1.881467 5.806106 2.080506 6.623703 5.466247
52 3 3 6.487560 2.044741 19.570997 1.996028 7.519827 3.053091 4.035507 2.634493 5.209067 5.397166
53 3 2 1.693332 1.031159 2.173866 2.634398 6.464912 2.752182 2.173018 2.869606 6.159311 8.223234
54 4 0 1.853362 1.142731 1.563643 1.790991 6.775316 2.527667 2.303480 2.052591 4.929328 5.733889
55 4 2 1.376641 1.166073 2.659623 6.989211 4.082363 2.216632 7.570482 1.677119 3.660612 2.767668
56 4 1 2.007301 1.381974 1.848951 1.556973 8.589000 2.261183 2.519841 1.949855 3.618353 4.867162
57 4 3 1.699753 1.332429 3.134779 2.798609 5.887694 2.181334 2.308083 1.590437 2.765997 2.518871
65 3 0 1.505403 0.982745 2.272262 1.559299 3.662271 1.865525 2.123732 1.833242 2.972099 3.012142
66 3 0 2.117110 1.128177 4.360419 1.811491 7.464208 2.293842 2.497047 1.976192 3.432743 5.053879
67 3 0 2.154054 1.095981 2.288203 1.514025 8.196733 2.416012 1.751327 1.652654 2.339723 3.411178
68 3 1 1.941233 0.999594 1.705899 1.671970 8.909673 2.208028 1.590584 1.714361 2.375012 3.110511
69 4 1 1.728947 1.074644 1.845603 2.018858 6.359041 2.419659 1.689073 2.073321 3.092104 4.246960
70 4 2 1.465107 3.702558 1.553448 3.529487 2.949020 9.487635 2.928028 6.906056 2.978877 12.488534
71 4 2 1.743830 1.589495 2.377443 2.319399 2.243466 5.845900 3.811943 2.390742 3.071082 2.087593
72 4 0 1.580285 1.343427 1.612525 1.727022 5.454769 3.116955 2.082155 2.253478 5.259572 9.012853
73 5 0 1.862732 1.427995 1.648003 1.797625 5.962461 2.582224 1.781198 1.821185 2.955797 3.966360
81 7 0 2.428265 1.854231 2.533427 1.833902 5.152199 3.005532 4.080513 1.993813 18.828284 9.164756
82 7 0 1.694570 1.269667 1.771768 3.481341 6.775430 2.514729 1.678581 2.003330 3.245407 5.681355
83 7 0 1.903977 0.990436 2.298498 2.031212 3.315876 1.748810 1.765989 1.821309 3.746560 7.152109
88 9 0 4.723518 11.482540 16.630395 20.768886 10.558610 29.982022 4.295865 8.419255 6.065297 4.408920
89 9 2 nan nan nan nan nan nan nan nan nan nan
90 9 0 12.722073 5.977765 3.938565 3.686884 21.869744 13.589199 47.856377 16.589527 106.895342 64.372513
91 9 1 6.160995 7.295582 14.950587 12.895789 5.967967 16.427199 5.913728 4.806819 7.478474 3.758953
92 10 0 2.796413 2.315604 2.776642 2.870239 7.081404 3.821402 1.865242 1.901132 1.808667 1.776727
93 10 0 9.765917 2.277000 14.627601 3.077693 15.349620 3.826092 4.505784 2.312375 3.561474 2.687328
94 10 0 1.684109 1.526459 1.356312 2.598675 6.101264 2.767311 2.067171 3.093446 3.245598 24.333299
98 7 2 2.373149 1.911670 3.748930 2.402071 3.501835 5.403100 2.978450 3.218257 10.811509 7.795578
99 7 2 1.679211 1.174496 1.762277 1.791057 7.165359 2.144886 1.679346 1.753601 4.872282 7.687033
100 7 1 1.318576 9.010730 2.130236 10.191228 4.108537 8.861107 1.577986 7.876643 2.969856 64.177789
105 9 1 3.977633 4.891569 15.808523 7.987213 3.441707 9.016796 5.689757 2.368126 10.703113 3.762817
106 9 3 2.516685 2.228238 1.680614 2.140462 8.858350 6.077929 2.881108 5.644528 5.880956 6.183925
107 9 0 4.877039 3.945707 23.591219 24.350784 7.992219 3.422799 3.702423 2.865659 4.838258 1.890227
108 9 1 2.169203 1.113187 2.015272 1.509458 7.346615 1.965516 1.812871 1.801104 3.660918 4.612610
109 10 1 2.632999 1.993712 1.890421 2.014023 4.515590 4.280361 1.730894 1.819307 2.601872 3.071020
110 10 1 1.338424 2.715426 1.948595 4.184386 4.309428 3.850512 2.436957 2.271281 3.901901 1.657132
111 10 1 1.605090 1.600736 2.025179 1.650295 5.834282 2.635588 1.814969 1.503616 2.333054 1.794328
112 10 2 1.211048 1.628257 3.364167 1.876774 2.037629 7.174240 1.641962 2.116760 2.460997 3.052761
116 7 2 1.205906 1.044595 2.013906 2.063351 3.219552 1.867639 1.585417 1.637290 5.808333 6.172460
117 7 3 nan nan nan nan nan nan nan nan nan nan
118 7 3 nan nan nan nan nan nan nan nan nan nan
119 7 1 1.718119 1.156194 1.676517 1.564090 7.322016 1.785097 1.537997 1.583043 3.931017 3.721676
124 9 2 nan nan nan nan nan nan nan nan nan nan
125 9 3 5.218764 4.313558 39.840847 11.529718 16.058409 5.851262 3.201585 2.080687 4.625301 2.025835
126 9 3 5.003974 5.510283 34.815843 8.532067 6.958086 16.475417 5.958561 4.545951 5.921902 4.684819
127 10 2 3.327001 2.800206 3.015448 2.124877 6.200700 7.975282 7.984096 8.082351 10.054437 12.430497
128 10 2 1.577788 1.118744 1.864009 1.536442 7.038238 2.181083 1.843685 1.547408 3.994051 3.856771
129 10 3 2.008294 1.052337 1.718871 1.407294 7.296967 2.074169 1.719017 1.687325 2.006846 2.842715
130 10 3 1.931669 2.128645 2.718294 14.421149 5.289519 3.090373 2.093315 2.275622 2.834676 1.884814
135 12 2 1.419395 1.229363 1.943552 1.569917 4.206009 2.277831 1.940675 1.642040 2.757517 2.234779
136 12 2 1.681730 1.452333 2.512329 1.660138 5.130720 2.846408 4.077810 2.847877 2.267356 5.330517
137 7 3 nan nan nan nan nan nan nan nan nan nan
138 7 1 2.202183 4.800379 1.883433 3.578588 9.512961 3.342427 2.023190 3.835463 12.624400 2.483019
140 13 2 1.826931 1.343764 1.823657 1.683727 6.654777 2.715009 2.093837 2.776320 2.930292 4.108142
141 13 2 7.692460 1.358418 16.545177 1.696925 24.010000 2.833208 18.047829 1.997471 40.736914 3.238375
142 13 2 2.930581 3.012942 5.246371 3.828274 5.475159 8.043060 2.520716 2.192222 3.999514 4.009357
143 14 2 1.912132 0.989651 2.449555 1.742202 7.709193 2.168595 1.873383 2.124713 2.910013 3.843276
144 14 3 2.268397 1.370542 1.312732 1.741056 7.411308 2.876525 1.527495 1.783815 2.672002 3.237512
145 14 3 3.786546 4.663986 14.235980 16.424354 12.557171 12.282380 3.361432 1.866125 2.675661 1.966931
150 15 1 5.862466 11.998630 28.805056 45.247736 11.523693 12.602101 4.227810 8.660228 2.809869 2.162935
155 12 2 2.959895 3.980649 8.468820 24.631736 3.687139 4.852620 3.027269 5.724088 3.066104 3.266044
156 12 3 2.248556 1.818357 1.618496 1.779248 6.635628 3.865444 1.889896 2.473292 2.329173 4.110282
157 12 3 2.121491 1.303141 1.563524 1.680123 6.737680 2.302526 1.881734 2.023856 2.251506 2.653434
158 12 3 2.037402 1.355435 2.874832 1.614497 8.404603 2.634890 2.336599 2.460680 5.298140 11.061216
160 13 3 4.248503 3.222420 9.344819 21.682034 8.378874 3.766295 3.965449 2.588337 3.821576 2.045130
161 13 0 3.848059 1.342386 2.990839 1.699551 9.737215 3.006277 3.256678 2.018499 4.295747 6.209378
162 13 0 1.736936 1.344049 1.749556 1.939599 5.800561 3.036147 2.464220 3.423931 3.096215 4.812329
163 14 2 2.190513 1.345807 1.683755 1.668809 9.487126 1.966444 2.305996 1.911524 3.404629 3.596205
164 14 2 2.015891 1.141688 1.668869 1.440748 7.974019 1.890864 2.089781 3.238579 3.794111 3.511120
165 14 0 3.915943 3.203496 7.727295 3.846502 12.304549 11.060696 4.342287 6.088028 8.380143 14.705661
166 14 0 4.409265 5.115691 2.035693 1.660894 5.460866 6.364306 8.164322 8.985285 5.100172 8.019717
167 15 1 4.725951 4.243859 4.409010 2.957029 6.332105 3.275387 2.758493 3.245537 2.897598 2.458737
168 15 2 1.900742 1.331732 3.211087 3.270740 2.668122 2.152644 4.125060 5.013857 5.399172 4.538883
169 15 2 1.593768 2.255920 1.778157 2.715782 3.493108 2.718513 3.414493 5.879828 6.049362 9.977107
170 15 2 2.089136 2.153681 2.892633 3.206164 7.097844 4.630768 4.121386 5.263254 4.914639 8.044364
176 12 0 1.269079 1.256796 1.680609 1.692007 3.142720 2.253934 2.043255 2.128901 3.399108 2.894576
177 12 0 1.369190 1.245554 1.703911 1.572282 4.291368 2.502933 1.923397 1.625273 2.832457 2.923853
178 12 0 1.836302 1.177135 1.535267 1.513079 7.179103 2.552009 1.706197 1.897253 2.788489 3.214934
179 12 1 2.151971 0.988244 1.663144 1.620988 7.795797 1.802513 6.492190 1.997410 5.834332 3.648573
180 13 3 4.054718 18.005773 15.706596 8.795713 3.467457 52.368830 2.810966 25.058237 2.711001 73.247303
181 13 3 2.219274 1.606866 1.805394 1.454599 7.850741 4.220652 1.608424 1.663370 2.152168 2.332710
182 13 0 1.798566 1.863186 1.925558 2.131457 5.185935 4.755552 2.364439 1.980809 3.851834 4.814474
183 13 1 2.164606 1.290399 1.692177 1.735756 8.869965 2.636901 1.796261 1.709907 2.262518 2.990474
184 14 0 1.892550 1.173988 1.518718 1.616456 7.088885 1.947726 2.061153 1.634642 3.412700 5.301055
185 14 1 nan nan nan nan nan nan nan nan nan nan
186 14 1 nan nan nan nan nan nan nan nan nan nan
187 14 1 nan nan nan nan nan nan nan nan nan nan
189 15 3 1.471016 1.312446 1.754081 1.566420 5.418924 2.573650 2.060105 1.846061 2.172665 2.126813
190 15 3 2.998192 3.677491 1.320281 1.209679 4.382866 3.758908 2.759932 2.773677 4.171697 4.686369
191 15 3 1.676308 1.283984 1.755070 1.714546 5.528313 2.336743 1.929327 1.652519 2.698568 2.579831
220 18 2 nan nan nan nan nan nan nan nan nan nan
221 18 2 nan nan nan nan nan nan nan nan nan nan
222 18 2 nan nan nan nan nan nan nan nan nan nan
320 3 2 6.336114 7.569348 7.342988 27.749540 6.629432 6.555018 9.004479 2.848535 6.426267 2.744884
321 2 3 1.850526 1.374609 1.775054 1.859206 3.798558 2.440510 1.722179 1.835989 2.325664 3.209023
323 2 3 1.291290 5.983656 3.029113 6.874899 2.272565 5.556510 1.831070 2.091891 2.007532 4.845813
324 4 3 1.033053 3.265244 4.113052 46.076185 1.714335 3.756074 1.834736 3.624579 1.751272 4.266964
329 12 1 1.210435 1.179369 1.777653 6.165201 2.138229 2.254710 1.519179 1.852512 2.089841 5.438547
333 12 1 1.200045 1.591607 1.878935 2.007907 3.444676 4.918724 1.598761 1.970629 2.185738 4.938787
In [16]:
csv_file = os.path.join(nb_outdir, 'ds_noise_ratio_{}.csv'.format(JD))
df.to_csv(csv_file, index=False)

Delay spectrum and autocorrelation plot per baseline per polarization for a given frequency (sub-)band¶

Left panel: time averaged delay spectum of autocorrelation in dB with 10*log10($|\tilde{V}|$) (blue) and noise from diff file representing the expected variance of the delay spectrum (red). The time-averaging is performed by 1. binning three time integrations of each even and odd visibility, 2. Fouier transform the binned even and odd visibilities, and 3. multiply the even and odd delay spectra at alternating time bin and average the squared delay spectrum along the time axis. This helps to reduce the noise bias. Both autocorrelation delay spectrum and diff delay spectrum are averaged in the same way

Right panel: time averaged autocorrelations w/o (orange) and w/ xRFI flags (blue). Flagged one is shifted from the unflagged one for clarity

In [17]:
utils.interactive_plots_dspec(bls, uvd, uvd_diff, JD)
In [ ]: