In [1]:
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData, UVFlag
import pyuvdata
import os
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, SkyCoord, AltAz, Angle
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import h5py
import importlib
from scipy import stats
import scipy
import pandas as pd
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
In [2]:
#get data location
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
nb_outdir = os.environ['NB_OUTDIR']
utc = Time(JD, format='jd').datetime
print(f'JD = {JD}')
print(f'Date = {utc.month}-{utc.day}-{utc.year}')
print(f'data_path = "{data_path}"')

JD = 2459766
Date = 7-5-2022
data_path = "/mnt/sn1/2459766"
In [3]:
# Load in data
HHfiles, difffiles, uvdx, uvdy = utils.load_data_ds(data_path,JD)
    
uvd = UVData()
uvd_diff = UVData()
uvd.read(HHfiles[0])
use_ants = [int(ant) for ant in uvd.get_ants()]
bls = [(ant, ant) for ant in use_ants]
uvd.read(HHfiles[::10], skip_bad_files=True, bls=bls)
uvd_diff.read(difffiles[::10], skip_bad_files=True, bls=bls)
lsts = uvd.lst_array

flagfile = glob.glob(os.path.join(HHfiles[0].split('zen')[0],'zen.{}*total_stage_1_threshold_flags.h5'.format(JD)))
uvf = UVFlag()
uvf.read(flagfile)
bls = [(ant, ant) for ant in uvd.get_ants()]
times_uvf = np.unique(uvf.time_array)
times_uvd = np.unique(uvd.time_array)
idx_times = [np.where(time_uvd == times_uvf)[0][0] for time_uvd in times_uvd]
uvd.flag_array[:,0,:,:] = np.repeat(uvf.flag_array[idx_times], len(bls), axis=0)

372 sum files found between JDs 2459766.25313 and 2459766.33612
372 diff files found between JDs 2459766.25313 and 2459766.33612

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [4]:
utils.plot_lst_coverage(uvd)

Delay spectrum¶

Delay spectrum CLEANed using uvtools.dspec.high_pass_fourier_filter with 7th-order Blackman-Harris window function. Odd/even visibilities are used to remove noise bias.

In [5]:
_data_cleaned_sq, d_even, d_odd = utils.clean_ds(bls, uvd, uvd_diff, N_threads=14)

Waterfalls of delay spectra for autocorrelation¶

These plots show autocorrelation delay spectrum waterfalls of each antenna that is active and whose status qualifies for this notebook. For nn/ee polarization, the autocorrelation delay spectrum is normalized by the max of the delay spectrum. For ne polarization, the autocorrelation delay spectrum is normalized by max(sqrt(|nn| |ee|)). ne and en are the same for autocorrelations, and thus only ne is shown here. The delay spectra are presented in dB with 10log10($|\tilde{V}|$).

For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

nn polarization¶

In [6]:
utils.plot_wfds(uvd, _data_cleaned_sq, 0)

ee polarization¶

In [7]:
utils.plot_wfds(uvd, _data_cleaned_sq, 1)

ne polarization¶

In [8]:
utils.plot_wfds(uvd, _data_cleaned_sq, 2)

Analysis of 2700ns features in delay spectra¶

This plot shows the relative amplitude at 2700 ns feature. The relative amplitude is calculated in dB with the mean amplitude at 2500-3000 ns compared to the mean amplitude at 2000-2500 ns. Larger values of relative feature amplitude indicate higher probability of detecting the peak at 2700 ns. Antennas in the same node are grouped by the shaded region.

In [9]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='nn')
In [10]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='ee')

This plot shows a matrix representing the 2700ns feature correlation of each baseline. The color bar indicates the amplitude of 2700ns (mean amplitude of 2500-3000ns delay spectrum) in dB which is the same as that in the above plot.

In [11]:
# utils.CorrMatrix_2700ns(uvd, HHfiles, difffiles, flagfile, JD, N_threads=14)

Analysis of noise floor in delay spectra¶

This plot shows the ratio of delay spectrum to noise floor (averaged over 1000-4000ns). Near 1 indicates the delay spectrum reaches to the noise floor, which may mean good.

In [12]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='nn')
In [13]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='ee')
In [14]:
# get the ratio of delay spectum to noise for different freqeuncy bands and pols
ds_noise_ratio = utils.get_ds_noise_ratio(uvd, uvd_diff, bls)

nodes, antDict, inclNodes = utils.generate_nodeDict(uvd)
ants = uvd.get_ants()
# build dataframe
to_show = {'Ant': ants, 'Node': [int(antDict[ant]['node']) for ant in ants], 'Snap': [int(antDict[ant]['snapLocs'][0]) for ant in ants]}
df = pd.DataFrame(to_show)
 
cols_ratio = []
for key in ds_noise_ratio.keys():
    if(key[0] == 40):
        col = r'Full '
    else:
        col = r'{}-{} MHz '.format(key[0], key[1])
    col += key[2]
    df[col] = ds_noise_ratio[key]
    cols_ratio.append(col)
    

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

ratio_cut = 3
# style dataframe
table = df.style.hide_index() \
          .applymap(lambda val: 'color: red' if val > ratio_cut else '', subset=cols_ratio) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

This table shows the ratio of the delay spectrum to the noise level from diff files for different frequency bands and pols. The ratio > 3 is colored in red

In [15]:
HTML(table.render())
Out[15]:
Ant Node Snap Full nn Full ee 50-85 MHz nn 50-85 MHz ee 120-155 MHz nn 120-155 MHz ee 155-190 MHz nn 155-190 MHz ee 190-225 MHz nn 190-225 MHz ee
3 1 2 1.263014 0.985733 1.760270 1.625594 2.260747 2.238956 1.913220 1.850223 1.880185 2.509254
4 1 2 1.134788 0.959478 2.472389 2.033629 2.381736 2.261790 2.264409 2.547360 2.802273 2.539115
5 1 2 1.131035 1.039616 1.759912 1.727067 1.898308 1.935211 2.081117 2.149009 7.590461 9.737889
7 2 0 1.155242 0.919241 2.237361 1.843491 1.962960 1.918283 1.918765 1.934938 2.334989 2.437366
8 2 0 1.061868 1.222273 2.327009 2.182216 2.428927 2.465814 2.181749 2.036580 2.790892 3.482585
9 2 0 0.969578 0.932526 1.715243 1.532619 1.837512 2.106281 2.224618 2.117566 2.527929 3.391198
10 2 1 1.053319 0.992788 1.935917 1.886067 2.052897 2.150825 2.135301 2.564303 2.339496 3.569258
15 1 3 nan nan nan nan nan nan nan nan nan nan
16 1 3 nan nan nan nan nan nan nan nan nan nan
17 1 3 nan nan nan nan nan nan nan nan nan nan
18 1 0 46.971828 13.322998 1.444183 1.584602 216.097311 55.417463 1.737379 1.570000 2.162728 1.692355
19 2 1 1.214505 1.122730 1.831714 1.899406 1.774921 2.227983 2.667814 2.275474 8.688479 10.922752
20 2 1 0.907990 1.036061 1.735724 1.669100 1.926964 2.089760 1.800450 1.713543 1.962088 2.419716
21 2 2 1.108326 0.883734 1.583089 1.454290 1.859511 1.935822 1.861355 1.906069 1.670308 2.423853
27 1 0 3.809870 3.285113 8.654918 9.627038 11.893336 6.646532 4.070901 2.291328 3.651238 1.815564
28 1 0 84.929542 0.989134 1.371692 3.074174 402.474898 2.269053 1.676476 2.305031 2.241610 3.761690
29 1 1 1.304031 1.137766 2.456138 1.769300 2.249016 2.089777 3.183543 2.899233 5.864387 5.104352
30 1 1 1.345244 1.007224 1.885467 1.728000 2.018912 2.133493 3.690339 4.648684 3.525235 4.529858
31 2 2 1.110802 1.339866 1.795811 1.815162 2.014661 2.440642 3.076031 7.925436 4.379349 15.777435
32 2 2 1.836119 1.794255 1.777998 1.726982 1.844330 2.373109 5.373537 8.009004 5.808103 9.332049
33 2 3 29.751105 0.946372 1.788406 1.716763 151.294057 2.152716 1.921748 2.098069 2.583716 2.372170
36 3 3 1.659091 1.367477 2.426941 1.929555 2.120975 1.980755 1.987811 1.766404 2.365753 2.567888
37 3 1 1.288666 1.089058 2.576809 1.726057 3.102996 2.275166 2.262258 1.890919 5.121119 3.662855
38 3 1 1.013545 1.191665 1.962634 1.739847 1.906384 2.496036 2.324092 2.396126 2.303075 2.516504
40 4 1 1.060034 1.006626 1.631365 1.596484 1.811635 2.186885 1.887787 2.297265 2.290945 3.408861
41 4 3 1.084835 1.280453 1.998039 2.260035 1.931872 3.228095 1.868051 1.838155 1.979524 2.051354
42 4 0 1.340622 1.062478 1.681702 1.615866 2.493189 2.415994 1.981086 2.156423 5.094598 11.180583
45 5 0 2.285768 1.728900 21.321081 12.245509 3.132336 3.314808 3.061197 1.944700 3.340750 1.917879
46 5 0 4.185121 2.865715 2.077467 1.931658 9.851655 6.911812 3.408489 3.748171 9.921567 23.940386
50 3 3 1.206293 1.442595 1.858659 1.662269 2.479428 2.200493 2.053879 2.130475 2.518514 2.272332
51 3 2 4.114318 0.991246 11.684268 1.982361 5.577843 2.092537 5.252842 2.001970 6.538862 4.106662
52 3 3 5.787157 1.318361 17.297447 1.768923 5.171221 2.115345 3.698919 2.293423 4.664094 3.945347
53 3 2 1.249952 1.055884 2.717290 2.658115 2.551997 2.734116 2.452494 2.522404 7.184760 6.301242
54 4 0 1.046641 0.971026 1.508413 1.697302 2.148711 2.299379 1.886417 1.916797 3.616768 3.856692
55 4 2 0.995982 0.950597 2.117574 1.449931 1.817874 2.285631 4.606537 2.007320 2.580041 1.825438
56 4 1 1.135185 1.122787 1.927098 2.086329 2.269652 2.962408 2.138147 2.292818 2.765525 3.965297
57 4 3 1.069842 1.009598 2.280458 1.710583 2.077060 2.182405 1.950958 1.946424 2.220948 2.159910
65 3 0 1.270440 0.922170 2.568710 1.897340 2.424348 2.056393 2.302131 2.300725 3.564997 3.469153
66 3 0 1.414371 1.030132 4.263487 2.082884 3.214796 2.503026 2.649797 1.869062 2.879129 2.890366
67 3 0 1.109706 0.947927 1.808310 1.879001 2.030251 2.142376 1.876730 1.777159 1.705547 2.082694
68 3 1 1.011858 0.862034 1.871128 1.703473 1.905686 2.112850 1.798079 1.855728 2.178657 2.596820
69 4 1 0.964557 0.910660 1.705667 1.858058 1.922422 1.939189 1.821911 1.976754 2.457734 2.301819
70 4 2 1.102681 2.200234 1.750364 2.839973 2.598917 3.634181 2.819034 3.435239 2.514542 8.709658
71 4 2 1.224477 1.043350 2.129298 2.524176 2.194536 2.069145 2.962921 2.158432 2.765990 1.997918
72 4 0 1.161245 1.112782 1.648073 2.052670 2.114947 3.064741 2.007931 2.025474 3.920758 4.475467
73 5 0 1.196409 1.348146 1.713018 1.549815 2.078986 2.413406 2.195610 2.166292 4.368463 4.513934
81 7 0 1.356516 2.336940 1.832636 1.659820 1.776151 4.144200 3.207875 8.736604 8.823028 11.698180
82 7 0 0.991043 1.049633 1.769358 2.294117 1.832839 2.319776 1.738255 1.888357 2.955734 2.888339
83 7 0 0.909620 0.868658 1.695775 1.982821 1.844335 1.872790 1.698477 1.784168 2.278241 2.304646
88 9 0 3.103832 7.665384 15.801345 37.778885 6.433561 23.815395 4.333604 3.669691 7.218848 3.881631
89 9 2 nan nan nan nan nan nan nan nan nan nan
90 9 0 9.410397 6.488301 3.243400 2.575534 14.592983 14.396437 40.432916 15.661971 76.672392 26.504760
91 9 1 3.277441 6.007381 13.014972 10.660464 5.103472 12.695875 5.720241 4.248555 7.694170 2.623555
92 10 0 1.813734 1.578344 1.888309 2.273616 2.357366 2.209978 1.881857 2.058255 1.835369 2.046977
93 10 0 4.517990 1.469411 8.763557 2.435519 5.252314 2.021121 3.876348 1.931642 3.042202 2.399426
94 10 0 1.008704 1.198397 1.596504 2.875431 2.036784 2.840943 2.329269 3.783408 2.932161 7.760631
98 7 2 1.779802 0.885848 2.201297 2.213736 3.544877 2.119325 3.619659 2.050905 12.010360 2.997860
99 7 2 0.970261 0.862330 1.641107 1.932678 1.841347 1.907794 1.746684 1.935254 3.797647 3.404280
100 7 1 0.928408 1.020098 2.107293 3.716461 1.739083 2.019128 1.933533 2.450380 2.588098 3.334243
105 9 1 2.837960 4.064927 18.932626 6.636376 3.953968 7.211918 5.274592 2.593211 9.716598 3.031463
106 9 3 1.713953 2.024673 1.731182 1.651794 3.741200 4.423830 3.447595 4.077766 5.773976 2.683568
107 9 0 8.335256 30.524919 39.155319 194.070537 10.786594 19.999803 12.839739 76.749218 5.204478 2.008499
108 9 1 1.492061 1.034185 1.968826 1.619109 2.089905 2.022803 1.868630 2.054611 3.203629 2.409036
109 10 1 1.872893 1.542191 2.087395 1.842221 3.553402 3.704224 1.938427 1.754513 2.495447 2.196450
110 10 1 0.931390 1.865393 2.255118 2.135379 2.329427 2.900448 1.941170 2.091282 2.318530 1.858286
111 10 1 1.131261 1.182912 2.582559 1.754107 2.730646 2.430700 2.116512 1.998121 2.135090 1.755980
112 10 2 0.929863 0.978391 2.637783 1.957824 1.800819 1.934007 1.718230 1.891691 1.941802 2.221116
116 7 2 0.928205 0.882995 1.933342 1.878357 2.164304 1.759845 1.739301 1.816247 4.374858 2.920008
117 7 3 1.276525 1.370218 2.096579 1.800485 1.867197 1.983765 2.065386 3.195702 3.360718 7.320752
118 7 3 1.098619 1.202641 1.874339 1.749516 1.921406 2.344518 1.980456 2.198253 3.639017 6.303206
119 7 1 1.010022 0.873832 1.966897 1.842608 1.934741 2.025972 1.905539 1.930104 3.568004 2.354136
124 9 2 nan nan nan nan nan nan nan nan nan nan
125 9 3 4.533656 2.320645 39.924995 12.966170 12.201797 5.051423 3.082129 2.390715 5.100060 2.440033
126 9 3 3.343855 2.850609 34.387218 11.802621 6.525649 10.481787 4.983590 4.782800 6.146058 4.682229
127 10 2 1.978382 1.944778 2.704816 1.914592 4.068949 5.379392 5.291395 4.661453 8.387089 9.409812
128 10 2 0.850589 1.004188 1.655679 2.187268 1.857424 2.044136 1.850318 2.014860 2.699260 2.866101
129 10 3 1.198960 0.903480 1.803520 1.634139 1.594713 2.223497 1.854684 1.853675 1.601520 2.589091
130 10 3 1.216354 1.704087 2.349437 14.446636 1.739008 2.136149 1.847312 2.083037 2.277676 1.996504
135 12 2 1.055878 1.059200 2.170849 1.656684 2.059169 2.321140 2.332681 2.033207 2.767321 2.719958
136 12 2 1.254629 1.133735 7.594962 1.990876 2.350089 2.414093 6.684948 3.588068 2.573102 3.745446
137 7 3 3.174849 1.114723 12.153673 3.601865 5.486558 2.610308 3.534448 1.941036 4.246521 2.502578
138 7 1 1.218024 4.012870 1.893296 3.741132 2.310233 3.240530 2.090148 2.106495 7.121300 2.402339
140 13 2 1.118513 0.984359 1.814330 1.847719 1.885304 2.042909 1.795875 2.045796 2.361458 2.000625
141 13 2 8.865765 1.069449 24.300444 1.850030 29.763444 2.578668 22.303254 1.778415 22.352004 2.874011
142 13 2 3.216156 1.636201 5.576823 3.446320 4.253064 3.185088 2.501008 1.505030 3.210529 2.869458
143 14 2 1.231649 0.863941 3.015718 1.600983 2.516008 1.951436 2.510468 2.014403 4.892193 9.342994
144 14 3 2.349922 1.182210 1.636713 1.638946 3.869066 2.554489 2.090034 1.836620 2.349986 2.298526
145 14 3 3.982754 4.483581 14.124307 20.338518 10.660441 11.487529 3.122683 2.008518 2.529267 1.913935
150 15 1 3.893463 11.361493 17.349002 47.459202 6.392498 11.389258 4.742306 18.609695 2.846283 1.747190
155 12 2 3.449878 3.019339 8.219548 20.442793 3.723885 5.291552 2.991789 5.143282 3.130411 2.431892
156 12 3 1.369884 1.276928 1.915856 1.869154 3.004541 3.098224 2.263314 2.008353 2.237792 2.226565
157 12 3 1.368367 1.124472 1.924909 2.019702 2.524952 2.720095 2.169235 2.328838 2.216490 2.342201
158 12 3 1.482304 1.094323 3.784420 1.851402 4.332766 2.070307 2.767617 2.377949 4.522505 9.724553
160 13 3 3.476941 10.391026 11.544857 73.173086 3.696347 7.768260 4.297743 23.079980 2.779047 2.000415
161 13 0 2.332106 1.189790 1.955992 1.827210 3.643980 3.315523 3.553060 2.121390 4.230472 5.229092
162 13 0 1.140872 1.022298 2.129650 2.060515 2.555336 2.381333 2.358434 2.094521 2.695840 3.072951
163 14 2 1.194195 1.034783 1.880477 1.982939 2.121525 1.948654 2.515678 1.861995 2.793972 4.186547
164 14 2 1.198038 0.960337 1.690523 1.947918 1.723775 2.282678 2.016711 3.224440 2.496008 3.764462
165 14 0 1.695271 1.219569 2.780158 2.052612 3.672860 2.881497 2.270300 2.350978 4.146868 3.072612
166 14 0 2.896701 3.050283 2.230600 1.606243 5.417461 5.676340 8.741143 6.695001 6.292873 6.966937
167 15 1 2.492646 2.094670 3.850829 1.708012 2.793645 2.479103 3.732343 1.997263 2.399493 2.219850
168 15 2 1.638374 1.225781 3.296166 2.078439 2.362610 2.015668 5.335211 5.699548 12.355588 8.030063
169 15 2 1.197677 1.246209 1.996492 2.201695 2.346892 1.985368 3.347004 5.260853 4.392151 6.374321
170 15 2 1.439044 1.204330 3.409462 2.316767 3.575924 3.105905 4.636957 4.577188 4.157994 4.951943
176 12 0 1.089740 1.015269 1.791570 1.958381 2.136808 2.231888 2.114966 2.381134 3.046556 2.468652
177 12 0 0.913245 1.044669 2.185479 1.810826 2.229763 2.569560 2.322032 1.976217 3.095331 2.556519
178 12 0 0.994792 1.052157 1.689904 1.734787 2.016996 2.159559 2.024487 2.229309 2.328727 2.179070
179 12 1 1.598201 0.894560 1.821603 1.879984 2.832355 2.058082 6.536237 2.537957 6.172647 3.628802
180 13 3 2.351952 10.408284 13.460732 4.885525 2.765936 31.456364 2.694570 11.326151 2.539958 22.406834
181 13 3 1.245614 1.043115 1.654128 1.483722 2.151705 2.664687 1.852107 1.639730 2.944643 1.726768
182 13 0 1.343056 1.426040 2.233904 2.274869 3.482616 4.396563 2.237917 2.296192 3.203325 3.475175
183 13 1 1.141335 1.036569 1.624032 1.849500 2.806274 2.442622 1.899842 2.225812 2.092432 2.791773
184 14 0 0.961724 0.884730 1.626828 1.738537 1.576839 2.072516 1.737111 1.750152 2.556768 3.989769
185 14 1 1.608472 1.177285 1.806082 2.247535 2.415598 2.168968 7.660227 2.903217 2.107019 2.476174
186 14 1 1.250383 1.068506 1.682676 1.942955 1.818821 1.956510 2.062638 2.650061 2.744718 6.047078
187 14 1 1.276812 1.173517 1.852831 1.794225 2.234667 2.110283 2.696299 3.789951 10.014520 20.278992
189 15 3 1.026479 1.251362 1.837590 1.839693 2.238812 2.589148 1.988403 2.581754 3.145143 5.867150
190 15 3 2.694529 3.096496 0.926779 1.332631 2.931220 2.824639 2.388169 2.829813 5.169305 5.987037
191 15 3 1.186819 1.105830 3.194073 2.285615 2.425634 2.587976 2.419403 2.309929 2.260208 2.922967
220 18 2 nan nan nan nan nan nan nan nan nan nan
221 18 2 nan nan nan nan nan nan nan nan nan nan
222 18 2 nan nan nan nan nan nan nan nan nan nan
320 3 2 4.166328 4.576423 8.112977 22.472946 4.338489 4.184008 6.137686 2.889555 5.713209 2.123963
321 2 3 1.535396 1.000371 1.820634 2.170898 1.835829 1.940322 1.656010 1.934968 1.884078 1.774095
323 2 3 0.920179 3.478201 2.482539 2.751264 1.921342 3.427644 1.791911 1.777547 2.098145 2.915111
324 4 3 0.924664 1.404888 2.398968 13.761088 1.619043 1.732717 1.886299 1.919102 1.780433 2.001310
329 12 1 0.880354 1.020437 1.825721 3.742179 1.545617 2.303349 1.799753 1.741733 2.037640 1.937997
333 12 1 0.850733 1.201653 2.017055 1.952662 1.993610 3.654404 1.834751 1.607952 2.109517 2.061794
In [16]:
csv_file = os.path.join(nb_outdir, 'ds_noise_ratio_{}.csv'.format(JD))
df.to_csv(csv_file, index=False)

Delay spectrum and autocorrelation plot per baseline per polarization for a given frequency (sub-)band¶

Left panel: time averaged delay spectum of autocorrelation in dB with 10*log10($|\tilde{V}|$) (blue) and noise from diff file representing the expected variance of the delay spectrum (red). The time-averaging is performed by 1. binning three time integrations of each even and odd visibility, 2. Fouier transform the binned even and odd visibilities, and 3. multiply the even and odd delay spectra at alternating time bin and average the squared delay spectrum along the time axis. This helps to reduce the noise bias. Both autocorrelation delay spectrum and diff delay spectrum are averaged in the same way

Right panel: time averaged autocorrelations w/o (orange) and w/ xRFI flags (blue). Flagged one is shifted from the unflagged one for clarity

In [17]:
utils.interactive_plots_dspec(bls, uvd, uvd_diff, JD)
In [ ]: