In [1]:
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData, UVFlag
import pyuvdata
import os
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, SkyCoord, AltAz, Angle
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import h5py
import importlib
from scipy import stats
import scipy
import pandas as pd
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
In [2]:
#get data location
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
nb_outdir = os.environ['NB_OUTDIR']
utc = Time(JD, format='jd').datetime
print(f'JD = {JD}')
print(f'Date = {utc.month}-{utc.day}-{utc.year}')
print(f'data_path = "{data_path}"')

JD = 2459770
Date = 7-9-2022
data_path = "/mnt/sn1/2459770"
In [3]:
# Load in data
HHfiles, difffiles, uvdx, uvdy = utils.load_data_ds(data_path,JD)
    
uvd = UVData()
uvd_diff = UVData()
uvd.read(HHfiles[0])
use_ants = [int(ant) for ant in uvd.get_ants()]
bls = [(ant, ant) for ant in use_ants]
uvd.read(HHfiles[::10], skip_bad_files=True, bls=bls)
uvd_diff.read(difffiles[::10], skip_bad_files=True, bls=bls)
lsts = uvd.lst_array

flagfile = glob.glob(os.path.join(HHfiles[0].split('zen')[0],'zen.{}*total_stage_1_threshold_flags.h5'.format(JD)))
uvf = UVFlag()
uvf.read(flagfile)
bls = [(ant, ant) for ant in uvd.get_ants()]
times_uvf = np.unique(uvf.time_array)
times_uvd = np.unique(uvd.time_array)
idx_times = [np.where(time_uvd == times_uvf)[0][0] for time_uvd in times_uvd]
uvd.flag_array[:,0,:,:] = np.repeat(uvf.flag_array[idx_times], len(bls), axis=0)

372 sum files found between JDs 2459770.25314 and 2459770.33613
372 diff files found between JDs 2459770.25314 and 2459770.33613

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [4]:
utils.plot_lst_coverage(uvd)

Delay spectrum¶

Delay spectrum CLEANed using uvtools.dspec.high_pass_fourier_filter with 7th-order Blackman-Harris window function. Odd/even visibilities are used to remove noise bias.

In [5]:
_data_cleaned_sq, d_even, d_odd = utils.clean_ds(bls, uvd, uvd_diff, N_threads=14)

Waterfalls of delay spectra for autocorrelation¶

These plots show autocorrelation delay spectrum waterfalls of each antenna that is active and whose status qualifies for this notebook. For nn/ee polarization, the autocorrelation delay spectrum is normalized by the max of the delay spectrum. For ne polarization, the autocorrelation delay spectrum is normalized by max(sqrt(|nn| |ee|)). ne and en are the same for autocorrelations, and thus only ne is shown here. The delay spectra are presented in dB with 10log10($|\tilde{V}|$).

For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

nn polarization¶

In [6]:
utils.plot_wfds(uvd, _data_cleaned_sq, 0)

ee polarization¶

In [7]:
utils.plot_wfds(uvd, _data_cleaned_sq, 1)

ne polarization¶

In [8]:
utils.plot_wfds(uvd, _data_cleaned_sq, 2)

Analysis of 2700ns features in delay spectra¶

This plot shows the relative amplitude at 2700 ns feature. The relative amplitude is calculated in dB with the mean amplitude at 2500-3000 ns compared to the mean amplitude at 2000-2500 ns. Larger values of relative feature amplitude indicate higher probability of detecting the peak at 2700 ns. Antennas in the same node are grouped by the shaded region.

In [9]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='nn')
In [10]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='ee')

This plot shows a matrix representing the 2700ns feature correlation of each baseline. The color bar indicates the amplitude of 2700ns (mean amplitude of 2500-3000ns delay spectrum) in dB which is the same as that in the above plot.

In [11]:
# utils.CorrMatrix_2700ns(uvd, HHfiles, difffiles, flagfile, JD, N_threads=14)

Analysis of noise floor in delay spectra¶

This plot shows the ratio of delay spectrum to noise floor (averaged over 1000-4000ns). Near 1 indicates the delay spectrum reaches to the noise floor, which may mean good.

In [12]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='nn')
In [13]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='ee')
In [14]:
# get the ratio of delay spectum to noise for different freqeuncy bands and pols
ds_noise_ratio = utils.get_ds_noise_ratio(uvd, uvd_diff, bls)

nodes, antDict, inclNodes = utils.generate_nodeDict(uvd)
ants = uvd.get_ants()
# build dataframe
to_show = {'Ant': ants, 'Node': [int(antDict[ant]['node']) for ant in ants], 'Snap': [int(antDict[ant]['snapLocs'][0]) for ant in ants]}
df = pd.DataFrame(to_show)
 
cols_ratio = []
for key in ds_noise_ratio.keys():
    if(key[0] == 40):
        col = r'Full '
    else:
        col = r'{}-{} MHz '.format(key[0], key[1])
    col += key[2]
    df[col] = ds_noise_ratio[key]
    cols_ratio.append(col)
    

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

ratio_cut = 3
# style dataframe
table = df.style.hide_index() \
          .applymap(lambda val: 'color: red' if val > ratio_cut else '', subset=cols_ratio) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

This table shows the ratio of the delay spectrum to the noise level from diff files for different frequency bands and pols. The ratio > 3 is colored in red

In [15]:
HTML(table.render())
Out[15]:
Ant Node Snap Full nn Full ee 50-85 MHz nn 50-85 MHz ee 120-155 MHz nn 120-155 MHz ee 155-190 MHz nn 155-190 MHz ee 190-225 MHz nn 190-225 MHz ee
3 1 2 1.290875 0.972155 1.478204 1.512284 2.785784 2.723060 1.738269 1.840245 1.778436 2.377795
4 1 2 1.200078 0.972905 1.620096 1.667642 2.254536 2.327867 1.802877 2.530034 2.356158 2.121764
5 1 2 1.139601 1.041798 1.914173 1.588793 2.407934 2.346008 2.396485 1.903765 6.877535 10.246489
7 2 0 1.188801 0.937176 1.849776 1.910695 3.098205 2.541814 1.837103 1.892159 2.205632 2.554011
8 2 0 1.314566 16.421995 2.767546 52.735476 3.328553 67.774664 2.446707 68.978127 2.588230 88.503327
9 2 0 1.002781 0.979216 1.719683 1.557000 2.371866 2.160024 2.114206 2.065145 1.589774 1.842614
10 2 1 nan nan nan nan nan nan nan nan nan nan
15 1 3 1.598762 1.223063 1.848473 1.847682 3.020514 2.748379 2.384678 2.743780 10.509221 10.681865
16 1 3 1.111973 0.909702 1.613172 1.581264 2.440374 1.889643 1.684507 1.637276 3.484743 3.227123
17 1 3 1.370227 0.979510 1.591364 1.633921 3.114093 2.031036 1.860052 1.705425 2.256662 2.996108
18 1 0 37.075688 10.832137 1.275513 1.610709 168.716403 46.469644 1.618404 1.931139 1.596930 1.859139
19 2 1 nan nan nan nan nan nan nan nan nan nan
20 2 1 nan nan nan nan nan nan nan nan nan nan
21 2 2 1.149153 0.808908 1.816790 1.442495 2.399434 2.024548 2.114072 1.845456 1.868069 1.841003
27 1 0 3.767555 4.008827 7.947019 17.745364 13.935419 5.940435 2.704714 2.063336 4.441974 1.919410
28 1 0 75.497611 4.634568 1.018920 18.258301 378.139758 13.879576 1.651025 7.236182 1.677720 23.106502
29 1 1 1.408634 1.034310 3.076404 1.765791 2.487963 1.957370 2.760900 3.419961 5.472571 5.153881
30 1 1 1.362379 1.247701 1.589023 1.549209 2.342018 1.794057 4.252279 4.990509 3.199691 8.006286
31 2 2 1.211184 1.429766 1.750197 1.487457 2.280129 2.275318 3.849374 9.416156 4.592563 12.983070
32 2 2 2.518043 2.966264 1.324767 1.472028 2.863840 3.275233 6.038633 7.336129 6.287141 8.962205
33 2 3 26.779366 0.893353 1.433651 1.618943 127.868021 2.030945 1.612071 2.014724 1.709554 2.503125
36 3 3 1.504295 1.331390 1.913969 1.799804 2.167782 2.378669 1.839094 1.730290 1.671925 1.824567
37 3 1 nan nan nan nan nan nan nan nan nan nan
38 3 1 nan nan nan nan nan nan nan nan nan nan
40 4 1 1.072189 1.001788 1.578372 1.547430 1.936023 2.186358 1.786497 2.035284 1.736271 2.692727
41 4 3 1.198996 1.103776 1.714557 1.573459 2.741956 2.428037 1.915540 1.704013 1.804344 1.663718
42 4 0 1.290388 0.983209 1.672717 1.531548 3.163262 2.539211 1.917016 2.092283 3.645395 6.366258
45 5 0 1.488887 0.989474 4.343049 1.656700 4.053935 1.948435 3.499383 1.870608 3.312307 1.617901
46 5 0 1.638716 1.040272 2.033020 1.626447 3.621405 2.295243 2.015098 1.876225 2.717037 2.687279
50 3 3 1.050982 0.948315 1.626206 1.407166 2.533446 2.316300 1.862565 1.741845 1.982178 1.644343
51 3 2 4.074700 0.955200 11.188406 2.149707 4.487804 2.164624 4.104555 2.308914 5.959181 4.011070
52 3 3 5.493806 1.152353 18.438875 1.821280 4.720838 2.221569 3.868498 2.021029 4.653755 3.411612
53 3 2 1.129445 1.114127 2.300858 3.312734 2.608155 3.037255 2.006271 2.719931 3.292559 4.879576
54 4 0 1.093988 0.902455 1.648529 1.930949 2.378111 2.329759 2.045437 1.990692 2.292382 2.756160
55 4 2 1.140386 0.832923 2.544789 1.388708 2.698884 2.102031 2.829028 1.916536 4.033113 1.621614
56 4 1 1.259341 1.096126 1.713247 1.602607 3.449507 2.621405 2.359172 2.282000 3.146663 3.529437
57 4 3 1.121520 0.982863 1.997581 1.592693 2.506419 2.194430 1.780417 1.901233 2.121957 2.016385
65 3 0 1.127860 0.828995 2.754863 1.813847 2.413848 1.849090 2.069124 1.774130 2.204649 2.037607
66 3 0 1.184441 0.938882 2.271712 1.888902 2.216345 2.418444 1.835550 1.845291 2.510425 3.058987
67 3 0 1.276788 0.919684 1.687444 1.780346 2.490195 1.968442 1.652222 1.777693 1.530490 1.801785
68 3 1 nan nan nan nan nan nan nan nan nan nan
69 4 1 1.098985 0.934773 1.774316 2.038926 2.623033 2.174136 2.400706 2.423278 2.793162 3.071703
70 4 2 2.302789 1.266137 3.294678 1.856233 6.470850 2.403339 4.545720 2.380289 3.425099 2.185552
71 4 2 1.234139 1.032928 3.036726 1.885810 2.865268 2.199342 2.581737 1.836241 2.933710 1.857232
72 4 0 1.399405 1.022465 2.062199 1.836379 3.362810 3.340152 2.076259 2.016564 3.693597 5.459719
73 5 0 1.360807 1.685305 1.900016 1.577803 3.297795 3.680361 2.281833 3.332835 4.030301 9.795986
81 7 0 1.304436 2.179175 1.782400 1.608048 2.020175 4.096777 2.475326 8.749612 6.273574 9.480236
82 7 0 1.067437 1.039585 1.476640 1.552524 2.101084 2.150608 1.597752 1.903135 1.955584 2.804002
83 7 0 1.258248 0.825581 2.740358 1.550433 2.840004 1.720206 2.356930 1.745917 2.532613 1.939425
88 9 0 2.371321 6.687678 16.218371 33.414020 5.949457 33.002049 4.572615 4.136561 7.134474 2.291361
89 9 2 nan nan nan nan nan nan nan nan nan nan
90 9 0 6.638531 3.840598 2.511735 2.157798 11.673120 9.367622 28.328276 12.321645 52.173978 24.807477
91 9 1 2.524448 5.708910 11.999675 11.963417 5.177036 12.388946 5.715056 3.707860 7.975352 2.616395
92 10 0 1.604069 1.354484 1.614728 1.866181 1.853277 1.804818 1.828096 1.693752 1.854306 1.846372
93 10 0 3.959251 1.521000 13.632251 1.524586 8.242666 1.720950 4.226861 1.746672 2.664636 1.919658
94 10 0 1.172330 1.132919 1.579426 2.143838 2.856646 2.036124 2.410804 2.691172 2.922042 8.574590
98 7 2 2.912935 13.846976 1.620983 45.091654 6.682711 50.444045 11.138148 46.879776 65.632039 70.527689
99 7 2 1.135730 0.892754 1.719063 1.778883 2.174776 2.014534 1.784116 1.856493 4.200714 2.859937
100 7 1 1.208650 23.501007 2.481484 89.576626 2.602059 76.651482 2.133768 69.873360 2.679559 100.182715
105 9 1 4.134205 3.254174 15.341008 7.025380 4.900230 11.242445 4.857002 2.271330 11.638248 2.761014
106 9 3 1.514074 1.471797 1.741073 1.678089 3.585712 3.443410 2.844434 4.385960 3.643226 2.634670
107 9 0 3.390114 2.789836 29.214678 43.066800 5.536279 3.287933 5.201794 6.334633 7.105401 6.921385
108 9 1 1.391918 0.901610 1.692094 1.568803 2.444684 1.932427 1.673479 1.624360 2.257613 1.786951
109 10 1 2.007052 1.763185 2.145164 1.725319 3.568566 4.216450 1.779857 1.812438 2.563390 2.227944
110 10 1 1.267139 1.741601 2.354478 1.131373 3.068274 2.418112 2.090791 2.038891 2.230320 1.822269
111 10 1 1.198588 1.138843 1.934022 1.620649 2.957702 2.241873 2.015306 1.644110 1.932630 1.633511
112 10 2 1.548777 1.191526 8.401063 2.607656 2.375483 2.830621 1.693426 2.147269 1.763966 2.090211
116 7 2 1.120631 0.777884 2.327522 1.623424 2.560956 1.919863 2.090531 1.526683 4.202109 2.195297
117 7 3 1.275090 1.351158 1.798730 1.958467 2.198926 1.937071 2.060532 3.136212 2.813786 5.167291
118 7 3 1.067421 1.083954 1.566782 2.079673 2.276803 2.013766 1.763714 2.352772 2.223227 4.156891
119 7 1 1.085400 0.815673 1.833696 1.623438 2.212916 2.027830 1.870913 1.853647 3.910590 2.204282
124 9 2 nan nan nan nan nan nan nan nan nan nan
125 9 3 3.402119 1.728594 39.448588 13.072811 12.210080 3.502608 2.655515 1.992725 5.058246 2.170609
126 9 3 2.756354 2.449936 36.571218 8.983305 5.604554 11.192405 3.905419 4.041994 5.655500 4.642449
127 10 2 2.455077 2.487081 3.461282 2.321877 6.068129 8.148263 6.779536 6.480052 8.174693 9.062797
128 10 2 0.887471 0.976970 1.662786 1.972157 2.150432 2.146815 1.839703 1.893161 2.241179 2.919861
129 10 3 1.159522 0.896742 1.716649 1.401299 2.296873 1.749576 1.816868 1.479383 1.825845 1.925725
130 10 3 1.265929 1.444221 2.642635 13.137507 1.967913 1.572769 2.357413 1.720414 3.214700 1.895656
135 12 2 1.118857 1.130959 1.648257 1.540174 1.988131 2.663121 1.959419 2.163122 1.908500 2.304264
136 12 2 1.399026 1.009401 3.922163 1.468199 2.875917 1.620824 6.115776 2.660104 4.219874 3.852006
137 7 3 2.395789 1.119346 12.244296 5.117024 5.810086 1.833589 3.663233 1.721442 4.768186 1.815655
138 7 1 1.294936 1.603959 1.685538 3.905343 2.876026 2.099830 1.993302 2.137226 8.207706 2.344742
140 13 2 1.270574 1.017421 1.698544 2.049908 3.177185 2.509153 2.044896 2.672384 1.953033 4.974911
141 13 2 6.156955 0.989011 19.850250 1.351105 25.356734 2.137807 14.067313 1.990300 11.149243 2.176273
142 13 2 3.899108 1.434263 6.602706 3.706350 3.971930 2.719853 2.651813 1.501759 3.549712 2.464548
143 14 2 1.239319 0.816440 2.459743 1.878973 2.901134 1.891343 2.619955 2.393676 2.515315 2.806513
144 14 3 1.604853 1.192589 1.591649 1.661423 3.454291 2.782447 1.860724 1.960975 2.768711 3.036820
145 14 3 2.795205 5.509954 16.463367 18.912816 15.103980 18.046905 2.782866 1.835904 2.596736 2.025775
150 15 1 2.492238 2.533862 15.271667 38.868596 4.683863 6.154456 3.191999 2.413244 2.776247 1.956657
155 12 2 3.025468 2.706402 8.376688 19.335070 4.175527 4.345095 3.143056 3.899803 3.879217 2.113429
156 12 3 1.481228 1.415642 1.678644 1.720501 3.928414 2.750655 2.514098 2.404723 3.701077 4.851161
157 12 3 1.478504 1.091314 1.456797 1.630875 2.473961 2.001613 1.713288 1.723640 1.707570 2.023895
158 12 3 1.289170 1.122808 2.168217 1.577430 2.944173 2.333673 2.485385 2.962931 3.896057 7.933495
160 13 3 1.997328 4.058812 6.408711 24.409788 3.296965 4.562325 2.684561 4.306772 3.009383 9.363168
161 13 0 1.660997 1.211503 2.373686 1.800356 5.287980 2.646430 3.191373 2.538477 4.311358 5.249620
162 13 0 1.069423 0.909768 1.729806 1.437316 2.480105 1.898265 1.795626 1.631027 1.711186 1.677216
163 14 2 1.202288 0.998267 1.694005 1.984663 1.976763 2.006045 2.485741 2.267958 2.462596 12.105242
164 14 2 1.164355 0.935672 1.849545 1.695868 2.495177 1.973484 2.176841 3.786370 2.539429 3.662102
165 14 0 5.810434 2.646264 6.474714 4.694532 22.345660 6.818703 4.306185 3.347162 8.086007 3.893959
166 14 0 1.265198 2.080625 1.827371 1.210907 3.439073 4.268739 2.840814 5.062932 2.528854 4.290793
167 15 1 2.120827 1.864229 2.085830 1.138201 3.097750 2.401429 3.661362 2.258238 2.755713 1.884898
168 15 2 1.700270 1.065144 3.557851 1.861856 3.185989 1.782548 4.064732 3.513640 5.959726 4.114993
169 15 2 1.017032 2.472663 2.174257 10.407943 2.684018 1.739832 2.659561 2.573028 3.266336 3.985114
170 15 2 1.256561 7.413300 2.675932 36.706447 3.049285 2.127498 2.706014 2.563164 3.147423 2.815462
176 12 0 0.971483 0.967715 1.643096 1.739512 1.975958 2.111769 1.811889 1.893096 1.830439 2.192594
177 12 0 1.024264 0.929715 1.654696 1.700465 2.279460 2.020300 1.864405 1.706169 2.267209 2.559655
178 12 0 1.183784 0.951390 1.635264 1.363060 2.610430 1.996775 1.975942 1.932641 2.030766 1.924333
179 12 1 1.575696 0.892406 1.555590 1.917344 3.155749 2.181525 6.076368 3.054274 3.757054 3.121167
180 13 3 1.862962 10.684909 11.755644 0.914231 2.880985 42.814406 2.293438 9.711020 2.260291 21.237213
181 13 3 1.274647 0.938645 1.568269 1.369020 2.813973 2.350078 1.838887 1.689834 1.698070 1.698833
182 13 0 3.783915 1.239015 6.702903 1.734887 11.503426 3.812436 1.924372 1.513880 2.046190 2.361070
183 13 1 1.266962 1.082248 1.761363 2.088145 3.503209 2.636214 2.007666 2.105102 2.296680 3.051721
184 14 0 1.144525 0.913023 1.563232 1.720521 2.164862 2.011452 1.619204 1.946919 1.857822 2.173715
185 14 1 1.421835 1.374135 1.802721 1.741605 2.246072 2.847165 2.421059 3.492539 4.122819 2.736286
186 14 1 1.329791 0.943796 1.893125 1.498072 2.438029 1.910938 2.180513 1.708901 3.161259 3.325996
187 14 1 1.287245 1.414605 1.724205 3.076572 2.478436 3.006300 2.995641 4.475140 6.113918 12.420795
189 15 3 0.955466 0.972179 1.771875 1.567091 2.311102 1.974769 1.715971 1.959067 1.826947 2.113449
190 15 3 2.705813 1.969826 11.545175 0.915413 3.676536 3.404837 3.168481 3.043899 3.318368 3.671014
191 15 3 1.196069 1.058391 1.624504 1.585089 2.755205 2.241246 2.133015 2.105420 2.076449 2.185682
220 18 2 nan nan nan nan nan nan nan nan nan nan
221 18 2 nan nan nan nan nan nan nan nan nan nan
222 18 2 nan nan nan nan nan nan nan nan nan nan
320 3 2 4.190385 3.952921 10.267511 20.390054 4.679885 3.069209 5.577271 6.029669 4.955454 7.235845
321 2 3 1.058995 0.983586 1.806267 2.000411 2.297533 2.035113 1.833569 1.850132 1.800203 1.864311
323 2 3 0.985948 1.480651 2.886853 4.208254 1.908450 4.266748 1.692197 1.880680 1.749654 2.043279
324 4 3 1.017065 0.994017 2.247567 1.942186 2.354403 2.153337 2.159940 1.755201 2.506987 1.879850
329 12 1 0.930821 0.920053 3.324977 16.944305 2.201863 2.572257 2.024017 1.637330 2.026145 2.015042
333 12 1 0.842846 1.490236 1.555370 1.735334 1.963337 6.497352 1.501721 1.590708 1.613546 1.786673
In [16]:
csv_file = os.path.join(nb_outdir, 'ds_noise_ratio_{}.csv'.format(JD))
df.to_csv(csv_file, index=False)

Delay spectrum and autocorrelation plot per baseline per polarization for a given frequency (sub-)band¶

Left panel: time averaged delay spectum of autocorrelation in dB with 10*log10($|\tilde{V}|$) (blue) and noise from diff file representing the expected variance of the delay spectrum (red). The time-averaging is performed by 1. binning three time integrations of each even and odd visibility, 2. Fouier transform the binned even and odd visibilities, and 3. multiply the even and odd delay spectra at alternating time bin and average the squared delay spectrum along the time axis. This helps to reduce the noise bias. Both autocorrelation delay spectrum and diff delay spectrum are averaged in the same way

Right panel: time averaged autocorrelations w/o (orange) and w/ xRFI flags (blue). Flagged one is shifted from the unflagged one for clarity

In [17]:
utils.interactive_plots_dspec(bls, uvd, uvd_diff, JD)
In [ ]: