In [1]:
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData, UVFlag
import pyuvdata
import os
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, SkyCoord, AltAz, Angle
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import h5py
import importlib
from scipy import stats
import scipy
import pandas as pd
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
In [2]:
#get data location
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
nb_outdir = os.environ['NB_OUTDIR']
utc = Time(JD, format='jd').datetime
print(f'JD = {JD}')
print(f'Date = {utc.month}-{utc.day}-{utc.year}')
print(f'data_path = "{data_path}"')

JD = 2459840
Date = 9-17-2022
data_path = "/mnt/sn1/2459840"
In [3]:
# Load in data
HHfiles, difffiles, uvdx, uvdy = utils.load_data_ds(data_path,JD)
    
uvd = UVData()
uvd_diff = UVData()
uvd.read(HHfiles[0])
use_ants = [int(ant) for ant in uvd.get_ants()]
bls = [(ant, ant) for ant in use_ants]
uvd.read(HHfiles[::10], skip_bad_files=True, bls=bls)
uvd_diff.read(difffiles[::10], skip_bad_files=True, bls=bls)
lsts = uvd.lst_array

flagfile = glob.glob(os.path.join(HHfiles[0].split('zen')[0],'zen.{}*total_stage_1_threshold_flags.h5'.format(JD)))
uvf = UVFlag()
uvf.read(flagfile)
bls = [(ant, ant) for ant in uvd.get_ants()]
times_uvf = np.unique(uvf.time_array)
times_uvd = np.unique(uvd.time_array)
idx_times = [np.where(time_uvd == times_uvf)[0][0] for time_uvd in times_uvd]
uvd.flag_array[:,0,:,:] = np.repeat(uvf.flag_array[idx_times], len(bls), axis=0)

372 sum files found between JDs 2459840.25319 and 2459840.33618
372 diff files found between JDs 2459840.25319 and 2459840.33618

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [4]:
utils.plot_lst_coverage(uvd)

Delay spectrum¶

Delay spectrum CLEANed using uvtools.dspec.high_pass_fourier_filter with 7th-order Blackman-Harris window function. Odd/even visibilities are used to remove noise bias.

In [5]:
_data_cleaned_sq, d_even, d_odd = utils.clean_ds(bls, uvd, uvd_diff, N_threads=14)

Waterfalls of delay spectra for autocorrelation¶

These plots show autocorrelation delay spectrum waterfalls of each antenna that is active and whose status qualifies for this notebook. For nn/ee polarization, the autocorrelation delay spectrum is normalized by the max of the delay spectrum. For ne polarization, the autocorrelation delay spectrum is normalized by max(sqrt(|nn| * |ee|)). ne and en are the same for autocorrelations, and thus only ne is shown here. The delay spectra are presented in dB with 10*log10($|\tilde{V}|$).

For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

nn polarization¶

In [6]:
utils.plot_wfds(uvd, _data_cleaned_sq, 0)

ee polarization¶

In [7]:
utils.plot_wfds(uvd, _data_cleaned_sq, 1)

ne polarization¶

In [8]:
utils.plot_wfds(uvd, _data_cleaned_sq, 2)

Analysis of 2700ns features in delay spectra¶

This plot shows the relative amplitude at 2700 ns feature. The relative amplitude is calculated in dB with the mean amplitude at 2500-3000 ns compared to the mean amplitude at 2000-2500 ns. Larger values of relative feature amplitude indicate higher probability of detecting the peak at 2700 ns. Antennas in the same node are grouped by the shaded region.

In [9]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='nn')
In [10]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='ee')

This plot shows a matrix representing the 2700ns feature correlation of each baseline. The color bar indicates the amplitude of 2700ns (mean amplitude of 2500-3000ns delay spectrum) in dB which is the same as that in the above plot.

In [11]:
# utils.CorrMatrix_2700ns(uvd, HHfiles, difffiles, flagfile, JD, N_threads=14)

Analysis of noise floor in delay spectra¶

This plot shows the ratio of delay spectrum to noise floor (averaged over 1000-4000ns). Near 1 indicates the delay spectrum reaches to the noise floor, which may mean good.

In [12]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='nn')
In [13]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='ee')
In [14]:
# get the ratio of delay spectum to noise for different freqeuncy bands and pols
ds_noise_ratio = utils.get_ds_noise_ratio(uvd, uvd_diff, bls)

nodes, antDict, inclNodes = utils.generate_nodeDict(uvd)
ants = uvd.get_ants()
# build dataframe
to_show = {'Ant': ants, 'Node': [int(antDict[ant]['node']) for ant in ants], 'Snap': [int(antDict[ant]['snapLocs'][0]) for ant in ants]}
df = pd.DataFrame(to_show)
 
cols_ratio = []
for key in ds_noise_ratio.keys():
    if(key[0] == 40):
        col = r'Full '
    else:
        col = r'{}-{} MHz '.format(key[0], key[1])
    col += key[2]
    df[col] = ds_noise_ratio[key]
    cols_ratio.append(col)
    

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

ratio_cut = 3
# style dataframe
table = df.style.hide_index() \
          .applymap(lambda val: 'color: red' if val > ratio_cut else '', subset=cols_ratio) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

This table shows the ratio of the delay spectrum to the noise level from diff files for different frequency bands and pols. The ratio > 3 is colored in red

In [15]:
HTML(table.render())
Out[15]:
Ant Node Snap Full nn Full ee 50-85 MHz nn 50-85 MHz ee 120-155 MHz nn 120-155 MHz ee 155-190 MHz nn 155-190 MHz ee 190-225 MHz nn 190-225 MHz ee
3 1 2 1.800717 2.587341 2.724247 9.370793 2.919676 3.835546 2.563240 3.219935 2.816706 6.395523
4 1 2 1.515776 1.684174 8.157016 4.408883 2.543187 1.989281 2.934813 2.061062 3.392954 2.115572
5 1 2 1.983508 1.123982 3.386199 5.683200 2.177272 2.353698 2.996121 2.008157 3.810457 2.172565
15 1 3 2.576142 1.572400 10.267338 3.353776 2.040678 2.591289 2.996178 2.227448 3.185880 2.178571
16 1 3 1.846630 1.301930 3.802484 5.108856 2.540077 2.313227 2.757860 2.120477 3.215886 2.172338
17 1 3 2.371311 1.120634 10.312283 2.777233 2.832916 2.236032 2.838648 2.150473 3.261303 2.234520
18 1 0 2.369463 2.308385 13.176607 6.075743 2.349137 2.448652 2.642252 2.143969 3.706475 2.153451
27 1 0 7.567772 3.039042 7.030155 19.643942 5.292195 6.513751 3.571410 2.457513 5.042953 2.392574
28 1 0 3.851047 1.028556 9.573163 5.434724 7.522445 1.938543 3.354911 1.969512 5.672614 2.137772
29 1 1 1.454348 2.900567 10.038473 12.224657 2.138608 4.786590 2.473696 2.091707 4.102801 2.034040
30 1 1 2.018312 1.284906 4.422918 4.037225 3.311780 2.475600 3.246981 1.998414 3.671844 2.044273
36 3 3 2.377497 1.735812 12.485848 13.869572 2.424664 1.915457 3.888423 2.034065 6.057584 2.362478
37 3 1 2.766711 2.719856 2.415238 21.571799 3.154291 2.935464 4.200892 2.416120 6.410889 2.284116
38 3 1 4.402128 2.140139 6.055912 8.176034 5.004722 3.063248 4.720674 2.118043 5.336567 2.237117
40 4 1 1.544545 2.960857 33.659185 6.213884 2.377869 3.348513 2.782667 1.994840 4.371950 2.195933
41 4 3 4.884225 24.183503 11.551250 4.464325 6.709153 15.938946 7.041804 50.235560 15.439234 97.894183
45 5 0 1.414876 2.007694 20.037063 13.531400 2.684332 2.886373 2.658511 2.164868 4.279329 2.019478
46 5 0 1.872721 1.301735 8.873105 2.159259 2.825185 2.146968 2.780073 2.263688 3.409212 2.167717
50 3 3 2.756095 1.740428 26.814305 6.324375 3.641211 2.153438 5.189435 2.204660 5.416085 2.184618
51 3 2 2.288051 2.908720 8.782757 19.451601 3.795846 4.528509 4.190520 2.464537 7.349106 2.290394
52 3 3 2.387828 5.811731 15.190096 19.327741 4.054380 4.697753 3.853123 3.698535 5.301687 3.099446
53 3 2 4.594508 31.976712 27.279711 5.411815 3.760892 20.114031 14.508598 85.296576 25.943108 123.689860
55 4 2 3.156581 1.311173 5.695623 4.501606 3.092440 2.732788 3.082967 2.555400 3.247550 2.526139
56 4 1 2.274162 2.001122 6.348242 8.895071 2.720656 2.397674 2.622641 3.676492 3.357616 4.402962
57 4 3 1.749765 1.812650 2.016047 2.489368 2.015388 2.535634 3.081747 2.809200 3.324629 3.450162
65 3 0 2.665222 1.714415 18.680768 12.965716 5.103281 3.444451 3.941443 2.205582 6.390798 2.239445
66 3 0 2.423232 2.395775 37.813013 11.154440 3.119137 2.604762 4.209321 2.102429 7.347887 2.137286
67 3 0 4.624803 1.710843 26.305950 7.654168 5.647926 2.346425 4.936929 2.299853 6.716134 2.147693
68 3 1 2.164529 5.125449 5.973162 13.533806 3.244464 5.286395 4.005087 20.046527 6.275569 71.563434
69 4 1 1.841305 1.352360 4.734220 12.114305 3.091018 3.150461 3.052309 2.257840 3.953841 2.294430
70 4 2 1.608379 2.640835 5.923364 10.078306 2.168234 6.261189 2.330836 2.451980 2.876131 2.397438
71 4 2 1.404557 1.310566 7.677717 3.621492 2.228599 2.291069 2.458108 2.127206 3.519935 2.091949
73 5 0 1.658061 2.935440 9.163108 5.655844 3.483805 4.193051 2.752837 2.412836 3.443710 2.219961
81 7 0 3.876225 2.157386 11.005202 2.621369 4.775192 2.450176 3.304416 2.069177 3.946266 2.004859
82 7 0 1.874227 4.718686 2.543056 6.691582 2.126842 3.667647 3.053026 2.368491 4.776614 2.423506
83 7 0 1.974350 1.487609 5.877443 8.833585 2.635840 2.265808 3.515089 2.385510 4.247621 2.123433
84 8 3 3.735355 38.474957 35.070704 21.677473 5.746278 28.118083 6.258052 68.367185 6.090754 55.124254
85 8 0 nan nan nan nan nan nan nan nan nan nan
86 8 0 nan nan nan nan nan nan nan nan nan nan
87 8 2 4.764911 26.236672 13.011278 9.904994 5.185604 18.472494 8.299018 40.305375 5.620233 29.702408
88 9 0 3.023072 7.391689 22.482550 42.838181 3.870822 14.421374 3.721834 4.101733 8.227538 2.631601
89 9 2 2.059876 2.674490 9.903635 15.368883 2.627557 3.518951 3.081180 2.650332 4.130156 2.468250
90 9 0 2.145933 1.403355 7.381748 11.775166 2.751635 2.077296 2.471778 2.307965 3.534078 2.303079
91 9 1 3.139532 4.652695 15.974521 20.007737 5.207571 10.217832 4.911494 4.387391 10.870962 2.948658
92 10 0 2.268311 1.234599 5.580463 6.423673 2.381239 2.408690 2.619712 2.318845 3.603565 2.281602
93 10 0 1.951455 1.382259 9.726243 9.613515 2.943592 2.924622 2.966535 2.037909 3.751639 1.884833
94 10 0 1.468742 1.364003 5.984551 7.184973 2.079461 2.391948 2.766411 2.211958 4.797365 2.325570
98 7 2 1.697546 1.819257 9.170382 7.747492 2.191041 2.144775 3.385378 2.210037 3.622086 2.318816
99 7 2 2.108034 1.282922 6.472599 7.117484 3.572648 2.683947 3.766326 2.035453 4.284418 2.078253
100 7 1 1.940459 4.666089 9.959563 7.055002 3.800592 4.006184 3.451547 5.414192 4.595921 11.093565
101 8 2 2.849480 5.243456 16.058651 14.886692 4.619752 3.757111 4.737942 3.259403 6.982354 3.051559
102 8 0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
103 8 1 2.904313 2.231949 7.098234 13.879196 4.089155 2.688068 5.958862 2.435524 6.825907 2.404656
104 8 1 13.973258 2.507680 6.797661 4.359029 1.976786 2.498748 4.051240 1.975321 12.026053 2.437941
105 9 1 1.659805 2.014710 15.605054 7.123461 2.435968 3.093176 3.948385 2.310254 6.816034 2.267138
106 9 3 2.081769 1.498515 13.473167 8.088462 3.218771 3.440399 3.144683 2.277249 7.187493 2.387456
107 9 0 3.821048 12.834179 41.228782 15.793554 6.747898 7.635927 6.193849 26.955693 11.646241 37.261761
108 9 1 2.644480 1.150459 9.643577 6.401825 3.036881 2.362585 3.955023 2.229291 4.745776 2.174020
109 10 1 1.866263 1.285460 8.229325 3.825647 2.727486 2.378285 2.471403 2.262249 2.820637 2.144988
110 10 1 2.463907 3.232080 7.694771 5.329437 2.921958 3.653754 3.835780 2.855194 5.669077 2.268575
111 10 1 1.467700 1.074053 6.722378 5.501764 2.493073 2.083269 2.708637 2.155648 3.610191 2.024575
112 10 2 1.599749 1.150879 4.425146 2.129245 2.710599 2.342362 2.742528 2.357635 3.670577 2.535125
116 7 2 3.216823 2.599190 3.127229 7.020591 3.340646 2.394740 3.957285 2.296253 3.972839 2.410977
117 7 3 2.997915 2.443788 3.983910 9.623332 4.069326 2.588801 4.475575 2.074717 5.181966 2.329971
118 7 3 1.675921 1.068123 5.684092 8.806099 3.299776 1.980772 3.762081 2.189766 4.431988 2.343758
119 7 1 2.502355 2.534513 4.981133 10.102338 3.478698 2.367203 3.959383 2.073801 5.395651 2.341295
120 8 1 3.836098 2.179664 12.876938 70.802009 7.533769 3.488725 10.686809 3.348535 27.391256 3.335807
121 8 3 3.035420 3.140998 18.614157 22.282070 4.376898 8.520634 4.407963 2.298957 5.886559 2.083460
122 8 2 2.952658 2.288675 29.487635 22.351301 3.626511 4.036135 5.316566 1.993283 5.189651 2.507729
123 8 3 2.598769 3.352084 12.322311 20.447475 3.671013 3.433274 4.040161 4.008106 6.656857 4.207269
124 9 2 8.124138 7.371599 3.291933 11.180016 12.047389 7.337112 3.837592 2.243457 9.048455 3.033226
125 9 3 3.770351 1.856148 51.892997 13.805868 5.778118 2.614240 2.797466 2.110060 4.850288 2.317010
126 9 3 1.942164 4.522727 33.867464 9.226691 2.897964 4.747539 3.561421 2.636571 4.938481 2.784941
127 10 2 1.622187 1.151887 10.947000 6.544659 2.151018 2.117737 2.626667 2.089785 3.818183 2.112774
128 10 2 2.304429 1.231149 10.477982 3.087085 3.643962 2.453846 2.530681 1.927865 2.449193 2.165953
129 10 3 1.468468 1.459160 10.451011 10.005124 2.274794 3.412162 2.528203 1.952948 2.965858 2.152141
130 10 3 1.782518 1.022064 7.734055 10.118894 2.757617 1.802480 3.136957 2.010521 3.048668 1.841475
135 12 2 19.973179 16.237112 56.733196 33.032466 16.028661 14.300685 2.995315 12.299884 2.174116 42.939137
136 12 2 6.495381 13.471304 10.430479 36.868451 1.034131 3.520599 1.648306 6.184295 0.848266 2.286847
137 7 3 2.241693 3.174716 9.900719 29.006686 4.443207 3.465073 3.426291 2.275249 6.166770 2.356616
138 7 1 1.506270 1.095245 3.792987 3.100099 2.047768 2.211908 2.600485 2.156891 3.716256 2.270681
143 14 2 21.985785 9.766301 50.200474 20.608145 1.842677 5.474147 1.500035 1.654353 1.451828 1.376304
144 14 3 7.155556 16.876207 13.371037 30.678753 1.135522 11.864763 1.350760 9.049030 0.394735 15.444605
145 14 3 18.718749 10.077954 40.870793 35.449681 4.055550 4.419933 1.302900 1.105702 1.920188 1.089328
155 12 2 0.000000 8.241687 0.000000 19.714469 0.000000 2.957804 0.000000 3.825409 0.000000 2.500042
156 12 3 56.789525 371.707313 24.815210 256.817135 47.382005 719.014416 39.386984 313.306062 323.123285 1391.362433
157 12 3 7.341777 31.192802 14.402166 43.239900 1.072646 22.315255 0.902602 24.562043 4.993707 117.487227
158 12 3 24.654521 11.258453 36.133633 31.368561 1.572500 2.353148 1.165425 1.121960 1.270480 1.271507
163 14 2 27.189372 517.622817 42.025075 349.211666 7.898595 397.738834 21.239456 455.953881 15.369512 283.646274
164 14 2 13.190732 17.424562 30.085070 49.016936 1.860554 8.807480 1.243818 3.060786 1.250898 5.935490
165 14 0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
166 14 0 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000 0.000000
176 12 0 23.668389 228.825669 20.253842 108.875890 19.895270 497.362207 18.767114 411.608410 57.841524 184.488131
177 12 0 31.375589 10.830461 33.376737 25.141358 3.974841 5.507901 2.042245 5.185723 1.804428 1.225611
178 12 0 18.687649 9.542239 38.440407 25.200850 2.066308 2.273044 1.363572 1.319613 1.393387 0.920146
179 12 1 0.000000 170.261316 0.000000 26.194181 0.000000 114.253642 0.000000 316.185194 0.000000 284.042990
184 14 0 0.000000 18.057647 0.000000 42.856687 0.000000 1.589633 0.000000 1.173393 0.000000 1.174144
185 14 1 213.754354 273.294405 37.420360 22.265298 194.684792 185.887005 291.154973 697.941001 175.252459 500.144898
186 14 1 14.108353 8.124395 30.319472 11.084799 3.938519 4.362038 5.157874 5.011459 2.342198 2.169888
187 14 1 8.698664 24.403641 13.283325 58.877263 0.745420 4.145249 0.569124 1.563419 0.696821 1.547024
320 3 2 4.346979 4.338193 14.167186 39.109674 5.864840 4.422055 6.783663 8.567666 9.463829 10.244995
324 4 3 2.302211 1.087353 7.282759 7.071967 2.645163 2.112718 3.373398 2.019904 4.077247 2.168881
325 9 2 1.690604 2.557102 2.758836 5.836447 2.205840 2.597857 3.365516 5.105271 5.343784 5.894204
329 12 1 20.245472 16.055110 37.504557 36.011754 6.623987 6.677117 2.786331 4.471803 2.987819 8.003710
333 12 1 23.121465 19.228756 40.570259 47.104222 2.124582 4.132028 1.752484 2.561432 1.956725 3.774487
In [16]:
csv_file = os.path.join(nb_outdir, 'ds_noise_ratio_{}.csv'.format(JD))
df.to_csv(csv_file, index=False)

Delay spectrum and autocorrelation plot per baseline per polarization for a given frequency (sub-)band¶

Left panel: time averaged delay spectum of autocorrelation in dB with 10*log10($|\tilde{V}|$) (blue) and noise from diff file representing the expected variance of the delay spectrum (red). The time-averaging is performed by 1. binning three time integrations of each even and odd visibility, 2. Fouier transform the binned even and odd visibilities, and 3. multiply the even and odd delay spectra at alternating time bin and average the squared delay spectrum along the time axis. This helps to reduce the noise bias. Both autocorrelation delay spectrum and diff delay spectrum are averaged in the same way

Right panel: time averaged autocorrelations w/o (orange) and w/ xRFI flags (blue). Flagged one is shifted from the unflagged one for clarity

In [17]:
utils.interactive_plots_dspec(bls, uvd, uvd_diff, JD)
In [ ]: