In [1]:
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData, UVFlag
import pyuvdata
import os
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, SkyCoord, AltAz, Angle
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import h5py
import importlib
from scipy import stats
import scipy
import pandas as pd
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
In [2]:
#get data location
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
nb_outdir = os.environ['NB_OUTDIR']
utc = Time(JD, format='jd').datetime
print(f'JD = {JD}')
print(f'Date = {utc.month}-{utc.day}-{utc.year}')
print(f'data_path = "{data_path}"')

JD = 2459842
Date = 9-19-2022
data_path = "/mnt/sn1/2459842"
In [3]:
# Load in data
HHfiles, difffiles, uvdx, uvdy = utils.load_data_ds(data_path,JD)
    
uvd = UVData()
uvd_diff = UVData()
uvd.read(HHfiles[0])
use_ants = [int(ant) for ant in uvd.get_ants()]
bls = [(ant, ant) for ant in use_ants]
uvd.read(HHfiles[::10], skip_bad_files=True, bls=bls)
uvd_diff.read(difffiles[::10], skip_bad_files=True, bls=bls)
lsts = uvd.lst_array

flagfile = glob.glob(os.path.join(HHfiles[0].split('zen')[0],'zen.{}*total_stage_1_threshold_flags.h5'.format(JD)))
uvf = UVFlag()
uvf.read(flagfile)
bls = [(ant, ant) for ant in uvd.get_ants()]
times_uvf = np.unique(uvf.time_array)
times_uvd = np.unique(uvd.time_array)
idx_times = [np.where(time_uvd == times_uvf)[0][0] for time_uvd in times_uvd]
uvd.flag_array[:,0,:,:] = np.repeat(uvf.flag_array[idx_times], len(bls), axis=0)

372 sum files found between JDs 2459842.35234 and 2459842.43533
372 diff files found between JDs 2459842.35234 and 2459842.43533

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [4]:
utils.plot_lst_coverage(uvd)

Delay spectrum¶

Delay spectrum CLEANed using uvtools.dspec.high_pass_fourier_filter with 7th-order Blackman-Harris window function. Odd/even visibilities are used to remove noise bias.

In [5]:
_data_cleaned_sq, d_even, d_odd = utils.clean_ds(bls, uvd, uvd_diff, N_threads=14)

Waterfalls of delay spectra for autocorrelation¶

These plots show autocorrelation delay spectrum waterfalls of each antenna that is active and whose status qualifies for this notebook. For nn/ee polarization, the autocorrelation delay spectrum is normalized by the max of the delay spectrum. For ne polarization, the autocorrelation delay spectrum is normalized by max(sqrt(|nn| * |ee|)). ne and en are the same for autocorrelations, and thus only ne is shown here. The delay spectra are presented in dB with 10*log10($|\tilde{V}|$).

For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

nn polarization¶

In [6]:
utils.plot_wfds(uvd, _data_cleaned_sq, 0)

ee polarization¶

In [7]:
utils.plot_wfds(uvd, _data_cleaned_sq, 1)

ne polarization¶

In [8]:
utils.plot_wfds(uvd, _data_cleaned_sq, 2)

Analysis of 2700ns features in delay spectra¶

This plot shows the relative amplitude at 2700 ns feature. The relative amplitude is calculated in dB with the mean amplitude at 2500-3000 ns compared to the mean amplitude at 2000-2500 ns. Larger values of relative feature amplitude indicate higher probability of detecting the peak at 2700 ns. Antennas in the same node are grouped by the shaded region.

In [9]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='nn')
In [10]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='ee')

This plot shows a matrix representing the 2700ns feature correlation of each baseline. The color bar indicates the amplitude of 2700ns (mean amplitude of 2500-3000ns delay spectrum) in dB which is the same as that in the above plot.

In [11]:
# utils.CorrMatrix_2700ns(uvd, HHfiles, difffiles, flagfile, JD, N_threads=14)

Analysis of noise floor in delay spectra¶

This plot shows the ratio of delay spectrum to noise floor (averaged over 1000-4000ns). Near 1 indicates the delay spectrum reaches to the noise floor, which may mean good.

In [12]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='nn')
In [13]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='ee')
In [14]:
# get the ratio of delay spectum to noise for different freqeuncy bands and pols
ds_noise_ratio = utils.get_ds_noise_ratio(uvd, uvd_diff, bls)

nodes, antDict, inclNodes = utils.generate_nodeDict(uvd)
ants = uvd.get_ants()
# build dataframe
to_show = {'Ant': ants, 'Node': [int(antDict[ant]['node']) for ant in ants], 'Snap': [int(antDict[ant]['snapLocs'][0]) for ant in ants]}
df = pd.DataFrame(to_show)
 
cols_ratio = []
for key in ds_noise_ratio.keys():
    if(key[0] == 40):
        col = r'Full '
    else:
        col = r'{}-{} MHz '.format(key[0], key[1])
    col += key[2]
    df[col] = ds_noise_ratio[key]
    cols_ratio.append(col)
    

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

ratio_cut = 3
# style dataframe
table = df.style.hide_index() \
          .applymap(lambda val: 'color: red' if val > ratio_cut else '', subset=cols_ratio) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

This table shows the ratio of the delay spectrum to the noise level from diff files for different frequency bands and pols. The ratio > 3 is colored in red

In [15]:
HTML(table.render())
Out[15]:
Ant Node Snap Full nn Full ee 50-85 MHz nn 50-85 MHz ee 120-155 MHz nn 120-155 MHz ee 155-190 MHz nn 155-190 MHz ee 190-225 MHz nn 190-225 MHz ee
3 1 2 1.710402 1.523604 1.964358 2.572027 2.242801 2.481535 2.259152 2.223403 5.404569 6.542106
4 1 2 1.317400 1.549510 15.246282 23.131970 2.278808 1.971845 2.182215 1.987526 3.059785 2.862397
5 1 2 1.579501 1.081339 2.574744 1.822341 2.179104 2.102280 2.150150 1.927832 3.397636 2.179558
15 1 3 1.640659 1.743866 3.137999 2.101472 2.332535 2.254504 2.107368 1.922225 2.172789 2.165223
16 1 3 2.112506 0.976107 3.386910 3.148853 2.411706 2.219571 2.552945 1.976564 2.701126 2.017102
17 1 3 2.000823 1.345821 2.587863 2.059439 2.373272 1.992767 2.454578 2.060183 2.161598 1.953675
18 1 0 119.933242 37.801101 2.066335 2.085176 563.933137 177.741861 1.966861 1.916082 2.455934 2.366250
22 6 0 1.465170 6.129356 2.049676 4.702648 1.916759 4.259973 1.968941 2.247207 1.936565 3.772325
27 1 0 4.505678 5.973273 10.922224 22.266937 18.477261 7.984186 5.389238 2.744657 5.217687 1.985475
28 1 0 82.452459 2.976126 2.371057 2.065561 406.307161 3.109502 2.675570 2.565894 4.000544 3.521559
29 1 1 1.692979 1.124497 4.723780 2.730218 2.565746 2.219931 2.425623 2.045773 2.884108 1.939753
30 1 1 1.978756 1.418274 3.812077 2.220549 2.812500 2.230464 2.500486 1.996487 2.461668 3.131205
34 6 0 1.349591 3.083991 2.159361 11.822986 2.070983 4.539903 2.284652 2.412332 2.056325 2.395934
35 6 0 1.574220 1.609487 2.006962 2.656985 1.869324 2.208968 2.052125 2.081810 1.992854 5.528206
47 6 1 1.821216 4.801299 2.254296 41.673280 2.512519 12.454272 2.505565 2.328986 2.480498 2.364939
48 6 2 0.972050 1.253823 1.909809 2.176074 2.172455 1.985997 2.395759 2.041695 1.973338 2.054547
49 6 2 1.329502 1.256539 1.700374 2.459153 1.638983 1.901955 1.855594 2.189009 1.618125 2.205392
61 6 1 2.543166 1.546742 2.940826 2.619813 2.690408 2.147507 2.758568 2.357842 2.257465 2.113759
62 6 2 1.096410 1.449175 2.003910 2.043722 1.889416 2.058135 2.066187 2.206251 1.997293 2.072153
63 6 3 4.735739 1.054983 19.106531 1.913696 3.720997 1.920769 6.518101 1.880359 5.104171 2.250147
64 6 3 1.029234 1.363502 2.105200 2.010452 1.986684 1.894903 2.190687 2.155963 2.084015 2.018072
77 6 1 4.624517 3.936011 4.044163 3.861572 4.699425 4.238211 2.222428 2.045028 2.952431 2.426340
78 6 3 1.219302 4.848672 2.574462 5.045731 2.307783 6.326090 2.465435 2.546505 2.208439 2.287342
84 8 3 6.562939 1.761656 29.048299 8.955914 9.622639 2.443738 9.245809 2.048817 5.992963 2.899435
85 8 0 1.477067 1.137447 6.164325 2.744630 2.885097 2.063311 2.248789 2.125255 2.907825 2.521048
86 8 0 2.507768 1.436916 4.569143 2.792397 2.663606 2.344348 2.999866 2.889876 3.175311 4.315400
87 8 2 2.403609 4.648858 3.422565 5.081712 2.137651 4.174261 2.397059 2.898766 3.086261 3.906921
88 9 0 2.330874 4.053896 5.427360 10.684773 1.998235 8.670056 1.936658 7.842246 2.541610 7.118786
89 9 2 1.277251 2.415522 2.190034 3.361435 2.001553 2.144857 1.925516 2.139571 2.449847 2.518199
90 9 0 1.626667 1.197923 3.083832 3.191226 2.713415 1.991396 2.085496 1.761826 2.707638 2.584762
91 9 1 2.827301 2.562162 2.512893 2.209670 7.024662 6.590064 2.197837 2.222654 2.652768 3.400451
92 10 0 6.206586 3.901890 1.983898 2.424320 2.838633 2.744898 5.330770 2.679517 6.590746 2.616102
93 10 0 1.279879 1.620802 2.197401 6.072234 2.085142 2.614161 2.182876 1.684248 2.183154 1.804899
94 10 0 1.771008 1.183533 2.876319 3.673287 2.187786 2.625626 2.321352 2.294975 2.566856 6.387998
101 8 2 2.457819 1.733775 2.501659 2.620051 2.197260 2.333321 1.986952 2.079554 2.151210 3.099057
102 8 0 2.490750 1.382920 2.280185 2.167182 13.541538 6.271227 2.605379 2.184742 2.493508 2.088303
103 8 1 2.330443 1.683733 4.317852 3.054512 2.064135 2.653140 2.089966 2.313764 2.372709 2.363308
104 8 1 6.813174 1.784373 1.944598 2.424493 4.447722 2.119713 5.843358 1.890992 6.137929 2.387265
105 9 1 2.422421 1.566852 2.555652 2.454429 2.150964 2.160100 2.102247 1.951846 2.772458 3.315155
106 9 3 1.736004 1.314318 2.594854 3.594577 1.856657 2.536607 2.057106 2.155378 2.315463 2.796541
107 9 0 2.226792 1.915240 3.487940 5.554739 3.028343 2.269392 5.706524 2.705638 77.560436 2.602434
108 9 1 1.849290 2.136848 2.580512 8.463024 2.165164 3.667398 2.229729 2.261690 3.354648 2.215832
109 10 1 3.063964 1.202383 8.938496 2.411096 4.044879 2.272631 3.397879 2.118341 3.034733 2.771408
110 10 1 5.531119 3.089801 4.876339 1.617630 5.363317 3.506528 3.403907 2.688889 5.233581 3.148397
111 10 1 2.971559 1.017004 8.131944 2.499107 4.420263 2.291627 3.207410 1.868068 3.229322 1.895888
112 10 2 1.429618 1.069678 2.460267 1.848852 2.260023 1.852056 2.209086 1.775756 1.941377 1.788565
120 8 1 6.230878 12.555468 15.180392 69.062961 12.412444 34.677697 12.587449 4.806869 33.685418 10.766929
121 8 3 2.122087 2.213903 3.168521 2.764950 2.434836 3.075795 2.285576 2.215131 3.474784 13.850784
122 8 2 2.403952 1.589850 3.657621 3.163779 2.115540 1.815868 2.274800 1.772736 3.170243 2.509703
123 8 3 2.058467 1.587249 3.027893 3.452363 2.175397 2.266211 1.906355 1.861291 3.150161 2.641886
124 9 2 5.595129 5.255088 9.075443 13.968820 11.809202 7.564127 5.349464 15.231902 13.603529 10.487320
125 9 3 2.115349 1.246971 2.607468 2.430965 2.285967 2.222164 2.625672 2.065251 5.150657 2.630344
126 9 3 2.432248 5.310022 3.110594 2.860710 2.295605 5.204293 1.865261 3.237736 2.490560 4.767315
127 10 2 1.880875 1.082280 4.075742 2.263170 3.309143 2.157312 2.634219 1.979483 2.725920 1.967988
128 10 2 2.151386 1.045294 2.569981 2.391722 2.298327 2.149628 2.388350 2.017878 1.945490 3.060917
129 10 3 1.417428 1.147095 2.693106 2.290645 2.049236 1.965027 1.967026 1.971133 2.003443 2.341766
130 10 3 1.993832 1.344035 4.297219 2.475947 3.369024 1.967338 2.946585 1.629643 2.505466 2.288293
135 12 2 2.047802 1.044245 2.869389 2.274904 2.445610 2.125456 2.239932 2.076549 3.499353 10.901730
136 12 2 2.899408 1.391056 7.716354 2.304058 3.364008 2.047908 2.745805 2.040329 2.479562 2.349346
140 13 2 3.703680 7.510421 12.143400 53.348961 5.254903 4.661455 3.653602 15.532197 3.110043 2.013017
141 13 2 12.424416 1.715209 57.167621 2.225280 43.740749 2.025622 35.406360 1.788762 36.649709 2.103614
142 13 2 4.494783 3.979673 9.953303 3.145177 8.315648 3.421343 4.794848 2.415860 5.712623 3.931447
143 14 2 1.411171 1.125860 2.338389 2.173809 1.818845 1.973633 1.850557 2.095398 2.040142 2.271250
144 14 3 2.006334 1.296905 3.854139 1.979637 2.674465 1.962797 2.549753 1.889036 2.382035 1.796324
145 14 3 6.480244 5.621796 33.512796 28.094421 18.801854 15.196963 5.081636 2.286628 4.897029 2.077342
147 15 0 1.701315 3.269695 2.457437 2.041681 2.507726 2.632171 2.442973 2.172594 2.374917 2.975464
148 15 0 1.414208 4.785725 2.432909 7.714333 2.517190 4.028873 2.464639 2.247514 2.260735 1.968749
149 15 0 3.510253 1.363528 19.682450 2.043665 4.996229 2.010520 3.991026 2.169439 4.033151 2.207532
150 15 1 9.234389 15.040785 35.023158 75.345037 18.262029 15.814270 5.545639 27.530430 3.313699 2.318520
155 12 2 2.896093 8.249950 10.586190 29.903709 3.480870 13.595714 3.691566 8.397684 4.525819 4.794416
156 12 3 1.687744 1.734702 3.114059 3.897657 1.982020 2.388555 2.553425 2.515889 5.544726 6.314429
157 12 3 1.780618 1.193278 2.028800 1.895939 1.939140 1.952467 1.870830 1.872161 2.317980 2.164892
158 12 3 2.614984 3.060210 7.979856 7.787837 4.806926 7.410429 3.729685 2.351978 3.464792 2.186285
160 13 3 4.608729 7.891268 15.575084 38.014521 16.698108 15.555844 3.905934 6.846487 4.103823 2.716623
161 13 0 5.792206 1.237703 4.603791 2.204609 7.565742 1.978248 3.481555 1.903002 3.481340 2.002443
162 13 0 2.450543 1.372018 2.534771 2.663839 2.464820 2.437991 2.318608 2.009772 1.913650 2.184030
163 14 2 1.575558 1.100855 3.693990 3.085489 1.974095 2.328972 2.040021 2.043744 2.108306 2.332668
164 14 2 1.650683 1.156867 2.704471 2.118756 2.169071 1.914023 2.169680 1.937844 2.002283 1.906646
165 14 0 1.554212 1.170889 2.917021 2.586421 2.423817 2.325217 2.332022 2.226296 2.217183 2.191852
166 14 0 4.387312 3.910359 1.860600 1.548349 3.656225 4.178498 3.046425 2.664027 2.863305 2.829908
167 15 1 7.939374 4.744061 5.579786 3.084500 6.949067 5.371960 2.323227 3.357029 2.762413 3.132938
168 15 2 1.247673 1.272810 2.353856 2.437113 2.108097 1.949923 2.532197 2.107606 2.344810 2.184890
169 15 2 1.780695 1.196241 2.521650 3.076480 2.125313 1.998510 2.321140 2.238297 2.507274 2.782250
170 15 2 2.258317 1.104456 2.677424 2.603663 2.256319 2.189919 2.268144 2.391724 3.008487 5.772953
179 12 1 1.807502 1.348387 2.843268 4.461462 2.376872 1.961089 2.186061 1.982855 2.215188 2.251933
180 13 3 3.437701 1.424425 18.291864 1.826423 6.120058 2.162559 4.408491 1.833615 4.126122 2.215369
181 13 3 5.340814 6.118061 1.652300 17.179539 5.749993 12.727927 2.459573 2.527490 2.728188 2.543522
182 13 0 6.422535 1.121470 20.000124 2.405098 21.103839 2.261972 4.634007 2.311527 4.524943 2.137914
183 13 1 1.680662 1.270510 2.516533 3.308266 2.390755 2.142028 2.301709 2.091603 2.081776 2.231436
184 14 0 1.648703 1.069076 2.413488 2.590196 2.185367 2.137854 2.380758 2.021911 2.371845 2.599472
185 14 1 1.397081 1.071136 2.382023 2.600290 2.079769 2.022144 2.246169 1.829822 2.025826 1.986374
186 14 1 1.608722 1.057445 2.284150 2.535144 2.191505 1.952302 2.304220 1.718258 1.884966 1.867908
187 14 1 2.076458 1.594492 3.218485 6.104572 2.616507 4.142170 2.421113 3.278668 2.294162 3.588298
189 15 3 1.152359 1.305846 2.997006 2.040557 3.014244 2.379000 2.882334 1.845019 2.690509 2.200713
190 15 3 9.883724 3.526626 18.113231 2.675964 8.726025 4.504047 4.252012 3.329739 5.052627 5.386766
191 15 3 2.489618 1.443237 3.886816 2.918419 2.298075 2.297583 2.341733 2.129444 2.146089 1.981237
325 9 2 1.521334 1.287359 2.169715 2.008399 2.172635 1.972096 1.893952 1.996237 2.231676 1.976663
329 12 1 2.749686 2.599271 13.262480 29.512020 2.291306 2.366517 2.127715 2.220660 2.460018 2.257488
333 12 1 2.356175 2.447934 2.980273 4.116622 2.816491 8.614491 2.672951 2.483506 2.199052 2.074121
In [16]:
csv_file = os.path.join(nb_outdir, 'ds_noise_ratio_{}.csv'.format(JD))
df.to_csv(csv_file, index=False)

Delay spectrum and autocorrelation plot per baseline per polarization for a given frequency (sub-)band¶

Left panel: time averaged delay spectum of autocorrelation in dB with 10*log10($|\tilde{V}|$) (blue) and noise from diff file representing the expected variance of the delay spectrum (red). The time-averaging is performed by 1. binning three time integrations of each even and odd visibility, 2. Fouier transform the binned even and odd visibilities, and 3. multiply the even and odd delay spectra at alternating time bin and average the squared delay spectrum along the time axis. This helps to reduce the noise bias. Both autocorrelation delay spectrum and diff delay spectrum are averaged in the same way

Right panel: time averaged autocorrelations w/o (orange) and w/ xRFI flags (blue). Flagged one is shifted from the unflagged one for clarity

In [17]:
utils.interactive_plots_dspec(bls, uvd, uvd_diff, JD)
In [ ]: