In [1]:
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData, UVFlag
import pyuvdata
import os
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, SkyCoord, AltAz, Angle
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import h5py
import importlib
from scipy import stats
import scipy
import pandas as pd
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
In [2]:
#get data location
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
nb_outdir = os.environ['NB_OUTDIR']
utc = Time(JD, format='jd').datetime
print(f'JD = {JD}')
print(f'Date = {utc.month}-{utc.day}-{utc.year}')
print(f'data_path = "{data_path}"')

JD = 2459765
Date = 7-4-2022
data_path = "/mnt/sn1/2459765"
In [3]:
# Load in data
HHfiles, difffiles, uvdx, uvdy = utils.load_data_ds(data_path,JD)
    
uvd = UVData()
uvd_diff = UVData()
uvd.read(HHfiles[0])
use_ants = [int(ant) for ant in uvd.get_ants()]
bls = [(ant, ant) for ant in use_ants]
uvd.read(HHfiles[::10], skip_bad_files=True, bls=bls)
uvd_diff.read(difffiles[::10], skip_bad_files=True, bls=bls)
lsts = uvd.lst_array

flagfile = glob.glob(os.path.join(HHfiles[0].split('zen')[0],'zen.{}*total_stage_1_threshold_flags.h5'.format(JD)))
uvf = UVFlag()
uvf.read(flagfile)
bls = [(ant, ant) for ant in uvd.get_ants()]
times_uvf = np.unique(uvf.time_array)
times_uvd = np.unique(uvd.time_array)
idx_times = [np.where(time_uvd == times_uvf)[0][0] for time_uvd in times_uvd]
uvd.flag_array[:,0,:,:] = np.repeat(uvf.flag_array[idx_times], len(bls), axis=0)

372 sum files found between JDs 2459765.25319 and 2459765.33618
372 diff files found between JDs 2459765.25319 and 2459765.33618

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [4]:
utils.plot_lst_coverage(uvd)

Delay spectrum¶

Delay spectrum CLEANed using uvtools.dspec.high_pass_fourier_filter with 7th-order Blackman-Harris window function. Odd/even visibilities are used to remove noise bias.

In [5]:
_data_cleaned_sq, d_even, d_odd = utils.clean_ds(bls, uvd, uvd_diff, N_threads=14)

Waterfalls of delay spectra for autocorrelation¶

These plots show autocorrelation delay spectrum waterfalls of each antenna that is active and whose status qualifies for this notebook. For nn/ee polarization, the autocorrelation delay spectrum is normalized by the max of the delay spectrum. For ne polarization, the autocorrelation delay spectrum is normalized by max(sqrt(|nn| |ee|)). ne and en are the same for autocorrelations, and thus only ne is shown here. The delay spectra are presented in dB with 10log10($|\tilde{V}|$).

For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

nn polarization¶

In [6]:
utils.plot_wfds(uvd, _data_cleaned_sq, 0)

ee polarization¶

In [7]:
utils.plot_wfds(uvd, _data_cleaned_sq, 1)

ne polarization¶

In [8]:
utils.plot_wfds(uvd, _data_cleaned_sq, 2)

Analysis of 2700ns features in delay spectra¶

This plot shows the relative amplitude at 2700 ns feature. The relative amplitude is calculated in dB with the mean amplitude at 2500-3000 ns compared to the mean amplitude at 2000-2500 ns. Larger values of relative feature amplitude indicate higher probability of detecting the peak at 2700 ns. Antennas in the same node are grouped by the shaded region.

In [9]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='nn')
In [10]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='ee')

This plot shows a matrix representing the 2700ns feature correlation of each baseline. The color bar indicates the amplitude of 2700ns (mean amplitude of 2500-3000ns delay spectrum) in dB which is the same as that in the above plot.

In [11]:
# utils.CorrMatrix_2700ns(uvd, HHfiles, difffiles, flagfile, JD, N_threads=14)

Analysis of noise floor in delay spectra¶

This plot shows the ratio of delay spectrum to noise floor (averaged over 1000-4000ns). Near 1 indicates the delay spectrum reaches to the noise floor, which may mean good.

In [12]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='nn')
In [13]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='ee')
In [14]:
# get the ratio of delay spectum to noise for different freqeuncy bands and pols
ds_noise_ratio = utils.get_ds_noise_ratio(uvd, uvd_diff, bls)

nodes, antDict, inclNodes = utils.generate_nodeDict(uvd)
ants = uvd.get_ants()
# build dataframe
to_show = {'Ant': ants, 'Node': [int(antDict[ant]['node']) for ant in ants], 'Snap': [int(antDict[ant]['snapLocs'][0]) for ant in ants]}
df = pd.DataFrame(to_show)
 
cols_ratio = []
for key in ds_noise_ratio.keys():
    if(key[0] == 40):
        col = r'Full '
    else:
        col = r'{}-{} MHz '.format(key[0], key[1])
    col += key[2]
    df[col] = ds_noise_ratio[key]
    cols_ratio.append(col)
    

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

ratio_cut = 3
# style dataframe
table = df.style.hide_index() \
          .applymap(lambda val: 'color: red' if val > ratio_cut else '', subset=cols_ratio) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

This table shows the ratio of the delay spectrum to the noise level from diff files for different frequency bands and pols. The ratio > 3 is colored in red

In [15]:
HTML(table.render())
Out[15]:
Ant Node Snap Full nn Full ee 50-85 MHz nn 50-85 MHz ee 120-155 MHz nn 120-155 MHz ee 155-190 MHz nn 155-190 MHz ee 190-225 MHz nn 190-225 MHz ee
3 1 2 1.167750 0.906989 1.761575 1.762110 2.960117 1.881063 1.697759 1.861138 1.906432 2.985306
4 1 2 1.241887 1.005263 5.563951 4.206415 2.839629 2.002953 2.345481 2.358686 3.453880 3.215334
5 1 2 1.110427 0.973552 2.103540 1.746939 2.772977 1.936678 2.603527 2.309523 8.696793 10.744947
7 2 0 1.046760 0.844821 1.916006 1.729886 2.472851 1.940890 1.738805 1.850307 2.291162 2.421260
8 2 0 2.046385 12.713144 7.669638 50.759621 4.121146 39.349260 6.802637 17.163872 8.516503 63.901538
9 2 0 1.026017 0.933368 1.612232 1.688642 2.449379 2.077362 2.361218 2.414169 2.037925 2.685475
10 2 1 1.036673 1.037063 1.837098 2.656132 2.414842 1.647741 1.718666 2.545982 2.179678 8.218601
15 1 3 1.318767 1.182459 1.928727 1.678935 2.636614 2.232967 2.225406 2.174772 7.182543 7.357795
16 1 3 1.331957 1.051992 1.731070 1.508016 2.590761 2.417504 1.883703 2.054429 15.604431 14.685080
17 1 3 1.148561 0.992458 1.859089 1.757536 3.256750 2.266644 1.928555 1.897046 2.874616 3.344009
18 1 0 55.857690 8.344239 1.231600 1.498341 250.114228 32.202305 1.461303 1.744131 1.859725 1.963402
19 2 1 1.146520 0.977365 2.187824 1.850956 2.133632 2.298482 2.398420 2.127108 5.259771 8.646503
20 2 1 0.836924 0.898042 1.571865 1.603909 2.102821 1.765216 1.494455 1.684226 1.871442 2.460102
21 2 2 1.022094 0.801951 1.425527 1.581314 2.649819 1.873590 1.869501 2.019903 1.745903 2.663620
27 1 0 3.753070 3.185019 7.734579 11.303829 10.803582 7.712300 4.274102 2.154078 3.812934 1.983344
28 1 0 128.534469 11.272243 1.438643 44.296969 634.655391 36.507805 1.591789 15.861111 2.246771 55.990943
29 1 1 1.425086 1.392994 2.509083 1.903561 2.541094 2.385884 3.539032 2.818810 4.177131 5.729562
30 1 1 1.406539 1.043665 1.887491 1.621099 2.485428 1.746818 4.223172 4.992496 3.153900 8.472734
31 2 2 1.129226 1.306229 1.674441 1.644761 2.439170 2.364365 3.546106 9.403544 4.421262 15.067192
32 2 2 2.052526 2.258028 2.939390 2.066394 2.625334 2.621300 6.339415 8.794687 9.259975 12.157828
33 2 3 36.890387 0.875485 1.765253 1.516950 183.058344 1.688521 1.801152 1.707431 2.207361 2.234846
36 3 3 1.503705 1.240284 2.244767 1.607716 2.956443 2.026553 1.991697 1.686598 2.192096 2.694624
37 3 1 1.274545 0.937223 2.068039 1.565996 3.297465 2.138932 2.171804 2.029277 4.418528 4.104387
38 3 1 1.124423 1.015526 2.204012 1.550217 2.505075 1.944353 2.445623 2.072045 2.434532 2.978861
40 4 1 1.003808 1.029175 1.616690 1.770461 2.032240 2.231457 1.986641 2.517207 2.317864 4.236167
41 4 3 1.013817 1.038702 1.690137 1.763316 2.427226 2.522147 1.852342 1.760672 1.990140 2.157559
42 4 0 1.113518 0.895371 1.663234 1.601457 2.746542 2.122523 1.837722 1.888380 4.396077 9.139988
45 5 0 2.615452 1.935766 17.323897 12.964374 3.473798 4.863226 2.965002 2.009762 2.956955 1.975175
46 5 0 2.435596 1.693341 4.357814 1.658999 5.515424 3.489653 2.426093 2.215572 3.565852 5.397591
50 3 3 1.119089 0.893592 1.817148 1.447613 3.178945 1.976638 1.948712 1.821743 2.274806 1.947241
51 3 2 3.592947 0.874276 12.237658 2.924236 5.702991 1.879372 5.390176 2.036305 6.885734 3.847369
52 3 3 4.601223 1.314940 18.491841 2.208687 3.991447 2.121196 3.670481 2.364588 4.947455 4.391053
53 3 2 1.138387 1.036083 2.870762 3.020759 2.850961 2.948296 2.354676 2.871421 4.289463 5.941845
54 4 0 1.111462 0.884381 1.741228 1.783721 2.806460 2.072277 2.105658 1.694104 3.540801 3.618524
55 4 2 1.510601 0.835556 4.865866 1.303833 2.675090 1.827542 4.563418 1.652990 5.835077 1.694044
56 4 1 1.127203 1.042963 1.775194 1.750146 2.952039 2.229246 2.358447 2.062588 2.901553 2.954423
57 4 3 1.101984 1.000138 2.571482 2.027216 2.541918 2.203688 2.108579 1.603754 2.377366 1.816839
65 3 0 1.158308 0.829367 2.559490 1.735646 2.991113 1.840808 2.053318 1.810470 2.144969 1.971100
66 3 0 1.291904 0.909227 3.164026 1.936826 3.361219 2.144421 2.192698 1.809521 2.908119 3.605101
67 3 0 1.129723 0.871118 1.821801 1.649186 3.375100 2.082926 1.794991 1.769068 1.847407 2.143787
68 3 1 0.992280 0.805568 1.598938 1.787268 2.515084 1.953895 1.753703 2.000364 2.072616 2.329895
69 4 1 1.043132 0.825075 1.710549 1.547928 2.260203 1.849102 2.391590 2.184506 3.288602 2.862539
70 4 2 0.925113 1.936579 1.431806 1.839118 2.029672 4.988001 2.213514 5.540508 2.070182 3.045938
71 4 2 0.990736 0.967346 1.837498 1.971434 1.914148 1.879476 1.679896 1.652447 2.151105 1.614934
72 4 0 1.197987 1.002482 2.068206 1.702943 3.020678 2.961696 2.017623 1.980607 3.123268 4.019579
73 5 0 1.165052 1.232795 1.748302 1.581009 2.921858 2.453326 2.145069 2.001330 3.951562 5.488674
81 7 0 1.831446 1.288951 1.746086 1.624185 2.757117 2.437499 2.781257 1.973744 13.458779 5.384450
82 7 0 1.073998 1.035442 1.658698 2.165271 2.381382 2.245667 1.761774 1.937423 2.869971 3.336061
83 7 0 1.584960 1.158576 5.771353 4.114508 2.454028 3.303555 4.837757 2.365434 6.401995 6.040505
88 9 0 2.304711 6.372535 14.869640 33.653618 5.643609 22.246528 4.562162 4.669981 6.741803 2.778732
89 9 2 nan nan nan nan nan nan nan nan nan nan
90 9 0 9.243233 3.218215 4.616699 3.226887 15.835938 7.303610 31.608821 11.735316 35.557656 15.264208
91 9 1 3.549921 3.427097 12.514259 10.694636 4.899399 9.647073 6.152980 3.999502 7.673974 2.578657
92 10 0 1.957406 1.532238 1.784719 2.282937 1.757944 1.748541 1.845615 1.841432 2.018185 1.940969
93 10 0 4.118513 1.572496 9.848501 2.023727 5.579582 1.634894 4.114293 1.542197 3.253827 2.205630
94 10 0 1.041459 1.191507 1.481803 3.337039 2.729761 2.903932 2.134431 3.528246 2.680603 18.752376
98 7 2 nan nan nan nan nan nan nan nan nan nan
99 7 2 nan nan nan nan nan nan nan nan nan nan
100 7 1 1.096403 23.327421 2.338464 101.371228 2.393739 67.131953 2.092579 27.083749 2.908351 100.030527
105 9 1 2.425492 2.395758 15.820709 7.327296 3.393358 6.899568 5.028685 2.242642 10.731773 2.761980
106 9 3 1.401581 1.474518 1.703937 1.818536 3.356277 3.355095 2.554087 4.043373 3.706461 2.904164
107 9 0 2.777936 2.718400 24.992992 30.086585 4.565607 2.651289 2.886934 4.215922 5.393582 9.250638
108 9 1 1.270690 0.929491 2.048384 1.473219 2.380878 1.805760 1.906491 1.837981 3.187570 2.221172
109 10 1 2.803025 2.081622 2.143420 1.981599 6.429175 3.870254 1.990312 1.912937 2.581719 2.608211
110 10 1 1.565292 1.257176 5.259062 1.455587 2.712556 2.072125 4.452954 2.137945 5.025276 2.014222
111 10 1 1.246068 1.046935 2.320061 1.867846 2.967334 2.176244 1.924271 1.865784 1.789399 1.869659
112 10 2 0.952965 0.957854 1.888001 1.972570 2.066353 2.343938 1.763485 2.187922 2.138756 2.391169
116 7 2 nan nan nan nan nan nan nan nan nan nan
117 7 3 nan nan nan nan nan nan nan nan nan nan
118 7 3 nan nan nan nan nan nan nan nan nan nan
119 7 1 0.993139 0.797649 2.017144 1.667280 2.201773 1.779426 1.950027 1.725066 3.757241 2.451479
124 9 2 nan nan nan nan nan nan nan nan nan nan
125 9 3 3.874725 2.246698 39.683359 12.688437 12.034920 5.489216 3.128976 2.175871 5.230378 2.300062
126 9 3 3.078559 2.709469 34.910574 9.241135 7.524593 10.210522 4.848700 4.432231 5.581626 3.990307
127 10 2 1.923202 1.780191 3.093143 1.982422 4.580370 5.621910 5.297801 5.316132 6.394811 7.403794
128 10 2 0.886405 1.014347 1.767521 2.494016 2.250741 2.394308 1.955310 2.126401 3.071956 7.518258
129 10 3 1.168824 0.823480 1.670273 1.633854 2.777423 2.009566 1.764059 1.877071 1.851953 2.553042
130 10 3 1.289098 1.403956 3.014329 13.855228 1.974484 1.595340 2.003393 1.760621 3.166742 1.733355
135 12 2 1.000920 0.869719 1.860090 1.727127 2.389154 2.039311 1.770722 1.920673 2.056574 2.105702
136 12 2 1.310108 0.987144 3.669852 1.577499 2.631336 1.791963 6.433624 2.927922 2.908050 4.955976
137 7 3 nan nan nan nan nan nan nan nan nan nan
138 7 1 1.253789 4.062198 1.606467 3.745297 2.800496 3.316818 1.778964 2.025031 9.587736 2.391312
140 13 2 1.299919 0.939906 1.946338 1.589311 2.910607 2.006362 2.032865 1.957031 2.409163 2.501699
141 13 2 6.687901 1.085098 17.341052 1.701903 23.729747 2.144765 18.002103 1.959026 13.777128 2.582415
142 13 2 3.319099 1.391347 5.601599 3.153402 3.518741 2.564878 2.370768 1.337726 3.516306 2.383174
143 14 2 1.208766 0.857722 2.112239 1.547038 2.888422 2.043631 2.124949 1.766144 3.459526 5.493035
144 14 3 1.384419 1.037414 1.477063 1.570725 2.862168 2.409532 1.804464 2.101027 1.979733 2.395473
145 14 3 4.897731 4.341502 15.947867 19.405214 9.902415 13.095948 3.428515 1.848991 2.658216 1.844566
150 15 1 3.371976 11.436010 17.369884 50.416287 3.914822 11.313766 4.410170 19.839279 4.521532 33.875752
155 12 2 2.599776 3.360708 7.610886 19.921874 3.691146 3.972405 3.378183 8.172699 3.836353 2.171114
156 12 3 1.289267 1.191815 1.961480 2.747189 3.187227 2.154582 1.738800 1.713157 2.115840 1.892664
157 12 3 1.335532 1.079545 1.726961 1.752523 2.965995 2.194492 1.957738 1.947966 1.959957 2.270267
158 12 3 1.387149 1.096536 2.697133 1.530793 3.798733 2.209905 2.264667 2.867367 4.133583 6.336578
160 13 3 2.004636 3.287092 6.484335 21.907777 3.371026 4.077131 2.568927 4.248533 3.049823 8.079053
161 13 0 2.069579 1.186377 1.279820 1.487992 3.955036 2.547413 3.276533 2.131931 6.817843 5.899620
162 13 0 1.045013 0.894526 1.698166 1.940442 2.295023 2.253294 1.890293 1.877572 1.779122 2.337935
163 14 2 1.226755 1.001160 1.767057 1.831487 2.880071 1.840043 2.201030 1.827309 2.332013 6.224830
164 14 2 1.182460 1.080939 2.066502 1.640166 3.419229 2.308138 1.971623 3.387029 3.381376 3.154573
165 14 0 3.086583 1.986508 3.890870 2.226221 8.856822 5.425418 3.999109 4.782740 8.060205 5.832725
166 14 0 2.404836 2.686857 1.626976 1.705469 3.448229 4.707603 5.285754 5.963896 3.069185 4.828939
167 15 1 1.870903 6.934579 2.586232 37.399707 3.414818 3.784878 2.519554 3.828818 2.874465 3.176721
168 15 2 2.012056 1.271641 5.499876 3.015084 3.626262 2.023657 7.150958 6.228729 9.849543 7.344189
169 15 2 1.235546 5.202067 2.568532 24.148004 2.837647 3.497713 3.265234 5.711625 3.762528 7.163460
170 15 2 1.578119 9.280254 2.886535 43.822655 3.928823 4.012082 4.094229 6.159142 3.947049 5.890052
176 12 0 1.079672 0.990508 1.750120 2.038351 2.927312 2.445031 2.180781 2.274744 2.689767 3.163170
177 12 0 1.002704 0.913829 1.839473 1.580915 2.274917 1.902256 2.008654 1.542236 2.502743 2.564934
178 12 0 1.093417 1.017355 1.787784 1.624863 2.840869 2.101731 1.971504 2.255598 2.316362 2.205817
179 12 1 1.418618 0.826777 1.849564 1.686148 3.133837 1.875982 4.884553 2.356123 4.071167 3.058284
180 13 3 2.347458 14.336396 11.990857 11.193717 2.534701 39.533762 3.106433 19.654950 2.528059 70.417127
181 13 3 1.135321 0.962590 1.734174 1.418059 2.450145 2.125532 1.804511 1.626196 2.624653 1.695442
182 13 0 1.204403 1.623084 2.046257 2.117884 3.059510 4.700889 1.900705 2.409378 2.041871 3.323002
183 13 1 1.127198 1.025754 1.559329 1.735571 3.114208 2.374881 1.825805 1.714217 2.478364 3.118416
184 14 0 1.022231 0.974739 1.503833 1.774914 2.094213 1.875821 1.875429 2.061941 4.039393 6.829180
185 14 1 1.271222 1.399624 1.900433 2.855825 2.352575 2.459438 2.498886 3.719198 3.982158 3.820462
186 14 1 1.230949 1.045783 1.863475 2.241802 2.385038 2.118985 2.083492 2.132024 3.436511 4.625497
187 14 1 1.937450 1.386770 6.214152 2.892281 5.606361 3.049992 4.706975 4.997290 6.757537 18.730614
189 15 3 1.041851 1.137177 1.949159 1.851563 2.489980 2.348837 1.938838 2.365510 2.753968 3.779606
190 15 3 2.176375 2.754027 1.009595 1.339928 3.533639 4.228202 2.730045 3.193227 4.328723 5.045704
191 15 3 1.105122 1.038928 1.817653 1.635921 2.772016 2.644544 1.986122 2.019233 2.127638 2.337892
220 18 2 nan nan nan nan nan nan nan nan nan nan
221 18 2 nan nan nan nan nan nan nan nan nan nan
222 18 2 nan nan nan nan nan nan nan nan nan nan
320 3 2 4.513421 5.460108 9.554809 25.656953 4.223786 6.322478 6.830888 4.303307 6.703058 4.006927
321 2 3 1.158448 0.914103 1.857056 4.274133 2.144688 2.139183 1.941189 1.745146 1.969509 1.976684
323 2 3 0.924331 2.704545 3.272295 6.174302 2.333695 4.025211 1.819555 2.048286 1.880958 3.408281
324 4 3 0.949442 1.639211 3.934979 29.903275 2.012404 1.716627 1.920767 1.777076 1.888942 1.849917
329 12 1 0.925430 1.189653 1.639371 2.938573 2.166729 2.216385 1.688648 1.808841 2.002007 2.455448
333 12 1 0.871726 1.085609 1.840599 2.048393 2.662242 3.616096 1.818127 1.658899 1.978422 2.092537
In [16]:
csv_file = os.path.join(nb_outdir, 'ds_noise_ratio_{}.csv'.format(JD))
df.to_csv(csv_file, index=False)

Delay spectrum and autocorrelation plot per baseline per polarization for a given frequency (sub-)band¶

Left panel: time averaged delay spectum of autocorrelation in dB with 10*log10($|\tilde{V}|$) (blue) and noise from diff file representing the expected variance of the delay spectrum (red). The time-averaging is performed by 1. binning three time integrations of each even and odd visibility, 2. Fouier transform the binned even and odd visibilities, and 3. multiply the even and odd delay spectra at alternating time bin and average the squared delay spectrum along the time axis. This helps to reduce the noise bias. Both autocorrelation delay spectrum and diff delay spectrum are averaged in the same way

Right panel: time averaged autocorrelations w/o (orange) and w/ xRFI flags (blue). Flagged one is shifted from the unflagged one for clarity

In [17]:
utils.interactive_plots_dspec(bls, uvd, uvd_diff, JD)
In [ ]: