In [1]:
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData, UVFlag
import pyuvdata
import os
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, SkyCoord, AltAz, Angle
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import h5py
import importlib
from scipy import stats
import scipy
import pandas as pd
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
In [2]:
#get data location
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
nb_outdir = os.environ['NB_OUTDIR']
utc = Time(JD, format='jd').datetime
print(f'JD = {JD}')
print(f'Date = {utc.month}-{utc.day}-{utc.year}')
print(f'data_path = "{data_path}"')

JD = 2459768
Date = 7-7-2022
data_path = "/mnt/sn1/2459768"
In [3]:
# Load in data
HHfiles, difffiles, uvdx, uvdy = utils.load_data_ds(data_path,JD)
    
uvd = UVData()
uvd_diff = UVData()
uvd.read(HHfiles[0])
use_ants = [int(ant) for ant in uvd.get_ants()]
bls = [(ant, ant) for ant in use_ants]
uvd.read(HHfiles[::10], skip_bad_files=True, bls=bls)
uvd_diff.read(difffiles[::10], skip_bad_files=True, bls=bls)
lsts = uvd.lst_array

flagfile = glob.glob(os.path.join(HHfiles[0].split('zen')[0],'zen.{}*total_stage_1_threshold_flags.h5'.format(JD)))
uvf = UVFlag()
uvf.read(flagfile)
bls = [(ant, ant) for ant in uvd.get_ants()]
times_uvf = np.unique(uvf.time_array)
times_uvd = np.unique(uvd.time_array)
idx_times = [np.where(time_uvd == times_uvf)[0][0] for time_uvd in times_uvd]
uvd.flag_array[:,0,:,:] = np.repeat(uvf.flag_array[idx_times], len(bls), axis=0)

372 sum files found between JDs 2459768.25303 and 2459768.33602
372 diff files found between JDs 2459768.25303 and 2459768.33602

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [4]:
utils.plot_lst_coverage(uvd)

Delay spectrum¶

Delay spectrum CLEANed using uvtools.dspec.high_pass_fourier_filter with 7th-order Blackman-Harris window function. Odd/even visibilities are used to remove noise bias.

In [5]:
_data_cleaned_sq, d_even, d_odd = utils.clean_ds(bls, uvd, uvd_diff, N_threads=14)

Waterfalls of delay spectra for autocorrelation¶

These plots show autocorrelation delay spectrum waterfalls of each antenna that is active and whose status qualifies for this notebook. For nn/ee polarization, the autocorrelation delay spectrum is normalized by the max of the delay spectrum. For ne polarization, the autocorrelation delay spectrum is normalized by max(sqrt(|nn| |ee|)). ne and en are the same for autocorrelations, and thus only ne is shown here. The delay spectra are presented in dB with 10log10($|\tilde{V}|$).

For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

nn polarization¶

In [6]:
utils.plot_wfds(uvd, _data_cleaned_sq, 0)

ee polarization¶

In [7]:
utils.plot_wfds(uvd, _data_cleaned_sq, 1)

ne polarization¶

In [8]:
utils.plot_wfds(uvd, _data_cleaned_sq, 2)

Analysis of 2700ns features in delay spectra¶

This plot shows the relative amplitude at 2700 ns feature. The relative amplitude is calculated in dB with the mean amplitude at 2500-3000 ns compared to the mean amplitude at 2000-2500 ns. Larger values of relative feature amplitude indicate higher probability of detecting the peak at 2700 ns. Antennas in the same node are grouped by the shaded region.

In [9]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='nn')
In [10]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='ee')

This plot shows a matrix representing the 2700ns feature correlation of each baseline. The color bar indicates the amplitude of 2700ns (mean amplitude of 2500-3000ns delay spectrum) in dB which is the same as that in the above plot.

In [11]:
# utils.CorrMatrix_2700ns(uvd, HHfiles, difffiles, flagfile, JD, N_threads=14)

Analysis of noise floor in delay spectra¶

This plot shows the ratio of delay spectrum to noise floor (averaged over 1000-4000ns). Near 1 indicates the delay spectrum reaches to the noise floor, which may mean good.

In [12]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='nn')
In [13]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='ee')
In [14]:
# get the ratio of delay spectum to noise for different freqeuncy bands and pols
ds_noise_ratio = utils.get_ds_noise_ratio(uvd, uvd_diff, bls)

nodes, antDict, inclNodes = utils.generate_nodeDict(uvd)
ants = uvd.get_ants()
# build dataframe
to_show = {'Ant': ants, 'Node': [int(antDict[ant]['node']) for ant in ants], 'Snap': [int(antDict[ant]['snapLocs'][0]) for ant in ants]}
df = pd.DataFrame(to_show)
 
cols_ratio = []
for key in ds_noise_ratio.keys():
    if(key[0] == 40):
        col = r'Full '
    else:
        col = r'{}-{} MHz '.format(key[0], key[1])
    col += key[2]
    df[col] = ds_noise_ratio[key]
    cols_ratio.append(col)
    

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

ratio_cut = 3
# style dataframe
table = df.style.hide_index() \
          .applymap(lambda val: 'color: red' if val > ratio_cut else '', subset=cols_ratio) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

This table shows the ratio of the delay spectrum to the noise level from diff files for different frequency bands and pols. The ratio > 3 is colored in red

In [15]:
HTML(table.render())
Out[15]:
Ant Node Snap Full nn Full ee 50-85 MHz nn 50-85 MHz ee 120-155 MHz nn 120-155 MHz ee 155-190 MHz nn 155-190 MHz ee 190-225 MHz nn 190-225 MHz ee
3 1 2 1.057823 0.834960 1.805109 1.763275 1.948097 2.126227 1.800664 1.846154 1.885027 3.067030
4 1 2 1.196273 0.968082 2.148017 1.765636 2.083826 1.739390 2.148232 2.195530 3.216390 2.655730
5 1 2 1.047767 0.928489 1.776380 1.624448 1.781646 1.804078 2.284716 2.066879 5.658277 7.699563
7 2 0 1.087905 0.845801 1.668730 1.825978 1.793817 1.573300 1.804526 1.880511 2.290117 2.672181
8 2 0 1.029446 20.487561 2.173823 82.365962 2.014712 55.326712 1.902315 20.827136 2.490379 84.914557
9 2 0 0.868687 0.777451 1.510215 1.550725 1.629661 1.707677 1.793360 1.803081 1.978640 1.850137
10 2 1 0.972258 1.141750 1.675495 1.838589 1.682843 1.969415 1.676887 2.880645 2.199018 10.822257
15 1 3 1.279568 1.123875 1.904110 1.627636 1.967328 2.216197 2.279150 2.099940 6.139366 6.538988
16 1 3 1.341610 1.081232 1.779069 1.476610 2.056060 2.278906 2.129801 2.551852 17.180738 19.238078
17 1 3 1.041083 0.905963 1.705523 1.651654 2.362454 1.811492 1.780151 1.750163 2.957067 4.399911
18 1 0 60.883448 41.523750 1.241952 1.612282 269.371624 211.831267 1.676312 1.808032 2.038544 1.813700
19 2 1 1.091928 1.027136 1.988337 1.676493 1.677436 2.048025 2.366071 2.221279 4.103368 6.517385
20 2 1 0.905477 0.891774 1.708787 1.498475 1.686292 1.647059 1.598169 1.660585 2.072973 2.541375
21 2 2 1.023362 0.834161 1.509714 1.519269 1.800944 1.869034 2.464379 2.414669 2.193929 2.764713
27 1 0 2.608704 3.512427 7.878214 14.051979 6.789823 7.203293 4.345473 2.049379 3.280446 1.980809
28 1 0 89.942089 13.987188 1.340189 49.551572 447.999084 42.966977 1.865057 16.145740 2.581892 62.670171
29 1 1 1.300795 1.046473 2.424581 1.780976 1.921216 1.871773 2.374082 3.665690 7.844640 4.533074
30 1 1 1.389132 0.959712 2.226362 1.721766 1.909686 1.777983 3.955576 4.524767 2.888807 4.165940
31 2 2 1.086906 1.375461 1.782809 1.673863 2.055861 2.275102 3.874224 11.134128 5.291933 20.157544
32 2 2 2.082273 1.847128 1.892226 1.951246 2.238519 2.254583 7.383452 9.595356 7.898373 12.981722
33 2 3 nan nan nan nan nan nan nan nan nan nan
36 3 3 1.549085 1.216649 1.986575 1.764738 2.048889 1.853505 1.884117 1.763296 2.234233 2.988506
37 3 1 1.361291 1.087167 2.210870 1.616200 2.901516 2.093213 2.348551 2.146685 6.455401 5.270929
38 3 1 0.982121 1.114200 1.958755 1.705050 1.751215 2.252057 2.291694 2.926961 2.105203 2.986841
40 4 1 0.975940 1.020540 1.580652 1.831993 1.721019 2.137524 1.840413 2.599609 2.380847 4.901885
41 4 3 0.928361 1.009768 1.683834 1.515489 1.723439 2.072038 1.769944 1.841080 1.983068 2.100312
42 4 0 1.122157 0.930469 1.815605 1.627446 2.320513 2.132684 2.004595 2.027455 4.579719 8.033011
45 5 0 1.300837 0.924983 3.375607 1.391990 3.016773 1.732070 3.234521 1.884108 3.967610 2.232546
46 5 0 2.007717 1.401820 1.725760 1.465956 4.086695 2.584952 1.963011 1.940768 2.972963 3.685984
50 3 3 1.029072 0.862920 1.839729 1.657732 2.335112 1.757408 2.132202 1.655656 2.822772 2.127649
51 3 2 4.048293 0.917081 12.073088 2.208829 5.439884 1.802717 5.297837 1.974869 6.318476 4.031891
52 3 3 5.652186 1.190724 17.612820 2.099295 3.467393 2.164259 4.160215 2.158368 4.456806 4.476351
53 3 2 1.084138 1.019285 3.197894 2.657193 2.386555 2.510261 2.315561 2.498726 4.363335 5.273902
54 4 0 1.119559 0.951591 1.689367 1.843665 2.336393 2.145560 1.957027 1.890302 3.508883 4.121293
55 4 2 0.833326 0.836513 1.817532 1.229652 1.577781 1.690812 2.089506 1.627240 2.901675 2.090692
56 4 1 1.187308 1.119686 1.588948 1.614207 2.055167 2.311403 2.123495 2.161569 2.542447 3.330399
57 4 3 1.197511 0.950884 2.961867 1.968825 2.822632 2.003353 2.338537 1.577619 2.486222 1.848243
65 3 0 1.028418 0.796835 2.079149 1.606662 1.764443 1.651147 1.863971 1.700123 2.185819 2.254238
66 3 0 1.312126 5.134797 3.666243 20.334646 2.841866 15.536640 2.372770 7.412759 2.786286 25.819517
67 3 0 0.984525 0.829909 1.759593 1.629633 1.857175 1.775037 1.908411 1.797172 1.853496 2.184152
68 3 1 0.916120 0.791447 1.662930 1.818716 1.614807 1.724588 1.703255 1.862218 1.882836 2.373732
69 4 1 1.002104 0.952488 1.758684 2.038854 1.904925 2.133751 2.309520 2.593857 3.344170 3.830742
70 4 2 1.673350 1.366997 2.323428 1.672774 4.382873 2.538747 2.461227 2.078065 2.312566 2.703551
71 4 2 0.825403 0.867802 1.922994 1.681795 1.715889 1.620208 1.834224 1.630335 1.960251 1.908684
72 4 0 1.041667 1.143861 1.792362 1.918909 2.393681 3.103573 1.725696 1.858043 2.301517 2.998599
73 5 0 1.069025 1.190236 1.740102 1.782225 2.518521 2.742016 1.937275 2.345527 2.937847 4.979543
81 7 0 1.941437 1.703242 2.139202 1.955526 2.459366 3.147893 3.820172 2.083267 15.283379 6.986796
82 7 0 0.944084 1.073812 1.805646 2.507289 1.753825 2.432467 1.840378 1.841067 2.609057 3.156582
83 7 0 0.858265 0.829683 2.074628 1.845410 1.780012 1.790087 2.002326 1.976391 2.403002 2.919430
88 9 0 3.219528 6.381525 14.205180 29.437040 8.222281 18.861844 4.414209 4.659854 6.208797 2.568862
89 9 2 nan nan nan nan nan nan nan nan nan nan
90 9 0 14.538209 5.177587 4.549915 4.324113 20.562194 11.111276 50.793861 19.460265 89.161391 36.930802
91 9 1 5.050891 3.499717 12.636838 11.085549 5.897278 7.119732 5.750515 5.694345 6.949982 2.393193
92 10 0 1.755274 1.362456 1.857396 2.465842 1.860824 2.108237 2.018352 2.190140 2.119304 2.032252
93 10 0 4.374394 1.472783 15.229503 2.274794 6.486722 1.738723 4.043454 1.936292 3.573016 2.280416
94 10 0 1.036490 1.167098 1.478376 2.754108 1.855466 2.584182 2.002710 3.179719 3.067327 19.851420
98 7 2 2.296171 15.174569 1.832365 61.965897 5.086734 41.714936 6.834067 15.891832 20.966392 65.006956
99 7 2 0.928927 0.897599 1.672655 1.590248 1.545386 1.831568 1.762387 1.823936 3.675206 7.592834
100 7 1 0.996638 24.580000 1.816577 97.093210 1.699231 63.637097 1.918242 23.757992 2.495059 84.846018
105 9 1 2.398006 2.562697 16.567044 7.300572 4.068631 8.269798 5.419562 2.189365 10.813850 2.667450
106 9 3 1.984935 2.947187 1.782949 2.049544 5.150427 9.821515 3.464105 6.381597 5.984462 5.173881
107 9 0 2.774119 3.617229 24.352801 21.325678 4.757917 2.361357 3.404323 3.724030 5.064298 8.103522
108 9 1 1.192768 0.884086 1.843146 1.614915 1.913027 1.752102 1.911107 1.977295 3.131973 2.591966
109 10 1 2.242088 1.812679 2.152579 1.816388 3.379852 3.965733 1.957050 1.760051 2.738950 2.273300
110 10 1 1.015805 1.453743 2.027970 1.348197 2.087007 2.800387 2.216105 2.191207 2.343424 2.003280
111 10 1 1.107151 1.044174 2.121192 1.562545 2.412348 1.766044 1.964049 1.590129 1.926546 1.526331
112 10 2 1.070012 0.978203 1.577775 1.974333 2.299795 2.001170 1.487832 1.774889 1.946845 2.575041
116 7 2 0.857869 0.806684 1.845545 1.659510 1.962851 1.758325 1.670667 1.649282 3.354177 2.465850
117 7 3 1.029336 23.458709 1.837425 100.732165 1.766875 60.353300 1.761916 24.283686 2.832664 90.104709
118 7 3 0.893971 22.466173 1.900049 91.741841 1.647178 48.762799 2.024472 18.203030 2.901390 77.202983
119 7 1 1.009830 0.871033 2.105429 2.121403 2.186777 1.999101 2.213562 1.937206 5.501075 2.650305
124 9 2 nan nan nan nan nan nan nan nan nan nan
125 9 3 3.854877 2.722547 38.355590 12.253783 8.703140 6.370016 3.124338 2.202446 4.965251 2.295102
126 9 3 2.767525 2.290526 32.647088 8.220306 7.198898 7.597543 5.151684 4.287432 5.883293 3.911327
127 10 2 2.295149 2.489284 3.465064 2.468876 5.054268 7.698408 8.665074 7.798701 8.977463 9.817037
128 10 2 0.799431 0.933815 1.781836 2.140990 1.809109 1.837133 1.795053 1.863787 2.495718 5.907218
129 10 3 1.183128 0.840441 1.752829 1.674063 1.827931 1.903376 1.953583 1.546690 1.902780 2.457765
130 10 3 1.354347 1.411678 2.976672 13.259870 2.006242 1.511946 2.900631 1.818117 4.034227 1.668747
135 12 2 1.026469 0.831259 2.179338 1.622170 2.046420 1.733786 2.128944 1.758296 2.243489 2.020626
136 12 2 1.303261 1.099212 4.427258 1.648562 2.262960 1.721114 6.612674 2.983638 2.855467 2.791858
137 7 3 2.340971 1.544882 12.752579 9.264727 6.108782 3.566752 4.276058 1.967391 5.098444 2.276370
138 7 1 1.116902 3.642524 1.833393 3.934237 2.288924 2.916789 2.075765 2.190317 8.988166 2.504102
140 13 2 1.129808 0.881241 1.759032 1.851707 2.011201 1.949444 1.900601 2.393695 2.175643 3.716706
141 13 2 8.040343 1.025252 20.570528 1.804249 28.816308 2.224519 20.538371 2.239023 17.592229 3.518103
142 13 2 2.257202 1.422282 6.111110 3.218300 6.108593 2.245853 2.692073 1.635526 3.847222 2.900140
143 14 2 1.148045 0.828566 2.710017 1.563844 2.324727 1.598130 2.321592 2.113020 2.432471 2.855576
144 14 3 1.416804 1.051931 1.779623 1.257134 2.627432 2.006372 1.745276 1.726844 2.298893 2.563023
145 14 3 4.369288 3.072450 14.829142 19.020784 11.653684 13.388039 3.131492 1.759920 2.735652 1.914218
150 15 1 nan nan nan nan nan nan nan nan nan nan
155 12 2 2.957928 2.547858 8.026980 19.626353 3.277304 3.478428 3.518608 4.477905 3.645806 2.101932
156 12 3 1.356916 1.278131 1.775954 1.522565 2.622129 2.767755 1.890538 1.863598 1.833077 2.233356
157 12 3 1.304813 1.053216 1.675418 1.616047 2.207916 2.025687 1.909696 1.899200 2.108281 2.358217
158 12 3 1.264679 1.042324 2.678505 1.759245 3.118601 2.521760 2.465341 2.677138 4.560774 8.065359
160 13 3 3.521767 5.567477 15.148842 49.441763 3.450011 4.007894 4.930206 13.925061 5.871106 21.029518
161 13 0 nan nan nan nan nan nan nan nan nan nan
162 13 0 nan nan nan nan nan nan nan nan nan nan
163 14 2 1.123346 1.037329 1.705473 1.746141 1.609712 1.795729 2.239027 1.954170 2.730497 6.034686
164 14 2 1.131901 0.871150 1.606892 1.677982 1.953643 1.642535 2.004411 3.049402 3.745867 2.971749
165 14 0 4.461597 3.877201 8.542693 3.935796 15.572291 14.515351 4.728097 7.751257 7.040672 11.272301
166 14 0 2.744484 2.830971 1.603569 1.451244 4.093245 4.982881 7.097994 8.037069 4.552293 7.053466
167 15 1 nan nan nan nan nan nan nan nan nan nan
168 15 2 1.288266 1.022052 3.059250 2.414723 1.812484 1.858889 3.880212 3.782082 5.095144 4.560324
169 15 2 1.085694 13.520522 1.757886 66.652024 1.914298 18.589000 3.248188 10.232573 4.456510 28.938193
170 15 2 1.526086 14.770195 2.647501 70.224952 3.703371 15.260753 4.794120 8.786268 4.398705 24.029515
176 12 0 0.945707 0.918218 1.741099 1.746324 1.882118 1.733699 2.143353 2.052726 2.682875 2.859075
177 12 0 0.963023 0.948131 1.768429 1.830812 1.809414 1.784372 1.711103 2.044529 2.617947 2.733900
178 12 0 1.031506 0.955731 1.657054 1.559139 1.894831 2.222869 1.885354 2.178299 2.397430 2.617008
179 12 1 1.451468 0.865599 1.521893 1.531239 2.532840 1.604808 6.998730 2.301300 4.623345 3.084738
180 13 3 2.020203 23.497977 13.124088 19.404658 2.710780 75.474541 2.526139 50.166213 2.421763 105.407403
181 13 3 1.292135 1.005630 2.098160 1.547403 2.409409 2.026121 2.161442 1.933168 2.892691 1.702590
182 13 0 nan nan nan nan nan nan nan nan nan nan
183 13 1 1.198213 1.046880 1.728637 1.640962 3.289164 2.295583 1.934511 1.923470 2.303312 2.757543
184 14 0 0.929867 0.852216 1.672724 1.589825 1.852114 1.872223 1.969152 1.697554 2.715038 3.564328
185 14 1 1.618057 1.507027 2.090969 1.961897 2.446664 2.549048 7.543954 3.389801 8.296647 11.609655
186 14 1 1.113620 0.925010 1.636589 1.839499 1.746046 1.710582 1.785629 1.904820 2.590426 3.240425
187 14 1 1.245610 1.334752 2.148473 2.772974 2.353543 2.622985 3.514766 4.876407 6.906197 15.224035
189 15 3 0.946675 0.948819 1.853166 1.436161 1.809497 1.676174 1.931570 1.754940 2.226533 2.262001
190 15 3 1.871798 2.168477 1.411289 1.593623 2.311430 2.806343 2.323262 2.702072 3.989809 4.273969
191 15 3 1.188889 1.083123 1.875146 1.614401 2.569267 2.304166 1.953134 2.083420 2.436300 2.094721
220 18 2 nan nan nan nan nan nan nan nan nan nan
221 18 2 nan nan nan nan nan nan nan nan nan nan
222 18 2 nan nan nan nan nan nan nan nan nan nan
320 3 2 3.794323 5.322185 6.247462 22.408906 4.173965 5.508928 5.242378 2.884749 5.897731 11.589286
321 2 3 nan nan nan nan nan nan nan nan nan nan
323 2 3 nan nan nan nan nan nan nan nan nan nan
324 4 3 0.830206 9.280052 4.324008 56.626328 1.506383 1.569730 1.623866 1.576621 1.749001 2.534883
329 12 1 0.890297 1.004356 1.985019 3.593016 1.853341 2.035602 1.722311 1.826370 2.061426 2.339384
333 12 1 0.857241 1.043259 2.009427 1.950213 2.490721 3.125351 1.779454 1.623829 2.213163 2.129801
In [16]:
csv_file = os.path.join(nb_outdir, 'ds_noise_ratio_{}.csv'.format(JD))
df.to_csv(csv_file, index=False)

Delay spectrum and autocorrelation plot per baseline per polarization for a given frequency (sub-)band¶

Left panel: time averaged delay spectum of autocorrelation in dB with 10*log10($|\tilde{V}|$) (blue) and noise from diff file representing the expected variance of the delay spectrum (red). The time-averaging is performed by 1. binning three time integrations of each even and odd visibility, 2. Fouier transform the binned even and odd visibilities, and 3. multiply the even and odd delay spectra at alternating time bin and average the squared delay spectrum along the time axis. This helps to reduce the noise bias. Both autocorrelation delay spectrum and diff delay spectrum are averaged in the same way

Right panel: time averaged autocorrelations w/o (orange) and w/ xRFI flags (blue). Flagged one is shifted from the unflagged one for clarity

In [17]:
utils.interactive_plots_dspec(bls, uvd, uvd_diff, JD)
In [ ]: