In [1]:
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData, UVFlag
import pyuvdata
import os
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, SkyCoord, AltAz, Angle
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import h5py
import importlib
from scipy import stats
import scipy
import pandas as pd
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
In [2]:
#get data location
JD = os.environ['JULIANDATE']
data_path = os.environ['DATA_PATH']
nb_outdir = os.environ['NB_OUTDIR']
utc = Time(JD, format='jd').datetime
print(f'JD = {JD}')
print(f'Date = {utc.month}-{utc.day}-{utc.year}')
print(f'data_path = "{data_path}"')

JD = 2459773
Date = 7-12-2022
data_path = "/mnt/sn1/2459773"
In [3]:
# Load in data
HHfiles, difffiles, uvdx, uvdy = utils.load_data_ds(data_path,JD)
    
uvd = UVData()
uvd_diff = UVData()
uvd.read(HHfiles[0])
use_ants = [int(ant) for ant in uvd.get_ants()]
bls = [(ant, ant) for ant in use_ants]
uvd.read(HHfiles[::10], skip_bad_files=True, bls=bls)
uvd_diff.read(difffiles[::10], skip_bad_files=True, bls=bls)
lsts = uvd.lst_array

flagfile = glob.glob(os.path.join(HHfiles[0].split('zen')[0],'zen.{}*total_stage_1_threshold_flags.h5'.format(JD)))
uvf = UVFlag()
uvf.read(flagfile)
bls = [(ant, ant) for ant in uvd.get_ants()]
times_uvf = np.unique(uvf.time_array)
times_uvd = np.unique(uvd.time_array)
idx_times = [np.where(time_uvd == times_uvf)[0][0] for time_uvd in times_uvd]
uvd.flag_array[:,0,:,:] = np.repeat(uvf.flag_array[idx_times], len(bls), axis=0)

372 sum files found between JDs 2459773.25320 and 2459773.33619
372 diff files found between JDs 2459773.25320 and 2459773.33619

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [4]:
utils.plot_lst_coverage(uvd)

Delay spectrum¶

Delay spectrum CLEANed using uvtools.dspec.high_pass_fourier_filter with 7th-order Blackman-Harris window function. Odd/even visibilities are used to remove noise bias.

In [5]:
_data_cleaned_sq, d_even, d_odd = utils.clean_ds(bls, uvd, uvd_diff, N_threads=14)

Waterfalls of delay spectra for autocorrelation¶

These plots show autocorrelation delay spectrum waterfalls of each antenna that is active and whose status qualifies for this notebook. For nn/ee polarization, the autocorrelation delay spectrum is normalized by the max of the delay spectrum. For ne polarization, the autocorrelation delay spectrum is normalized by max(sqrt(|nn| |ee|)). ne and en are the same for autocorrelations, and thus only ne is shown here. The delay spectra are presented in dB with 10log10($|\tilde{V}|$).

For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

nn polarization¶

In [6]:
utils.plot_wfds(uvd, _data_cleaned_sq, 0)

ee polarization¶

In [7]:
utils.plot_wfds(uvd, _data_cleaned_sq, 1)

ne polarization¶

In [8]:
utils.plot_wfds(uvd, _data_cleaned_sq, 2)

Analysis of 2700ns features in delay spectra¶

This plot shows the relative amplitude at 2700 ns feature. The relative amplitude is calculated in dB with the mean amplitude at 2500-3000 ns compared to the mean amplitude at 2000-2500 ns. Larger values of relative feature amplitude indicate higher probability of detecting the peak at 2700 ns. Antennas in the same node are grouped by the shaded region.

In [9]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='nn')
In [10]:
utils.plot_antFeatureMap_2700ns(uvd, _data_cleaned_sq, JD, pol='ee')

This plot shows a matrix representing the 2700ns feature correlation of each baseline. The color bar indicates the amplitude of 2700ns (mean amplitude of 2500-3000ns delay spectrum) in dB which is the same as that in the above plot.

In [11]:
# utils.CorrMatrix_2700ns(uvd, HHfiles, difffiles, flagfile, JD, N_threads=14)

Analysis of noise floor in delay spectra¶

This plot shows the ratio of delay spectrum to noise floor (averaged over 1000-4000ns). Near 1 indicates the delay spectrum reaches to the noise floor, which may mean good.

In [12]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='nn')
In [13]:
utils.plot_antFeatureMap_noise(uvd_diff, d_even, d_odd, JD, pol='ee')
In [14]:
# get the ratio of delay spectum to noise for different freqeuncy bands and pols
ds_noise_ratio = utils.get_ds_noise_ratio(uvd, uvd_diff, bls)

nodes, antDict, inclNodes = utils.generate_nodeDict(uvd)
ants = uvd.get_ants()
# build dataframe
to_show = {'Ant': ants, 'Node': [int(antDict[ant]['node']) for ant in ants], 'Snap': [int(antDict[ant]['snapLocs'][0]) for ant in ants]}
df = pd.DataFrame(to_show)
 
cols_ratio = []
for key in ds_noise_ratio.keys():
    if(key[0] == 40):
        col = r'Full '
    else:
        col = r'{}-{} MHz '.format(key[0], key[1])
    col += key[2]
    df[col] = ds_noise_ratio[key]
    cols_ratio.append(col)
    

# sort by node number and then by antenna number within nodes
df.sort_values(['Node', 'Ant'], ascending=True)

ratio_cut = 3
# style dataframe
table = df.style.hide_index() \
          .applymap(lambda val: 'color: red' if val > ratio_cut else '', subset=cols_ratio) \
          .set_table_styles([dict(selector="th",props=[('max-width', f'70pt')])])

This table shows the ratio of the delay spectrum to the noise level from diff files for different frequency bands and pols. The ratio > 3 is colored in red

In [15]:
HTML(table.render())
Out[15]:
Ant Node Snap Full nn Full ee 50-85 MHz nn 50-85 MHz ee 120-155 MHz nn 120-155 MHz ee 155-190 MHz nn 155-190 MHz ee 190-225 MHz nn 190-225 MHz ee
3 1 2 1.077437 0.838294 1.688610 1.565602 2.087783 1.856515 1.777485 1.762838 1.942774 2.436286
4 1 2 1.188372 0.857808 3.123855 2.787046 2.179764 1.729043 2.094825 2.435880 3.075045 2.822183
5 1 2 1.005153 1.043122 1.848724 1.733681 1.945077 1.949524 2.306745 2.092706 7.879874 9.022135
7 2 0 0.977707 0.818092 1.627658 1.486003 1.817195 1.743430 1.811650 1.743461 2.444844 2.328447
8 2 0 1.042164 25.219266 2.654228 73.151157 2.246119 88.969107 2.074107 81.984348 2.557827 109.185182
9 2 0 0.923465 0.926184 1.743048 1.463470 1.831918 1.933271 2.439328 2.380064 1.991518 2.257837
10 2 1 nan nan nan nan nan nan nan nan nan nan
15 1 3 1.293100 1.067806 2.217535 1.600310 1.962738 2.062143 2.175993 2.102900 5.428904 6.738785
16 1 3 1.263312 1.170780 1.706612 1.657609 2.182646 2.389301 2.031474 2.389245 11.221304 15.110069
17 1 3 1.159420 0.956567 1.685594 1.759885 2.386618 1.926559 1.950552 1.816321 3.410875 3.510916
18 1 0 52.250783 49.122827 1.256378 1.441283 274.996191 253.051551 1.818060 1.841413 2.067689 1.827820
19 2 1 nan nan nan nan nan nan nan nan nan nan
20 2 1 nan nan nan nan nan nan nan nan nan nan
21 2 2 0.975354 0.794185 1.453866 1.543847 1.765098 1.764114 2.138872 2.053385 1.912361 2.497433
27 1 0 3.770306 3.177502 8.439371 9.945755 16.074748 8.287870 4.203647 1.918813 3.849381 1.800304
28 1 0 195.588073 14.690649 0.979809 54.073628 1011.302476 46.314461 1.600113 17.949901 2.516751 70.297128
29 1 1 1.407163 1.410198 2.246989 1.720339 2.031334 2.388483 3.115755 2.457451 4.127534 5.775834
30 1 1 1.292660 0.991382 2.340447 1.642172 2.338126 1.800743 2.474412 2.842730 3.477678 4.053940
31 2 2 1.129081 1.370754 1.708014 1.652053 2.076066 2.381711 3.354446 8.096741 4.710989 15.419728
32 2 2 2.000483 2.587373 1.335171 1.410970 2.741588 2.245674 5.565455 6.398737 6.615405 8.363131
33 2 3 36.025671 0.834545 1.519374 1.792238 185.981183 1.849553 1.753932 1.937523 2.179954 2.646183
36 3 3 1.490150 1.190540 1.801215 1.730175 1.991099 2.117122 2.153897 1.631577 2.325745 2.710520
37 3 1 1.365798 1.186545 1.848599 1.845987 3.068115 2.558156 2.172250 2.387600 5.711561 5.403542
38 3 1 0.965533 0.985760 1.998488 1.569044 1.968519 2.035304 2.725335 2.708495 2.353424 2.873545
40 4 1 0.995622 0.989125 1.594533 2.017911 1.893920 2.085481 2.183902 2.455167 2.535056 5.098507
41 4 3 0.986415 1.034849 1.414698 1.791018 1.804482 2.394165 1.828214 2.033329 1.937329 2.347216
42 4 0 1.201345 0.936302 1.656503 1.931912 2.316238 2.524616 1.858809 2.278253 4.835478 9.538532
45 5 0 1.563405 0.912469 4.310966 1.669854 4.017057 1.880308 3.469269 1.878624 3.590991 2.053714
46 5 0 1.680632 1.079426 1.666243 1.581444 3.302029 2.085325 2.027757 1.727047 2.743767 2.852952
50 3 3 1.036634 0.797799 1.690591 1.617425 2.139349 1.738704 1.919063 1.767002 2.288997 1.872205
51 3 2 6.041156 0.839854 14.622875 2.023659 5.824796 1.557236 4.340525 1.905570 5.842749 4.241970
52 3 3 6.373874 1.096203 19.888105 2.174860 3.965053 2.152322 3.705511 2.378162 4.556878 4.729415
53 3 2 1.049468 1.065868 2.366098 2.619520 2.281536 2.823185 2.240646 2.778610 4.275633 5.455668
54 4 0 1.146623 0.928683 1.769089 1.846827 2.608156 2.421133 2.080327 2.122482 3.138739 3.655277
55 4 2 0.844499 0.815143 1.677912 1.297874 1.633080 1.644795 2.241563 1.600099 3.136292 1.804383
56 4 1 1.244059 1.211900 1.670274 1.510949 2.650599 2.494431 2.448166 2.231820 3.253944 3.951316
57 4 3 1.244767 0.900522 3.353555 1.616244 3.130549 1.887739 2.703783 1.543833 3.249972 1.771614
65 3 0 1.091898 0.830791 2.157257 1.761028 1.954778 1.953719 1.824920 1.972698 2.187297 2.228698
66 3 0 1.340000 3.569709 3.891282 13.916139 3.129321 10.752116 2.712790 5.081032 2.611265 17.924898
67 3 0 1.086474 0.887897 1.666027 1.538486 1.736478 1.763556 1.785526 1.660822 1.802678 2.106031
68 3 1 0.948411 0.827325 1.538255 1.763998 1.787876 1.855013 1.617129 2.011686 2.575110 3.192952
69 4 1 1.003384 0.959575 1.720764 2.112056 2.216667 2.135383 2.498286 2.233585 3.045296 2.744564
70 4 2 2.528229 1.347990 2.360649 1.820716 7.175940 2.795397 4.766936 2.470201 3.071903 2.646586
71 4 2 0.918263 0.926642 1.818010 1.680731 1.721147 1.699821 1.651478 1.770885 1.912702 2.140582
72 4 0 1.191132 0.977927 1.786307 1.824790 2.372108 2.579545 2.005169 2.432810 5.035465 6.646261
73 5 0 1.201948 1.288469 1.677324 1.614440 1.858568 2.721851 2.021437 2.271941 4.029743 4.845984
81 7 0 1.496389 1.259944 2.147204 1.730306 2.254720 2.054299 2.918238 2.079900 8.264696 5.323656
82 7 0 1.028895 1.066008 1.496289 1.592176 1.591443 2.456987 1.827330 2.062681 3.137655 3.340844
83 7 0 0.850017 0.902089 1.831870 2.391252 1.748401 1.790735 1.767495 1.849397 2.064787 2.916005
88 9 0 4.551051 19.613218 21.776681 31.269022 6.336047 30.112864 4.287852 3.049643 7.028207 3.279863
89 9 2 nan nan nan nan nan nan nan nan nan nan
90 9 0 7.791924 3.404621 3.119862 2.936071 10.628955 8.346937 25.313479 9.315978 40.126862 23.145462
91 9 1 3.091203 3.229162 13.301804 10.761176 5.423538 7.468121 5.771829 3.774230 7.534612 2.359542
92 10 0 1.644290 1.365485 1.822932 1.819523 1.867924 1.662486 2.012297 1.661864 1.992403 1.826894
93 10 0 4.048659 1.334492 15.854631 1.554235 7.019970 1.678182 3.082235 1.701317 3.633901 1.961650
94 10 0 1.017280 1.218612 1.599230 2.811149 1.907384 2.654261 2.385383 2.898398 2.822949 15.754551
98 7 2 3.299083 17.114704 1.893143 54.716461 7.937260 56.297093 13.810753 48.524609 50.745335 73.878866
99 7 2 1.001521 0.927343 1.514395 1.639261 1.553042 1.672318 1.603857 1.860877 3.093191 3.314153
100 7 1 1.057099 25.171051 2.004637 87.841592 1.733811 81.186325 1.862383 71.685327 2.421308 101.207990
105 9 1 2.335354 2.470737 15.380488 6.648840 3.912722 8.889136 4.840656 2.268039 9.423023 2.636882
106 9 3 1.957683 2.575189 1.908876 1.774966 4.191400 7.455530 3.406272 4.860120 5.354015 3.915850
107 9 0 4.068697 4.753052 26.862460 33.959473 4.328601 2.843745 5.704884 6.925764 7.472309 7.628418
108 9 1 1.320423 1.150402 1.546140 1.636009 2.334078 2.369114 1.784637 1.993702 2.581104 2.573638
109 10 1 2.270202 1.704595 1.969613 2.044118 4.730484 3.304778 1.913890 2.158064 2.627012 2.744144
110 10 1 0.992436 1.618702 2.019941 1.572356 2.055318 3.660708 1.747011 2.812657 2.067812 2.242465
111 10 1 1.067145 0.928567 2.435842 1.706097 2.215097 1.811683 2.114410 1.621275 2.014009 1.681930
112 10 2 1.364722 0.940140 8.262667 2.015081 1.941930 1.772988 1.717936 1.963561 2.013755 2.235868
116 7 2 0.936707 1.045785 1.617260 1.882450 1.790068 2.889295 1.687530 2.715125 3.553587 3.555806
117 7 3 1.233219 1.262372 1.866154 1.856045 1.716345 1.798175 2.076055 2.877659 3.181145 5.429293
118 7 3 1.057679 1.045583 1.733634 1.849154 1.863725 1.935903 2.001969 2.122272 2.923865 4.626300
119 7 1 1.006105 0.788209 1.956599 1.820783 2.067835 1.747502 2.030099 1.841160 4.695345 2.454730
124 9 2 nan nan nan nan nan nan nan nan nan nan
125 9 3 3.904699 2.377949 36.981416 11.627894 9.527064 5.986548 3.033167 1.915471 4.972355 2.093390
126 9 3 2.864319 2.533825 35.065821 9.532451 8.413760 9.302194 4.672617 4.685823 5.663543 3.922425
127 10 2 2.456566 2.931050 3.748995 2.527325 5.710020 9.797364 7.280897 6.714712 8.417638 11.300969
128 10 2 0.819173 0.873150 1.791162 2.133352 1.693019 1.838468 1.849826 1.913440 4.614024 7.384192
129 10 3 1.150357 0.839218 1.674052 1.605833 1.789172 1.781147 1.779866 1.753196 1.767834 2.449079
130 10 3 1.375708 1.766440 2.791921 14.699263 2.020280 1.854730 3.764583 1.919179 4.288233 2.066045
135 12 2 0.946750 0.914157 2.072117 1.598610 1.875365 1.851133 2.147533 1.821744 2.332387 1.835248
136 12 2 1.322942 1.062244 4.170248 1.736914 2.194713 1.810117 5.486210 3.320283 2.219584 2.936312
137 7 3 2.088796 2.921163 11.477163 12.448878 5.906843 2.876126 3.707789 1.724525 4.453065 1.871075
138 7 1 1.178200 4.128617 1.670141 3.827955 2.257554 3.324014 2.073728 1.990011 7.929217 2.240151
140 13 2 1.028088 0.928933 1.846648 1.922744 1.842364 2.159653 2.155966 2.532876 2.504921 3.636924
141 13 2 6.304059 1.024308 17.496825 1.948910 23.922928 2.139786 14.655687 1.947717 12.354891 3.125458
142 13 2 2.320273 1.807942 5.843726 3.088236 6.218107 2.717976 2.528142 1.628428 3.128477 3.499349
143 14 2 1.325728 0.887936 2.342774 1.685329 2.033075 1.655277 4.285486 2.548215 2.194277 2.646453
144 14 3 1.438584 1.120405 1.668680 1.410384 2.819095 1.953476 1.919864 1.570903 2.406736 2.827007
145 14 3 4.186974 3.560243 15.655635 21.718970 14.669369 16.223442 3.034185 1.995245 2.742089 1.971938
150 15 1 3.592366 11.225859 26.147453 48.881105 5.968374 10.761141 5.412311 19.052757 6.575300 34.956938
155 12 2 3.047975 2.276935 10.082293 18.967797 4.893343 4.143692 3.665144 2.699518 3.867726 1.952350
156 12 3 1.264125 1.221124 1.774219 1.624475 2.634201 2.379158 1.913513 1.994188 2.177219 2.318412
157 12 3 1.278337 1.073948 1.493076 1.641747 1.973024 2.113228 1.836991 1.931059 1.861924 2.313282
158 12 3 1.496225 1.012638 3.811098 1.619191 3.786634 2.090684 3.159746 2.595463 4.203644 7.404652
160 13 3 2.167082 3.160563 6.887059 21.490789 3.426637 3.296687 2.776225 3.739560 3.188729 8.282537
161 13 0 1.545763 1.214598 0.959241 1.663386 3.931102 1.943021 3.168797 2.154046 4.523912 4.857140
162 13 0 1.069009 0.920978 1.712885 1.859440 1.799483 1.957482 2.261158 1.938192 2.126595 2.389953
163 14 2 1.157045 0.969608 1.722832 1.673018 1.766176 1.698334 2.423283 1.957705 2.868782 2.798175
164 14 2 1.006360 0.981996 1.828411 1.544187 2.063826 1.828793 2.167915 3.123788 3.576091 2.914006
165 14 0 6.643042 6.980265 9.322618 7.070415 27.372707 18.558500 4.857431 4.460383 14.913580 6.153794
166 14 0 1.533695 1.859456 1.972470 1.559361 2.903731 3.610907 3.366533 4.398861 2.472530 4.618887
167 15 1 2.904854 11.596910 2.548618 59.834478 4.178967 2.642891 2.690112 2.564811 3.657898 2.767616
168 15 2 1.576910 1.322159 3.097709 2.659379 2.577691 2.052548 5.912704 7.435919 11.105685 9.822962
169 15 2 1.008782 9.062539 1.920227 44.969876 1.841244 12.186586 3.107971 8.078036 4.208580 20.050971
170 15 2 1.548908 10.968568 2.661574 56.757436 3.264545 15.351661 3.970125 12.188503 4.046569 25.010784
176 12 0 0.877032 0.876954 1.716504 1.776362 1.849713 1.822079 2.016133 2.033110 2.519618 2.400942
177 12 0 1.021258 0.917066 1.603189 1.733854 1.673578 1.853150 1.837159 1.804611 2.284357 2.661968
178 12 0 1.014580 0.936463 1.762489 1.568156 2.089362 1.937945 2.039627 2.143940 2.288294 2.461018
179 12 1 1.432702 0.864052 1.621097 1.738729 2.335333 1.680913 5.763494 2.469270 3.726513 2.835380
180 13 3 1.840468 1.620494 11.335657 2.569975 3.125255 3.249195 2.295158 2.536476 2.275477 4.665204
181 13 3 1.255982 1.828713 3.172668 1.545446 3.799070 2.949160 1.596100 1.608449 2.098063 2.012774
182 13 0 1.246570 1.339153 1.852106 2.201013 2.816888 4.628166 2.252464 2.172127 2.785130 3.811229
183 13 1 1.216952 1.070850 1.669502 1.887043 3.187079 2.240855 1.977312 2.023559 2.259105 2.851905
184 14 0 0.964286 0.836175 1.695694 1.709195 1.684439 1.666672 2.240609 1.844399 2.995406 4.298036
185 14 1 1.121980 1.283529 1.636156 2.356009 2.193575 2.391811 2.708023 4.028822 4.097895 3.653899
186 14 1 1.190538 0.861149 1.509496 1.713659 1.689405 1.789902 2.071892 1.931391 2.127661 2.583500
187 14 1 1.176126 1.150192 1.968232 1.966321 2.190506 2.115089 3.383927 4.135710 8.474390 17.071173
189 15 3 0.908021 1.052080 1.771936 1.691520 1.742005 2.188508 1.725951 2.155031 2.147116 3.623171
190 15 3 3.084129 2.984643 9.641528 1.251235 3.293513 3.735047 3.109266 2.925117 2.808943 5.002946
191 15 3 1.152420 1.176707 1.624558 1.798950 2.464149 2.305931 2.330038 2.140853 2.449019 2.402474
220 18 2 nan nan nan nan nan nan nan nan nan nan
221 18 2 nan nan nan nan nan nan nan nan nan nan
222 18 2 nan nan nan nan nan nan nan nan nan nan
320 3 2 3.936161 5.220042 8.331219 23.777616 4.714084 5.485128 6.056390 5.335173 6.442600 6.735052
321 2 3 0.973896 1.225135 1.754381 2.196509 1.729901 2.212122 1.782646 1.699145 1.974408 1.843952
323 2 3 0.934377 1.519489 3.947209 6.876179 1.826762 3.441918 1.673494 1.955046 1.838521 3.129008
324 4 3 0.724093 7.086739 1.674013 33.189938 1.622660 1.639629 1.641724 1.666792 1.833767 2.552638
329 12 1 1.213202 1.226132 2.768625 11.219831 1.755454 1.886064 1.816508 1.770338 2.032739 2.203026
333 12 1 0.902834 1.094305 1.906157 1.874773 2.215180 3.058345 1.732044 1.577202 1.913326 2.100515
In [16]:
csv_file = os.path.join(nb_outdir, 'ds_noise_ratio_{}.csv'.format(JD))
df.to_csv(csv_file, index=False)

Delay spectrum and autocorrelation plot per baseline per polarization for a given frequency (sub-)band¶

Left panel: time averaged delay spectum of autocorrelation in dB with 10*log10($|\tilde{V}|$) (blue) and noise from diff file representing the expected variance of the delay spectrum (red). The time-averaging is performed by 1. binning three time integrations of each even and odd visibility, 2. Fouier transform the binned even and odd visibilities, and 3. multiply the even and odd delay spectra at alternating time bin and average the squared delay spectrum along the time axis. This helps to reduce the noise bias. Both autocorrelation delay spectrum and diff delay spectrum are averaged in the same way

Right panel: time averaged autocorrelations w/o (orange) and w/ xRFI flags (blue). Flagged one is shifted from the unflagged one for clarity

In [17]:
utils.interactive_plots_dspec(bls, uvd, uvd_diff, JD)
In [ ]: