In [1]:
import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, AltAz, Angle
from astropy.coordinates import SkyCoord as sc
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import importlib
from scipy import stats
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))
In [2]:
#get data location
data_path = os.environ['DATA_PATH']
print(f'DATA_PATH = "{data_path}"')
statuses = os.environ['APRIORI_STATUSES']
print(f'APRIORI_STATUSES = {statuses}')
JD = os.environ['JULIANDATE']
print(f'JULIANDATE = {JD}')
utc = Time(JD, format='jd').datetime
print(f'Date = {utc.month}-{utc.day}-{utc.year}')

DATA_PATH = "/mnt/sn1/data1/2460555"
APRIORI_STATUSES = dish_maintenance,dish_ok,RF_maintenance,RF_ok,digital_ok,digital_maintenance,calibration_maintenance,calibration_triage,calibration_ok
JULIANDATE = 2460555
Date = 9-1-2024
In [3]:
# Load in data
HHfiles, difffiles, HHautos, diffautos, uvdx, uvdy = utils.load_data(data_path,JD)
    
uvd = UVData()
unread = True
readInd=0
while unread and readInd<len(HHautos):
    try:
        uvd.read(HHautos[readInd])
        unread = False
    except:
        readInd += 1
        continue
use_ants = utils.get_use_ants(uvd,statuses,JD)
print(f'This day contains {len(use_ants)} antennas of the given status category.')
uvd.read(HHautos[::10], skip_bad_files=True, antenna_nums = use_ants)
lsts = uvd.lst_array
uvdx.select(antenna_nums=use_ants)
uvdy.select(antenna_nums=use_ants)

1571 sum files found between JDs 2460555.16921 and 2460555.52041
1571 diff files found between JDs 2460555.16921 and 2460555.52041
1571 sum auto files found between JDs 2460555.16921 and 2460555.52041
1571 diff auto files found between JDs 2460555.16921 and 2460555.52041

This day contains 252 antennas of the given status category.

Sky Coverage Map¶

Map of the sky (made using the Haslam 408MHz map). The RA/DEC range covered by this night of observation is shaded based on a 12 degree FWHM of the beam. Horizontal dashed lines represent the stripe that HERA can observe, while the shaded region is what was observed on this night. Vertical lines represent the beginning and ending LSTs of this observation. Selected sources are labelled, sources included are those in the GLEAM 4Jy catalog with a flux >10.9 Jy. Note that the map is clipped at the northern horizon.

In [4]:
sources = utils.gather_source_list()
utils.plot_sky_map(uvd,dec_pad=55,ra_pad=55,clip=False,sources=sources)
No description has been provided for this image

LST Coverage¶

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

In [5]:
utils.plot_lst_coverage(uvd)
No description has been provided for this image

Autocorrelations for a single file¶

This plot shows autocorrelations for one timestamp of each antenna that is active and each polarization. For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

In [6]:
### plot autos
utils.plot_autos(uvdx, uvdy)

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[6], line 2
      1 ### plot autos
----> 2 utils.plot_autos(uvdx, uvdy)

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/hera_notebook_templates/utils.py:557, in plot_autos(uvdx, uvdy)
    555 ax.set_xlim(xlim)
    556 ax.set_ylim(ylim)
--> 557 px, = ax.plot(freqs, 10*np.log10(np.abs(uvdx.get_data((a, a))[t_index])), color='r', alpha=0.75, linewidth=1)
    558 py, = ax.plot(freqs, 10*np.log10(np.abs(uvdy.get_data((a, a))[t_index])), color='b', alpha=0.75, linewidth=1)
    559 ax.grid(False, which='both')

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/matplotlib/axes/_axes.py:1724, in Axes.plot(self, scalex, scaley, data, *args, **kwargs)
   1481 """
   1482 Plot y versus x as lines and/or markers.
   1483 
   (...)
   1721 (``'green'``) or hex strings (``'#008000'``).
   1722 """
   1723 kwargs = cbook.normalize_kwargs(kwargs, mlines.Line2D)
-> 1724 lines = [*self._get_lines(self, *args, data=data, **kwargs)]
   1725 for line in lines:
   1726     self.add_line(line)

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/matplotlib/axes/_base.py:303, in _process_plot_var_args.__call__(self, axes, data, *args, **kwargs)
    301     this += args[0],
    302     args = args[1:]
--> 303 yield from self._plot_args(
    304     axes, this, kwargs, ambiguous_fmt_datakey=ambiguous_fmt_datakey)

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/matplotlib/axes/_base.py:499, in _process_plot_var_args._plot_args(self, axes, tup, kwargs, return_kwargs, ambiguous_fmt_datakey)
    496     axes.yaxis.update_units(y)
    498 if x.shape[0] != y.shape[0]:
--> 499     raise ValueError(f"x and y must have same first dimension, but "
    500                      f"have shapes {x.shape} and {y.shape}")
    501 if x.ndim > 2 or y.ndim > 2:
    502     raise ValueError(f"x and y can be no greater than 2D, but have "
    503                      f"shapes {x.shape} and {y.shape}")

ValueError: x and y must have same first dimension, but have shapes (1,) and (1536,)
No description has been provided for this image

Waterfalls of Autocorrelation Amplitudes for each Antenna and Each polarization¶

These plots show autocorrelation waterfalls of each antenna that is active and whose status qualifies for this notebook. For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

In [7]:
utils.plot_wfs(uvd, pol = 0)

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[7], line 1
----> 1 utils.plot_wfs(uvd, pol = 0)

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/hera_notebook_templates/utils.py:583, in plot_wfs(uvd, pol, mean_sub, save, jd, auto_scale, vmin, vmax)
    582 def plot_wfs(uvd, pol, mean_sub=False, save=False, jd='',auto_scale=True,vmin=6.5,vmax=8):
--> 583     amps = np.abs(uvd.data_array[:, :, :, pol].reshape(uvd.Ntimes, uvd.Nants_data, uvd.Nfreqs, 1))
    584     nodes, antDict, inclNodes = generate_nodeDict(uvd)
    585     ants = uvd.get_ants()

IndexError: too many indices for array: array is 3-dimensional, but 4 were indexed
In [8]:
utils.plot_wfs(uvd, pol = 1)

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[8], line 1
----> 1 utils.plot_wfs(uvd, pol = 1)

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/hera_notebook_templates/utils.py:583, in plot_wfs(uvd, pol, mean_sub, save, jd, auto_scale, vmin, vmax)
    582 def plot_wfs(uvd, pol, mean_sub=False, save=False, jd='',auto_scale=True,vmin=6.5,vmax=8):
--> 583     amps = np.abs(uvd.data_array[:, :, :, pol].reshape(uvd.Ntimes, uvd.Nants_data, uvd.Nfreqs, 1))
    584     nodes, antDict, inclNodes = generate_nodeDict(uvd)
    585     ants = uvd.get_ants()

IndexError: too many indices for array: array is 3-dimensional, but 4 were indexed

Correlation Metrics¶

The first plot shows the correlation metric (described below) for a set of baseline types, as calculated at several times throughout the night. It is expected that longer baselines (darker color) will exhibit lower values than the short baselines.

The matrices show the phase correlation between antennas. Using the even and odd visibilities, each pixel is calculated as (even/abs(even)) * (conj(odd)/abs(odd)), and then averaged across time and frequency. If the phases are noise-like, this value will average down to zero. If the antennas are well correlated, the phases should not be noise-like, and this value should average to 1. The lines denoting node boundaries are intended to help confirm that inter-node correlations are functioning - if they aren't, this plot will appear block-diagonal.

This metric has shown to be LST locked - when comparing to other nights, be sure to compare for the same LST. It is expected that some LSTs will look much better or worse than others.

Note: Within each node, the order of antennas is determined by snap, and within that by snap input number.

In [9]:
badAnts = []
badAnts = utils.plotNodeAveragedSummary(uvd,HHfiles,JD,use_ants,mat_pols=['xx','yy','xy','yx'])
No description has been provided for this image
No description has been provided for this image
No description has been provided for this image 
WARNING:matplotlib.legend:No artists with labels found to put in legend.  Note that artists whose label start with an underscore are ignored when legend() is called with no argument.
No description has been provided for this image

Visibility amplitude spectra for a set of redundant baselines, labeled by inter vs. intranode baselines. The red and blue should exhibit the same bandpass shape - if the red are consistently different from the blue, this indicates an issue with internode correlations.

Note: antennas that were identified as bad by the correlation matrix have been removed from this plot.

In [10]:
utils.plotVisibilitySpectra(HHfiles[len(HHfiles)//2+1], JD, use_ants, badAnts=[])

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
Cell In[10], line 1
----> 1 utils.plotVisibilitySpectra(HHfiles[len(HHfiles)//2+1], JD, use_ants, badAnts=[])

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/hera_notebook_templates/utils.py:920, in plotVisibilitySpectra(file, jd, use_ants, badAnts, pols)
    918 if n1 == n2:
    919     if intra is False:
--> 920         axs[j][p].plot(freqs,dat,color='blue',label='intranode')
    921         intra=True
    922     else:

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/matplotlib/axes/_axes.py:1724, in Axes.plot(self, scalex, scaley, data, *args, **kwargs)
   1481 """
   1482 Plot y versus x as lines and/or markers.
   1483 
   (...)
   1721 (``'green'``) or hex strings (``'#008000'``).
   1722 """
   1723 kwargs = cbook.normalize_kwargs(kwargs, mlines.Line2D)
-> 1724 lines = [*self._get_lines(self, *args, data=data, **kwargs)]
   1725 for line in lines:
   1726     self.add_line(line)

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/matplotlib/axes/_base.py:303, in _process_plot_var_args.__call__(self, axes, data, *args, **kwargs)
    301     this += args[0],
    302     args = args[1:]
--> 303 yield from self._plot_args(
    304     axes, this, kwargs, ambiguous_fmt_datakey=ambiguous_fmt_datakey)

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/matplotlib/axes/_base.py:499, in _process_plot_var_args._plot_args(self, axes, tup, kwargs, return_kwargs, ambiguous_fmt_datakey)
    496     axes.yaxis.update_units(y)
    498 if x.shape[0] != y.shape[0]:
--> 499     raise ValueError(f"x and y must have same first dimension, but "
    500                      f"have shapes {x.shape} and {y.shape}")
    501 if x.ndim > 2 or y.ndim > 2:
    502     raise ValueError(f"x and y can be no greater than 2D, but have "
    503                      f"shapes {x.shape} and {y.shape}")

ValueError: x and y must have same first dimension, but have shapes (1,) and (1536,)
<Figure size 640x480 with 0 Axes>
No description has been provided for this image

Even and Odd File Checks¶

A waterfall showing the ratio between the even and odd visibilities. The purpose of this is to highlight xengine failures, which will cause this value to fall to zero or go to infinity. If things are working properly, this value should be stable at 1. The boundaries between different x-engines are shown by the vertical white lines.

In [11]:
if len(HHautos) == len(diffautos):
    uvd_diff = UVData()
    uvd_diff.read(diffautos[::10], skip_bad_files=True, antenna_nums=use_ants)
    rat = utils.plotEvenOddWaterfalls(uvd,uvd_diff)
else:
    uvd_diff = UVData()
    use_diffs = [f for f in diffautos if '%s/zen.%s.%s.sum.autos.uvh5' % (data_path,f.split('.')[1],f.split('.')[2]) in HHautos[::10]]
    uvd_diff.read(use_diffs, skip_bad_files=True, antenna_nums = use_ants)
    uvd_sum = uvd.select(times=np.unique(uvd_diff.time_array),inplace=False)
    rat = utils.plotEvenOddWaterfalls(uvd_sum,uvd_diff)

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
Cell In[11], line 4
      2     uvd_diff = UVData()
      3     uvd_diff.read(diffautos[::10], skip_bad_files=True, antenna_nums=use_ants)
----> 4     rat = utils.plotEvenOddWaterfalls(uvd,uvd_diff)
      5 else:
      6     uvd_diff = UVData()

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/hera_notebook_templates/utils.py:1111, in plotEvenOddWaterfalls(uvd_sum, uvd_diff)
   1109 nants = len(uvd_sum.get_ants())
   1110 freqs = uvd_sum.freq_array[0]*1e-6
-> 1111 nfreqs = len(freqs)
   1112 lsts = np.unique(uvd_sum.lst_array*3.819719)
   1113 sm = np.abs(uvd_sum.data_array[:,0,:,0])

TypeError: object of type 'numpy.float64' has no len()

Crossed Antenna Check¶

These are differences between different panels of the correlation matrices shown above (see panel titles for specifics). Antennas showing as consistently blue are ones which are correlating stronger in the cross pols than in the auto pols, indicating that the antenna polarizations are likely crossed.

In [12]:
crossedAnts = utils.plotNodeAveragedSummary(uvd,HHfiles,JD,use_ants,mat_pols=['xx','yy','xy','yx'],plotRatios=True,
                                       plotSummary=False)
No description has been provided for this image

Antenna Positions¶

Antennas outlined in black here have been identified by the correlation matrix as bad antennas. Antennas with a colorful outline correspond to their status as identified by ant_metrics (see above plot). Faded antennas are those not meeting the apriori status requirement for this notebook run. Gold stars are node box locations.

In [13]:
uvd1 = UVData()
uvd1.read(HHfiles[readInd], skip_bad_files=True)
utils.plot_antenna_positions(uvd1, badAnts=badAnts,use_ants=use_ants)
No description has been provided for this image

Observer Inspection Plots¶

Antennas of status digital_OK or better that are flagged as bad by any of the above metrics are plotted here so observers can inspect their failures in more detail. Additionally, a 'good' template has been used to identify outliers. The upper line plots are averages over the whole observation, and the lower line plots are slices of a single time in the middle of the observation. These plots are recommended diagnostics for demoting antennas to lower statuses or reporting issues. If the plots below look OK, check other plots in notebook to hunt why the antenna was flagged. NOTE: The colorbar/power scales in these plots are NOT locked between antennas OR polarizations so that the detail will be visible on all plots. Be sure to check for reasonable power levels, as this may be the reason the antenna was flagged for inspection.

In [14]:
d, tempAnts = utils.flag_by_template(uvd,HHautos,JD,use_ants=use_ants,pols=['XX','YY'],plotMap=False)

No template for lst=0.01 - skipping
In [15]:
inspectAnts = utils.plot_inspect_ants(uvd,JD,badAnts=badAnts,use_ants=use_ants,
                                     tempAnts=tempAnts,crossedAnts=crossedAnts)

Antennas that require further inspection are:
[ 15  22  27  28  77  86  88  91 105 106 107 170 171 176 177 178 241 242
 243 325 328]

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[15], line 1
----> 1 inspectAnts = utils.plot_inspect_ants(uvd,JD,badAnts=badAnts,use_ants=use_ants,
      2                                      tempAnts=tempAnts,crossedAnts=crossedAnts)

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/hera_notebook_templates/utils.py:408, in plot_inspect_ants(uvd1, jd, badAnts, flaggedAnts, tempAnts, crossedAnts, use_ants)
    405 print(inspectAnts)
    407 for ant in inspectAnts:
--> 408     auto_waterfall_lineplot(uvd1,ant,jd,title=inspectTitles[ant])
    410 return inspectAnts

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/hera_notebook_templates/utils.py:458, in auto_waterfall_lineplot(uv, ant, jd, pols, colorbar_min, colorbar_max, title)
    456 abb = status_abbreviations[status]
    457 waterfall.set_title(f'{pol_dirs[p]} pol')
--> 458 freqs = uv.freq_array[0, :] / 1000000
    459 xticks = np.arange(0, len(freqs), 120)
    460 plt.xticks(xticks, labels =np.around(freqs[xticks],2))

IndexError: too many indices for array: array is 1-dimensional, but 2 were indexed
No description has been provided for this image

Mean-Subtracted Waterfalls¶

Here the mean value in each frequency bin has been subtracted out. This effectively subtracts out the bandpass shape, making time variations more visible.

In [16]:
utils.plot_wfs(uvd,0,mean_sub=True,jd=JD)
utils.plot_wfs(uvd,1,mean_sub=True,jd=JD)

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In[16], line 1
----> 1 utils.plot_wfs(uvd,0,mean_sub=True,jd=JD)
      2 utils.plot_wfs(uvd,1,mean_sub=True,jd=JD)

File ~/mambaforge/envs/RTP/lib/python3.12/site-packages/hera_notebook_templates/utils.py:583, in plot_wfs(uvd, pol, mean_sub, save, jd, auto_scale, vmin, vmax)
    582 def plot_wfs(uvd, pol, mean_sub=False, save=False, jd='',auto_scale=True,vmin=6.5,vmax=8):
--> 583     amps = np.abs(uvd.data_array[:, :, :, pol].reshape(uvd.Ntimes, uvd.Nants_data, uvd.Nfreqs, 1))
    584     nodes, antDict, inclNodes = generate_nodeDict(uvd)
    585     ants = uvd.get_ants()

IndexError: too many indices for array: array is 3-dimensional, but 4 were indexed
In [ ]: