Loading [MathJax]/jax/output/CommonHTML/fonts/TeX/fontdata.js

import os
os.environ['HDF5_USE_FILE_LOCKING'] = 'FALSE'
import h5py
import hdf5plugin
import matplotlib
import matplotlib.pyplot as plt
from matplotlib.ticker import FormatStrFormatter
import matplotlib.patches as mpatches
import matplotlib.gridspec as gridspec
import numpy as np
from pyuvdata import UVCal, UVData
import sys
import glob
import uvtools as uvt
from astropy.time import Time
from astropy.coordinates import EarthLocation, AltAz, Angle
from astropy.coordinates import SkyCoord as sc
import pandas
import warnings 
import copy
from hera_notebook_templates import utils
import hera_qm
from hera_mc import cm_hookup
import importlib
from scipy import stats
from IPython.display import display, HTML
#warnings.filterwarnings('ignore')

%matplotlib inline
%config InlineBackend.figure_format = 'retina'
display(HTML("<style>.container { width:100% !important; }</style>"))

#get data location
data_path = os.environ['DATA_PATH']
print(f'DATA_PATH = "{data_path}"')
statuses = os.environ['APRIORI_STATUSES']
print(f'APRIORI_STATUSES = {statuses}')
JD = os.environ['JULIANDATE']
print(f'JULIANDATE = {JD}')
utc = Time(JD, format='jd').datetime
print(f'Date = {utc.month}-{utc.day}-{utc.year}')

DATA_PATH = "/mnt/sn1/2459927"
APRIORI_STATUSES = dish_maintenance,dish_ok,RF_maintenance,RF_ok,digital_ok,digital_maintenance,calibration_maintenance,calibration_triage,calibration_ok
JULIANDATE = 2459927
Date = 12-13-2022

# Load in data
HHfiles, difffiles, HHautos, diffautos, uvdx, uvdy = utils.load_data(data_path,JD)
    
uvd = UVData()
unread = True
readInd=0
while unread and readInd<len(HHautos):
    try:
        uvd.read(HHautos[readInd])
        unread = False
    except:
        readInd += 1
        continue
use_ants = utils.get_use_ants(uvd,statuses,JD)
print(f'This day contains {len(use_ants)} antennas of the given status category.')
uvd.read(HHautos[::10], skip_bad_files=True, antenna_nums = use_ants)
lsts = uvd.lst_array
uvdx.select(antenna_nums=use_ants)
uvdy.select(antenna_nums=use_ants)

149 sum files found between JDs 2459927.59352 and 2459927.62663
149 diff files found between JDs 2459927.59352 and 2459927.62663
149 sum auto files found between JDs 2459927.59352 and 2459927.62663
149 diff auto files found between JDs 2459927.59352 and 2459927.62663

---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:64, in is_valid_database(base, session)
     63 try:  # This tries thrice with 5sec sleeps in between
---> 64     iengine = inspect(engine)
     65 except OperationalError:  # pragma: no cover

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:70, in is_valid_database(base, session)
     69 try:
---> 70     iengine = inspect(engine)
     71 except OperationalError:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
Cell In [3], line 14
     12         readInd += 1
     13         continue
---> 14 use_ants = utils.get_use_ants(uvd,statuses,JD)
     15 print(f'This day contains {len(use_ants)} antennas of the given status category.')
     16 uvd.read(HHautos[::10], skip_bad_files=True, antenna_nums = use_ants)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:81, in get_use_ants(uvd, statuses, jd)
     79 ants = np.unique(np.concatenate((uvd.ant_1_array, uvd.ant_2_array)))
     80 use_ants = []
---> 81 h = cm_active.get_active(at_date=jd, float_format="jd")
     82 for ant_name in h.apriori:
     83     ant = int("".join(filter(str.isdigit, ant_name)))

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/cm_active.py:41, in get_active(at_date, at_time, float_format, loading, testing)
     13 def get_active(
     14     at_date="now", at_time=None, float_format=None, loading=["apriori"], testing=False
     15 ):
     16     """
     17     Return an ActiveData object with specified loading.
     18 
   (...)
     39         ActiveData objects with loading parameters as specified.
     40     """
---> 41     with mc.MCSessionWrapper(session=None, testing=testing) as session:
     42         active = ActiveData(
     43             session=session, at_date=at_date, at_time=at_time, float_format=float_format
     44         )
     45         for param in loading:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:116, in MCSessionWrapper.__init__(self, session, testing)
    114     db = connect_to_mc_testing_db()
    115 else:
--> 116     db = connect_to_mc_db(None)
    117 self.session = db.sessionmaker()
    118 self.close_when_done = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:288, in connect_to_mc_db(args, forced_db_name, check_connect)
    286     db = DeclarativeDB(db_url)
    287 elif db_mode == "production":
--> 288     db = AutomappedDB(db_url)
    289 else:
    290     raise RuntimeError(
    291         "cannot connect to M&C database: unrecognized mode "
    292         "{0!r} for the DB named {1!r} in {2!r}".format(
    293             db_mode, db_name, config_path
    294         )
    295     )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:92, in AutomappedDB.__init__(self, db_url)
     89 from .db_check import is_valid_database
     91 with self.sessionmaker() as session:
---> 92     if not is_valid_database(MCDeclarativeBase, session):
     93         raise RuntimeError(
     94             "database {0} does not match expected schema".format(db_url)
     95         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:73, in is_valid_database(base, session)
     71     except OperationalError:
     72         time.sleep(5)
---> 73         iengine = inspect(engine)
     75 errors = False
     77 tables = iengine.get_table_names()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     62 if reg is True:
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:
     66     break

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    114     cls = bind.dialect.inspector
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    126 def _init_engine(self, engine):
    127     self.bind = self.engine = engine
--> 128     engine.connect().close()
    129     self._op_context_requires_connect = True
    130     self.dialect = self.engine.dialect

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3300 def connect(self, close_with_result=False):
   3301     """Return a new :class:`_engine.Connection` object.
   3302 
   3303     The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313     """
-> 3315     return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     91     self._has_events = _branch_from._has_events
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None
    100     self.__savepoint_seq = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3372 def raw_connection(self, _connection=None):
   3373     """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375     The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393     """
-> 3394     return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:
-> 3364         Connection._handle_dbapi_exception_noconnection(
   3365             e, dialect, self
   3366         )
   3367     else:
   3368         util.raise_(
   3369             sys.exc_info()[1], with_traceback=sys.exc_info()[2]
   3370         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2196     util.raise_(newraise, with_traceback=exc_info[2], from_=e)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:
   2202     util.raise_(exc_info[1], with_traceback=exc_info[2])

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3359 dialect = self.dialect
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    312 def connect(self):
    313     """Return a DBAPI connection from the pool.
    314 
    315     The connection is instrumented such that when its
   (...)
    318 
    319     """
--> 320     return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    881 @classmethod
    882 def _checkout(cls, pool, threadconns=None, fairy=None):
    883     if not fairy:
--> 884         fairy = _ConnectionRecord.checkout(pool)
    886         fairy._pool = pool
    887         fairy._counter = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:
    488         dbapi_connection = rec.get_connection()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    143         return self._create_connection()
    144     except:
--> 145         with util.safe_reraise():
    146             self._dec_overflow()
    147 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    141 if self._inc_overflow():
    142     try:
--> 143         return self._create_connection()
    144     except:
    145         with util.safe_reraise():

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    263 def _create_connection(self):
    264     """Called by subclasses to create a new ConnectionRecord."""
--> 266     return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    379 self.__pool = pool
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    675     self.fresh = True
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)
    679 else:
    680     # in SQLAlchemy 1.4 the first_connect event is not used by
    681     # the engine, so this will usually not be set

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    671 try:
    672     self.starttime = time.time()
--> 673     self.dbapi_connection = connection = pool._invoke_creator(self)
    674     pool.logger.debug("Created new connection %r", connection)
    675     self.fresh = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    576         if connection is not None:
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    119     kwasync['async_'] = kwargs.pop('async_')
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:
    124     conn.cursor_factory = cursor_factory

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

Sky Coverage Map

Map of the sky (made using the Haslam 408MHz map). The RA/DEC range covered by this night of observation is shaded based on a 12 degree FWHM of the beam. Horizontal dashed lines represent the stripe that HERA can observe, while the shaded region is what was observed on this night. Vertical lines represent the beginning and ending LSTs of this observation. Selected sources are labelled, sources included are those in the GLEAM 4Jy catalog with a flux >10.9 Jy. Note that the map is clipped at the northern horizon.

sources = utils.gather_source_list()
utils.plot_sky_map(uvd,dec_pad=55,ra_pad=55,clip=False,sources=sources)

LST Coverage

Shows the LSTs (in hours) and JDs for which data is collected. Green represents data, red means no data.

utils.plot_lst_coverage(uvd)

---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
Cell In [5], line 1
----> 1 utils.plot_lst_coverage(uvd)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:1064, in plot_lst_coverage(uvd)
   1062 lsts = uvd.lst_array*3.819719
   1063 jds = np.unique(uvd.time_array)
-> 1064 alltimes = np.arange(np.floor(jds[0]),np.ceil(jds[0]),jds[2]-jds[1])
   1065 df = jds[2]-jds[1]
   1066 truetimes = [np.min(np.abs(jds-jd))<=df*0.6 for jd in alltimes]

IndexError: index 2 is out of bounds for axis 0 with size 2

Autocorrelations for a single file

This plot shows autocorrelations for one timestamp of each antenna that is active and each polarization. For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

### plot autos
utils.plot_autos(uvdx, uvdy)

---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:64, in is_valid_database(base, session)
     63 try:  # This tries thrice with 5sec sleeps in between
---> 64     iengine = inspect(engine)
     65 except OperationalError:  # pragma: no cover

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:70, in is_valid_database(base, session)
     69 try:
---> 70     iengine = inspect(engine)
     71 except OperationalError:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
Cell In [6], line 2
      1 ### plot autos
----> 2 utils.plot_autos(uvdx, uvdy)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:515, in plot_autos(uvdx, uvdy)
    514 def plot_autos(uvdx, uvdy):
--> 515     nodes, antDict, inclNodes = generate_nodeDict(uvdx)
    516     ants = uvdx.get_ants()
    517     sorted_ants = sort_antennas(uvdx)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:1741, in generate_nodeDict(uv)
   1722 """
   1723 Generates dictionaries containing node and antenna information.
   1724 
   (...)
   1737     Nodes that have hooked up antennas.
   1738 """
   1740 antnums = uv.get_ants()
-> 1741 x = cm_hookup.get_hookup('default')
   1742 nodes = {}
   1743 antDict = {}

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/cm_hookup.py:69, in get_hookup(hpn, pol, at_date, at_time, float_format, exact_match, hookup_type, testing)
     18 def get_hookup(
     19     hpn,
     20     pol="all",
   (...)
     26     testing=False,
     27 ):
     28     """
     29     Return a single hookup dossier.
     30 
   (...)
     67 
     68     """
---> 69     with mc.MCSessionWrapper(session=None, testing=testing) as session:
     70         hookup = Hookup(session)
     71         return hookup.get_hookup_from_db(
     72             hpn=hpn,
     73             pol=pol,
   (...)
     78             hookup_type=hookup_type,
     79         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:116, in MCSessionWrapper.__init__(self, session, testing)
    114     db = connect_to_mc_testing_db()
    115 else:
--> 116     db = connect_to_mc_db(None)
    117 self.session = db.sessionmaker()
    118 self.close_when_done = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:288, in connect_to_mc_db(args, forced_db_name, check_connect)
    286     db = DeclarativeDB(db_url)
    287 elif db_mode == "production":
--> 288     db = AutomappedDB(db_url)
    289 else:
    290     raise RuntimeError(
    291         "cannot connect to M&C database: unrecognized mode "
    292         "{0!r} for the DB named {1!r} in {2!r}".format(
    293             db_mode, db_name, config_path
    294         )
    295     )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:92, in AutomappedDB.__init__(self, db_url)
     89 from .db_check import is_valid_database
     91 with self.sessionmaker() as session:
---> 92     if not is_valid_database(MCDeclarativeBase, session):
     93         raise RuntimeError(
     94             "database {0} does not match expected schema".format(db_url)
     95         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:73, in is_valid_database(base, session)
     71     except OperationalError:
     72         time.sleep(5)
---> 73         iengine = inspect(engine)
     75 errors = False
     77 tables = iengine.get_table_names()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     62 if reg is True:
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:
     66     break

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    114     cls = bind.dialect.inspector
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    126 def _init_engine(self, engine):
    127     self.bind = self.engine = engine
--> 128     engine.connect().close()
    129     self._op_context_requires_connect = True
    130     self.dialect = self.engine.dialect

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3300 def connect(self, close_with_result=False):
   3301     """Return a new :class:`_engine.Connection` object.
   3302 
   3303     The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313     """
-> 3315     return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     91     self._has_events = _branch_from._has_events
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None
    100     self.__savepoint_seq = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3372 def raw_connection(self, _connection=None):
   3373     """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375     The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393     """
-> 3394     return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:
-> 3364         Connection._handle_dbapi_exception_noconnection(
   3365             e, dialect, self
   3366         )
   3367     else:
   3368         util.raise_(
   3369             sys.exc_info()[1], with_traceback=sys.exc_info()[2]
   3370         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2196     util.raise_(newraise, with_traceback=exc_info[2], from_=e)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:
   2202     util.raise_(exc_info[1], with_traceback=exc_info[2])

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3359 dialect = self.dialect
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    312 def connect(self):
    313     """Return a DBAPI connection from the pool.
    314 
    315     The connection is instrumented such that when its
   (...)
    318 
    319     """
--> 320     return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    881 @classmethod
    882 def _checkout(cls, pool, threadconns=None, fairy=None):
    883     if not fairy:
--> 884         fairy = _ConnectionRecord.checkout(pool)
    886         fairy._pool = pool
    887         fairy._counter = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:
    488         dbapi_connection = rec.get_connection()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    143         return self._create_connection()
    144     except:
--> 145         with util.safe_reraise():
    146             self._dec_overflow()
    147 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    141 if self._inc_overflow():
    142     try:
--> 143         return self._create_connection()
    144     except:
    145         with util.safe_reraise():

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    263 def _create_connection(self):
    264     """Called by subclasses to create a new ConnectionRecord."""
--> 266     return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    379 self.__pool = pool
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    675     self.fresh = True
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)
    679 else:
    680     # in SQLAlchemy 1.4 the first_connect event is not used by
    681     # the engine, so this will usually not be set

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    671 try:
    672     self.starttime = time.time()
--> 673     self.dbapi_connection = connection = pool._invoke_creator(self)
    674     pool.logger.debug("Created new connection %r", connection)
    675     self.fresh = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    576         if connection is not None:
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    119     kwasync['async_'] = kwargs.pop('async_')
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:
    124     conn.cursor_factory = cursor_factory

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

Waterfalls of Autocorrelation Amplitudes for each Antenna and Each polarization

These plots show autocorrelation waterfalls of each antenna that is active and whose status qualifies for this notebook. For each node, antennas are ordered by SNAP number, and within that by SNAP input number. The antenna number label color corresponds to the a priori status of that antenna.

utils.plot_wfs(uvd, pol = 0)

---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:64, in is_valid_database(base, session)
     63 try:  # This tries thrice with 5sec sleeps in between
---> 64     iengine = inspect(engine)
     65 except OperationalError:  # pragma: no cover

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:70, in is_valid_database(base, session)
     69 try:
---> 70     iengine = inspect(engine)
     71 except OperationalError:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
Cell In [7], line 1
----> 1 utils.plot_wfs(uvd, pol = 0)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:584, in plot_wfs(uvd, pol, mean_sub, save, jd, auto_scale, vmin, vmax)
    582 def plot_wfs(uvd, pol, mean_sub=False, save=False, jd='',auto_scale=True,vmin=6.5,vmax=8):
    583     amps = np.abs(uvd.data_array[:, :, :, pol].reshape(uvd.Ntimes, uvd.Nants_data, uvd.Nfreqs, 1))
--> 584     nodes, antDict, inclNodes = generate_nodeDict(uvd)
    585     ants = uvd.get_ants()
    586     sorted_ants = sort_antennas(uvd)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:1741, in generate_nodeDict(uv)
   1722 """
   1723 Generates dictionaries containing node and antenna information.
   1724 
   (...)
   1737     Nodes that have hooked up antennas.
   1738 """
   1740 antnums = uv.get_ants()
-> 1741 x = cm_hookup.get_hookup('default')
   1742 nodes = {}
   1743 antDict = {}

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/cm_hookup.py:69, in get_hookup(hpn, pol, at_date, at_time, float_format, exact_match, hookup_type, testing)
     18 def get_hookup(
     19     hpn,
     20     pol="all",
   (...)
     26     testing=False,
     27 ):
     28     """
     29     Return a single hookup dossier.
     30 
   (...)
     67 
     68     """
---> 69     with mc.MCSessionWrapper(session=None, testing=testing) as session:
     70         hookup = Hookup(session)
     71         return hookup.get_hookup_from_db(
     72             hpn=hpn,
     73             pol=pol,
   (...)
     78             hookup_type=hookup_type,
     79         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:116, in MCSessionWrapper.__init__(self, session, testing)
    114     db = connect_to_mc_testing_db()
    115 else:
--> 116     db = connect_to_mc_db(None)
    117 self.session = db.sessionmaker()
    118 self.close_when_done = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:288, in connect_to_mc_db(args, forced_db_name, check_connect)
    286     db = DeclarativeDB(db_url)
    287 elif db_mode == "production":
--> 288     db = AutomappedDB(db_url)
    289 else:
    290     raise RuntimeError(
    291         "cannot connect to M&C database: unrecognized mode "
    292         "{0!r} for the DB named {1!r} in {2!r}".format(
    293             db_mode, db_name, config_path
    294         )
    295     )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:92, in AutomappedDB.__init__(self, db_url)
     89 from .db_check import is_valid_database
     91 with self.sessionmaker() as session:
---> 92     if not is_valid_database(MCDeclarativeBase, session):
     93         raise RuntimeError(
     94             "database {0} does not match expected schema".format(db_url)
     95         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:73, in is_valid_database(base, session)
     71     except OperationalError:
     72         time.sleep(5)
---> 73         iengine = inspect(engine)
     75 errors = False
     77 tables = iengine.get_table_names()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     62 if reg is True:
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:
     66     break

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    114     cls = bind.dialect.inspector
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    126 def _init_engine(self, engine):
    127     self.bind = self.engine = engine
--> 128     engine.connect().close()
    129     self._op_context_requires_connect = True
    130     self.dialect = self.engine.dialect

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3300 def connect(self, close_with_result=False):
   3301     """Return a new :class:`_engine.Connection` object.
   3302 
   3303     The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313     """
-> 3315     return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     91     self._has_events = _branch_from._has_events
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None
    100     self.__savepoint_seq = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3372 def raw_connection(self, _connection=None):
   3373     """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375     The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393     """
-> 3394     return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:
-> 3364         Connection._handle_dbapi_exception_noconnection(
   3365             e, dialect, self
   3366         )
   3367     else:
   3368         util.raise_(
   3369             sys.exc_info()[1], with_traceback=sys.exc_info()[2]
   3370         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2196     util.raise_(newraise, with_traceback=exc_info[2], from_=e)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:
   2202     util.raise_(exc_info[1], with_traceback=exc_info[2])

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3359 dialect = self.dialect
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    312 def connect(self):
    313     """Return a DBAPI connection from the pool.
    314 
    315     The connection is instrumented such that when its
   (...)
    318 
    319     """
--> 320     return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    881 @classmethod
    882 def _checkout(cls, pool, threadconns=None, fairy=None):
    883     if not fairy:
--> 884         fairy = _ConnectionRecord.checkout(pool)
    886         fairy._pool = pool
    887         fairy._counter = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:
    488         dbapi_connection = rec.get_connection()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    143         return self._create_connection()
    144     except:
--> 145         with util.safe_reraise():
    146             self._dec_overflow()
    147 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    141 if self._inc_overflow():
    142     try:
--> 143         return self._create_connection()
    144     except:
    145         with util.safe_reraise():

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    263 def _create_connection(self):
    264     """Called by subclasses to create a new ConnectionRecord."""
--> 266     return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    379 self.__pool = pool
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    675     self.fresh = True
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)
    679 else:
    680     # in SQLAlchemy 1.4 the first_connect event is not used by
    681     # the engine, so this will usually not be set

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    671 try:
    672     self.starttime = time.time()
--> 673     self.dbapi_connection = connection = pool._invoke_creator(self)
    674     pool.logger.debug("Created new connection %r", connection)
    675     self.fresh = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    576         if connection is not None:
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    119     kwasync['async_'] = kwargs.pop('async_')
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:
    124     conn.cursor_factory = cursor_factory

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

utils.plot_wfs(uvd, pol = 1)

---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:64, in is_valid_database(base, session)
     63 try:  # This tries thrice with 5sec sleeps in between
---> 64     iengine = inspect(engine)
     65 except OperationalError:  # pragma: no cover

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:70, in is_valid_database(base, session)
     69 try:
---> 70     iengine = inspect(engine)
     71 except OperationalError:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
Cell In [8], line 1
----> 1 utils.plot_wfs(uvd, pol = 1)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:584, in plot_wfs(uvd, pol, mean_sub, save, jd, auto_scale, vmin, vmax)
    582 def plot_wfs(uvd, pol, mean_sub=False, save=False, jd='',auto_scale=True,vmin=6.5,vmax=8):
    583     amps = np.abs(uvd.data_array[:, :, :, pol].reshape(uvd.Ntimes, uvd.Nants_data, uvd.Nfreqs, 1))
--> 584     nodes, antDict, inclNodes = generate_nodeDict(uvd)
    585     ants = uvd.get_ants()
    586     sorted_ants = sort_antennas(uvd)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:1741, in generate_nodeDict(uv)
   1722 """
   1723 Generates dictionaries containing node and antenna information.
   1724 
   (...)
   1737     Nodes that have hooked up antennas.
   1738 """
   1740 antnums = uv.get_ants()
-> 1741 x = cm_hookup.get_hookup('default')
   1742 nodes = {}
   1743 antDict = {}

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/cm_hookup.py:69, in get_hookup(hpn, pol, at_date, at_time, float_format, exact_match, hookup_type, testing)
     18 def get_hookup(
     19     hpn,
     20     pol="all",
   (...)
     26     testing=False,
     27 ):
     28     """
     29     Return a single hookup dossier.
     30 
   (...)
     67 
     68     """
---> 69     with mc.MCSessionWrapper(session=None, testing=testing) as session:
     70         hookup = Hookup(session)
     71         return hookup.get_hookup_from_db(
     72             hpn=hpn,
     73             pol=pol,
   (...)
     78             hookup_type=hookup_type,
     79         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:116, in MCSessionWrapper.__init__(self, session, testing)
    114     db = connect_to_mc_testing_db()
    115 else:
--> 116     db = connect_to_mc_db(None)
    117 self.session = db.sessionmaker()
    118 self.close_when_done = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:288, in connect_to_mc_db(args, forced_db_name, check_connect)
    286     db = DeclarativeDB(db_url)
    287 elif db_mode == "production":
--> 288     db = AutomappedDB(db_url)
    289 else:
    290     raise RuntimeError(
    291         "cannot connect to M&C database: unrecognized mode "
    292         "{0!r} for the DB named {1!r} in {2!r}".format(
    293             db_mode, db_name, config_path
    294         )
    295     )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:92, in AutomappedDB.__init__(self, db_url)
     89 from .db_check import is_valid_database
     91 with self.sessionmaker() as session:
---> 92     if not is_valid_database(MCDeclarativeBase, session):
     93         raise RuntimeError(
     94             "database {0} does not match expected schema".format(db_url)
     95         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:73, in is_valid_database(base, session)
     71     except OperationalError:
     72         time.sleep(5)
---> 73         iengine = inspect(engine)
     75 errors = False
     77 tables = iengine.get_table_names()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     62 if reg is True:
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:
     66     break

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    114     cls = bind.dialect.inspector
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    126 def _init_engine(self, engine):
    127     self.bind = self.engine = engine
--> 128     engine.connect().close()
    129     self._op_context_requires_connect = True
    130     self.dialect = self.engine.dialect

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3300 def connect(self, close_with_result=False):
   3301     """Return a new :class:`_engine.Connection` object.
   3302 
   3303     The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313     """
-> 3315     return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     91     self._has_events = _branch_from._has_events
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None
    100     self.__savepoint_seq = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3372 def raw_connection(self, _connection=None):
   3373     """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375     The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393     """
-> 3394     return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:
-> 3364         Connection._handle_dbapi_exception_noconnection(
   3365             e, dialect, self
   3366         )
   3367     else:
   3368         util.raise_(
   3369             sys.exc_info()[1], with_traceback=sys.exc_info()[2]
   3370         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2196     util.raise_(newraise, with_traceback=exc_info[2], from_=e)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:
   2202     util.raise_(exc_info[1], with_traceback=exc_info[2])

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3359 dialect = self.dialect
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    312 def connect(self):
    313     """Return a DBAPI connection from the pool.
    314 
    315     The connection is instrumented such that when its
   (...)
    318 
    319     """
--> 320     return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    881 @classmethod
    882 def _checkout(cls, pool, threadconns=None, fairy=None):
    883     if not fairy:
--> 884         fairy = _ConnectionRecord.checkout(pool)
    886         fairy._pool = pool
    887         fairy._counter = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:
    488         dbapi_connection = rec.get_connection()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    143         return self._create_connection()
    144     except:
--> 145         with util.safe_reraise():
    146             self._dec_overflow()
    147 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    141 if self._inc_overflow():
    142     try:
--> 143         return self._create_connection()
    144     except:
    145         with util.safe_reraise():

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    263 def _create_connection(self):
    264     """Called by subclasses to create a new ConnectionRecord."""
--> 266     return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    379 self.__pool = pool
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    675     self.fresh = True
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)
    679 else:
    680     # in SQLAlchemy 1.4 the first_connect event is not used by
    681     # the engine, so this will usually not be set

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    671 try:
    672     self.starttime = time.time()
--> 673     self.dbapi_connection = connection = pool._invoke_creator(self)
    674     pool.logger.debug("Created new connection %r", connection)
    675     self.fresh = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    576         if connection is not None:
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    119     kwasync['async_'] = kwargs.pop('async_')
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:
    124     conn.cursor_factory = cursor_factory

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

Correlation Metrics

The first plot shows the correlation metric (described below) for a set of baseline types, as calculated at several times throughout the night. It is expected that longer baselines (darker color) will exhibit lower values than the short baselines.

The matrices show the phase correlation between antennas. Using the even and odd visibilities, each pixel is calculated as (even/abs(even)) * (conj(odd)/abs(odd)), and then averaged across time and frequency. If the phases are noise-like, this value will average down to zero. If the antennas are well correlated, the phases should not be noise-like, and this value should average to 1. The lines denoting node boundaries are intended to help confirm that inter-node correlations are functioning - if they aren't, this plot will appear block-diagonal.

This metric has shown to be LST locked - when comparing to other nights, be sure to compare for the same LST. It is expected that some LSTs will look much better or worse than others.

Note: Within each node, the order of antennas is determined by snap, and within that by snap input number.

badAnts = []
badAnts = utils.plotNodeAveragedSummary(uvd,HHfiles,JD,use_ants,mat_pols=['xx','yy','xy','yx'])

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In [9], line 2
      1 badAnts = []
----> 2 badAnts = utils.plotNodeAveragedSummary(uvd,HHfiles,JD,use_ants,mat_pols=['xx','yy','xy','yx'])

NameError: name 'use_ants' is not defined

Visibility amplitude spectra for a set of redundant baselines, labeled by inter vs. intranode baselines. The red and blue should exhibit the same bandpass shape - if the red are consistently different from the blue, this indicates an issue with internode correlations.

Note: antennas that were identified as bad by the correlation matrix have been removed from this plot.

utils.plotVisibilitySpectra(HHfiles[len(HHfiles)//2+1], JD, use_ants, badAnts=[])

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In [10], line 1
----> 1 utils.plotVisibilitySpectra(HHfiles[len(HHfiles)//2+1], JD, use_ants, badAnts=[])

NameError: name 'use_ants' is not defined

Even and Odd File Checks

A waterfall showing the ratio between the even and odd visibilities. The purpose of this is to highlight xengine failures, which will cause this value to fall to zero or go to infinity. If things are working properly, this value should be stable at 1. The boundaries between different x-engines are shown by the vertical white lines.

if len(HHautos) == len(diffautos):
    uvd_diff = UVData()
    uvd_diff.read(diffautos[::10], skip_bad_files=True, antenna_nums=use_ants)
    rat = utils.plotEvenOddWaterfalls(uvd,uvd_diff)
else:
    uvd_diff = UVData()
    use_diffs = [f for f in diffautos if '%s/zen.%s.%s.sum.autos.uvh5' % (data_path,f.split('.')[1],f.split('.')[2]) in HHautos[::10]]
    uvd_diff.read(use_diffs, skip_bad_files=True, antenna_nums = use_ants)
    uvd_sum = uvd.select(times=np.unique(uvd_diff.time_array),inplace=False)
    rat = utils.plotEvenOddWaterfalls(uvd_sum,uvd_diff)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In [11], line 3
      1 if len(HHautos) == len(diffautos):
      2     uvd_diff = UVData()
----> 3     uvd_diff.read(diffautos[::10], skip_bad_files=True, antenna_nums=use_ants)
      4     rat = utils.plotEvenOddWaterfalls(uvd,uvd_diff)
      5 else:

NameError: name 'use_ants' is not defined

Crossed Antenna Check

These are differences between different panels of the correlation matrices shown above (see panel titles for specifics). Antennas showing as consistently blue are ones which are correlating stronger in the cross pols than in the auto pols, indicating that the antenna polarizations are likely crossed.

crossedAnts = utils.plotNodeAveragedSummary(uvd,HHfiles,JD,use_ants,mat_pols=['xx','yy','xy','yx'],plotRatios=True,
                                       plotSummary=False)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In [12], line 1
----> 1 crossedAnts = utils.plotNodeAveragedSummary(uvd,HHfiles,JD,use_ants,mat_pols=['xx','yy','xy','yx'],plotRatios=True,
      2                                        plotSummary=False)

NameError: name 'use_ants' is not defined

Antenna Positions

Antennas outlined in black here have been identified by the correlation matrix as bad antennas. Antennas with a colorful outline correspond to their status as identified by ant_metrics (see above plot). Faded antennas are those not meeting the apriori status requirement for this notebook run. Gold stars are node box locations.

uvd1 = UVData()
uvd1.read(HHfiles[readInd], skip_bad_files=True)
utils.plot_antenna_positions(uvd1, badAnts=badAnts,use_ants=use_ants)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In [13], line 3
      1 uvd1 = UVData()
      2 uvd1.read(HHfiles[readInd], skip_bad_files=True)
----> 3 utils.plot_antenna_positions(uvd1, badAnts=badAnts,use_ants=use_ants)

NameError: name 'use_ants' is not defined

Observer Inspection Plots

Antennas of status digital_OK or better that are flagged as bad by any of the above metrics are plotted here so observers can inspect their failures in more detail. Additionally, a 'good' template has been used to identify outliers. The upper line plots are averages over the whole observation, and the lower line plots are slices of a single time in the middle of the observation. These plots are recommended diagnostics for demoting antennas to lower statuses or reporting issues. If the plots below look OK, check other plots in notebook to hunt why the antenna was flagged. NOTE: The colorbar/power scales in these plots are NOT locked between antennas OR polarizations so that the detail will be visible on all plots. Be sure to check for reasonable power levels, as this may be the reason the antenna was flagged for inspection.

d, tempAnts = utils.flag_by_template(uvd,HHautos,JD,use_ants=use_ants,pols=['XX','YY'],plotMap=False)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In [14], line 1
----> 1 d, tempAnts = utils.flag_by_template(uvd,HHautos,JD,use_ants=use_ants,pols=['XX','YY'],plotMap=False)

NameError: name 'use_ants' is not defined

inspectAnts = utils.plot_inspect_ants(uvd,JD,badAnts=badAnts,use_ants=use_ants,
                                     tempAnts=tempAnts,crossedAnts=crossedAnts)

---------------------------------------------------------------------------
NameError                                 Traceback (most recent call last)
Cell In [15], line 1
----> 1 inspectAnts = utils.plot_inspect_ants(uvd,JD,badAnts=badAnts,use_ants=use_ants,
      2                                      tempAnts=tempAnts,crossedAnts=crossedAnts)

NameError: name 'use_ants' is not defined

Mean-Subtracted Waterfalls

Here the mean value in each frequency bin has been subtracted out. This effectively subtracts out the bandpass shape, making time variations more visible.

utils.plot_wfs(uvd,0,mean_sub=True,jd=JD)
utils.plot_wfs(uvd,1,mean_sub=True,jd=JD)

---------------------------------------------------------------------------
OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:64, in is_valid_database(base, session)
     63 try:  # This tries thrice with 5sec sleeps in between
---> 64     iengine = inspect(engine)
     65 except OperationalError:  # pragma: no cover

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:70, in is_valid_database(base, session)
     69 try:
---> 70     iengine = inspect(engine)
     71 except OperationalError:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    127 self.bind = self.engine = engine
--> 128 engine.connect().close()
    129 self._op_context_requires_connect = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3301 """Return a new :class:`_engine.Connection` object.
   3302 
   3303 The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313 """
-> 3315 return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3373 """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375 The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393 """
-> 3394 return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3363 if connection is None:
-> 3364     Connection._handle_dbapi_exception_noconnection(
   3365         e, dialect, self
   3366     )
   3367 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)

During handling of the above exception, another exception occurred:

OperationalError                          Traceback (most recent call last)
File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    313 """Return a DBAPI connection from the pool.
    314 
    315 The connection is instrumented such that when its
   (...)
    318 
    319 """
--> 320 return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    883 if not fairy:
--> 884     fairy = _ConnectionRecord.checkout(pool)
    886     fairy._pool = pool

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    144 except:
--> 145     with util.safe_reraise():
    146         self._dec_overflow()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    142 try:
--> 143     return self._create_connection()
    144 except:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    264 """Called by subclasses to create a new ConnectionRecord."""
--> 266 return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    672 self.starttime = time.time()
--> 673 self.dbapi_connection = connection = pool._invoke_creator(self)
    674 pool.logger.debug("Created new connection %r", connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:

OperationalError: connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?


The above exception was the direct cause of the following exception:

OperationalError                          Traceback (most recent call last)
Cell In [16], line 1
----> 1 utils.plot_wfs(uvd,0,mean_sub=True,jd=JD)
      2 utils.plot_wfs(uvd,1,mean_sub=True,jd=JD)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:584, in plot_wfs(uvd, pol, mean_sub, save, jd, auto_scale, vmin, vmax)
    582 def plot_wfs(uvd, pol, mean_sub=False, save=False, jd='',auto_scale=True,vmin=6.5,vmax=8):
    583     amps = np.abs(uvd.data_array[:, :, :, pol].reshape(uvd.Ntimes, uvd.Nants_data, uvd.Nfreqs, 1))
--> 584     nodes, antDict, inclNodes = generate_nodeDict(uvd)
    585     ants = uvd.get_ants()
    586     sorted_ants = sort_antennas(uvd)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_notebook_templates/utils.py:1741, in generate_nodeDict(uv)
   1722 """
   1723 Generates dictionaries containing node and antenna information.
   1724 
   (...)
   1737     Nodes that have hooked up antennas.
   1738 """
   1740 antnums = uv.get_ants()
-> 1741 x = cm_hookup.get_hookup('default')
   1742 nodes = {}
   1743 antDict = {}

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/cm_hookup.py:69, in get_hookup(hpn, pol, at_date, at_time, float_format, exact_match, hookup_type, testing)
     18 def get_hookup(
     19     hpn,
     20     pol="all",
   (...)
     26     testing=False,
     27 ):
     28     """
     29     Return a single hookup dossier.
     30 
   (...)
     67 
     68     """
---> 69     with mc.MCSessionWrapper(session=None, testing=testing) as session:
     70         hookup = Hookup(session)
     71         return hookup.get_hookup_from_db(
     72             hpn=hpn,
     73             pol=pol,
   (...)
     78             hookup_type=hookup_type,
     79         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:116, in MCSessionWrapper.__init__(self, session, testing)
    114     db = connect_to_mc_testing_db()
    115 else:
--> 116     db = connect_to_mc_db(None)
    117 self.session = db.sessionmaker()
    118 self.close_when_done = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:288, in connect_to_mc_db(args, forced_db_name, check_connect)
    286     db = DeclarativeDB(db_url)
    287 elif db_mode == "production":
--> 288     db = AutomappedDB(db_url)
    289 else:
    290     raise RuntimeError(
    291         "cannot connect to M&C database: unrecognized mode "
    292         "{0!r} for the DB named {1!r} in {2!r}".format(
    293             db_mode, db_name, config_path
    294         )
    295     )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/mc.py:92, in AutomappedDB.__init__(self, db_url)
     89 from .db_check import is_valid_database
     91 with self.sessionmaker() as session:
---> 92     if not is_valid_database(MCDeclarativeBase, session):
     93         raise RuntimeError(
     94             "database {0} does not match expected schema".format(db_url)
     95         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/hera_mc/db_check.py:73, in is_valid_database(base, session)
     71     except OperationalError:
     72         time.sleep(5)
---> 73         iengine = inspect(engine)
     75 errors = False
     77 tables = iengine.get_table_names()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/inspection.py:64, in inspect(subject, raiseerr)
     62 if reg is True:
     63     return subject
---> 64 ret = reg(subject)
     65 if ret is not None:
     66     break

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:182, in Inspector._engine_insp(bind)
    180 @inspection._inspects(Engine)
    181 def _engine_insp(bind):
--> 182     return Inspector._construct(Inspector._init_engine, bind)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:117, in Inspector._construct(cls, init, bind)
    114     cls = bind.dialect.inspector
    116 self = cls.__new__(cls)
--> 117 init(self, bind)
    118 return self

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/reflection.py:128, in Inspector._init_engine(self, engine)
    126 def _init_engine(self, engine):
    127     self.bind = self.engine = engine
--> 128     engine.connect().close()
    129     self._op_context_requires_connect = True
    130     self.dialect = self.engine.dialect

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3315, in Engine.connect(self, close_with_result)
   3300 def connect(self, close_with_result=False):
   3301     """Return a new :class:`_engine.Connection` object.
   3302 
   3303     The :class:`_engine.Connection` object is a facade that uses a DBAPI
   (...)
   3312 
   3313     """
-> 3315     return self._connection_cls(self, close_with_result=close_with_result)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:96, in Connection.__init__(self, engine, connection, close_with_result, _branch_from, _execution_options, _dispatch, _has_events, _allow_revalidate)
     91     self._has_events = _branch_from._has_events
     92 else:
     93     self._dbapi_connection = (
     94         connection
     95         if connection is not None
---> 96         else engine.raw_connection()
     97     )
     99     self._transaction = self._nested_transaction = None
    100     self.__savepoint_seq = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3394, in Engine.raw_connection(self, _connection)
   3372 def raw_connection(self, _connection=None):
   3373     """Return a "raw" DBAPI connection from the connection pool.
   3374 
   3375     The returned object is a proxied version of the DBAPI
   (...)
   3392 
   3393     """
-> 3394     return self._wrap_pool_connect(self.pool.connect, _connection)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3364, in Engine._wrap_pool_connect(self, fn, connection)
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:
-> 3364         Connection._handle_dbapi_exception_noconnection(
   3365             e, dialect, self
   3366         )
   3367     else:
   3368         util.raise_(
   3369             sys.exc_info()[1], with_traceback=sys.exc_info()[2]
   3370         )

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:2198, in Connection._handle_dbapi_exception_noconnection(cls, e, dialect, engine)
   2196     util.raise_(newraise, with_traceback=exc_info[2], from_=e)
   2197 elif should_wrap:
-> 2198     util.raise_(
   2199         sqlalchemy_exception, with_traceback=exc_info[2], from_=e
   2200     )
   2201 else:
   2202     util.raise_(exc_info[1], with_traceback=exc_info[2])

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/base.py:3361, in Engine._wrap_pool_connect(self, fn, connection)
   3359 dialect = self.dialect
   3360 try:
-> 3361     return fn()
   3362 except dialect.dbapi.Error as e:
   3363     if connection is None:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:320, in Pool.connect(self)
    312 def connect(self):
    313     """Return a DBAPI connection from the pool.
    314 
    315     The connection is instrumented such that when its
   (...)
    318 
    319     """
--> 320     return _ConnectionFairy._checkout(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:884, in _ConnectionFairy._checkout(cls, pool, threadconns, fairy)
    881 @classmethod
    882 def _checkout(cls, pool, threadconns=None, fairy=None):
    883     if not fairy:
--> 884         fairy = _ConnectionRecord.checkout(pool)
    886         fairy._pool = pool
    887         fairy._counter = 0

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:486, in _ConnectionRecord.checkout(cls, pool)
    484 @classmethod
    485 def checkout(cls, pool):
--> 486     rec = pool._do_get()
    487     try:
    488         dbapi_connection = rec.get_connection()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:145, in QueuePool._do_get(self)
    143         return self._create_connection()
    144     except:
--> 145         with util.safe_reraise():
    146             self._dec_overflow()
    147 else:

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/impl.py:143, in QueuePool._do_get(self)
    141 if self._inc_overflow():
    142     try:
--> 143         return self._create_connection()
    144     except:
    145         with util.safe_reraise():

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:266, in Pool._create_connection(self)
    263 def _create_connection(self):
    264     """Called by subclasses to create a new ConnectionRecord."""
--> 266     return _ConnectionRecord(self)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:381, in _ConnectionRecord.__init__(self, pool, connect)
    379 self.__pool = pool
    380 if connect:
--> 381     self.__connect()
    382 self.finalize_callback = deque()

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:677, in _ConnectionRecord.__connect(self)
    675     self.fresh = True
    676 except Exception as e:
--> 677     with util.safe_reraise():
    678         pool.logger.debug("Error on connect(): %s", e)
    679 else:
    680     # in SQLAlchemy 1.4 the first_connect event is not used by
    681     # the engine, so this will usually not be set

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/langhelpers.py:70, in safe_reraise.__exit__(self, type_, value, traceback)
     68     self._exc_info = None  # remove potential circular references
     69     if not self.warn_only:
---> 70         compat.raise_(
     71             exc_value,
     72             with_traceback=exc_tb,
     73         )
     74 else:
     75     if not compat.py3k and self._exc_info and self._exc_info[1]:
     76         # emulate Py3K's behavior of telling us when an exception
     77         # occurs in an exception handler.

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/util/compat.py:208, in raise_(***failed resolving arguments***)
    205     exception.__cause__ = replace_context
    207 try:
--> 208     raise exception
    209 finally:
    210     # credit to
    211     # https://cosmicpercolator.com/2016/01/13/exception-leaks-in-python-2-and-3/
    212     # as the __traceback__ object creates a cycle
    213     del exception, replace_context, from_, with_traceback

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/pool/base.py:673, in _ConnectionRecord.__connect(self)
    671 try:
    672     self.starttime = time.time()
--> 673     self.dbapi_connection = connection = pool._invoke_creator(self)
    674     pool.logger.debug("Created new connection %r", connection)
    675     self.fresh = True

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/create.py:578, in create_engine.<locals>.connect(connection_record)
    576         if connection is not None:
    577             return connection
--> 578 return dialect.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/sqlalchemy/engine/default.py:598, in DefaultDialect.connect(self, *cargs, **cparams)
    596 def connect(self, *cargs, **cparams):
    597     # inherits the docstring from interfaces.Dialect.connect
--> 598     return self.dbapi.connect(*cargs, **cparams)

File ~/mambaforge/envs/RTP/lib/python3.10/site-packages/psycopg2/__init__.py:122, in connect(dsn, connection_factory, cursor_factory, **kwargs)
    119     kwasync['async_'] = kwargs.pop('async_')
    121 dsn = _ext.make_dsn(dsn, **kwargs)
--> 122 conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
    123 if cursor_factory is not None:
    124     conn.cursor_factory = cursor_factory

OperationalError: (psycopg2.OperationalError) connection to server at "qmaster" (10.80.10.1), port 5432 failed: Connection refused
	Is the server running on that host and accepting TCP/IP connections?

(Background on this error at: https://sqlalche.me/e/14/e3q8)