{
"cells": [
{
"cell_type": "markdown",
"id": "504e14fa",
"metadata": {},
"source": [
"# File Inspection (Experimental)\n",
"\n",
"**by Josh Dillon & Aaron Parsons**, last updated August 8, 2022"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "668f7418",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T19:44:38.539183Z",
"start_time": "2022-08-08T19:44:38.528103Z"
},
"execution": {
"iopub.execute_input": "2022-08-14T19:28:28.092208Z",
"iopub.status.busy": "2022-08-14T19:28:28.090670Z",
"iopub.status.idle": "2022-08-14T19:28:31.504987Z",
"shell.execute_reply": "2022-08-14T19:28:31.504209Z"
}
},
"outputs": [
{
"data": {
"text/html": [
""
],
"text/plain": [
""
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"import numpy as np\n",
"import os\n",
"from hera_qm import ant_metrics, ant_class\n",
"from hera_cal import io, utils\n",
"from IPython.display import display, HTML\n",
"display(HTML(\"\"))"
]
},
{
"cell_type": "markdown",
"id": "ab0f8167",
"metadata": {},
"source": [
"## Parse inputs\n",
"\n",
"To use this notebook interactively, you will have to provide a sum filename path if none exists as an environment variable. All other parameters have reasonable default values.\n"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "4b88cc02",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T19:45:12.866506Z",
"start_time": "2022-08-08T19:45:12.853701Z"
},
"execution": {
"iopub.execute_input": "2022-08-14T19:28:31.511055Z",
"iopub.status.busy": "2022-08-14T19:28:31.509652Z",
"iopub.status.idle": "2022-08-14T19:28:31.518436Z",
"shell.execute_reply": "2022-08-14T19:28:31.517694Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"SUM_FILE = '/mnt/sn1/2459803/zen.2459803.25316.sum.uvh5'\n"
]
}
],
"source": [
"# get file names\n",
"SUM_FILE = os.environ.get(\"SUM_FILE\", None)\n",
"# SUM_FILE = '/mnt/sn1/zen.2459797.30001.sum.uvh5' # If sum_file is not defined in the environment variables, define it here.\n",
"DIFF_FILE = SUM_FILE.replace('sum', 'diff')\n",
"print(f\"SUM_FILE = '{SUM_FILE}'\")"
]
},
{
"cell_type": "markdown",
"id": "5596c041",
"metadata": {},
"source": [
"### Parse Bounds"
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "9ebbfd85",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T19:47:02.245642Z",
"start_time": "2022-08-08T19:47:02.224052Z"
},
"execution": {
"iopub.execute_input": "2022-08-14T19:28:31.524104Z",
"iopub.status.busy": "2022-08-14T19:28:31.522693Z",
"iopub.status.idle": "2022-08-14T19:28:31.541380Z",
"shell.execute_reply": "2022-08-14T19:28:31.540636Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"am_corr_bad = (0, 0.2)\n",
"am_corr_suspect = (0.2, 0.4)\n",
"am_xpol_bad = (-1, -0.1)\n",
"am_xpol_suspect = (-0.1, 0.0)\n",
"auto_power_good = (5.0, 30.0)\n",
"auto_power_suspect = (1.0, 80.0)\n",
"auto_slope_good = (-0.2, 0.2)\n",
"auto_slope_suspect = (-0.4, 0.4)\n"
]
}
],
"source": [
"# ant_metrics bounds for low correlation / dead antennas\n",
"AM_CORR_BAD = float(os.environ.get(\"AM_CORR_BAD\", 0.2))\n",
"AM_CORR_SUSPECT = float(os.environ.get(\"AM_CORR_SUSPECT\", 0.4))\n",
"am_corr_bad = (0, AM_CORR_BAD)\n",
"am_corr_suspect = (AM_CORR_BAD, AM_CORR_SUSPECT)\n",
"\n",
"# ant_metrics bounds for cross-polarized antennas\n",
"AM_XPOL_BAD = float(os.environ.get(\"AM_XPOL_BAD\", -0.1))\n",
"AM_XPOL_SUSPECT = float(os.environ.get(\"AM_XPOL_SUSPECT\", 0))\n",
"am_xpol_bad = (-1, AM_XPOL_BAD)\n",
"am_xpol_suspect = (AM_XPOL_BAD, AM_XPOL_SUSPECT)\n",
"\n",
"# bounds on autocorrelation power\n",
"AUTO_POWER_GOOD_LOW = float(os.environ.get(\"AUTO_POWER_GOOD_LOW\", 5))\n",
"AUTO_POWER_GOOD_HIGH = float(os.environ.get(\"AUTO_POWER_GOOD_HIGH\", 30))\n",
"auto_power_good = (AUTO_POWER_GOOD_LOW, AUTO_POWER_GOOD_HIGH)\n",
"AUTO_POWER_SUSPECT_LOW = float(os.environ.get(\"AUTO_POWER_SUSPECT_LOW\", 1))\n",
"AUTO_POWER_SUSPECT_HIGH = float(os.environ.get(\"AUTO_POWER_SUSPECT_HIGH\", 80))\n",
"auto_power_suspect = (AUTO_POWER_SUSPECT_LOW, AUTO_POWER_SUSPECT_HIGH)\n",
"\n",
"# bounds on autocorrelation slope\n",
"AUTO_SLOPE_GOOD_LOW = float(os.environ.get(\"AUTO_SLOPE_GOOD_LOW\", -0.2))\n",
"AUTO_SLOPE_GOOD_HIGH = float(os.environ.get(\"AUTO_SLOPE_GOOD_HIGH\", 0.2))\n",
"auto_slope_good = (AUTO_SLOPE_GOOD_LOW, AUTO_SLOPE_GOOD_HIGH)\n",
"AUTO_SLOPE_SUSPECT_LOW = float(os.environ.get(\"AUTO_SLOPE_SUSPECT_LOW\", -0.4))\n",
"AUTO_SLOPE_SUSPECT_HIGH = float(os.environ.get(\"AUTO_SLOPE_SUSPECT_HIGH\", 0.4))\n",
"auto_slope_suspect = (AUTO_SLOPE_SUSPECT_LOW, AUTO_SLOPE_SUSPECT_HIGH)\n",
"\n",
"for bound in ['am_corr_bad', 'am_corr_suspect', 'am_xpol_bad', 'am_xpol_suspect', \n",
" 'auto_power_good', 'auto_power_suspect', 'auto_slope_good', 'auto_slope_suspect']:\n",
" print(f'{bound} = {eval(bound)}')"
]
},
{
"cell_type": "markdown",
"id": "105dc915",
"metadata": {},
"source": [
"## Load sum and diff data"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "e5d3725b",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T18:00:29.144034Z",
"start_time": "2022-08-08T18:00:26.045578Z"
},
"execution": {
"iopub.execute_input": "2022-08-14T19:28:31.546937Z",
"iopub.status.busy": "2022-08-14T19:28:31.545543Z",
"iopub.status.idle": "2022-08-14T19:28:50.662153Z",
"shell.execute_reply": "2022-08-14T19:28:50.661316Z"
}
},
"outputs": [],
"source": [
"hd = io.HERADataFastReader(SUM_FILE)\n",
"data, _, _ = hd.read(read_flags=False, read_nsamples=False)\n",
"hd_diff = io.HERADataFastReader(DIFF_FILE)\n",
"diff_data, _, _ = hd_diff.read(read_flags=False, read_nsamples=False)"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "61f26bc2",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T18:00:29.170365Z",
"start_time": "2022-08-08T18:00:29.145133Z"
},
"execution": {
"iopub.execute_input": "2022-08-14T19:28:50.668214Z",
"iopub.status.busy": "2022-08-14T19:28:50.666794Z",
"iopub.status.idle": "2022-08-14T19:28:50.773217Z",
"shell.execute_reply": "2022-08-14T19:28:50.772427Z"
}
},
"outputs": [],
"source": [
"ants = sorted(set([ant for bl in hd.bls for ant in utils.split_bl(bl)]))\n",
"auto_bls = [bl for bl in data if (bl[0] == bl[1]) and (utils.split_pol(bl[2])[0] == utils.split_pol(bl[2])[1])]"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "e945a0ec",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T19:55:27.745442Z",
"start_time": "2022-08-08T19:55:27.735910Z"
},
"execution": {
"iopub.execute_input": "2022-08-14T19:28:50.779149Z",
"iopub.status.busy": "2022-08-14T19:28:50.777756Z",
"iopub.status.idle": "2022-08-14T19:28:50.789030Z",
"shell.execute_reply": "2022-08-14T19:28:50.788291Z"
},
"scrolled": true
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"File: /mnt/sn1/2459803/zen.2459803.25316.sum.uvh5\n",
"JDs: [2459803.25310807 2459803.25321992] (9.66368 s integrations)\n",
"LSTS: [16.84868847 16.85138018] hours\n",
"Frequencies: 1536 0.12207 MHz channels from 46.92078 to 234.29871 MHz\n",
"Antennas: 147\n",
"Polarizations: ['nn', 'ee', 'ne', 'en']\n"
]
}
],
"source": [
"print(f'File: {SUM_FILE}')\n",
"print(f'JDs: {hd.times} ({np.median(np.diff(hd.times)) * 24 * 3600:.5f} s integrations)')\n",
"print(f'LSTS: {hd.lsts * 12 / np.pi } hours')\n",
"print(f'Frequencies: {len(hd.freqs)} {np.median(np.diff(hd.freqs)) / 1e6:.5f} MHz channels from {hd.freqs[0] / 1e6:.5f} to {hd.freqs[-1] / 1e6:.5f} MHz')\n",
"print(f'Antennas: {len(hd.data_ants)}')\n",
"print(f'Polarizations: {hd.pols}')"
]
},
{
"cell_type": "markdown",
"id": "f0450fb4",
"metadata": {},
"source": [
"## Run `ant_metrics`"
]
},
{
"cell_type": "code",
"execution_count": 7,
"id": "4bbb0ac4",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T19:55:44.190609Z",
"start_time": "2022-08-08T19:55:37.646653Z"
},
"execution": {
"iopub.execute_input": "2022-08-14T19:28:50.794595Z",
"iopub.status.busy": "2022-08-14T19:28:50.793205Z",
"iopub.status.idle": "2022-08-14T19:29:26.608874Z",
"shell.execute_reply": "2022-08-14T19:29:26.607967Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:129: RuntimeWarning: invalid value encountered in divide\n",
" odd /= np.abs(odd)\n",
"/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:129: RuntimeWarning: invalid value encountered in divide\n",
" odd /= np.abs(odd)\n",
"/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:128: RuntimeWarning: invalid value encountered in divide\n",
" even /= np.abs(even)\n",
"/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:128: RuntimeWarning: invalid value encountered in divide\n",
" even /= np.abs(even)\n",
"/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:128: RuntimeWarning: invalid value encountered in divide\n",
" even /= np.abs(even)\n",
"/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:129: RuntimeWarning: invalid value encountered in divide\n",
" odd /= np.abs(odd)\n",
"/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:129: RuntimeWarning: invalid value encountered in divide\n",
" odd /= np.abs(odd)\n",
"/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:128: RuntimeWarning: invalid value encountered in divide\n",
" even /= np.abs(even)\n"
]
}
],
"source": [
"am = ant_metrics.AntennaMetrics(SUM_FILE, DIFF_FILE, sum_data=data, diff_data=diff_data)\n",
"am.iterative_antenna_metrics_and_flagging(crossCut=am_xpol_bad[1], deadCut=am_corr_bad[1])"
]
},
{
"cell_type": "code",
"execution_count": 8,
"id": "d8e52c30",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T19:55:44.200663Z",
"start_time": "2022-08-08T19:55:44.192150Z"
},
"code_folding": [],
"execution": {
"iopub.execute_input": "2022-08-14T19:29:26.615257Z",
"iopub.status.busy": "2022-08-14T19:29:26.613842Z",
"iopub.status.idle": "2022-08-14T19:29:26.642864Z",
"shell.execute_reply": "2022-08-14T19:29:26.642132Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"ant_metrics Classification:\n",
"\n",
" Jee:\n",
"----------\n",
"good (90 antpols):\n",
"7, 8, 9, 10, 15, 16, 17, 18, 19, 20, 21, 28, 29, 30, 31, 32, 33, 36, 37, 38, 40, 41, 42, 45, 46, 50, 51, 52, 53, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 81, 82, 83, 84, 85, 86, 87, 94, 98, 99, 100, 101, 102, 103, 104, 109, 110, 111, 112, 116, 117, 118, 119, 120, 121, 122, 123, 127, 128, 129, 130, 135, 136, 141, 161, 162, 167, 168, 169, 170, 179, 180, 182, 183, 189, 190, 191, 321, 323, 324\n",
"\n",
"suspect (4 antpols):\n",
"92, 142, 329, 333\n",
"\n",
"bad (53 antpols):\n",
"3, 4, 5, 27, 57, 73, 88, 90, 91, 93, 105, 106, 107, 108, 125, 126, 137, 138, 140, 143, 144, 145, 150, 155, 156, 157, 158, 160, 163, 164, 165, 166, 176, 177, 178, 181, 184, 185, 186, 187, 203, 205, 206, 207, 220, 221, 222, 223, 224, 241, 242, 243, 320\n",
"\n",
"\n",
"Jnn:\n",
"----------\n",
"good (79 antpols):\n",
"7, 8, 9, 10, 15, 16, 17, 19, 20, 21, 29, 30, 31, 32, 36, 37, 38, 40, 41, 42, 45, 46, 50, 51, 52, 53, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 73, 81, 82, 83, 84, 85, 86, 87, 94, 98, 99, 100, 101, 102, 103, 104, 109, 110, 111, 112, 116, 117, 119, 121, 122, 123, 127, 128, 129, 130, 138, 141, 161, 162, 167, 168, 169, 170, 179, 183, 189, 191\n",
"\n",
"suspect (14 antpols):\n",
"18, 28, 33, 57, 92, 135, 136, 180, 181, 321, 323, 324, 329, 333\n",
"\n",
"bad (54 antpols):\n",
"3, 4, 5, 27, 88, 90, 91, 93, 105, 106, 107, 108, 118, 120, 125, 126, 137, 140, 142, 143, 144, 145, 150, 155, 156, 157, 158, 160, 163, 164, 165, 166, 176, 177, 178, 182, 184, 185, 186, 187, 190, 203, 205, 206, 207, 220, 221, 222, 223, 224, 241, 242, 243, 320\n"
]
}
],
"source": [
"totally_dead_ants = [ant for ant, i in am.removal_iteration.items() if i == -1]\n",
"am_totally_dead = ant_class.AntennaClassification(good=[ant for ant in ants if ant not in totally_dead_ants], bad=totally_dead_ants)\n",
"am_corr = ant_class.antenna_bounds_checker(am.final_metrics['corr'], bad=[am_corr_bad], suspect=[am_corr_suspect], good=[(0, 1)])\n",
"am_xpol = ant_class.antenna_bounds_checker(am.final_metrics['corrXPol'], bad=[am_xpol_bad], suspect=[am_xpol_suspect], good=[(-1, 1)])\n",
"ant_metrics_class = am_totally_dead + am_corr + am_xpol\n",
"print('ant_metrics Classification:\\n\\n', ant_metrics_class)"
]
},
{
"cell_type": "markdown",
"id": "102587ce",
"metadata": {},
"source": [
"## Examine and classify autocorrelations"
]
},
{
"cell_type": "code",
"execution_count": 9,
"id": "ae2e2234",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T19:55:44.388506Z",
"start_time": "2022-08-08T19:55:44.202111Z"
},
"execution": {
"iopub.execute_input": "2022-08-14T19:29:26.648465Z",
"iopub.status.busy": "2022-08-14T19:29:26.647052Z",
"iopub.status.idle": "2022-08-14T19:29:27.022795Z",
"shell.execute_reply": "2022-08-14T19:29:27.022048Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"Autocorrelation Classification:\n",
"\n",
" Jee:\n",
"----------\n",
"good (80 antpols):\n",
"7, 9, 10, 15, 16, 17, 18, 19, 20, 21, 28, 29, 30, 31, 33, 37, 38, 40, 41, 42, 45, 46, 50, 51, 53, 54, 55, 56, 65, 66, 67, 68, 69, 71, 72, 81, 82, 83, 85, 86, 93, 94, 98, 100, 103, 109, 110, 111, 112, 116, 117, 118, 120, 121, 127, 128, 129, 130, 135, 136, 141, 143, 144, 161, 162, 163, 164, 165, 179, 180, 183, 184, 185, 186, 187, 189, 191, 206, 329, 333\n",
"\n",
"suspect (24 antpols):\n",
"8, 36, 52, 70, 84, 87, 99, 101, 102, 104, 119, 122, 123, 166, 168, 169, 170, 182, 205, 207, 223, 321, 323, 324\n",
"\n",
"bad (43 antpols):\n",
"3, 4, 5, 27, 32, 57, 73, 88, 90, 91, 92, 105, 106, 107, 108, 125, 126, 137, 138, 140, 142, 145, 150, 155, 156, 157, 158, 160, 167, 176, 177, 178, 181, 190, 203, 220, 221, 222, 224, 241, 242, 243, 320\n",
"\n",
"\n",
"Jnn:\n",
"----------\n",
"good (75 antpols):\n",
"7, 9, 10, 15, 16, 17, 18, 19, 20, 21, 29, 30, 31, 33, 37, 38, 40, 41, 42, 45, 46, 51, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 73, 81, 82, 83, 86, 93, 94, 99, 100, 102, 109, 110, 111, 112, 116, 117, 119, 127, 128, 129, 130, 135, 136, 138, 141, 143, 144, 162, 163, 164, 165, 166, 179, 183, 184, 185, 186, 187, 189, 191, 205, 223\n",
"\n",
"suspect (24 antpols):\n",
"8, 36, 50, 52, 53, 57, 84, 85, 98, 101, 121, 122, 123, 167, 168, 169, 170, 206, 207, 321, 323, 324, 329, 333\n",
"\n",
"bad (48 antpols):\n",
"3, 4, 5, 27, 28, 32, 87, 88, 90, 91, 92, 103, 104, 105, 106, 107, 108, 118, 120, 125, 126, 137, 140, 142, 145, 150, 155, 156, 157, 158, 160, 161, 176, 177, 178, 180, 181, 182, 190, 203, 220, 221, 222, 224, 241, 242, 243, 320\n"
]
}
],
"source": [
"auto_power_class = ant_class.auto_power_checker(data, good=auto_power_good, suspect=auto_power_suspect)\n",
"auto_slope_class = ant_class.auto_slope_checker(data, good=auto_slope_good, suspect=auto_slope_suspect, edge_cut=100, filt_size=17)\n",
"auto_class = auto_power_class + auto_slope_class\n",
"print('Autocorrelation Classification:\\n\\n', auto_class)"
]
},
{
"cell_type": "markdown",
"id": "f52a30f9",
"metadata": {},
"source": [
"## Metadata"
]
},
{
"cell_type": "code",
"execution_count": 10,
"id": "78d18d26",
"metadata": {
"ExecuteTime": {
"end_time": "2022-08-08T19:55:44.392152Z",
"start_time": "2022-08-08T19:55:44.390236Z"
},
"execution": {
"iopub.execute_input": "2022-08-14T19:29:27.028517Z",
"iopub.status.busy": "2022-08-14T19:29:27.027101Z",
"iopub.status.idle": "2022-08-14T19:29:27.035794Z",
"shell.execute_reply": "2022-08-14T19:29:27.035062Z"
}
},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"hera_cal: 3.1.4.dev3+g68bd8c3\n",
"hera_qm: 2.0.3.dev44+g7d4aa18\n"
]
}
],
"source": [
"from hera_cal import __version__\n",
"print('hera_cal:', __version__)\n",
"from hera_qm import __version__\n",
"print('hera_qm:', __version__)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "581b70cd",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.9.7"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": true,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {
"height": "calc(100% - 180px)",
"left": "10px",
"top": "150px",
"width": "248.391px"
},
"toc_section_display": true,
"toc_window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 5
}