{ "cells": [ { "cell_type": "markdown", "id": "504e14fa", "metadata": {}, "source": [ "# File Inspection (Experimental)\n", "\n", "**by Josh Dillon & Aaron Parsons**, last updated August 8, 2022" ] }, { "cell_type": "code", "execution_count": 1, "id": "668f7418", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T19:44:38.539183Z", "start_time": "2022-08-08T19:44:38.528103Z" }, "execution": { "iopub.execute_input": "2022-08-14T19:28:28.092208Z", "iopub.status.busy": "2022-08-14T19:28:28.090670Z", "iopub.status.idle": "2022-08-14T19:28:31.504987Z", "shell.execute_reply": "2022-08-14T19:28:31.504209Z" } }, "outputs": [ { "data": { "text/html": [ "" ], "text/plain": [ "" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import numpy as np\n", "import os\n", "from hera_qm import ant_metrics, ant_class\n", "from hera_cal import io, utils\n", "from IPython.display import display, HTML\n", "display(HTML(\"\"))" ] }, { "cell_type": "markdown", "id": "ab0f8167", "metadata": {}, "source": [ "## Parse inputs\n", "\n", "To use this notebook interactively, you will have to provide a sum filename path if none exists as an environment variable. All other parameters have reasonable default values.\n" ] }, { "cell_type": "code", "execution_count": 2, "id": "4b88cc02", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T19:45:12.866506Z", "start_time": "2022-08-08T19:45:12.853701Z" }, "execution": { "iopub.execute_input": "2022-08-14T19:28:31.511055Z", "iopub.status.busy": "2022-08-14T19:28:31.509652Z", "iopub.status.idle": "2022-08-14T19:28:31.518436Z", "shell.execute_reply": "2022-08-14T19:28:31.517694Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "SUM_FILE = '/mnt/sn1/2459803/zen.2459803.25316.sum.uvh5'\n" ] } ], "source": [ "# get file names\n", "SUM_FILE = os.environ.get(\"SUM_FILE\", None)\n", "# SUM_FILE = '/mnt/sn1/zen.2459797.30001.sum.uvh5' # If sum_file is not defined in the environment variables, define it here.\n", "DIFF_FILE = SUM_FILE.replace('sum', 'diff')\n", "print(f\"SUM_FILE = '{SUM_FILE}'\")" ] }, { "cell_type": "markdown", "id": "5596c041", "metadata": {}, "source": [ "### Parse Bounds" ] }, { "cell_type": "code", "execution_count": 3, "id": "9ebbfd85", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T19:47:02.245642Z", "start_time": "2022-08-08T19:47:02.224052Z" }, "execution": { "iopub.execute_input": "2022-08-14T19:28:31.524104Z", "iopub.status.busy": "2022-08-14T19:28:31.522693Z", "iopub.status.idle": "2022-08-14T19:28:31.541380Z", "shell.execute_reply": "2022-08-14T19:28:31.540636Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "am_corr_bad = (0, 0.2)\n", "am_corr_suspect = (0.2, 0.4)\n", "am_xpol_bad = (-1, -0.1)\n", "am_xpol_suspect = (-0.1, 0.0)\n", "auto_power_good = (5.0, 30.0)\n", "auto_power_suspect = (1.0, 80.0)\n", "auto_slope_good = (-0.2, 0.2)\n", "auto_slope_suspect = (-0.4, 0.4)\n" ] } ], "source": [ "# ant_metrics bounds for low correlation / dead antennas\n", "AM_CORR_BAD = float(os.environ.get(\"AM_CORR_BAD\", 0.2))\n", "AM_CORR_SUSPECT = float(os.environ.get(\"AM_CORR_SUSPECT\", 0.4))\n", "am_corr_bad = (0, AM_CORR_BAD)\n", "am_corr_suspect = (AM_CORR_BAD, AM_CORR_SUSPECT)\n", "\n", "# ant_metrics bounds for cross-polarized antennas\n", "AM_XPOL_BAD = float(os.environ.get(\"AM_XPOL_BAD\", -0.1))\n", "AM_XPOL_SUSPECT = float(os.environ.get(\"AM_XPOL_SUSPECT\", 0))\n", "am_xpol_bad = (-1, AM_XPOL_BAD)\n", "am_xpol_suspect = (AM_XPOL_BAD, AM_XPOL_SUSPECT)\n", "\n", "# bounds on autocorrelation power\n", "AUTO_POWER_GOOD_LOW = float(os.environ.get(\"AUTO_POWER_GOOD_LOW\", 5))\n", "AUTO_POWER_GOOD_HIGH = float(os.environ.get(\"AUTO_POWER_GOOD_HIGH\", 30))\n", "auto_power_good = (AUTO_POWER_GOOD_LOW, AUTO_POWER_GOOD_HIGH)\n", "AUTO_POWER_SUSPECT_LOW = float(os.environ.get(\"AUTO_POWER_SUSPECT_LOW\", 1))\n", "AUTO_POWER_SUSPECT_HIGH = float(os.environ.get(\"AUTO_POWER_SUSPECT_HIGH\", 80))\n", "auto_power_suspect = (AUTO_POWER_SUSPECT_LOW, AUTO_POWER_SUSPECT_HIGH)\n", "\n", "# bounds on autocorrelation slope\n", "AUTO_SLOPE_GOOD_LOW = float(os.environ.get(\"AUTO_SLOPE_GOOD_LOW\", -0.2))\n", "AUTO_SLOPE_GOOD_HIGH = float(os.environ.get(\"AUTO_SLOPE_GOOD_HIGH\", 0.2))\n", "auto_slope_good = (AUTO_SLOPE_GOOD_LOW, AUTO_SLOPE_GOOD_HIGH)\n", "AUTO_SLOPE_SUSPECT_LOW = float(os.environ.get(\"AUTO_SLOPE_SUSPECT_LOW\", -0.4))\n", "AUTO_SLOPE_SUSPECT_HIGH = float(os.environ.get(\"AUTO_SLOPE_SUSPECT_HIGH\", 0.4))\n", "auto_slope_suspect = (AUTO_SLOPE_SUSPECT_LOW, AUTO_SLOPE_SUSPECT_HIGH)\n", "\n", "for bound in ['am_corr_bad', 'am_corr_suspect', 'am_xpol_bad', 'am_xpol_suspect', \n", " 'auto_power_good', 'auto_power_suspect', 'auto_slope_good', 'auto_slope_suspect']:\n", " print(f'{bound} = {eval(bound)}')" ] }, { "cell_type": "markdown", "id": "105dc915", "metadata": {}, "source": [ "## Load sum and diff data" ] }, { "cell_type": "code", "execution_count": 4, "id": "e5d3725b", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T18:00:29.144034Z", "start_time": "2022-08-08T18:00:26.045578Z" }, "execution": { "iopub.execute_input": "2022-08-14T19:28:31.546937Z", "iopub.status.busy": "2022-08-14T19:28:31.545543Z", "iopub.status.idle": "2022-08-14T19:28:50.662153Z", "shell.execute_reply": "2022-08-14T19:28:50.661316Z" } }, "outputs": [], "source": [ "hd = io.HERADataFastReader(SUM_FILE)\n", "data, _, _ = hd.read(read_flags=False, read_nsamples=False)\n", "hd_diff = io.HERADataFastReader(DIFF_FILE)\n", "diff_data, _, _ = hd_diff.read(read_flags=False, read_nsamples=False)" ] }, { "cell_type": "code", "execution_count": 5, "id": "61f26bc2", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T18:00:29.170365Z", "start_time": "2022-08-08T18:00:29.145133Z" }, "execution": { "iopub.execute_input": "2022-08-14T19:28:50.668214Z", "iopub.status.busy": "2022-08-14T19:28:50.666794Z", "iopub.status.idle": "2022-08-14T19:28:50.773217Z", "shell.execute_reply": "2022-08-14T19:28:50.772427Z" } }, "outputs": [], "source": [ "ants = sorted(set([ant for bl in hd.bls for ant in utils.split_bl(bl)]))\n", "auto_bls = [bl for bl in data if (bl[0] == bl[1]) and (utils.split_pol(bl[2])[0] == utils.split_pol(bl[2])[1])]" ] }, { "cell_type": "code", "execution_count": 6, "id": "e945a0ec", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T19:55:27.745442Z", "start_time": "2022-08-08T19:55:27.735910Z" }, "execution": { "iopub.execute_input": "2022-08-14T19:28:50.779149Z", "iopub.status.busy": "2022-08-14T19:28:50.777756Z", "iopub.status.idle": "2022-08-14T19:28:50.789030Z", "shell.execute_reply": "2022-08-14T19:28:50.788291Z" }, "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "File: /mnt/sn1/2459803/zen.2459803.25316.sum.uvh5\n", "JDs: [2459803.25310807 2459803.25321992] (9.66368 s integrations)\n", "LSTS: [16.84868847 16.85138018] hours\n", "Frequencies: 1536 0.12207 MHz channels from 46.92078 to 234.29871 MHz\n", "Antennas: 147\n", "Polarizations: ['nn', 'ee', 'ne', 'en']\n" ] } ], "source": [ "print(f'File: {SUM_FILE}')\n", "print(f'JDs: {hd.times} ({np.median(np.diff(hd.times)) * 24 * 3600:.5f} s integrations)')\n", "print(f'LSTS: {hd.lsts * 12 / np.pi } hours')\n", "print(f'Frequencies: {len(hd.freqs)} {np.median(np.diff(hd.freqs)) / 1e6:.5f} MHz channels from {hd.freqs[0] / 1e6:.5f} to {hd.freqs[-1] / 1e6:.5f} MHz')\n", "print(f'Antennas: {len(hd.data_ants)}')\n", "print(f'Polarizations: {hd.pols}')" ] }, { "cell_type": "markdown", "id": "f0450fb4", "metadata": {}, "source": [ "## Run `ant_metrics`" ] }, { "cell_type": "code", "execution_count": 7, "id": "4bbb0ac4", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T19:55:44.190609Z", "start_time": "2022-08-08T19:55:37.646653Z" }, "execution": { "iopub.execute_input": "2022-08-14T19:28:50.794595Z", "iopub.status.busy": "2022-08-14T19:28:50.793205Z", "iopub.status.idle": "2022-08-14T19:29:26.608874Z", "shell.execute_reply": "2022-08-14T19:29:26.607967Z" } }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:129: RuntimeWarning: invalid value encountered in divide\n", " odd /= np.abs(odd)\n", "/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:129: RuntimeWarning: invalid value encountered in divide\n", " odd /= np.abs(odd)\n", "/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:128: RuntimeWarning: invalid value encountered in divide\n", " even /= np.abs(even)\n", "/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:128: RuntimeWarning: invalid value encountered in divide\n", " even /= np.abs(even)\n", "/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:128: RuntimeWarning: invalid value encountered in divide\n", " even /= np.abs(even)\n", "/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:129: RuntimeWarning: invalid value encountered in divide\n", " odd /= np.abs(odd)\n", "/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:129: RuntimeWarning: invalid value encountered in divide\n", " odd /= np.abs(odd)\n", "/home/obs/anaconda/envs/RTP/lib/python3.9/site-packages/hera_qm/ant_metrics.py:128: RuntimeWarning: invalid value encountered in divide\n", " even /= np.abs(even)\n" ] } ], "source": [ "am = ant_metrics.AntennaMetrics(SUM_FILE, DIFF_FILE, sum_data=data, diff_data=diff_data)\n", "am.iterative_antenna_metrics_and_flagging(crossCut=am_xpol_bad[1], deadCut=am_corr_bad[1])" ] }, { "cell_type": "code", "execution_count": 8, "id": "d8e52c30", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T19:55:44.200663Z", "start_time": "2022-08-08T19:55:44.192150Z" }, "code_folding": [], "execution": { "iopub.execute_input": "2022-08-14T19:29:26.615257Z", "iopub.status.busy": "2022-08-14T19:29:26.613842Z", "iopub.status.idle": "2022-08-14T19:29:26.642864Z", "shell.execute_reply": "2022-08-14T19:29:26.642132Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "ant_metrics Classification:\n", "\n", " Jee:\n", "----------\n", "good (90 antpols):\n", "7, 8, 9, 10, 15, 16, 17, 18, 19, 20, 21, 28, 29, 30, 31, 32, 33, 36, 37, 38, 40, 41, 42, 45, 46, 50, 51, 52, 53, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 81, 82, 83, 84, 85, 86, 87, 94, 98, 99, 100, 101, 102, 103, 104, 109, 110, 111, 112, 116, 117, 118, 119, 120, 121, 122, 123, 127, 128, 129, 130, 135, 136, 141, 161, 162, 167, 168, 169, 170, 179, 180, 182, 183, 189, 190, 191, 321, 323, 324\n", "\n", "suspect (4 antpols):\n", "92, 142, 329, 333\n", "\n", "bad (53 antpols):\n", "3, 4, 5, 27, 57, 73, 88, 90, 91, 93, 105, 106, 107, 108, 125, 126, 137, 138, 140, 143, 144, 145, 150, 155, 156, 157, 158, 160, 163, 164, 165, 166, 176, 177, 178, 181, 184, 185, 186, 187, 203, 205, 206, 207, 220, 221, 222, 223, 224, 241, 242, 243, 320\n", "\n", "\n", "Jnn:\n", "----------\n", "good (79 antpols):\n", "7, 8, 9, 10, 15, 16, 17, 19, 20, 21, 29, 30, 31, 32, 36, 37, 38, 40, 41, 42, 45, 46, 50, 51, 52, 53, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 73, 81, 82, 83, 84, 85, 86, 87, 94, 98, 99, 100, 101, 102, 103, 104, 109, 110, 111, 112, 116, 117, 119, 121, 122, 123, 127, 128, 129, 130, 138, 141, 161, 162, 167, 168, 169, 170, 179, 183, 189, 191\n", "\n", "suspect (14 antpols):\n", "18, 28, 33, 57, 92, 135, 136, 180, 181, 321, 323, 324, 329, 333\n", "\n", "bad (54 antpols):\n", "3, 4, 5, 27, 88, 90, 91, 93, 105, 106, 107, 108, 118, 120, 125, 126, 137, 140, 142, 143, 144, 145, 150, 155, 156, 157, 158, 160, 163, 164, 165, 166, 176, 177, 178, 182, 184, 185, 186, 187, 190, 203, 205, 206, 207, 220, 221, 222, 223, 224, 241, 242, 243, 320\n" ] } ], "source": [ "totally_dead_ants = [ant for ant, i in am.removal_iteration.items() if i == -1]\n", "am_totally_dead = ant_class.AntennaClassification(good=[ant for ant in ants if ant not in totally_dead_ants], bad=totally_dead_ants)\n", "am_corr = ant_class.antenna_bounds_checker(am.final_metrics['corr'], bad=[am_corr_bad], suspect=[am_corr_suspect], good=[(0, 1)])\n", "am_xpol = ant_class.antenna_bounds_checker(am.final_metrics['corrXPol'], bad=[am_xpol_bad], suspect=[am_xpol_suspect], good=[(-1, 1)])\n", "ant_metrics_class = am_totally_dead + am_corr + am_xpol\n", "print('ant_metrics Classification:\\n\\n', ant_metrics_class)" ] }, { "cell_type": "markdown", "id": "102587ce", "metadata": {}, "source": [ "## Examine and classify autocorrelations" ] }, { "cell_type": "code", "execution_count": 9, "id": "ae2e2234", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T19:55:44.388506Z", "start_time": "2022-08-08T19:55:44.202111Z" }, "execution": { "iopub.execute_input": "2022-08-14T19:29:26.648465Z", "iopub.status.busy": "2022-08-14T19:29:26.647052Z", "iopub.status.idle": "2022-08-14T19:29:27.022795Z", "shell.execute_reply": "2022-08-14T19:29:27.022048Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Autocorrelation Classification:\n", "\n", " Jee:\n", "----------\n", "good (80 antpols):\n", "7, 9, 10, 15, 16, 17, 18, 19, 20, 21, 28, 29, 30, 31, 33, 37, 38, 40, 41, 42, 45, 46, 50, 51, 53, 54, 55, 56, 65, 66, 67, 68, 69, 71, 72, 81, 82, 83, 85, 86, 93, 94, 98, 100, 103, 109, 110, 111, 112, 116, 117, 118, 120, 121, 127, 128, 129, 130, 135, 136, 141, 143, 144, 161, 162, 163, 164, 165, 179, 180, 183, 184, 185, 186, 187, 189, 191, 206, 329, 333\n", "\n", "suspect (24 antpols):\n", "8, 36, 52, 70, 84, 87, 99, 101, 102, 104, 119, 122, 123, 166, 168, 169, 170, 182, 205, 207, 223, 321, 323, 324\n", "\n", "bad (43 antpols):\n", "3, 4, 5, 27, 32, 57, 73, 88, 90, 91, 92, 105, 106, 107, 108, 125, 126, 137, 138, 140, 142, 145, 150, 155, 156, 157, 158, 160, 167, 176, 177, 178, 181, 190, 203, 220, 221, 222, 224, 241, 242, 243, 320\n", "\n", "\n", "Jnn:\n", "----------\n", "good (75 antpols):\n", "7, 9, 10, 15, 16, 17, 18, 19, 20, 21, 29, 30, 31, 33, 37, 38, 40, 41, 42, 45, 46, 51, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 73, 81, 82, 83, 86, 93, 94, 99, 100, 102, 109, 110, 111, 112, 116, 117, 119, 127, 128, 129, 130, 135, 136, 138, 141, 143, 144, 162, 163, 164, 165, 166, 179, 183, 184, 185, 186, 187, 189, 191, 205, 223\n", "\n", "suspect (24 antpols):\n", "8, 36, 50, 52, 53, 57, 84, 85, 98, 101, 121, 122, 123, 167, 168, 169, 170, 206, 207, 321, 323, 324, 329, 333\n", "\n", "bad (48 antpols):\n", "3, 4, 5, 27, 28, 32, 87, 88, 90, 91, 92, 103, 104, 105, 106, 107, 108, 118, 120, 125, 126, 137, 140, 142, 145, 150, 155, 156, 157, 158, 160, 161, 176, 177, 178, 180, 181, 182, 190, 203, 220, 221, 222, 224, 241, 242, 243, 320\n" ] } ], "source": [ "auto_power_class = ant_class.auto_power_checker(data, good=auto_power_good, suspect=auto_power_suspect)\n", "auto_slope_class = ant_class.auto_slope_checker(data, good=auto_slope_good, suspect=auto_slope_suspect, edge_cut=100, filt_size=17)\n", "auto_class = auto_power_class + auto_slope_class\n", "print('Autocorrelation Classification:\\n\\n', auto_class)" ] }, { "cell_type": "markdown", "id": "f52a30f9", "metadata": {}, "source": [ "## Metadata" ] }, { "cell_type": "code", "execution_count": 10, "id": "78d18d26", "metadata": { "ExecuteTime": { "end_time": "2022-08-08T19:55:44.392152Z", "start_time": "2022-08-08T19:55:44.390236Z" }, "execution": { "iopub.execute_input": "2022-08-14T19:29:27.028517Z", "iopub.status.busy": "2022-08-14T19:29:27.027101Z", "iopub.status.idle": "2022-08-14T19:29:27.035794Z", "shell.execute_reply": "2022-08-14T19:29:27.035062Z" } }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "hera_cal: 3.1.4.dev3+g68bd8c3\n", "hera_qm: 2.0.3.dev44+g7d4aa18\n" ] } ], "source": [ "from hera_cal import __version__\n", "print('hera_cal:', __version__)\n", "from hera_qm import __version__\n", "print('hera_qm:', __version__)" ] }, { "cell_type": "code", "execution_count": null, "id": "581b70cd", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.9.7" }, "toc": { "base_numbering": 1, "nav_menu": {}, "number_sections": true, "sideBar": true, "skip_h1_title": true, "title_cell": "Table of Contents", "title_sidebar": "Contents", "toc_cell": false, "toc_position": { "height": "calc(100% - 180px)", "left": "10px", "top": "150px", "width": "248.391px" }, "toc_section_display": true, "toc_window_display": false } }, "nbformat": 4, "nbformat_minor": 5 }