{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "504e14fa",
   "metadata": {},
   "source": [
    "# File Inspection (Experimental)\n",
    "\n",
    "**by Josh Dillon & Aaron Parsons**, last updated August 8, 2022"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "668f7418",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T19:44:38.539183Z",
     "start_time": "2022-08-08T19:44:38.528103Z"
    },
    "execution": {
     "iopub.execute_input": "2022-08-09T19:07:40.942718Z",
     "iopub.status.busy": "2022-08-09T19:07:40.942280Z",
     "iopub.status.idle": "2022-08-09T19:07:44.059830Z",
     "shell.execute_reply": "2022-08-09T19:07:44.059033Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import os\n",
    "from hera_qm import ant_metrics, ant_class\n",
    "from hera_cal import io, utils\n",
    "from IPython.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab0f8167",
   "metadata": {},
   "source": [
    "## Parse inputs\n",
    "\n",
    "To use this notebook interactively, you will have to provide a sum filename path if none exists as an environment variable. All other parameters have reasonable default values.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "4b88cc02",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T19:45:12.866506Z",
     "start_time": "2022-08-08T19:45:12.853701Z"
    },
    "execution": {
     "iopub.execute_input": "2022-08-09T19:07:44.066065Z",
     "iopub.status.busy": "2022-08-09T19:07:44.065701Z",
     "iopub.status.idle": "2022-08-09T19:07:44.072086Z",
     "shell.execute_reply": "2022-08-09T19:07:44.071288Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SUM_FILE = '/mnt/sn1/2459800/zen.2459800.25309.sum.uvh5'\n"
     ]
    }
   ],
   "source": [
    "# get file names\n",
    "SUM_FILE = os.environ.get(\"SUM_FILE\", None)\n",
    "# SUM_FILE = '/mnt/sn1/zen.2459797.30001.sum.uvh5'  # If sum_file is not defined in the environment variables, define it here.\n",
    "DIFF_FILE = SUM_FILE.replace('sum', 'diff')\n",
    "print(f\"SUM_FILE = '{SUM_FILE}'\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5596c041",
   "metadata": {},
   "source": [
    "### Parse Bounds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9ebbfd85",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T19:47:02.245642Z",
     "start_time": "2022-08-08T19:47:02.224052Z"
    },
    "execution": {
     "iopub.execute_input": "2022-08-09T19:07:44.077209Z",
     "iopub.status.busy": "2022-08-09T19:07:44.076952Z",
     "iopub.status.idle": "2022-08-09T19:07:44.091594Z",
     "shell.execute_reply": "2022-08-09T19:07:44.090844Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "am_corr_bad = (0, 0.2)\n",
      "am_corr_suspect = (0.2, 0.4)\n",
      "am_xpol_bad = (-1, -0.1)\n",
      "am_xpol_suspect = (-0.1, 0.0)\n",
      "auto_power_good = (5.0, 30.0)\n",
      "auto_power_suspect = (1.0, 80.0)\n",
      "auto_slope_good = (-0.2, 0.2)\n",
      "auto_slope_suspect = (-0.4, 0.4)\n"
     ]
    }
   ],
   "source": [
    "# ant_metrics bounds for low correlation / dead antennas\n",
    "AM_CORR_BAD = float(os.environ.get(\"AM_CORR_BAD\", 0.2))\n",
    "AM_CORR_SUSPECT = float(os.environ.get(\"AM_CORR_SUSPECT\", 0.4))\n",
    "am_corr_bad = (0, AM_CORR_BAD)\n",
    "am_corr_suspect = (AM_CORR_BAD, AM_CORR_SUSPECT)\n",
    "\n",
    "# ant_metrics bounds for cross-polarized antennas\n",
    "AM_XPOL_BAD = float(os.environ.get(\"AM_XPOL_BAD\", -0.1))\n",
    "AM_XPOL_SUSPECT = float(os.environ.get(\"AM_XPOL_SUSPECT\", 0))\n",
    "am_xpol_bad = (-1, AM_XPOL_BAD)\n",
    "am_xpol_suspect = (AM_XPOL_BAD, AM_XPOL_SUSPECT)\n",
    "\n",
    "# bounds on autocorrelation power\n",
    "AUTO_POWER_GOOD_LOW = float(os.environ.get(\"AUTO_POWER_GOOD_LOW\", 5))\n",
    "AUTO_POWER_GOOD_HIGH = float(os.environ.get(\"AUTO_POWER_GOOD_HIGH\", 30))\n",
    "auto_power_good = (AUTO_POWER_GOOD_LOW, AUTO_POWER_GOOD_HIGH)\n",
    "AUTO_POWER_SUSPECT_LOW = float(os.environ.get(\"AUTO_POWER_SUSPECT_LOW\", 1))\n",
    "AUTO_POWER_SUSPECT_HIGH = float(os.environ.get(\"AUTO_POWER_SUSPECT_HIGH\", 80))\n",
    "auto_power_suspect = (AUTO_POWER_SUSPECT_LOW, AUTO_POWER_SUSPECT_HIGH)\n",
    "\n",
    "# bounds on autocorrelation slope\n",
    "AUTO_SLOPE_GOOD_LOW = float(os.environ.get(\"AUTO_SLOPE_GOOD_LOW\", -0.2))\n",
    "AUTO_SLOPE_GOOD_HIGH = float(os.environ.get(\"AUTO_SLOPE_GOOD_HIGH\", 0.2))\n",
    "auto_slope_good = (AUTO_SLOPE_GOOD_LOW, AUTO_SLOPE_GOOD_HIGH)\n",
    "AUTO_SLOPE_SUSPECT_LOW = float(os.environ.get(\"AUTO_SLOPE_SUSPECT_LOW\", -0.4))\n",
    "AUTO_SLOPE_SUSPECT_HIGH = float(os.environ.get(\"AUTO_SLOPE_SUSPECT_HIGH\", 0.4))\n",
    "auto_slope_suspect = (AUTO_SLOPE_SUSPECT_LOW, AUTO_SLOPE_SUSPECT_HIGH)\n",
    "\n",
    "for bound in ['am_corr_bad', 'am_corr_suspect', 'am_xpol_bad', 'am_xpol_suspect', \n",
    "             'auto_power_good', 'auto_power_suspect', 'auto_slope_good', 'auto_slope_suspect']:\n",
    "    print(f'{bound} = {eval(bound)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "105dc915",
   "metadata": {},
   "source": [
    "## Load sum and diff data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e5d3725b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T18:00:29.144034Z",
     "start_time": "2022-08-08T18:00:26.045578Z"
    },
    "execution": {
     "iopub.execute_input": "2022-08-09T19:07:44.095760Z",
     "iopub.status.busy": "2022-08-09T19:07:44.095503Z",
     "iopub.status.idle": "2022-08-09T19:07:56.439632Z",
     "shell.execute_reply": "2022-08-09T19:07:56.438314Z"
    }
   },
   "outputs": [],
   "source": [
    "hd = io.HERADataFastReader(SUM_FILE)\n",
    "data, _, _ = hd.read(read_flags=False, read_nsamples=False)\n",
    "hd_diff = io.HERADataFastReader(DIFF_FILE)\n",
    "diff_data, _, _ = hd_diff.read(read_flags=False, read_nsamples=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "61f26bc2",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T18:00:29.170365Z",
     "start_time": "2022-08-08T18:00:29.145133Z"
    },
    "execution": {
     "iopub.execute_input": "2022-08-09T19:07:56.445517Z",
     "iopub.status.busy": "2022-08-09T19:07:56.445126Z",
     "iopub.status.idle": "2022-08-09T19:07:56.543846Z",
     "shell.execute_reply": "2022-08-09T19:07:56.542776Z"
    }
   },
   "outputs": [],
   "source": [
    "ants = sorted(set([ant for bl in hd.bls for ant in utils.split_bl(bl)]))\n",
    "auto_bls = [bl for bl in data if (bl[0] == bl[1]) and (utils.split_pol(bl[2])[0] == utils.split_pol(bl[2])[1])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e945a0ec",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T19:55:27.745442Z",
     "start_time": "2022-08-08T19:55:27.735910Z"
    },
    "execution": {
     "iopub.execute_input": "2022-08-09T19:07:56.548757Z",
     "iopub.status.busy": "2022-08-09T19:07:56.548319Z",
     "iopub.status.idle": "2022-08-09T19:07:56.558518Z",
     "shell.execute_reply": "2022-08-09T19:07:56.557665Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "File: /mnt/sn1/2459800/zen.2459800.25309.sum.uvh5\n",
      "JDs: [2459800.25303886 2459800.25315071] (9.66368 s integrations)\n",
      "LSTS: [16.64988596 16.65257766] hours\n",
      "Frequencies: 1536 0.12207 MHz channels from 46.92078 to 234.29871 MHz\n",
      "Antennas: 147\n",
      "Polarizations: ['nn', 'ee', 'ne', 'en']\n"
     ]
    }
   ],
   "source": [
    "print(f'File: {SUM_FILE}')\n",
    "print(f'JDs: {hd.times} ({np.median(np.diff(hd.times)) * 24 * 3600:.5f} s integrations)')\n",
    "print(f'LSTS: {hd.lsts * 12 / np.pi } hours')\n",
    "print(f'Frequencies: {len(hd.freqs)} {np.median(np.diff(hd.freqs)) / 1e6:.5f} MHz channels from {hd.freqs[0] / 1e6:.5f} to {hd.freqs[-1] / 1e6:.5f} MHz')\n",
    "print(f'Antennas: {len(hd.data_ants)}')\n",
    "print(f'Polarizations: {hd.pols}')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f0450fb4",
   "metadata": {},
   "source": [
    "## Run `ant_metrics`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4bbb0ac4",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T19:55:44.190609Z",
     "start_time": "2022-08-08T19:55:37.646653Z"
    },
    "execution": {
     "iopub.execute_input": "2022-08-09T19:07:56.563180Z",
     "iopub.status.busy": "2022-08-09T19:07:56.562756Z",
     "iopub.status.idle": "2022-08-09T19:08:37.215453Z",
     "shell.execute_reply": "2022-08-09T19:08:37.213837Z"
    }
   },
   "outputs": [],
   "source": [
    "am = ant_metrics.AntennaMetrics(SUM_FILE, DIFF_FILE, sum_data=data, diff_data=diff_data)\n",
    "am.iterative_antenna_metrics_and_flagging(crossCut=am_xpol_bad[1], deadCut=am_corr_bad[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d8e52c30",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T19:55:44.200663Z",
     "start_time": "2022-08-08T19:55:44.192150Z"
    },
    "code_folding": [],
    "execution": {
     "iopub.execute_input": "2022-08-09T19:08:37.222151Z",
     "iopub.status.busy": "2022-08-09T19:08:37.221629Z",
     "iopub.status.idle": "2022-08-09T19:08:37.258541Z",
     "shell.execute_reply": "2022-08-09T19:08:37.257506Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ant_metrics Classification:\n",
      "\n",
      " Jee:\n",
      "----------\n",
      "good (78 antpols):\n",
      "3, 4, 5, 7, 8, 9, 15, 16, 17, 18, 21, 29, 30, 31, 32, 33, 36, 37, 38, 40, 41, 42, 45, 46, 50, 51, 52, 53, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 81, 82, 83, 98, 99, 100, 116, 117, 118, 119, 135, 136, 141, 143, 144, 156, 157, 158, 161, 162, 163, 164, 167, 168, 169, 170, 176, 177, 178, 179, 182, 183, 185, 186, 187, 189, 191, 321, 324\n",
      "\n",
      "suspect (15 antpols):\n",
      "28, 94, 109, 110, 111, 112, 127, 128, 129, 130, 142, 190, 323, 329, 333\n",
      "\n",
      "bad (54 antpols):\n",
      "10, 19, 20, 27, 57, 73, 84, 85, 86, 87, 88, 90, 91, 92, 93, 101, 102, 103, 104, 105, 106, 107, 108, 120, 121, 122, 123, 125, 126, 137, 138, 140, 145, 150, 155, 160, 165, 166, 180, 181, 184, 203, 205, 206, 207, 220, 221, 222, 223, 224, 241, 242, 243, 320\n",
      "\n",
      "\n",
      "Jnn:\n",
      "----------\n",
      "good (74 antpols):\n",
      "3, 4, 5, 7, 8, 9, 15, 16, 17, 21, 29, 30, 31, 32, 36, 37, 38, 40, 41, 42, 45, 46, 50, 51, 52, 53, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 73, 81, 82, 83, 98, 99, 100, 116, 117, 118, 119, 135, 136, 138, 141, 143, 144, 156, 157, 158, 162, 163, 164, 168, 169, 170, 176, 177, 178, 179, 182, 183, 185, 186, 187, 189, 191\n",
      "\n",
      "suspect (20 antpols):\n",
      "18, 28, 33, 57, 94, 109, 110, 111, 112, 127, 128, 129, 130, 161, 167, 321, 323, 324, 329, 333\n",
      "\n",
      "bad (53 antpols):\n",
      "10, 19, 20, 27, 84, 85, 86, 87, 88, 90, 91, 92, 93, 101, 102, 103, 104, 105, 106, 107, 108, 120, 121, 122, 123, 125, 126, 137, 140, 142, 145, 150, 155, 160, 165, 166, 180, 181, 184, 190, 203, 205, 206, 207, 220, 221, 222, 223, 224, 241, 242, 243, 320\n"
     ]
    }
   ],
   "source": [
    "totally_dead_ants = [ant for ant, i in am.removal_iteration.items() if i == -1]\n",
    "am_totally_dead = ant_class.AntennaClassification(good=[ant for ant in ants if ant not in totally_dead_ants], bad=totally_dead_ants)\n",
    "am_corr = ant_class.antenna_bounds_checker(am.final_metrics['corr'], bad=[am_corr_bad], suspect=[am_corr_suspect], good=[(0, 1)])\n",
    "am_xpol = ant_class.antenna_bounds_checker(am.final_metrics['corrXPol'], bad=[am_xpol_bad], suspect=[am_xpol_suspect], good=[(-1, 1)])\n",
    "ant_metrics_class = am_totally_dead + am_corr + am_xpol\n",
    "print('ant_metrics Classification:\\n\\n', ant_metrics_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "102587ce",
   "metadata": {},
   "source": [
    "## Examine and classify autocorrelations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ae2e2234",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T19:55:44.388506Z",
     "start_time": "2022-08-08T19:55:44.202111Z"
    },
    "execution": {
     "iopub.execute_input": "2022-08-09T19:08:37.263549Z",
     "iopub.status.busy": "2022-08-09T19:08:37.263177Z",
     "iopub.status.idle": "2022-08-09T19:08:37.951606Z",
     "shell.execute_reply": "2022-08-09T19:08:37.949349Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Autocorrelation Classification:\n",
      "\n",
      " Jee:\n",
      "----------\n",
      "good (84 antpols):\n",
      "3, 4, 5, 7, 9, 15, 16, 17, 18, 21, 28, 29, 30, 31, 33, 37, 38, 40, 41, 42, 45, 46, 51, 53, 54, 55, 56, 65, 66, 67, 68, 69, 71, 72, 81, 82, 83, 85, 86, 87, 93, 94, 98, 100, 103, 109, 110, 111, 112, 116, 117, 118, 120, 121, 127, 128, 129, 130, 135, 136, 141, 143, 144, 156, 157, 158, 161, 162, 163, 164, 176, 177, 178, 179, 183, 185, 186, 187, 189, 191, 206, 223, 329, 333\n",
      "\n",
      "suspect (22 antpols):\n",
      "8, 36, 52, 70, 84, 99, 101, 102, 104, 119, 122, 123, 167, 168, 169, 170, 182, 205, 207, 321, 323, 324\n",
      "\n",
      "bad (41 antpols):\n",
      "10, 19, 20, 27, 32, 50, 57, 73, 88, 90, 91, 92, 105, 106, 107, 108, 125, 126, 137, 138, 140, 142, 145, 150, 155, 160, 165, 166, 180, 181, 184, 190, 203, 220, 221, 222, 224, 241, 242, 243, 320\n",
      "\n",
      "\n",
      "Jnn:\n",
      "----------\n",
      "good (81 antpols):\n",
      "3, 4, 5, 7, 9, 15, 16, 17, 18, 21, 29, 30, 31, 33, 37, 38, 40, 41, 42, 45, 46, 50, 51, 54, 55, 56, 65, 66, 67, 68, 69, 70, 71, 72, 73, 81, 82, 83, 85, 86, 93, 94, 99, 100, 102, 109, 110, 111, 112, 116, 117, 118, 119, 127, 128, 129, 130, 135, 136, 138, 141, 143, 144, 156, 157, 158, 162, 163, 164, 176, 177, 178, 179, 182, 183, 185, 186, 187, 189, 191, 205\n",
      "\n",
      "suspect (20 antpols):\n",
      "8, 36, 52, 53, 57, 98, 121, 122, 167, 168, 169, 170, 206, 207, 223, 321, 323, 324, 329, 333\n",
      "\n",
      "bad (46 antpols):\n",
      "10, 19, 20, 27, 28, 32, 84, 87, 88, 90, 91, 92, 101, 103, 104, 105, 106, 107, 108, 120, 123, 125, 126, 137, 140, 142, 145, 150, 155, 160, 161, 165, 166, 180, 181, 184, 190, 203, 220, 221, 222, 224, 241, 242, 243, 320\n"
     ]
    }
   ],
   "source": [
    "auto_power_class = ant_class.auto_power_checker(data, good=auto_power_good, suspect=auto_power_suspect)\n",
    "auto_slope_class = ant_class.auto_slope_checker(data, good=auto_slope_good, suspect=auto_slope_suspect, edge_cut=100, filt_size=17)\n",
    "auto_class = auto_power_class + auto_slope_class\n",
    "print('Autocorrelation Classification:\\n\\n', auto_class)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f52a30f9",
   "metadata": {},
   "source": [
    "## Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "78d18d26",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2022-08-08T19:55:44.392152Z",
     "start_time": "2022-08-08T19:55:44.390236Z"
    },
    "execution": {
     "iopub.execute_input": "2022-08-09T19:08:37.957341Z",
     "iopub.status.busy": "2022-08-09T19:08:37.956474Z",
     "iopub.status.idle": "2022-08-09T19:08:37.967620Z",
     "shell.execute_reply": "2022-08-09T19:08:37.966750Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hera_cal: 3.1.2\n",
      "hera_qm: 2.0.3.dev44+g7d4aa18\n"
     ]
    }
   ],
   "source": [
    "from hera_cal import __version__\n",
    "print('hera_cal:', __version__)\n",
    "from hera_qm import __version__\n",
    "print('hera_qm:', __version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "581b70cd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "248.391px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}