good_quality_selection_sashabrownsberger.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "# LSST stack imports\n",
    "from lsst.daf.persistence import Butler\n",
    "import lsst.afw.display as afw_display\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import math\n",
    "import pandas as pd\n",
    "\n",
    "# Firefly client imports\n",
    "from firefly_client import FireflyClient\n",
    "\n",
    "# Standard libraries in support of Firefly display\n",
    "from urllib.parse import urlparse, urlunparse, ParseResult\n",
    "from IPython.display import display, Markdown\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "##### If you want to edit the data set, tract, and patch yourself, change the numbers below appropriately###\n",
    "data_set = 'UDEEP'# can be WIDE, DEEP, UDEEP\n",
    "datadir = '/datasets/hsc/repo/rerun/DM-13666/' + data_set \n",
    "data_set = 'deepCoadd_forced_src'\n",
    "butler = Butler(sample_datadir)\n",
    "tract = 8765 #8766\n",
    "patch = '1,2' #'8,3'\n",
    "#All subsequent data structures will be stored in sets, with the elements of the sets \n",
    "# corresponding to the filters you specify here.  So, probably a good idea to remember the order! \n",
    "filters = ['HSC-I','HSC-Y'] \n",
    "n_filters = len(filters)\n",
    "data_set = 'deepCoadd_forced_src'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of objects in filter HSC-I = 25266\n",
      "Number of objects in filter HSC-Y = 25266\n",
      "(Note that those numbers should all be the same.)\n"
     ]
    }
   ],
   "source": [
    "deep_coadds = [butler.get(data_set, tract = tract, patch = patch, dataId={'filter': filter}) for filter in filters]\n",
    "n_raw_objects = len(deep_coadds[0])\n",
    "for i in range(len(filters)):\n",
    "    print ('Number of objects in filter ' + filters[i] + ' = ' + str(len(deep_coadds[i])))\n",
    "print ('(Note that those numbers should all be the same.)')\n",
    "#print (deep_coadd_I)\n",
    "#print (deep_coadd_Y)\n",
    "deep_coadd_tables = [deep_coadd.asAstropy() for deep_coadd in deep_coadds]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define all possible flags, and our recommended flags \n",
    "flags_to_reject_on_recommended= ['base_PixelFlags_flag_bad', 'base_PixelFlags_flag_cr', 'base_PixelFlags_flag_offimage','base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 'base_PixelFlags_flag_rejected', 'base_PixelFlags_flag_interpolated']\n",
    "flags_to_reject_on_all = [ (colname, deep_coadd_tables[0][colname].description) for colname in deep_coadd_tables[0].colnames if '_flag_' in colname]\n",
    "#print (flags_to_reject_on_all)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Choose the flags on which you wish to reject observations. \n",
      "Here are your options: \n",
      "['base_PixelFlags_flag_bad', 'base_PixelFlags_flag_cr', 'base_PixelFlags_flag_offimage', 'base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 'base_PixelFlags_flag_rejected', 'base_PixelFlags_flag_interpolated']\n",
      "Here are our suggestions:\n",
      "['base_PixelFlags_flag_bad', 'base_PixelFlags_flag_cr', 'base_PixelFlags_flag_offimage', 'base_PixelFlags_flag_edge', 'base_PixelFlags_flag_saturated', 'base_PixelFlags_flag_rejected', 'base_PixelFlags_flag_interpolated']\n"
     ]
    }
   ],
   "source": [
    "print ('Choose the flags on which you wish to reject observations. ')\n",
    "print ('Here are your options: ')\n",
    "print (flags_to_reject_on_all)\n",
    "print ('Here are our suggestions:')\n",
    "print (flags_to_reject_on_recommended)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "######ENTER YOUR CHOICE OF REJECTION FLAGS HERE##### \n",
    "flags_to_reject_on = [] #Say ['all'] if you want to use all possible flags \n",
    "if len(flags_to_reject_on) == 0: \n",
    "    flags_to_reject_on = flags_to_reject_on_recommended[:]\n",
    "if flags_to_reject_on[0] is 'all':\n",
    "    flags_to_reject_on_all "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For filter HSC-I, 15105 good objects were found, based on your chosen flags.\n",
      "For filter HSC-Y, 10497 good objects were found, based on your chosen flags.\n"
     ]
    }
   ],
   "source": [
    "#Do or operation on all flags in individual filter, and then between filter.  \n",
    "# So if an object has ANY of the desired flags high, that object will be rejected by the master flag \n",
    "total_rej_flags_by_filters = [[any([deep_coadd_table[flag][i] for flag in flags_to_reject_on]) for i in range(n_raw_objects)] \n",
    "                                              for deep_coadd_table in deep_coadd_tables] \n",
    "for i in range(n_filters):\n",
    "    print ('For filter ' + str(filters[i]) + ', ' + str(len([flag for flag in total_rej_flags_by_filters[i] if flag == False])) + ' good objects were found, based on your chosen flags.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Combining them all, we are left with 8242 good objects. \n"
     ]
    }
   ],
   "source": [
    "master_rejection_flags = [any([total_rej_flags[i] for total_rej_flags in total_rej_flags_by_filters]) for i in range(n_raw_objects)]\n",
    "good_objects_by_filter = [deep_coadd_table[[not(flag) for flag in master_rejection_flags]] \n",
    "                         for deep_coadd_table in deep_coadd_tables]\n",
    "print ('Combining them all, we are left with ' + str(len(good_objects_by_filter[0])) + ' good objects. ') \n",
    "n_good_objects = len(good_objects_by_filter[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have now parsed the given object lists by the specified flags.\n",
      "If you JUST want objects sliced by the given flags, take the good_objects_by_filter array. \n",
      "But if you want to also cut based on whether an object is a star or a galaxy, then continue onward!\n"
     ]
    }
   ],
   "source": [
    "print ('We have now parsed the given object lists by the specified flags.')\n",
    "print  ('If you JUST want objects sliced by the given flags, take the good_objects_by_filter array. ')\n",
    "print ('But if you want to also cut based on whether an object is a star or a galaxy, then continue onward!') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "extendedness_by_filter = [good_objects['base_ClassificationExtendedness_value'] \n",
    "                          for good_objects in good_objects_by_filter]\n",
    "#Note that is_star is NOT the opposite of the is_galaxy\n",
    "# This is because the extendness is sometimes nan, which we treat as neither \n",
    "#   a star nor a galaxy.\n",
    "\n",
    "is_star_by_filter = [[extendedness <= 0 for extendedness in extendednesses] \n",
    "                      for extendednesses in extendedness_by_filter] \n",
    "is_galaxy_by_filter = [[extendedness > 0 for extendedness in extendednesses] \n",
    "                      for extendednesses in extendedness_by_filter] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For filter HSC-I, there are: \n",
      "2148 good stars.\n",
      "4523 good galaxies.\n",
      "For filter HSC-Y, there are: \n",
      "2016 good stars.\n",
      "4650 good galaxies.\n"
     ]
    }
   ],
   "source": [
    "for i in range(n_filters):\n",
    "    print ('For filter ' + str(filters[i]) + ', there are: ')\n",
    "    print (str(len([is_star for is_star in is_star_by_filter[i] if is_star == True])) + ' good stars.')\n",
    "    print (str(len([is_galaxy for is_galaxy in is_galaxy_by_filter[i] if is_galaxy == True])) + ' good galaxies.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "master_is_star = [all([is_star[i] for is_star in is_star_by_filter]) for i in range(n_good_objects)]\n",
    "master_is_galaxy = [all([is_star[i] for is_star in is_star_by_filter]) for i in range(n_good_objects)]\n",
    "good_stars_by_filter = [good_objects_by_filter[i][master_is_star] for i in range(n_filters)] \n",
    "good_galaxy_by_filter = [good_objects_by_filter[i][master_is_galaxy] for i in range(n_filters)] \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "After looking only for those that are stars or galaxies in ALL filters, we find that there are: \n",
      "946 good stars. \n",
      "4523 good galaxies. \n"
     ]
    }
   ],
   "source": [
    "print ('After looking only for those that are deemed to be stars or galaxies in ALL filters, we find that there are: ')\n",
    "print (str(len(good_stars_by_filter[0])) + ' good stars. ')\n",
    "print (str(len(good_galaxies_by_filter[0])) + ' good galaxies. ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Flag</th>\n",
       "      <th>HSC-I</th>\n",
       "      <th>HSC-Y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>base_SdssCentroid_flag_edge</td>\n",
       "      <td>3%</td>\n",
       "      <td>3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>base_SdssCentroid_flag_noSecondDerivative</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>base_SdssCentroid_flag_almostNoSecondDerivative</td>\n",
       "      <td>3%</td>\n",
       "      <td>3%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>base_SdssCentroid_flag_notAtMaximum</td>\n",
       "      <td>35%</td>\n",
       "      <td>38%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>base_SdssCentroid_flag_resetToPeak</td>\n",
       "      <td>2%</td>\n",
       "      <td>1%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>base_CircularApertureFlux_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>base_GaussianFlux_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>base_InputCount_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>base_LocalBackground_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>base_PsfFlux_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>base_SdssCentroid_flag_badInitialCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>base_SdssShape_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>base_Variance_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_0_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_1_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_2_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_3_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>ext_photometryKron_KronFlux_flag_badInitialCen...</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>modelfit_DoubleShapeletPsfApprox_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>undeblended_base_CircularApertureFlux_flag_bad...</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>undeblended_base_PsfFlux_flag_badCentroid</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>undeblended_ext_photometryKron_KronFlux_flag_b...</td>\n",
       "      <td>15%</td>\n",
       "      <td>15%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>base_InputCount_flag_noInputs</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>base_SdssShape_flag_unweightedBad</td>\n",
       "      <td>31%</td>\n",
       "      <td>43%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>base_SdssShape_flag_unweighted</td>\n",
       "      <td>36%</td>\n",
       "      <td>31%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>base_SdssShape_flag_shift</td>\n",
       "      <td>12%</td>\n",
       "      <td>9%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>base_SdssShape_flag_maxIter</td>\n",
       "      <td>1%</td>\n",
       "      <td>2%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>base_SdssShape_flag_psf</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>base_GaussianFlux_flag_badShape</td>\n",
       "      <td>47%</td>\n",
       "      <td>47%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>undeblended_ext_photometryKron_KronFlux_flag_s...</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>undeblended_ext_photometryKron_KronFlux_flag_b...</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_0_6_0_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_0_4_5_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>modelfit_CModel_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160</th>\n",
       "      <td>slot_ModelFlux_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>161</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_1_6_0_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>162</th>\n",
       "      <td>modelfit_CModel_initial_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>slot_ModelFlux_initial_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_3_kron_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_2_3_3_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>166</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_1_4_5_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>167</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_3_6_0_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>168</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_1_kron_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169</th>\n",
       "      <td>modelfit_CModel_exp_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>170</th>\n",
       "      <td>slot_ModelFlux_exp_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>171</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_3_4_5_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>172</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_0_3_3_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>173</th>\n",
       "      <td>base_GaussianFlux_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>174</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_3_3_3_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>175</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_2_4_5_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176</th>\n",
       "      <td>base_PsfFlux_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177</th>\n",
       "      <td>slot_PsfFlux_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>178</th>\n",
       "      <td>modelfit_CModel_dev_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>slot_ModelFlux_dev_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>ext_photometryKron_KronFlux_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_2_6_0_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_0_kron_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_2_kron_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>ext_convolved_ConvolvedFlux_1_3_3_flag_apCorr</td>\n",
       "      <td>0%</td>\n",
       "      <td>0%</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>185 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  Flag HSC-I HSC-Y\n",
       "0                          base_SdssCentroid_flag_edge    3%    3%\n",
       "1            base_SdssCentroid_flag_noSecondDerivative    0%    0%\n",
       "2      base_SdssCentroid_flag_almostNoSecondDerivative    3%    3%\n",
       "3                  base_SdssCentroid_flag_notAtMaximum   35%   38%\n",
       "4                   base_SdssCentroid_flag_resetToPeak    2%    1%\n",
       "5           base_CircularApertureFlux_flag_badCentroid   15%   15%\n",
       "6                   base_GaussianFlux_flag_badCentroid   15%   15%\n",
       "7                     base_InputCount_flag_badCentroid   15%   15%\n",
       "8                base_LocalBackground_flag_badCentroid   15%   15%\n",
       "9                        base_PsfFlux_flag_badCentroid   15%   15%\n",
       "10           base_SdssCentroid_flag_badInitialCentroid   15%   15%\n",
       "11                     base_SdssShape_flag_badCentroid   15%   15%\n",
       "12                      base_Variance_flag_badCentroid   15%   15%\n",
       "13      ext_convolved_ConvolvedFlux_0_flag_badCentroid   15%   15%\n",
       "14      ext_convolved_ConvolvedFlux_1_flag_badCentroid   15%   15%\n",
       "15      ext_convolved_ConvolvedFlux_2_flag_badCentroid   15%   15%\n",
       "16      ext_convolved_ConvolvedFlux_3_flag_badCentroid   15%   15%\n",
       "17        ext_convolved_ConvolvedFlux_flag_badCentroid   15%   15%\n",
       "18   ext_photometryKron_KronFlux_flag_badInitialCen...   15%   15%\n",
       "19   modelfit_DoubleShapeletPsfApprox_flag_badCentroid   15%   15%\n",
       "20   undeblended_base_CircularApertureFlux_flag_bad...   15%   15%\n",
       "21           undeblended_base_PsfFlux_flag_badCentroid   15%   15%\n",
       "22   undeblended_ext_photometryKron_KronFlux_flag_b...   15%   15%\n",
       "23                       base_InputCount_flag_noInputs    0%    0%\n",
       "24                   base_SdssShape_flag_unweightedBad   31%   43%\n",
       "25                      base_SdssShape_flag_unweighted   36%   31%\n",
       "26                           base_SdssShape_flag_shift   12%    9%\n",
       "27                         base_SdssShape_flag_maxIter    1%    2%\n",
       "28                             base_SdssShape_flag_psf    0%    0%\n",
       "29                     base_GaussianFlux_flag_badShape   47%   47%\n",
       "..                                                 ...   ...   ...\n",
       "155  undeblended_ext_photometryKron_KronFlux_flag_s...    0%    0%\n",
       "156  undeblended_ext_photometryKron_KronFlux_flag_b...    0%    0%\n",
       "157      ext_convolved_ConvolvedFlux_0_6_0_flag_apCorr    0%    0%\n",
       "158      ext_convolved_ConvolvedFlux_0_4_5_flag_apCorr    0%    0%\n",
       "159                        modelfit_CModel_flag_apCorr    0%    0%\n",
       "160                         slot_ModelFlux_flag_apCorr    0%    0%\n",
       "161      ext_convolved_ConvolvedFlux_1_6_0_flag_apCorr    0%    0%\n",
       "162                modelfit_CModel_initial_flag_apCorr    0%    0%\n",
       "163                 slot_ModelFlux_initial_flag_apCorr    0%    0%\n",
       "164     ext_convolved_ConvolvedFlux_3_kron_flag_apCorr    0%    0%\n",
       "165      ext_convolved_ConvolvedFlux_2_3_3_flag_apCorr    0%    0%\n",
       "166      ext_convolved_ConvolvedFlux_1_4_5_flag_apCorr    0%    0%\n",
       "167      ext_convolved_ConvolvedFlux_3_6_0_flag_apCorr    0%    0%\n",
       "168     ext_convolved_ConvolvedFlux_1_kron_flag_apCorr    0%    0%\n",
       "169                    modelfit_CModel_exp_flag_apCorr    0%    0%\n",
       "170                     slot_ModelFlux_exp_flag_apCorr    0%    0%\n",
       "171      ext_convolved_ConvolvedFlux_3_4_5_flag_apCorr    0%    0%\n",
       "172      ext_convolved_ConvolvedFlux_0_3_3_flag_apCorr    0%    0%\n",
       "173                      base_GaussianFlux_flag_apCorr    0%    0%\n",
       "174      ext_convolved_ConvolvedFlux_3_3_3_flag_apCorr    0%    0%\n",
       "175      ext_convolved_ConvolvedFlux_2_4_5_flag_apCorr    0%    0%\n",
       "176                           base_PsfFlux_flag_apCorr    0%    0%\n",
       "177                           slot_PsfFlux_flag_apCorr    0%    0%\n",
       "178                    modelfit_CModel_dev_flag_apCorr    0%    0%\n",
       "179                     slot_ModelFlux_dev_flag_apCorr    0%    0%\n",
       "180            ext_photometryKron_KronFlux_flag_apCorr    0%    0%\n",
       "181      ext_convolved_ConvolvedFlux_2_6_0_flag_apCorr    0%    0%\n",
       "182     ext_convolved_ConvolvedFlux_0_kron_flag_apCorr    0%    0%\n",
       "183     ext_convolved_ConvolvedFlux_2_kron_flag_apCorr    0%    0%\n",
       "184      ext_convolved_ConvolvedFlux_1_3_3_flag_apCorr    0%    0%\n",
       "\n",
       "[185 rows x 3 columns]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Now we want to know the fraction of objects rejected by each flag:\n",
    "\n",
    "#tables = [catalog.asAstropy() for catalog in catalogs]\n",
    "\n",
    "def format_fraction(table, colname):\n",
    "    size = len(table)\n",
    "    fraction = \"{}%\".format(int(len(table[table[colname]==True])/size*100))\n",
    "    return fraction\n",
    "\n",
    "fraction_rejected = [[colname] +  [format_fraction(table, colname) for table in deep_coadd_tables] \n",
    "                     for colname, description in flags_to_reject_on_all]\n",
    "pd.DataFrame(fraction_rejected, columns=['Flag'] + filters)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#The data products of interest are:\n",
    "#    good_objects_by_filter = all objects that pass flags for all filters.\n",
    "#    good_stars_by_filter = all objects that pass flags for all filters and are deemed to be stars. \n",
    "#    good_galaxies_by_filter = all objects that pass flags for all filters and are deemed to be galaxies.\n",
    "#    All data products are lists of astropy tables.  \n",
    "#    The length and sequences of the lists corresponds to the specified filters.  "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "LSST",
   "language": "python",
   "name": "lsst"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}