Extract summary.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook contains functions to extract the following information from a document:\n",
    "* Keywords\n",
    "* Top relevant sentences\n",
    "\n",
    "** Input **\n",
    "Reads in data stored in pickle format as extracted using Kinshuk's notebook\n",
    "\n",
    "\n",
    "** Output **\n",
    "`list_of_keywords`,`top_10_relevant_sentences`\n",
    "\n",
    "\n",
    "** Usage **\n",
    "Call `extract_summary` function with the document text as a parameter.\n",
    "\n",
    "\n",
    "** Notes **\n",
    "\n",
    "\n",
    "* I tried to use Texttiling in order to tokenize the text by topics and then use it to extract keywords, themes, etc. However, it did not result in any better quality keywords. A new challenge was that of increased number of keywords, hence beating the purpose of summarizing the text. We decided not to use the algorithm in our processing pipeline.\n",
    "* The function takes into account only unigrams and bigrams while extracting top relevant sentences."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Imports **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from pickle import dump, load\n",
    "import nltk\n",
    "from nltk import word_tokenize,FreqDist\n",
    "import re\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.util import ngrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_document_text(raw_text):\n",
    "    \"\"\" This function takes in raw document text as input which we receive from the API and returns a clean text \n",
    "    of the associated document. It cleans up any HTML code in the text, newline characters, and extracts supplemental\n",
    "    information part of the document.\n",
    "    \n",
    "    INPUT: string\n",
    "    OUTPUT: string\n",
    "    \"\"\"\n",
    "    raw_text = raw_text.replace('\\n',' ')\n",
    "    raw_text = raw_text.replace('*','') # added\n",
    "    raw_text = raw_text.replace('\\r',' ') # added\n",
    "    supp_info_idx = raw_text.find(\"SUPPLEMENTARY INFORMATION:\")\n",
    "    summary_idx = raw_text.find(\"SUMMARY:\")\n",
    "    dates_idx = raw_text.find(\"DATES:\")\n",
    "    suppl_info = raw_text[supp_info_idx+26:] # To leave out the string 'Supplementary Information'\n",
    "    summary = raw_text[summary_idx+8:dates_idx]\n",
    "    # Remove any residual HTML tags in text\n",
    "    cleanr = re.compile('<.*?>')\n",
    "    cleantext = re.sub(cleanr, '', suppl_info)\n",
    "    cleansummary = re.sub(cleanr, '', summary)\n",
    "    return cleantext, cleansummary\n",
    "\n",
    "# def get_keywords(clean_corpus):\n",
    "#     \"\"\" This function takes in a clean corpus as input and extracts most important keywords and top 10% of relevant \n",
    "#     sentences from the text.\n",
    "    \n",
    "#     INPUT: string\n",
    "#     OUTPUT: List of tuples: [(list_of_keywords,list_of_sentences)]\n",
    "#     \"\"\"\n",
    "    \n",
    "#     tagged_tokens = tag_my_text(tokenize_text_sent(clean_corpus))\n",
    "#     grand_list = get_top_np(noun_phrase_finder(tagged_tokens))\n",
    "#     return grand_list\n",
    "\n",
    "def tokenize_text(corpus):\n",
    "    pattern = r'''(?x)    # set flag to allow verbose regexps\n",
    "    (([A-Z]\\.)+)       # abbreviations, e.g. B.C.\n",
    "    |(\\w+([-']\\w+)*)       # words with optional internal hyphens e.g. after-ages or author's\n",
    "    '''\n",
    "    tokens = nltk.regexp_tokenize(corpus,pattern)\n",
    "    all_token = [word.lower() for token in tokens for word in token if word != \"\" \n",
    "                 and word[0] != \"'\" and word[0] != \"-\"]\n",
    "    return all_token\n",
    "\n",
    "def tokenize_text_sent(corpus):\n",
    "    sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "    raw_sents = sent_tokenizer.tokenize(corpus) # Split text into sentences    \n",
    "    return [tokenize_text(sent) for sent in raw_sents]\n",
    "\n",
    "def tag_my_text(sents):\n",
    "    return [nltk.pos_tag(sent) for sent in sents]\n",
    "\n",
    "#Chunk noun phrases in tree \n",
    "def noun_phrase_chunker():\n",
    "    grammar = r\"\"\"\n",
    "    NP: {<DT|PP\\$>?<JJ>*<NN>}   # chunk determiner/possessive, adjectives and noun\n",
    "    \"\"\"\n",
    "    cp = nltk.RegexpParser(grammar)\n",
    "    return cp\n",
    "\n",
    "#Extract only the NP marked phrases from the parse tree, that is the chunk we defined\n",
    "def noun_phrase_extractor(sentences, chunker):\n",
    "    res = []\n",
    "    for sent in sentences:\n",
    "        tree = chunker.parse(sent)\n",
    "        for subtree in tree.subtrees():\n",
    "            if subtree.label() == 'NP' : \n",
    "                res.append(subtree[0:len(subtree)])\n",
    "                #res.append(subtree[0])\n",
    "                #print(subtree)\n",
    "    return res\n",
    "\n",
    "def noun_phrase_finder(tagged_text):\n",
    "    all_proper_noun = noun_phrase_extractor(tagged_text,noun_phrase_chunker()) \n",
    "    #does not literally mean proper noun. Chunker only extracts common noun\n",
    "    noun_phrase_list = []                                                      \n",
    "    #noun_phrase_string_list =[]\n",
    "    for noun_phrase in all_proper_noun:\n",
    "        if len(noun_phrase) > 0: #this means where the size of the phrase is greater than 1\n",
    "            small_list =[]\n",
    "            for (word,tag) in noun_phrase:\n",
    "                small_list.append(word)\n",
    "            noun_phrase_list.append(small_list)\n",
    "            #noun_phrase_string_list.append(' '.join(small_list))\n",
    "    return noun_phrase_list\n",
    "\n",
    "#get frequency dist of different length in all the noun phrases extracted. \n",
    "#Something of the form {1:45,2:23} - how many 1phrased and 2 phrased chunks I have etc.\n",
    "def get_length_np(nounPhrase):\n",
    "    np_length={}\n",
    "    for inner_np in nounPhrase:\n",
    "        np_length[len(inner_np)] = np_length.get(len(inner_np),0) + 1\n",
    "    return np_length\n",
    "\n",
    "#get freq dist obj for noun phrase of different lengths\n",
    "def find_freq(nested_list,nest_len):\n",
    "    #from nltk.probability import FreqDist\n",
    "    fdist_list =[]\n",
    "    for inner_np in nested_list:\n",
    "        if len(inner_np) == nest_len:\n",
    "            fdist_list.append(' '.join(inner_np))\n",
    "    fdist = FreqDist(fdist_list)\n",
    "    return fdist\n",
    "\n",
    "#make a grand list of top occuring noun phrases of different sizes --- For testing purpose only. Wont be used\n",
    "def get_top_np(np):\n",
    "    master_common_list=[]\n",
    "    len_list =get_length_np(np).keys()\n",
    "    for item in len_list:\n",
    "        fdist_np = find_freq(np,item)\n",
    "        master_common_list.append(fdist_np.most_common(15))\n",
    "    return master_common_list\n",
    "\n",
    "def get_top_unigrams(np):\n",
    "    unigrams = []\n",
    "    for item in np:\n",
    "        if len(item) ==  1:\n",
    "            unigrams.append(item)\n",
    "    fdist_uni = find_freq(np,1)\n",
    "    uni_list = fdist_uni.most_common()\n",
    "    threshold = 0.3 * len(unigrams)\n",
    "    top = []\n",
    "    s = 0\n",
    "    for word,count in uni_list:\n",
    "        if(len(word)>3):\n",
    "            top.append(word)\n",
    "            s += count\n",
    "            if s > threshold:\n",
    "                break      \n",
    "    return top\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_top_sents(corpus,keywords_list):\n",
    "    sentence_list = get_sentences(corpus)\n",
    "    indexed_sents = sentence_indexing(sentence_list) # This is so that we can re-order most relevant sentences later\n",
    "    \n",
    "    table_scores = handle_tables(sentence_list)\n",
    "    sentence_length_scores = get_sentence_lengths(sentence_list)\n",
    "    keyphrase_scores = get_keyphrase_scores(corpus,sentence_list, keywords_list)\n",
    "    #stepped length\n",
    "    stepped_sentence_length =[]\n",
    "    for each_score in sentence_length_scores:\n",
    "        s = each_score//10\n",
    "        if s>10:\n",
    "            s = 10\n",
    "        stepped_sentence_length.append(s)\n",
    "        \n",
    "    #sent_scores = [s+c for s,c in zip(sentence_length_scores,keyphrase_scores)] #original score = keyphrase +length\n",
    "    #sent_scores = [c/s for s,c in zip(sentence_length_scores,keyphrase_scores)] #score = ratio of keyphrase /length\n",
    "    sent_scores = [c+s+t for s,c,t in zip(stepped_sentence_length,keyphrase_scores,table_scores)] #score = key phrase + stepped length\n",
    "    #sent_scores = [s+(k/l)*100 for s,l,k in zip(stepped_sentence_length,sentence_length_scores,keyphrase_scores)] #score = key phrase + stepped length\n",
    "    \n",
    "    idx_sent_scores = [(s,c) for s,c in zip(indexed_sents,sent_scores)]\n",
    "    sorted_sents = sorted(idx_sent_scores,key=lambda sent: sent[1],reverse=True)\n",
    "    \n",
    "    # Keep top 10% of the sentences, or top 10 whichever is less\n",
    "    top_10 = int(len(sorted_sents) * 0.1)\n",
    "    if top_10 > 4: # changed from 10 to 4\n",
    "        top_10 = 4\n",
    "    x = sorted_sents[:top_10]\n",
    "    top_list = [item[0] for item in x]\n",
    "    sorted_top_list = sorted(top_list,key=lambda sent:sent[1],reverse=False)\n",
    "    sorted_top_list = [sent[0] for sent in sorted_top_list]\n",
    "    \n",
    "    return sorted_top_list\n",
    "    \n",
    "def get_sentences(corpus):\n",
    "    # First, tokenize the corpus into sentences\n",
    "    sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "    raw_sents = sent_tokenizer.tokenize(corpus)\n",
    "    clean_sent =[]\n",
    "    for sent in raw_sents:\n",
    "        clean_sent.append(re.sub(r\"\\[+Page\\s*\\d+\\]+|\\\\+\\d+\\\\+|-+\",'',sent))\n",
    "    return clean_sent\n",
    "    #return raw_sents\n",
    "\n",
    "def sentence_indexing(sent_list):\n",
    "    indexed_sents = []\n",
    "    for idx,sent in enumerate(sent_list):\n",
    "        indexed_sents.append((sent,idx))\n",
    "    return indexed_sents    \n",
    "\n",
    "def get_sentence_lengths(sent_list):\n",
    "    sent_length = []\n",
    "    for s in sent_list:\n",
    "        sent_length.append(len(s.split(' ')))\n",
    "    \n",
    "    return sent_length\n",
    "\n",
    "def get_keyphrase_scores(corpus,sent_list, keywords):\n",
    "    #keywords = get_keywords(corpus) # This gives us a list containing unigrams at index 0 and bigrams at index 1,etc\n",
    "    \n",
    "    unigrams = [item[0] for item in keywords[0]]\n",
    "    bigrams = [item[0] for item in keywords[1]]\n",
    "\n",
    "    unigram_scores = get_unigram_scores(unigrams,sent_list)\n",
    "    bigram_scores = get_bigram_scores(bigrams,sent_list)\n",
    "\n",
    "    sent_feature_import = [a+b for a,b in zip(unigram_scores,bigram_scores)]\n",
    "    \n",
    "    return sent_feature_import\n",
    "\n",
    "def get_unigram_scores(unigram_list,sent_list):\n",
    "    occurence_list = []\n",
    "    for s in sent_list:\n",
    "        words = s.split(' ')\n",
    "        occurence_count = 0\n",
    "        for w in words:\n",
    "            if w.lower() in unigram_list or w.lower() in ['complaint','concern','documented','evidence','warn']:\n",
    "                occurence_count += 1\n",
    "        occurence_list.append(occurence_count)\n",
    "        \n",
    "    return occurence_list\n",
    "\n",
    "def handle_tables(sent_list):\n",
    "    scores = []\n",
    "    for sent in sent_list:\n",
    "        ss = re.sub(r\"\\s+\",' ',sent)\n",
    "        dots = ss.count('.')\n",
    "        numbers = ss.count('1') +sent.count('2') +sent.count('3')+ sent.count('4') +sent.count('5')+sent.count('6')+sent.count('7')+sent.count('8')+sent.count('9')+sent.count('10')\n",
    "        sent_len = len(ss)\n",
    "        if (dots+numbers)/sent_len >= 0.09:\n",
    "            scores.append(-100)\n",
    "        else:\n",
    "            scores.append(0)\n",
    "    return scores\n",
    "            \n",
    "\n",
    "def get_bigram_scores(bigram_list,sent_list):\n",
    "    occurence_list = []\n",
    "    for s in sent_list:\n",
    "        # create bigrams\n",
    "        token=nltk.word_tokenize(s)\n",
    "        bigram_phrases = ngrams(token,2)\n",
    "        occurence_count = 0\n",
    "        for w in bigram_phrases:\n",
    "            w = [word.lower() for word in w]\n",
    "            if ' '.join(w) in bigram_list:\n",
    "                occurence_count += 1\n",
    "        occurence_list.append(occurence_count)\n",
    "        \n",
    "    return occurence_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def extract_summary(text):\n",
    "    clean_text, clean_summary = get_document_text(text)\n",
    "    tagged_tokens = tag_my_text(tokenize_text_sent(clean_text))\n",
    "    np_list = noun_phrase_finder(tagged_tokens)\n",
    "    keywords = get_top_np(np_list)\n",
    "    top_np = get_top_unigrams(np_list)  \n",
    "    #keywords = get_keywords(clean_text)\n",
    "    top_sent = get_top_sents(clean_text,keywords)\n",
    "    \n",
    "    return top_np,top_sent,clean_summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Test **\n",
    "\n",
    "Testing out the function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Test\n",
    "doc_list =load(open(\"data/Master_doc_content\",'rb'))\n",
    "document = doc_list[3]\n",
    "document_text = []\n",
    "for item in document['text']:\n",
    "    document_text.append(str(item))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "key,sentences, summary = extract_summary(document_text[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "** Output **"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['room',\n",
       " 'research',\n",
       " 'usage',\n",
       " 'labor',\n",
       " 'part',\n",
       " 'nara',\n",
       " 'washington',\n",
       " 'march',\n",
       " 'college',\n",
       " 'monday',\n",
       " 'amend']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "key"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Research Room Hours in DC Area Facilities        Our research center and Central Research Room in the National   Archives Building and the research rooms at the National Archives at   College Park facility are currently open for research Monday through   Friday from 8:45 a.m. to 5 p.m.; on Tuesday, Thursday and Friday   evenings from 5 p.m. to 9 p.m.; and Saturdays from 8:45 a.m. to 4:45   p.m.\n",
      "\n",
      "\n",
      "Researchers who conduct research in original archival records in the   evening or on Saturday currently must make a reference request in  person before 3:30 on weekdays to have the records identified and   retrieved from the stack areas for their research use; no records are   retrieved during those extended hours.\n",
      "\n",
      "\n",
      "Currently the National Archives Experience (our Washington DC   museum) including the Rotunda for the Charters of Freedom (displaying   the Declaration of Independence, Constitution, and Bill of Rights) is   open to the public as follows:       The day after Labor Day through March 31, 10 a.m. to 5:30   p.m. (closed on December 25);       April 1 through the Friday before Memorial Day, 10 a.m. to   7 p.m.;       Memorial Day weekend through Labor Day, 10 a.m. to 9 p.m.    We are revising Sec.\n",
      "\n",
      "\n",
      "There are many actions   that must be carried out in order to provide access to archival records   in NARA research rooms; to assist researchers via online descriptions   in our Archival Research Catalog or via written reference requests; and   to develop public programs and exhibits.\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for sent in sentences:\n",
    "    print(sent)\n",
    "    print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "' This document provides an additional 15 days for interested   persons to submit comments on the proposed rule to amend the Customs   and Border Protection (CBP) regulations pertaining to pilots of any   private aircraft arriving in the United States from a foreign port or   location or departing the United States for a foreign port or location.   The proposed rule was published in the Federal Register on September   18, 2007, and the comment period was scheduled to expire on November   19, 2007.    '"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Writing to a fle as JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc_list1 =load(open(\"data/Master_doc_content\",'rb'))\n",
    "doc_list2 = load(open(\"data/Master2_doc_content\",'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "doc_id1 = [\"FAA-2010-1127-0001\",\"USCBP-2007-0064-1986\",\"FMCSA-2015-0419-0001\",\"NARA-06-0007-0001\",\"APHIS-2006-0041-0001\",\"EBSA-2012-0031-0001\",\"IRS-2010-0009-0001\",\"BOR-2008-0004-0001\",\"OSHA-2013-0023-1443\",\"DOL-2016-0001-0001\",\"NRC-2015-0057-0086\",\"CMS-2010-0259-0001\",\"CMS-2009-0008-0003\",\"CMS-2009-0038-0002\",\"NPS-2014-0005-000\",\"BIS-2015-0011-0001\",\"HUD-2011-0056-0019\",\"HUD-2011-0014-0001\",\"OCC-2011-0002-0001\",\"ACF-2015-0008-0124\",\"ETA-2008-0003-0001\",\"CMS-2012-0152-0004\",\"CFPB-2013-0033-0001\",\"USCIS-2016-0001-0001\",\"FMCSA-2011-0146-0001\",\"USCG-2013-0915-0001\",\"NHTSA-2012-0177-0001\",\"USCBP-2005-0005-0001\"]\n",
    "doc_id2 = [\"HUD-2015-0101-0001\",\"ACF-2010-0003-0001\",\"NPS-2015-0008-0001\",\"FAR-2014-0025-0026\",\"CFPB-2013-0002-0001\",\"DOS-2010-0035-0001\",\"USCG-2013-0915-0001\",\"SBA-2010-0001-0001\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc_title1 = [\"Photo Requirements for Pilot Certificates\",\n",
    "             \"Advance Information on Private Aircraft Arriving and Departing the United States\",\n",
    "             \"Evaluation of Safety Sensitive Personnel for Moderate-to-Severe Obstructive Sleep Apnea\",\n",
    "             \"Changes in NARA Research Room and Museum Hours\",\n",
    "             \"Bovine Spongiform Encephalopathy; Minimal-Risk Regions; Importation of Live Bovines and Products Derived From Bovines\",\n",
    "             \"Incentives for Nondiscriminatory Wellness Programs in Group Health Plans\",\n",
    "             \"Furnishing Identifying Number of Tax Return Preparer\",\n",
    "             \"Use of Bureau of Reclamation Land, Facilities, and Waterbodies\",\n",
    "             \"Improve Tracking of Workplace Injuries and Illnesses\",\n",
    "             \"Implementation of the Nondiscrimination and Equal Opportunity Provisions of the Workforce Innovation and Opportunity Act\",\n",
    "             \"Linear No-Threshold Model and Standards for Protection Against Radiation; Extension of Comment Period\",\n",
    "             \"Medicare Program: Accountable Care Organizations and the Medicare Shared Saving Program\",\n",
    "             \"Medicare Program: Changes to the Competitive Acquisition of Certain Durable Medical Equipment, Prosthetics, Orthotics and Supplies (DMEPOS) by Certain Provisions of the Medicare Improvements for Patients and Providers Act of 2008 (MIPPA)\",\n",
    "             \"Medicare Program: Inpatient Rehabilitation Facility Prospective Payment System for Federal Fiscal Year 2010 \",\n",
    "             \"Special Regulations: Areas of the National Park System, Cuyahoga Valley National Park, Bicycling\",\n",
    "             \"Wassenaar Arrangement Plenary Agreements Implementation; Intrusion and Surveillance Items\",\n",
    "             \"Credit Risk Retention 2\",\n",
    "             \"FR 5359–P–01 Equal Access to Housing in HUD Programs Regardless of Sexual Orientation or Gender Identity \",\n",
    "             \"Credit Risk Retention\",\n",
    "             \"Head Start Performance Standards; Extension of Comment Period\",\n",
    "             \"Senior Community Service Employment Program\",\n",
    "             \"Patient Protection and Affordable Care Act: Benefit and Payment Parameters for 2014\",\n",
    "             \"Debt Collection (Regulation F)\",\n",
    "             \"U.S. Citizenship and Immigration Services Fee Schedule\",\n",
    "             \"Applicability of Regulations to Operators of Certain Farm Vehicles and Off-Road Agricultural Equipment\",\n",
    "             \"Carriage of Conditionally Permitted Shale Gas Extraction Waste Water in Bulk\",\n",
    "             \"Federal Motor Vehicle Safety Standards: Event Data Recorders\",\n",
    "             \"Documents Required for Travel Within the Western Hemisphere\"]\n",
    "doc_title2 = [\"FR 5597-P-02 Instituting Smoke- Free Public Housing\",\n",
    "             \"Head Start Program\",\n",
    "             \"Off-Road Vehicle Management: Cape Lookout National Seashore\",\n",
    "             \"Federal Acquisition Regulations: Fair Pay and Safe Workplaces; Second Extension of Time for Comments (FAR Case 2014-025)\",\n",
    "             \"Ability to Repay Standards under Truth in Lending Act (Regulation Z)\",\n",
    "             \"Schedule of Fees for Consular Services, Department of State and Overseas Embassies and Consulates\",\n",
    "             \"Carriage of Conditionally Permitted Shale Gas Extraction Waste Water in Bulk\",\n",
    "             \"Women-Owned Small Business Federal Contract Program\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "9\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "19\n",
      "20\n",
      "21\n",
      "22\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "23\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "Warning: parsing empty text\n",
      "24\n",
      "25\n",
      "26\n",
      "27\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(doc_list1)):\n",
    "    print(i)\n",
    "    info_dic = {}\n",
    "    doc_text = str(doc_list1[i]['text'][0])\n",
    "    info_dic[\"keywords\"],info_dic[\"sentences\"], info_dic[\"summary\"] = extract_summary(doc_text)\n",
    "    info_dic[\"doc_id\"], info_dic[\"doc_title\"] = doc_id1[i], doc_title1[i]\n",
    "    data.append(info_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(doc_list2)):\n",
    "    print(i)\n",
    "    info_dic = {}\n",
    "    doc_text = str(doc_list2[i]['text'][0])\n",
    "    info_dic[\"keywords\"],info_dic[\"sentences\"], info_dic[\"summary\"] = extract_summary(doc_text)\n",
    "    info_dic[\"doc_id\"], info_dic[\"doc_title\"] = doc_id2[i], doc_title2[i]\n",
    "    data.append(info_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top_obj ={}\n",
    "top_obj[\"data\"] = data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'doc_id': 'HUD-2015-0101-0001',\n",
       " 'doc_title': 'FR 5597-P-02 Instituting Smoke- Free Public Housing',\n",
       " 'keywords': ['housing',\n",
       "  'phas',\n",
       "  'tobacco',\n",
       "  'smoking',\n",
       "  'smoke',\n",
       "  'exposure',\n",
       "  'rule',\n",
       "  'health',\n",
       "  'implementation',\n",
       "  'state',\n",
       "  'multiunit',\n",
       "  'community',\n",
       "  'fire',\n",
       "  'percent',\n",
       "  'enforcement',\n",
       "  'information',\n",
       "  'unit',\n",
       "  'damage',\n",
       "  'order',\n",
       "  'research',\n",
       "  'section',\n",
       "  'lease',\n",
       "  'part',\n",
       "  'control',\n",
       "  'notice'],\n",
       " 'sentences': ['Surgeon General estimates that exposure to secondhand tobacco  smoke (i.e., the smoke that comes from burning tobacco products and is  exhaled by smokers) is responsible for the death of 41,000 adults non smokers in the United States each year from lung cancer and heart  disease. Secondhand smoke (SHS) contains hundreds of toxic chemicals  and is designated as a known human carcinogen by the U.S. Environmental  Protection Agency, the U.S. National Toxicology Program, and the  International Agency for Research on Cancer. Exposure to SHS can  also cause sudden infant death syndrome and respiratory symptoms such  as cough and wheeze, middle ear infections, and slowed lung growth and  reduced lung function in children, and increased risk of stroke in  adults. The Surgeon General has concluded that there is no riskfree  level of exposure to SHS, and that eliminating smoking in indoor spaces  fully protects nonsmokers from exposure to secondhand smoke.',\n",
       "  'SHS can move both from  external hallways into apartments and between adjacent units. A  study of public housing documented lower concentrations of SHS  contaminants in buildings covered by smokefree policies (i.e.,  policies prohibiting the smoking of tobacco products in all indoor  spaces) compared to buildings without these policies. Analysis of  data from the National Health and Nutrition Examination Survey (NHANES)  demonstrated evidence of greater SHS exposure among children (aged 6 to  18) living in multiunit housing through measurements of cotinine (a  metabolite of nicotine) in their blood. The study demonstrated that  children living in nonsmoking households in apartments had 45 percent  higher levels of cotinine in their blood compared to children living in  nonsmoking households in detached homes.',\n",
       "  \"      In October 2012, HUD also published a Federal Register notice that  solicited feedback on the HUD's smokefree housing initiative,  specifically seeking information on topics such as best practices and  practical strategies from housing providers who have implemented smoke free policies, potential obstacles to policy implementation and how  these could be overcome, suggestions for supporting housing providers  and residents to facilitate policy implementation, and feedback from  housing providers who have decided not to implement smokefree  policies. HUD received many comments in response to this  solicitation, largely from public health organizations and State and  local health departments, expressing support for the concept and citing  the great health risks posed by smoking and SHS.        77 FR 60712, ``Request for Information on Adopting Smoke Free Policies in PHAs and Multifamily Housing'' (October 4, 2012).\",\n",
       "  \"Rather, HUD is proposing a  prohibition on smoking inside public housing living units and indoor  common areas, public housing administrative office buildings, public  housing community rooms or community facilities, public housing day  care centers and laundry rooms, in outdoor areas within 25 feet of the  housing and administrative office buildings, and in other areas  designated by a PHA as smokefree (collectively, ``restricted areas'').\"],\n",
       " 'summary': ' This proposed rule would require each public housing agency  (PHA) administering public housing to implement a smoke-free policy.  Specifically, this rule proposes that no later than 18 months from the  effective date of the final rule, each PHA must implement a policy  prohibiting lit tobacco products in all living units, indoor common  areas in public housing, and in PHA administrative office buildings (in  brief, a smoke-free policy for all public housing indoor areas). The  smoke-free policy must also extend to all outdoor areas up to 25 feet  from the housing and administrative office buildings. HUD proposes  implementation of smoke-free public housing to improve indoor air  quality in the housing, benefit the health of public housing residents  and PHA staff, reduce the risk of catastrophic fires, and lower overall  maintenance costs.  '}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[28]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "with open('data/text_data.json', 'w') as outfile:\n",
    "    json.dump(top_obj, outfile)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### for debugging from down here, do not execute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "with open('data/text_data.json') as comment_data_file:    \n",
    "    op_data = json.load(comment_data_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "36"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(op_data[\"data\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Photo Requirements for Pilot Certificates\n",
      "Advance Information on Private Aircraft Arriving and Departing the United States\n",
      "Evaluation of Safety Sensitive Personnel for Moderate-to-Severe Obstructive Sleep Apnea\n",
      "Changes in NARA Research Room and Museum Hours\n",
      "Bovine Spongiform Encephalopathy; Minimal-Risk Regions; Importation of Live Bovines and Products Derived From Bovines\n",
      "Incentives for Nondiscriminatory Wellness Programs in Group Health Plans\n",
      "Furnishing Identifying Number of Tax Return Preparer\n",
      "Use of Bureau of Reclamation Land, Facilities, and Waterbodies\n",
      "Improve Tracking of Workplace Injuries and Illnesses\n",
      "Implementation of the Nondiscrimination and Equal Opportunity Provisions of the Workforce Innovation and Opportunity Act\n",
      "Linear No-Threshold Model and Standards for Protection Against Radiation; Extension of Comment Period\n",
      "Medicare Program: Accountable Care Organizations and the Medicare Shared Saving Program\n",
      "Medicare Program: Changes to the Competitive Acquisition of Certain Durable Medical Equipment, Prosthetics, Orthotics and Supplies (DMEPOS) by Certain Provisions of the Medicare Improvements for Patients and Providers Act of 2008 (MIPPA)\n",
      "Medicare Program: Inpatient Rehabilitation Facility Prospective Payment System for Federal Fiscal Year 2010 \n",
      "Special Regulations: Areas of the National Park System, Cuyahoga Valley National Park, Bicycling\n",
      "Wassenaar Arrangement Plenary Agreements Implementation; Intrusion and Surveillance Items\n",
      "Credit Risk Retention 2\n",
      "FR 5359–P–01 Equal Access to Housing in HUD Programs Regardless of Sexual Orientation or Gender Identity \n",
      "Credit Risk Retention\n",
      "Head Start Performance Standards; Extension of Comment Period\n",
      "Senior Community Service Employment Program\n",
      "Patient Protection and Affordable Care Act: Benefit and Payment Parameters for 2014\n",
      "Debt Collection (Regulation F)\n",
      "U.S. Citizenship and Immigration Services Fee Schedule\n",
      "Applicability of Regulations to Operators of Certain Farm Vehicles and Off-Road Agricultural Equipment\n",
      "Carriage of Conditionally Permitted Shale Gas Extraction Waste Water in Bulk\n",
      "Federal Motor Vehicle Safety Standards: Event Data Recorders\n",
      "Documents Required for Travel Within the Western Hemisphere\n",
      "FR 5597-P-02 Instituting Smoke- Free Public Housing\n",
      "Head Start Program\n",
      "Off-Road Vehicle Management: Cape Lookout National Seashore\n"
     ]
    }
   ],
   "source": [
    "for item in data:\n",
    "    print(item[\"doc_title\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clean_text, clean_summary= get_document_text(str(doc_list[1]['text'][0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "keywords = get_keywords(clean_text) # This gives us a list containing unigrams at index 0 and bigrams at index 1,etc\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tagged_tokens = tag_my_text(tokenize_text_sent(clean_text))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[('posed', 'VBN'),\n",
       "  ('rule', 'NN'),\n",
       "  ('published', 'VBN'),\n",
       "  ('at', 'IN'),\n",
       "  ('72', 'CD'),\n",
       "  ('fr', 'JJ'),\n",
       "  ('53394', 'CD'),\n",
       "  ('september', 'NN'),\n",
       "  ('18', 'CD'),\n",
       "  ('2007', 'CD'),\n",
       "  ('must', 'MD'),\n",
       "  ('be', 'VB'),\n",
       "  ('received', 'VBN'),\n",
       "  ('on', 'IN'),\n",
       "  ('or', 'CC'),\n",
       "  ('before', 'IN'),\n",
       "  ('december', 'JJ'),\n",
       "  ('4', 'CD'),\n",
       "  ('2007', 'CD')],\n",
       " [('addresses', 'NNS'),\n",
       "  ('you', 'PRP'),\n",
       "  ('may', 'MD'),\n",
       "  ('submit', 'VB'),\n",
       "  ('comments', 'NNS'),\n",
       "  ('identified', 'VBN'),\n",
       "  ('by', 'IN'),\n",
       "  ('docket', 'NN'),\n",
       "  ('number', 'NN'),\n",
       "  ('by', 'IN'),\n",
       "  ('one', 'CD'),\n",
       "  ('of', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('following', 'JJ'),\n",
       "  ('methods', 'NNS'),\n",
       "  ('federal', 'JJ'),\n",
       "  ('erulemaking', 'VBG'),\n",
       "  ('portal', 'JJ'),\n",
       "  ('http', 'NN'),\n",
       "  ('www', 'JJ'),\n",
       "  ('regulations', 'NNS'),\n",
       "  ('gov', 'VBP')],\n",
       " [('follow', 'VB'),\n",
       "  ('the', 'DT'),\n",
       "  ('instructions', 'NNS'),\n",
       "  ('for', 'IN'),\n",
       "  ('submitting', 'VBG'),\n",
       "  ('comments', 'NNS'),\n",
       "  ('via', 'IN'),\n",
       "  ('docket', 'NN'),\n",
       "  ('number', 'NN'),\n",
       "  ('uscbp-2007-0064', 'JJ')],\n",
       " [('mail', 'NN'),\n",
       "  ('border', 'NN'),\n",
       "  ('security', 'NN'),\n",
       "  ('regulations', 'NNS'),\n",
       "  ('branch', 'NN'),\n",
       "  ('office', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('international', 'JJ'),\n",
       "  ('trade', 'NN'),\n",
       "  ('customs', 'NNS'),\n",
       "  ('and', 'CC'),\n",
       "  ('border', 'NN'),\n",
       "  ('protection', 'NN'),\n",
       "  ('1300', 'CD'),\n",
       "  ('pennsylvania', 'NN'),\n",
       "  ('ave', 'VBP'),\n",
       "  ('nw', 'VBN'),\n",
       "  ('mint', 'NN'),\n",
       "  ('annex', 'IN'),\n",
       "  ('washington', 'NN'),\n",
       "  ('dc', 'NN'),\n",
       "  ('20229', 'CD')],\n",
       " [('instructions', 'NNS'),\n",
       "  ('all', 'DT'),\n",
       "  ('submissions', 'NNS'),\n",
       "  ('received', 'VBD'),\n",
       "  ('must', 'MD'),\n",
       "  ('include', 'VB'),\n",
       "  ('the', 'DT'),\n",
       "  ('agency', 'NN'),\n",
       "  ('name', 'NN'),\n",
       "  ('and', 'CC'),\n",
       "  ('document', 'NN'),\n",
       "  ('number', 'NN'),\n",
       "  ('for', 'IN'),\n",
       "  ('this', 'DT'),\n",
       "  ('rulemaking', 'NN')],\n",
       " [('all', 'DT'),\n",
       "  ('comments', 'NNS'),\n",
       "  ('received', 'VBN'),\n",
       "  ('will', 'MD'),\n",
       "  ('be', 'VB'),\n",
       "  ('posted', 'VBN'),\n",
       "  ('without', 'IN'),\n",
       "  ('change', 'NN'),\n",
       "  ('to', 'TO'),\n",
       "  ('http', 'VB'),\n",
       "  ('www', 'JJ'),\n",
       "  ('regulations', 'NNS'),\n",
       "  ('gov', 'VBP'),\n",
       "  ('including', 'VBG'),\n",
       "  ('any', 'DT'),\n",
       "  ('personal', 'JJ'),\n",
       "  ('information', 'NN'),\n",
       "  ('provided', 'VBD')],\n",
       " [('docket', 'NN'),\n",
       "  ('for', 'IN'),\n",
       "  ('access', 'NN'),\n",
       "  ('to', 'TO'),\n",
       "  ('the', 'DT'),\n",
       "  ('docket', 'NN'),\n",
       "  ('to', 'TO'),\n",
       "  ('read', 'VB'),\n",
       "  ('background', 'NN'),\n",
       "  ('documents', 'NNS'),\n",
       "  ('or', 'CC'),\n",
       "  ('comments', 'NNS'),\n",
       "  ('received', 'VBN'),\n",
       "  ('go', 'VBP'),\n",
       "  ('to', 'TO'),\n",
       "  ('http', 'VB'),\n",
       "  ('www', 'JJ'),\n",
       "  ('regulations', 'NNS'),\n",
       "  ('gov', 'VBP')],\n",
       " [('submitted', 'VBN'),\n",
       "  ('comments', 'NNS'),\n",
       "  ('may', 'MD'),\n",
       "  ('also', 'RB'),\n",
       "  ('be', 'VB'),\n",
       "  ('inspected', 'VBN'),\n",
       "  ('during', 'IN'),\n",
       "  ('regular', 'JJ'),\n",
       "  ('business', 'NN'),\n",
       "  ('days', 'NNS'),\n",
       "  ('between', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('hours', 'NNS'),\n",
       "  ('of', 'IN'),\n",
       "  ('9', 'CD'),\n",
       "  ('a', 'DT'),\n",
       "  ('m', 'NN'),\n",
       "  ('and', 'CC'),\n",
       "  ('4', 'CD'),\n",
       "  ('30', 'CD'),\n",
       "  ('p', 'NN'),\n",
       "  ('m', 'NN'),\n",
       "  ('at', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('office', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('international', 'JJ'),\n",
       "  ('trade', 'NN'),\n",
       "  ('customs', 'NNS'),\n",
       "  ('and', 'CC'),\n",
       "  ('border', 'NN'),\n",
       "  ('protection', 'NN'),\n",
       "  ('799', 'CD'),\n",
       "  ('9th', 'CD'),\n",
       "  ('street', 'NN'),\n",
       "  ('nw', 'JJ'),\n",
       "  ('5th', 'JJ'),\n",
       "  ('floor', 'NN'),\n",
       "  ('washington', 'NN'),\n",
       "  ('dc', 'NN')],\n",
       " [('arrangements', 'NNS'),\n",
       "  ('to', 'TO'),\n",
       "  ('inspect', 'VB'),\n",
       "  ('submitted', 'VBN'),\n",
       "  ('comments', 'NNS'),\n",
       "  ('should', 'MD'),\n",
       "  ('be', 'VB'),\n",
       "  ('made', 'VBN'),\n",
       "  ('in', 'IN'),\n",
       "  ('advance', 'NN'),\n",
       "  ('by', 'IN'),\n",
       "  ('calling', 'VBG'),\n",
       "  ('mr', 'NN'),\n",
       "  ('joseph', 'NN'),\n",
       "  ('clark', 'NN'),\n",
       "  ('at', 'IN'),\n",
       "  ('202', 'CD'),\n",
       "  ('572-8768', 'JJ')],\n",
       " [('for', 'IN'),\n",
       "  ('further', 'JJ'),\n",
       "  ('information', 'NN'),\n",
       "  ('contact', 'NN'),\n",
       "  ('for', 'IN'),\n",
       "  ('operational', 'JJ'),\n",
       "  ('matters', 'NNS'),\n",
       "  ('michael', 'VBP'),\n",
       "  ('kaneris', 'NNP'),\n",
       "  ('office', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('field', 'NN'),\n",
       "  ('operations', 'NNS'),\n",
       "  ('customs', 'NNS'),\n",
       "  ('and', 'CC'),\n",
       "  ('border', 'NN'),\n",
       "  ('protection', 'NN'),\n",
       "  ('202-344-1584', 'JJ')],\n",
       " [('for', 'IN'),\n",
       "  ('legal', 'JJ'),\n",
       "  ('matters', 'NNS'),\n",
       "  ('glen', 'VBN'),\n",
       "  ('vereb', 'JJ'),\n",
       "  ('branch', 'NN'),\n",
       "  ('chief', 'JJ'),\n",
       "  ('office', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('international', 'JJ'),\n",
       "  ('trade', 'NN'),\n",
       "  ('regulations', 'NNS'),\n",
       "  ('amp', 'VBP'),\n",
       "  ('rulings', 'NNS'),\n",
       "  ('customs', 'NNS'),\n",
       "  ('and', 'CC'),\n",
       "  ('border', 'NN'),\n",
       "  ('protection', 'NN'),\n",
       "  ('202-572-8700', 'JJ')],\n",
       " [('supplementary', 'JJ'),\n",
       "  ('information', 'NN'),\n",
       "  ('background', 'NN'),\n",
       "  ('cbp', 'NN'),\n",
       "  ('published', 'VBD'),\n",
       "  ('a', 'DT'),\n",
       "  ('notice', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('proposed', 'VBN'),\n",
       "  ('rulemaking', 'NN'),\n",
       "  ('in', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('federal', 'JJ'),\n",
       "  ('register', 'NN'),\n",
       "  ('72', 'CD'),\n",
       "  ('fr', 'NN'),\n",
       "  ('53394', 'CD'),\n",
       "  ('on', 'IN'),\n",
       "  ('september', 'NN'),\n",
       "  ('18', 'CD'),\n",
       "  ('2007', 'CD'),\n",
       "  ('proposing', 'NN'),\n",
       "  ('to', 'TO'),\n",
       "  ('amend', 'VB'),\n",
       "  ('the', 'DT'),\n",
       "  ('cbp', 'JJ'),\n",
       "  ('regulations', 'NNS'),\n",
       "  ('pertaining', 'VBG'),\n",
       "  ('to', 'TO'),\n",
       "  ('private', 'JJ'),\n",
       "  ('aircraft', 'NN'),\n",
       "  ('arriving', 'VBG'),\n",
       "  ('in', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('united', 'JJ'),\n",
       "  ('states', 'NNS'),\n",
       "  ('from', 'IN'),\n",
       "  ('a', 'DT'),\n",
       "  ('foreign', 'JJ'),\n",
       "  ('port', 'NN'),\n",
       "  ('or', 'CC'),\n",
       "  ('location', 'NN'),\n",
       "  ('or', 'CC'),\n",
       "  ('departing', 'VBG'),\n",
       "  ('the', 'DT'),\n",
       "  ('united', 'JJ'),\n",
       "  ('states', 'NNS'),\n",
       "  ('for', 'IN'),\n",
       "  ('a', 'DT'),\n",
       "  ('foreign', 'JJ'),\n",
       "  ('port', 'NN'),\n",
       "  ('or', 'CC'),\n",
       "  ('location', 'NN')],\n",
       " [('the', 'DT'),\n",
       "  ('proposed', 'VBN'),\n",
       "  ('rule', 'NN'),\n",
       "  ('would', 'MD'),\n",
       "  ('require', 'VB'),\n",
       "  ('any', 'DT'),\n",
       "  ('pilot', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('a', 'DT'),\n",
       "  ('private', 'JJ'),\n",
       "  ('aircraft', 'NN'),\n",
       "  ('to', 'TO'),\n",
       "  ('submit', 'VB'),\n",
       "  ('advance', 'JJ'),\n",
       "  ('electronic', 'JJ'),\n",
       "  ('information', 'NN'),\n",
       "  ('regarding', 'VBG'),\n",
       "  ('each', 'DT'),\n",
       "  ('individual', 'NN'),\n",
       "  ('traveling', 'VBG'),\n",
       "  ('onboard', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('aircraft', 'NN'),\n",
       "  ('no', 'RB'),\n",
       "  ('later', 'RB'),\n",
       "  ('than', 'IN'),\n",
       "  ('60', 'CD'),\n",
       "  ('minutes', 'NNS'),\n",
       "  ('before', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('arriving', 'VBG'),\n",
       "  ('private', 'JJ'),\n",
       "  ('aircraft', 'NN'),\n",
       "  ('departs', 'NNS'),\n",
       "  ('from', 'IN'),\n",
       "  ('a', 'DT'),\n",
       "  ('foreign', 'JJ'),\n",
       "  ('location', 'NN'),\n",
       "  ('for', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('united', 'JJ'),\n",
       "  ('states', 'NNS'),\n",
       "  ('and', 'CC'),\n",
       "  ('no', 'DT'),\n",
       "  ('later', 'JJR'),\n",
       "  ('than', 'IN'),\n",
       "  ('60', 'CD'),\n",
       "  ('minutes', 'NNS'),\n",
       "  ('before', 'IN'),\n",
       "  ('a', 'DT'),\n",
       "  ('private', 'JJ'),\n",
       "  ('aircraft', 'NN'),\n",
       "  ('departs', 'VBZ'),\n",
       "  ('a', 'DT'),\n",
       "  ('united', 'JJ'),\n",
       "  ('states', 'NNS'),\n",
       "  ('airport', 'VBP'),\n",
       "  ('or', 'CC'),\n",
       "  ('location', 'NN'),\n",
       "  ('for', 'IN'),\n",
       "  ('a', 'DT'),\n",
       "  ('foreign', 'JJ'),\n",
       "  ('location', 'NN')],\n",
       " [('the', 'DT'),\n",
       "  ('proposed', 'VBN'),\n",
       "  ('rule', 'NN'),\n",
       "  ('would', 'MD'),\n",
       "  ('also', 'RB'),\n",
       "  ('amend', 'VB'),\n",
       "  ('cbp', 'JJ'),\n",
       "  ('regulations', 'NNS'),\n",
       "  ('to', 'TO'),\n",
       "  ('add', 'VB'),\n",
       "  ('data', 'NNS'),\n",
       "  ('elements', 'NNS'),\n",
       "  ('to', 'TO'),\n",
       "  ('the', 'DT'),\n",
       "  ('existing', 'VBG'),\n",
       "  ('notice', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('arrival', 'NN'),\n",
       "  ('requirements', 'NNS'),\n",
       "  ('add', 'VBP'),\n",
       "  ('a', 'DT'),\n",
       "  ('notice', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('departure', 'NN'),\n",
       "  ('requirement', 'NN'),\n",
       "  ('and', 'CC'),\n",
       "  ('clarify', 'VB'),\n",
       "  ('landing', 'VBG'),\n",
       "  ('rights', 'NNS'),\n",
       "  ('procedures', 'NNS')],\n",
       " [('the', 'DT'),\n",
       "  ('notice', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('proposed', 'VBN'),\n",
       "  ('rulemaking', 'NN'),\n",
       "  ('invited', 'VBD'),\n",
       "  ('the', 'DT'),\n",
       "  ('public', 'NN'),\n",
       "  ('to', 'TO'),\n",
       "  ('comment', 'VB'),\n",
       "  ('on', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('proposal', 'NN')],\n",
       " [('comments', 'NNS'),\n",
       "  ('on', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('proposed', 'VBN'),\n",
       "  ('rule', 'NN'),\n",
       "  ('were', 'VBD'),\n",
       "  ('requested', 'VBN'),\n",
       "  ('on', 'IN'),\n",
       "  ('or', 'CC'),\n",
       "  ('before', 'IN'),\n",
       "  ('november', 'JJ'),\n",
       "  ('19', 'CD'),\n",
       "  ('2007', 'CD')],\n",
       " [('extension', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('comment', 'NN'),\n",
       "  ('period', 'NN'),\n",
       "  ('in', 'IN'),\n",
       "  ('response', 'NN'),\n",
       "  ('to', 'TO'),\n",
       "  ('the', 'DT'),\n",
       "  ('proposed', 'VBN'),\n",
       "  ('rule', 'NN'),\n",
       "  ('published', 'VBN'),\n",
       "  ('in', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('federal', 'JJ'),\n",
       "  ('register', 'NN'),\n",
       "  ('cbp', 'NN'),\n",
       "  ('has', 'VBZ'),\n",
       "  ('received', 'VBN'),\n",
       "  ('correspondence', 'NN'),\n",
       "  ('from', 'IN'),\n",
       "  ('various', 'JJ'),\n",
       "  ('parties', 'NNS'),\n",
       "  ('requesting', 'VBG'),\n",
       "  ('an', 'DT'),\n",
       "  ('extension', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('the', 'DT'),\n",
       "  ('comment', 'NN'),\n",
       "  ('period', 'NN')],\n",
       " [('a', 'DT'),\n",
       "  ('decision', 'NN'),\n",
       "  ('has', 'VBZ'),\n",
       "  ('been', 'VBN'),\n",
       "  ('made', 'VBN'),\n",
       "  ('to', 'TO'),\n",
       "  ('grant', 'VB'),\n",
       "  ('an', 'DT'),\n",
       "  ('extension', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('15', 'CD'),\n",
       "  ('days', 'NNS')],\n",
       " [('comments', 'NNS'),\n",
       "  ('are', 'VBP'),\n",
       "  ('now', 'RB'),\n",
       "  ('due', 'JJ'),\n",
       "  ('on', 'IN'),\n",
       "  ('or', 'CC'),\n",
       "  ('before', 'IN'),\n",
       "  ('december', 'JJ'),\n",
       "  ('4', 'CD'),\n",
       "  ('2007', 'CD')],\n",
       " [('dated', 'JJ'), ('november', 'RB'), ('9', 'CD'), ('2007', 'CD')],\n",
       " [('sandra', 'JJ'),\n",
       "  ('l.', 'NN'),\n",
       "  ('l.', 'NN'),\n",
       "  ('bell', 'NNP'),\n",
       "  ('executive', 'NN'),\n",
       "  ('director', 'NN'),\n",
       "  ('regulations', 'NNS'),\n",
       "  ('amp', 'VBP'),\n",
       "  ('rulings', 'NNS'),\n",
       "  ('office', 'NN'),\n",
       "  ('of', 'IN'),\n",
       "  ('international', 'JJ'),\n",
       "  ('trade', 'NN')],\n",
       " [('fr', 'NN'), ('doc', 'NN')],\n",
       " [('e7-22309', 'JJ'),\n",
       "  ('filed', 'VBD'),\n",
       "  ('11-13-07', 'JJ'),\n",
       "  ('8', 'CD'),\n",
       "  ('45', 'CD'),\n",
       "  ('am', 'VBP'),\n",
       "  ('billing', 'VBG'),\n",
       "  ('code', 'NN'),\n",
       "  ('9111-14-p', 'CD')]]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tagged_tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#grand_list = get_top_np(noun_phrase_finder(tagged_tokens))\n",
    "np_list = noun_phrase_finder(tagged_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "grand = get_top_np(np_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[('border', 5),\n",
       "  ('rule', 5),\n",
       "  ('protection', 4),\n",
       "  ('location', 3),\n",
       "  ('office', 3),\n",
       "  ('number', 3),\n",
       "  ('docket', 3),\n",
       "  ('dc', 2),\n",
       "  ('washington', 2),\n",
       "  ('period', 2),\n",
       "  ('rulemaking', 2),\n",
       "  ('background', 2),\n",
       "  ('september', 2),\n",
       "  ('fr', 2),\n",
       "  ('cbp', 2)],\n",
       " [('international trade', 4),\n",
       "  ('an extension', 2),\n",
       "  ('private aircraft', 2),\n",
       "  ('a notice', 2),\n",
       "  ('this rulemaking', 1),\n",
       "  ('regular business', 1),\n",
       "  ('any pilot', 1),\n",
       "  ('the agency', 1),\n",
       "  ('the notice', 1),\n",
       "  ('vereb branch', 1),\n",
       "  ('the docket', 1),\n",
       "  ('the comment', 1),\n",
       "  ('the office', 1),\n",
       "  ('supplementary information', 1),\n",
       "  ('a m', 1)],\n",
       " [('a private aircraft', 2),\n",
       "  ('a foreign location', 2),\n",
       "  ('the federal register', 2),\n",
       "  ('a foreign port', 2),\n",
       "  ('any personal information', 1),\n",
       "  ('advance electronic information', 1),\n",
       "  ('nw 5th floor', 1)]]"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grand"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### doc_id and title -1 \n",
    "0. FAA-2010-1127-0001\n",
    "1. USCBP-2007-0064-1986\n",
    "2. FMCSA-2015-0419-0001\n",
    "3. NARA-06-0007-0001\n",
    "4. APHIS-2006-0041-0001\n",
    "5. EBSA-2012-0031-0001\n",
    "6. IRS-2010-0009-0001\n",
    "7. BOR-2008-0004-0001\n",
    "8. OSHA-2013-0023-1443\n",
    "9. DOL-2016-0001-0001\n",
    "10. NRC-2015-0057-0086\n",
    "11. CMS-2010-0259-0001\n",
    "12. CMS-2009-0008-0003\n",
    "13. CMS-2009-0038-0002\n",
    "14. NPS-2014-0005-0001\n",
    "15. BIS-2015-0011-0001\n",
    "16. HUD-2011-0056-0019\n",
    "17. HUD-2011-0014-0001\n",
    "18. OCC-2011-0002-0001\n",
    "19. ACF-2015-0008-0124\n",
    "20. ETA-2008-0003-0001\n",
    "21. CMS-2012-0152-0004\n",
    "22. CFPB-2013-0033-0001\n",
    "23. USCIS-2016-0001-0001\n",
    "24. FMCSA-2011-0146-0001\n",
    "25. USCG-2013-0915-0001\n",
    "26. NHTSA-2012-0177-0001\n",
    "27. USCBP-2005-0005-0001\n",
    "\n",
    "\n",
    "#### doc_id and title -2\n",
    "1. HUD-2015-0101-0001\n",
    "2. ACF-2010-0003-0001\n",
    "3. NPS-2015-0008-0001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "s = str(doc_list2[2]['text'][0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "title_end = s.find(\"AGENCY:\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "title_start = s.rfind(\"\\n\", 0, title_end - 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nCape Lookout National Seashore, Off-Road Vehicle Management\\n\\n'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "s[title_start:title_end]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<pre>\n",
       "[Federal Register Volume 80, Number 243 (Friday, December 18, 2015)]\n",
       "[Proposed Rules]\n",
       "[Pages 79013-79020]\n",
       "From the Federal Register Online via the Government Publishing Office [<a href=\"http://www.gpo.gov\">www.gpo.gov</a>]\n",
       "[FR Doc No: 2015-31793]\n",
       "\n",
       "\n",
       "\n",
       "[[Page 79013]]\n",
       "\n",
       "=======================================================================\n",
       "-----------------------------------------------------------------------\n",
       "\n",
       "DEPARTMENT OF THE INTERIOR\n",
       "\n",
       "National Park Service\n",
       "\n",
       "36 CFR Part 7\n",
       "\n",
       "[NPS-CALO-19111; PPWONRADE2, PMP00EI05.YP]\n",
       "RIN 1024-AE24\n",
       "\n",
       "\n",
       "Cape Lookout National Seashore, Off-Road Vehicle Management\n",
       "\n",
       "AGENCY: National Park Service, Interior\n",
       "\n",
       "ACTION: Proposed rule.\n",
       "\n",
       "-----------------------------------------------------------------------\n",
       "\n",
       "SUMMARY: The National Park Service proposes to designate routes for, \n",
       "and manage off-road vehicle use within Cape Lookout National Seashore, \n",
       "North Carolina. Under the National Park Service general regulations, \n",
       "the operation of motor vehicles off roads is prohibited unless \n",
       "authorized by special regulation. The proposed rule would authorize \n",
       "off-road vehicle use at the Seashore through a permit system and \n",
       "establish operational and vehicle requirements.\n",
       "\n",
       "DATES: Comments must be received by February 16, 2016.\n",
       "\n",
       "ADDRESSES: You may submit comments, identified by the Regulation \n",
       "Identifier Number (RIN) 1024-AE24, by any of the following methods:\n",
       "    <bullet> Electronically: Federal eRulemaking Portal: <a href=\"http://www.regulations.gov\">http://www.regulations.gov</a>. Follow the instructions for submitting comments.\n",
       "    <bullet> Hardcopy: Mail or hand-deliver to: Superintendent, Cape \n",
       "Lookout National Seashore, 131 Charles St., Harkers Island, North \n",
       "Carolina 28531.\n",
       "    Instructions: All comments received must include the agency name \n",
       "and RIN for this rulemaking: 1024-AE24. Comments submitted through \n",
       "Federal eRulemaking Portal: <a href=\"http://www.regulations.gov\">http://www.regulations.gov</a> or submitted by \n",
       "mail must be entered or postmarked before midnight (Eastern Daylight \n",
       "Time) February 16, 2016 Comments submitted by hand delivery must be \n",
       "received by the close of business hours (5 p.m. Eastern Daylight Time) \n",
       "February 16, 2016.\n",
       "    If you commented on the Draft ORV Management Plan/Draft \n",
       "Environmental Impact Statement, (plan/DEIS) your comments have been \n",
       "considered in drafting the proposed rule. Comments should focus on this \n",
       "proposed rule; comments that refer back to the draft Plan/DEIS will be \n",
       "untimely and will not be considered.\n",
       "    Comments will not be accepted by fax, email, or in any way other \n",
       "than those specified above, and bulk comments in any format (hard copy \n",
       "or electronic) submitted on behalf of others will not be accepted. \n",
       "Comments received will be posted without change to <a href=\"http://www.regulations.gov\">http://www.regulations.gov</a>, including any personal information provided.\n",
       "\n",
       "FOR FURTHER INFORMATION CONTACT: Patrick Kenney, Superintendent, Cape \n",
       "Lookout National Seashore, 131 Charles St., Harkers Island, North \n",
       "Carolina 28531; phone 252-728-2250, extension 3014.\n",
       "\n",
       "SUPPLEMENTARY INFORMATION: \n",
       "\n",
       "Background\n",
       "\n",
       "Description of Cape Lookout National Seashore\n",
       "\n",
       "    Authorized by Congress in 1966 (Pub. L. 89-366), and established as \n",
       "a unit of the National Park System in 1976 (41 FR 39363), Cape Lookout \n",
       "National Seashore (Seashore) is located approximately three miles off \n",
       "the mainland in the central coastal area of North Carolina. Consisting \n",
       "of more than 29,000 acres of land and water from Ocracoke Inlet to \n",
       "Beaufort Inlet, the 56 miles of barrier islands consist mostly of wide, \n",
       "bare beaches with low dunes covered by scattered grasses, flat \n",
       "grasslands bordered by dense vegetation, maritime forests, and large \n",
       "expanses of salt marsh alongside the sounds. The Seashore serves as a \n",
       "popular recreation destination where visitors participate in a variety \n",
       "of recreational activities. The Seashore also contains important \n",
       "habitat for wildlife created by the Seashore's dynamic environmental \n",
       "processes. Several species listed under the Endangered Species Act, \n",
       "including the piping plover, seabeach amaranth, red knot, and four \n",
       "species of sea turtles, are found within the Seashore.\n",
       "\n",
       "Authority and Jurisdiction To Promulgate Regulations\n",
       "\n",
       "    In the statute commonly known as the NPS Organic Act (54 U.S.C. \n",
       "100101), Congress granted the National Park Service (NPS) broad \n",
       "authority to regulate the use of areas under its jurisdiction. The \n",
       "Organic Act authorizes the Secretary of the Interior (Secretary), \n",
       "acting through the NPS, to ``prescribe such regulations as the \n",
       "Secretary considers necessary or proper for the use and management of \n",
       "[National Park] System units.'' 54 U.S.C. 100751(a). In the Seashore's \n",
       "enabling act, Congress directed the Secretary to administer the \n",
       "Seashore ``for the general purposes of public outdoor recreation, \n",
       "including conservation of natural features contributing to public \n",
       "enjoyment.'' 16 U.S.C. 459g-4.\n",
       "\n",
       "Off-Road Motor Vehicle Regulation\n",
       "\n",
       "    Executive Order 11644, Use of Off-Road Vehicles on the Public \n",
       "Lands, was issued in 1972 in response to the widespread and rapidly \n",
       "increasing off-road driving on public lands ``often for legitimate \n",
       "purposes but also in frequent conflict with wise land and resource \n",
       "management practices, environmental values, and other types of \n",
       "recreational activity.'' Executive Order 11644 was amended by Executive \n",
       "Order 11989 in 1977, and together they are collectively referred to in \n",
       "this rule as ``E.O.'' The E.O. requires Federal agencies that allow \n",
       "motorized vehicle use in off-road areas to designate specific areas and \n",
       "routes on public lands where the use of motorized vehicles may be \n",
       "permitted. Specifically, section three of the E.O. requires agencies to \n",
       "develop and issue regulations that designate the specific areas and \n",
       "trails on public lands where off-road vehicle (ORV) use is permitted, \n",
       "and areas where ORV use is prohibited. The regulations must ensure that \n",
       "the designation of such areas and trails will be based upon the \n",
       "protection of the resources of the public lands, promotion of the \n",
       "safety of all users of those lands, and minimization of conflicts among \n",
       "the various uses of those lands. The regulations must also require that \n",
       "the designation of such areas and trails shall be in accordance with \n",
       "the following:\n",
       "    (1) Areas and trails shall be located to minimize damage to soil, \n",
       "watershed, vegetation, or other resources of the public lands.\n",
       "    (2) Areas and trails shall be located to minimize harassment of \n",
       "wildlife or significant disruption of wildlife habitats.\n",
       "    (3) Areas and trails shall be located to minimize conflicts between \n",
       "off-road vehicle use and other existing or proposed recreational uses \n",
       "of the same or neighboring public lands, and to ensure the \n",
       "compatibility of such uses with existing conditions in populated areas, \n",
       "taking into account noise and other factors.\n",
       "    (4) Areas and trails shall not be located in officially designated \n",
       "Wilderness Areas or Primitive Areas. Areas and trails shall be located \n",
       "in areas of the National Park System, Natural Areas, or National \n",
       "Wildlife Refuges and Game Ranges only if the respective agency head \n",
       "determines that off-road vehicle use in such locations will not \n",
       "adversely affect their natural, aesthetic, or scenic values.\n",
       "    The NPS regulation at 36 CFR 4.10(b) implements the E.O. and \n",
       "requires that routes and areas designated for ORV use be promulgated as \n",
       "special regulations and that the designation of routes and\n",
       "\n",
       "[[Page 79014]]\n",
       "\n",
       "areas must comply with the E.O. It also states that ORV routes and use-\n",
       "areas may be designated only in national recreation areas, national \n",
       "seashores, national lakeshores, and national preserves. The proposed \n",
       "rule is consistent with these authorities and with Section 8.2.3.1 \n",
       "(Motorized Off-road Vehicle Use) of NPS Management Policies 2006, \n",
       "available at: <a href=\"http://www.nps.gov/policy/mp/policies.html\">http://www.nps.gov/policy/mp/policies.html</a>.\n",
       "\n",
       "ORV Use at Cape Lookout National Seashore\n",
       "\n",
       "    ORV use at the Seashore predates the authorization and \n",
       "establishment of the Seashore. Beginning in the 1930s, vehicles were \n",
       "transported to the islands by shallow draft ferries and were used to \n",
       "access fishing grounds.\n",
       "    Today, ORVs provide vehicular access to the Seashore beaches for \n",
       "recreational purposes. ORV routes have been designated and ORV use has \n",
       "been managed through the Superintendent's Compendium, which currently \n",
       "allows for ORV use from March 16 to December 31 (with a closure of the \n",
       "Seashore to ORVs from January 1 through March 15). The Cape Lookout \n",
       "National Seashore General Management Plan (NPS 1980) identified 47 of \n",
       "the 56 miles of the Seashore as appropriate for controlled ORV use. The \n",
       "remaining nine miles on Shackleford Banks is a proposed wilderness area \n",
       "under the Wilderness Act (Pub. L. 88-577) and is managed to preserve \n",
       "its wilderness character and is closed to recreational vehicle use. \n",
       "Currently, of the 47 miles identified as appropriate for ORV use, 2.2 \n",
       "miles are closed to ORV use year-round. Additional areas may be \n",
       "periodically closed to ORV use for resource protection during the bird \n",
       "nesting and fledgling season or turtle nesting and hatching season.\n",
       "\n",
       "Draft ORV Management Plan/Draft Environmental Impact Statement\n",
       "\n",
       "    The plan/DEIS was released for public comment on May 23, 2014, with \n",
       "the public comment period extended twice through September 19, 2014. \n",
       "This long-term ORV planning effort is based on recognition by the NPS \n",
       "that if allowed, ORVs must be regulated in a manner that is consistent \n",
       "with applicable law and in a manner that appropriately addresses \n",
       "resource protection, potential conflicts among the various Seashore \n",
       "users, and visitor safety.\n",
       "    The plan/DEIS and other supporting documentation can be found \n",
       "online at <a href=\"http://www.parkplanning.nps.gov/calo\">http://www.parkplanning.nps.gov/calo</a>.\n",
       "\n",
       "The Proposed Rule\n",
       "\n",
       "    This proposed rule would establish a special regulation pursuant to \n",
       "the E.O. and 36 CFR 4.10(b) that would implement portions of the \n",
       "preferred alternative as described in the plan/DEIS. The proposed rule \n",
       "would:\n",
       "    <bullet> Designate ORV routes and pedestrian-only areas at the \n",
       "Seashore;\n",
       "    <bullet> Implement a permit system for ORVs with limits on the \n",
       "number of permits;\n",
       "    <bullet> Impose date and time restrictions on the use of ORVs to \n",
       "protect resources and enhance visitor experience; and\n",
       "    <bullet> Set vehicle and equipment standards, phasing out sport \n",
       "model All-Terrain Vehicles (ATVs) and Utility Vehicles (UTVs), and\n",
       "    <bullet> Allow an additional four ramps on North Core Banks and \n",
       "five ramps on South Core Banks where vehicle traffic could cross \n",
       "between the Beach Route and the Back Route could be constructed.\n",
       "    Based on review of public comments on the plan/DEIS (received \n",
       "through compliance with the National Environmental Policy Act), the \n",
       "proposed rule reflects input received during this planning effort, \n",
       "including the following:\n",
       "    <bullet> Closure dates would be consistent to eliminate confusion;\n",
       "    <bullet> Seven miles of existing pedestrian only areas would be \n",
       "changed from year-round to seasonal closures (Memorial Day-Labor Day);\n",
       "    <bullet> Creation of a designated route for ORVs in front of the \n",
       "Long Point and Great Island cabin camps;\n",
       "    <bullet> ORV permits would be valid for ORV use on both North and \n",
       "South Core Banks;\n",
       "    <bullet> An annual limit on the number of ORV permits that would be \n",
       "issued, would be determined based on 5 years of data instead of 3 \n",
       "years, and data from years with significant ORV closure events in \n",
       "excess of 14 days (such as a hurricane) would not be used to establish \n",
       "a vehicle cap;\n",
       "    <bullet> Night driving on beach ORV routes would not be allowed \n",
       "from 9:00 p.m.-6:00 a.m. from May 1 to September 14. However, driving \n",
       "on the back routes would be allowed from 5:00 a.m.-10:00 p.m., and at \n",
       "any time in the Great Island and Long Point cabin camps as defined on \n",
       "the map available at the office of the Superintendent and for review on \n",
       "the Seashore's Web site at <a href=\"http://www.nps.gov/calo\">http://www.nps.gov/calo</a>;\n",
       "    <bullet> Vehicle length restriction would be removed and replaced \n",
       "with a wheelbase limit not to exceed 180 inches;\n",
       "    <bullet> Vehicles with a two-stroke engine would be prohibited \n",
       "immediately;\n",
       "    <bullet> ATV trailer length limit would be removed and made \n",
       "consistent with vehicle trailer lengths; and\n",
       "    <bullet> New prohibitions (including use of sport-model ATV/UTVs, \n",
       "trailers exceeding 30 feet in length and vehicles with wheel base \n",
       "exceeding 180 inches) would be phased in after a one year grace period.\n",
       "    The NPS intends to recover the costs of administering the ORV \n",
       "special use permit program under 54 U.S.C. 103104. In order to obtain a \n",
       "special use permit required to operate a motor vehicle on designated \n",
       "ORV routes in the Seashore, the proposed rule would require ORV \n",
       "operators to pay a permit fee.\n",
       "    The following explains some of the principal elements of the \n",
       "proposed rule in a question and answer format:\n",
       "\n",
       "What is an ``Off-Road Vehicle'' (ORV)?\n",
       "\n",
       "    For the purposes of this regulation, an ``off-road vehicle'' or \n",
       "``ORV'' means a motor vehicle used off of Seashore roads (off-road). \n",
       "ORVs authorized for use at the Seashore are subject to the vehicle \n",
       "requirements, prohibitions, and permitting requirements described in \n",
       "this proposed rule. However, certain ORVs would be prohibited at the \n",
       "Seashore by this rule; these include motorcycles, tracked vehicles, \n",
       "farm vehicles, vehicles with two-stroke engines, and amphibious ATVs.\n",
       "    In addition, although the Seashore allows ATV and UTV use for \n",
       "transportation within its boundaries, the seashore has determined that \n",
       "the use of these vehicles for performance riding is not consistent with \n",
       "the Seashore's purpose or NPS Management Policies 2006 and has proposed \n",
       "phasing out the specific type of ATVs and UTVs that are designed and \n",
       "classified by manufacturers as high performance vehicles. These high \n",
       "performance vehicles are by design faster, noisier, and less safe than \n",
       "utility models. They are by design made for racing, jumping, and moving \n",
       "at high speeds, instead of used as a tool for access to the Seashore. \n",
       "The NPS will not issue ORV permits for these specific types of ATV and \n",
       "UTV vehicles after the expiration of the grace period.\n",
       "\n",
       "Do I need a permit to operate a vehicle off road?\n",
       "\n",
       "    Yes. To obtain an ORV permit, you must complete a short educational \n",
       "program, acknowledge in writing that you understand and agree to abide \n",
       "by the rules governing ORV use at the Seashore, and pay the applicable \n",
       "permit fee. A permit fee schedule would be developed by the \n",
       "Superintendent. Implementation of the permit system would begin the \n",
       "first full calendar year after the final regulation becomes effective.\n",
       "\n",
       "[[Page 79015]]\n",
       "\n",
       " Is there a limit to the number of ORV permits that will be issued?\n",
       "\n",
       "    Yes. Initially, the maximum number of ORV permits that may be \n",
       "issued is 5,500. This number is based on current use levels. In order \n",
       "to ensure that ORV use does not exceed current levels, ORV use would be \n",
       "monitored for the first five years that permits are issued and \n",
       "thereafter capped at the five-year average use level (not to exceed \n",
       "5,500). Any year in which there is a significant ORV closure (more than \n",
       "14 days) will not be counted and an additional year of data would be \n",
       "collected and averaged to determine the cap. A permit would allow \n",
       "access to all routes where ORVs are permitted within the Seashore. \n",
       "Permits would be established on a first-come, first-served basis. An \n",
       "annual lottery may be established to equitably allocate permits.\n",
       "\n",
       "My family has several ORVs that we would like to use on Seashore \n",
       "beaches. Do we need to get a permit for each vehicle?\n",
       "\n",
       "    Yes. You would need to get a permit for each vehicle you plan to \n",
       "use on ORV routes in the Seashore. You would also need to affix the \n",
       "permit to each vehicle in a manner specified by the Superintendent.\n",
       "\n",
       "Where and when may I operate my ORV?\n",
       "\n",
       "    Once you obtain an ORV permit and an education certificate, you may \n",
       "operate your motor vehicle off road only on the routes described in the \n",
       "tables in the proposed rule. The Seashore would be closed to motor \n",
       "vehicles from December 16 through March 15 of each year. The tables in \n",
       "this rule also provide dates for seasonal restrictions for driving on \n",
       "designated routes. If deemed necessary by the Superintendent, seasonal \n",
       "closures for resource management could occur from March 16 through \n",
       "December 15. Some pedestrian only areas would be established from May \n",
       "1-September 14, however some are year round. Maps of designated ORV \n",
       "routes will be available at the Office of the Superintendent and for \n",
       "review on the Seashore Web site at: <a href=\"http://www.nps.gov/calo\">http://www.nps.gov/calo</a>.\n",
       "\n",
       "Does the ORV permit guarantee that all designated ORV routes will be \n",
       "open for me to use?\n",
       "\n",
       "    No. In addition to the seasonal pedestrian-only restrictions from \n",
       "May 1-September 14, ORV routes may be subject to temporary resource and \n",
       "visitor safety closures. This authority would be exercised independent \n",
       "of the Superintendent's authority under 36 CFR 1.5 and would provide \n",
       "the park with greater flexibility to respond to the impacts of ORV use \n",
       "in designated routes and areas to prevent `unacceptable impacts'. \n",
       "Public notice of any action taken under this authority would be given \n",
       "pursuant to one or more of the methods set forth in 36 CFR 1.7.\n",
       "\n",
       "Are there any requirements for my ORV?\n",
       "\n",
       "    Yes. To receive a permit to operate a vehicle on designated ORV \n",
       "routes (except for ATVs and UTVs), your vehicle must be registered, \n",
       "licensed, and insured for highway use and comply with inspection \n",
       "regulations within the state, province or country, where the vehicle is \n",
       "registered. ORV operators (except for ATVs and UTVs) would be required \n",
       "to carry a low-pressure tire gauge, shovel, jack, and jack stand/board \n",
       "in the vehicle.\n",
       "\n",
       "Can I drive my two-wheel-drive vehicle on designated ORV routes?\n",
       "\n",
       "    Yes. Four-wheel drive vehicles are recommended, but two-wheel drive \n",
       "vehicles would be allowed if, in the judgment of the vehicle operator, \n",
       "the vehicle is capable of over-sand travel.\n",
       "\n",
       "Can I tow a trailer with my vehicle on designated ORV routes?\n",
       "\n",
       "    Yes. Trailers with one or two axles would be allowed. Trailers with \n",
       "more than two axles would be prohibited. Trailers cannot exceed 30 feet \n",
       "in length including all attachments. Restrictions on trailers would be \n",
       "implemented after a one-year grace period. Transporting passengers in a \n",
       "trailer would be prohibited.\n",
       "\n",
       "Is there a vehicle length restriction?\n",
       "\n",
       "    No. The proposed rule would not institute a vehicle length \n",
       "restriction, but the NPS wants to ensure that all vehicles on the \n",
       "Seashore can safely navigate the back routes and properly stay in the \n",
       "track on designated routes. Therefore, the NPS would not issue ORV \n",
       "permits for vehicles with a wheelbase measurement that exceeds 180 \n",
       "inches. There would be a one year grace period before the wheelbase \n",
       "restriction is implemented.\n",
       "\n",
       "May I ride my all-terrain vehicle (ATV) or utility vehicle (UTV) at the \n",
       "Seashore?\n",
       "\n",
       "    After a one year grace period, only non-sport model ATVs and UTVs \n",
       "would be allowed from March 16 through December 15 on designated routes \n",
       "within the Seashore provided the non-sport ATVs and UTVs have not been \n",
       "modified and still meet the manufacturer's original specifications for \n",
       "a non-sport or utility model. ATV/UTVs would be allowed a maximum of 3 \n",
       "axles, and would not need to be registered, licensed, insured for \n",
       "highway use, or comply with inspection regulations. ATV/UTV operators \n",
       "would not be required to carry in or on the vehicle a low-pressure tire \n",
       "gauge, shovel, jack, or jack stand/board. ATV operators must wear a \n",
       "U.S. Department of Transportation approved helmet and eye protection. \n",
       "All high-performance sport-model ATVs and UTVs would be prohibited \n",
       "after a one year grace period and can no longer receive the required \n",
       "ORV permit to ride at the seashore. While these high-performance sports \n",
       "model are generally identified by the manufactures, any questions \n",
       "concerning the models subject to this prohibition will be resolved by \n",
       "the Superintendent.\n",
       "\n",
       "What is the speed limit on designated ORV routes?\n",
       "\n",
       "    The speed limit would be 25 miles per hour (unless otherwise \n",
       "designated). The speed limit would be reduced to 15 miles per hour when \n",
       "operating a vehicle within 100 feet of any person, another vehicle, a \n",
       "campsite, any structure or while towing a trailer.\n",
       "\n",
       "May I drive on designated ORV routes at night?\n",
       "\n",
       "    Yes, but not at all times on all routes. Night driving on beach ORV \n",
       "routes would not be allowed from 9:00 p.m. until 6:00 a.m. from May 1 \n",
       "to September 14 to reduce impacts on wildlife. In addition, driving on \n",
       "the back routes would be allowed from 5:00 a.m.-10:00 p.m., and at any \n",
       "time in the Great Island and Long Point cabin camps (as defined by the \n",
       "map available at the office of the Superintendent and for review on the \n",
       "Seashore's Web site at <a href=\"http://www.nps.gov/calo\">http://www.nps.gov/calo</a>.\n",
       "\n",
       "May my ORV be parked on the beach if I don't drive it between 9 p.m. \n",
       "and 6 a.m. during the dates night driving restrictions are in effect?\n",
       "\n",
       "    Yes, if the ORV is attended and not driven on the beach during that \n",
       "time.\n",
       "\n",
       "I have a family member who is disabled or mobility-impaired. Can I use \n",
       "my ORV to drive that family member to the beach where we are gathering, \n",
       "even if it is not designated as an ORV route?\n",
       "\n",
       "    Yes, such use could be accommodated on a case-by-case basis and \n",
       "would be subject to the conditions of a special use permit issued by \n",
       "the Superintendent.\n",
       "\n",
       "Do concessionaires and commercial use authorization holders need a \n",
       "separate ORV permit?\n",
       "\n",
       "    No. Use of ORVs associated with businesses would be controlled \n",
       "through an NPS concession contract or commercial use authorization.\n",
       "\n",
       "[[Page 79016]]\n",
       "\n",
       "What is the relationship of NC Motor Vehicle Law to these Regulations?\n",
       "\n",
       "    NPS specifically adopts state motor vehicle laws at 36 CFR 4.2(a), \n",
       "and can deviate from these state laws by special regulation. Under N.C. \n",
       "General Statute 20-4.01(32)(b), beach areas are considered public \n",
       "vehicular areas, and are not necessarily subject to the broader set of \n",
       "registration and operation laws for state roads. This rule would \n",
       "specify that notwithstanding the definition of public vehicular area in \n",
       "North Carolina State law, the operator of any motor vehicle at the \n",
       "Seashore, whether the ORV is moving or parked, must at all times comply \n",
       "with North Carolina traffic laws that would apply as if operating on a \n",
       "North Carolina highway. This requirement would apply to ATVs/UTVs even \n",
       "though they are, for example, not required to be registered, inspected \n",
       "or display a license plate, nor allowed on highways under North \n",
       "Carolina law. However, this special regulation would allow ATV/UTV use \n",
       "as ORVs the same as other vehicles within the Seashore; the operation \n",
       "of all these vehicles must be in compliance with this and other \n",
       "applicable regulations such as operation and safety requirements, and \n",
       "non-conflicting North Carolina law.\n",
       "\n",
       "Compliance With Other Laws, Executive Orders, and Department Policy\n",
       "\n",
       "Use of Off-Road Vehicles on the Public Lands (Executive Order 11644)\n",
       "\n",
       "    Executive Order 11644, as amended by Executive Order 11989, was \n",
       "adopted to address impacts on public lands from ORV use. The Executive \n",
       "Order applies to ORV use on federal public lands that is not authorized \n",
       "under a valid lease, permit, contract, or license. Section 3(4) of E.O. \n",
       "11644 provides that ORV ``areas and trails shall be located in areas of \n",
       "the National Park system, Natural Areas, or National Wildlife Refuges \n",
       "and Game Ranges only if the respective agency head determines that off-\n",
       "road vehicle use in such locations will not adversely affect their \n",
       "natural, aesthetic, or scenic values.'' Since the E.O. clearly was not \n",
       "intended to prohibit all ORV use everywhere in these units, the term \n",
       "``adversely affect'' does not have the same meaning as the somewhat \n",
       "similar terms ``adverse impact'' or ``adverse effect'' commonly used in \n",
       "the National Environmental Policy Act of 1969 (NEPA). Under NEPA, a \n",
       "procedural statute that provides for the study of environmental \n",
       "impacts, the term ``adverse effect'' refers to any effect, no matter \n",
       "how minor or negligible.\n",
       "    Section 3(4) of the E.O., by contrast, does not prescribe \n",
       "procedures or any particular means of analysis. It concerns substantive \n",
       "management decisions, and must instead be read in the context of the \n",
       "authorities applicable to such decisions. The Seashore is an area of \n",
       "the National Park System. Therefore, the NPS interprets the E.O. term \n",
       "``adversely affect'' consistent with its NPS Management Policies 2006. \n",
       "Those policies require that NPS only allows ``appropriate use'' of \n",
       "parks, and avoids ``unacceptable impacts.''\n",
       "    Specifically, this rule would not impede the attainment of the \n",
       "Seashore's desired future conditions for natural and cultural resources \n",
       "as identified in the plan/DEIS. The NPS has determined this rule would \n",
       "not unreasonably interfere with the atmosphere of peace and \n",
       "tranquility, or the natural soundscape maintained in natural locations \n",
       "within the Seashore. Therefore, within the context of the E.O., ORV use \n",
       "on the ORV routes designated by this rule (which are also subject to \n",
       "safety and resource closures and other species management measures that \n",
       "would be implemented under the proposed rule) would not adversely \n",
       "affect the natural, aesthetic, or scenic values of the Seashore.\n",
       "    Section 8(a) of the E.O. requires the respective agency head to \n",
       "monitor the effects of the use of off-road vehicles on lands under \n",
       "their jurisdictions. On the basis of the information gathered, such \n",
       "agency head shall from time to time amend or rescind designations of \n",
       "areas or other actions taken pursuant to the E.O. as necessary to \n",
       "further the policy of the E.O. The plan/DEIS identifies monitoring and \n",
       "resource protection procedures, and desired future conditions to \n",
       "provide for the ongoing and future evaluation of impacts of ORV use on \n",
       "protected resources. The Park Superintendent would have authority under \n",
       "both this rule and under 36 CFR 1.5 to close portions of the Seashore \n",
       "as needed to protect park resources and values, and public health and \n",
       "safety.\n",
       "\n",
       "Regulatory Planning and Review (Executive Orders 12866 and 13563)\n",
       "\n",
       "    Executive Order 12866 provides that the Office of Information and \n",
       "Regulatory Affairs in the Office of Management and Budget will review \n",
       "all significant rules. The Office of Information and Regulatory Affairs \n",
       "has determined that this rule is not significant.\n",
       "    Executive Order 13563 reaffirms the principles of E.O. 12866 while \n",
       "calling for improvements in the nation's regulatory system to promote \n",
       "predictability, to reduce uncertainty, and to use the best, most \n",
       "innovative, and least burdensome tools for achieving regulatory ends. \n",
       "The executive order directs agencies to consider regulatory approaches \n",
       "that reduce burdens and maintain flexibility and freedom of choice for \n",
       "the public where these approaches are relevant, feasible, and \n",
       "consistent with regulatory objectives. E.O. 13563 emphasizes further \n",
       "that regulations must be based on the best available science and that \n",
       "the rulemaking process must allow for public participation and an open \n",
       "exchange of ideas. We have developed this rule in a manner consistent \n",
       "with these requirements.\n",
       "\n",
       "Regulatory Flexibility Act (RFA)\n",
       "\n",
       "    This rule will not have a significant economic effect on a \n",
       "substantial number of small entities under the RFA (5 U.S.C. 601 et \n",
       "seq.). This certification is based on information contained in a report \n",
       "entitled, ``Benefit Cost Analysis of Proposed ORV Use Regulation at \n",
       "Cape Lookout National Seashore, available for public review at: <a href=\"http://parkplanning.nps.gov/projectHome.cfm?projectID=15978\">http://parkplanning.nps.gov/projectHome.cfm?projectID=15978</a>.\n",
       "\n",
       "Small Business Regulatory Enforcement Fairness Act (SBREFA)\n",
       "\n",
       "    This rule is not a major rule under 5 U.S.C. 804(2) of the SBREFA. \n",
       "This rule:\n",
       "    (a) Does not have an annual effect on the economy of $100 million \n",
       "or more.\n",
       "    (b) Will not cause a major increase in costs or prices for \n",
       "consumers, individual industries, Federal, State, or local government \n",
       "agencies, or geographic regions.\n",
       "    (c) Does not have significant adverse effects on competition, \n",
       "employment, investment, productivity, innovation, or the ability of \n",
       "U.S.-based enterprises to compete with foreign-based enterprises.\n",
       "\n",
       "Unfunded Mandates Reform Act (UMRA)\n",
       "\n",
       "    This rule does not impose an unfunded mandate on state, local, or \n",
       "tribal governments or the private sector of more than $100 million per \n",
       "year. The rule does not have a significant or unique effect on state, \n",
       "local, or tribal governments or the private sector. The designated ORV \n",
       "routes are located entirely within the Seashore, and will not result in \n",
       "direct expenditure by State, local, or tribal governments. This rule \n",
       "addresses public use of NPS lands, and imposes no requirements on other \n",
       "agencies or governments. Therefore, a statement containing the \n",
       "information required by the UMRA (2 U.S.C. 1531 et seq.) is not \n",
       "required.\n",
       "\n",
       "[[Page 79017]]\n",
       "\n",
       "Takings (Executive Order 12630)\n",
       "\n",
       "    This rule does not affect a taking of private property or otherwise \n",
       "have taking implications under Executive Order 12630. Access to private \n",
       "property located within or adjacent to the Seashore will not be \n",
       "affected, and this rule does not regulate uses of private property. \n",
       "Therefore, a takings implication assessment is not required.\n",
       "\n",
       "Federalism (Executive Order 13132)\n",
       "\n",
       "    Under the criteria in section 1 of Executive Order 13132, this rule \n",
       "does not have sufficient federalism implications to warrant the \n",
       "preparation of a federalism summary impact statement. This rule only \n",
       "affects use of NPS-administered lands and imposes no requirements on \n",
       "other agencies or governments. A federalism summary impact statement is \n",
       "not required.\n",
       "\n",
       "Civil Justice Reform (Executive Order 12988)\n",
       "\n",
       "    This rule complies with the requirements of Executive Order. 12988. \n",
       "Specifically, this rule:\n",
       "    (a) Meets the criteria of section 3(a) requiring that all \n",
       "regulations be reviewed to eliminate errors and ambiguity and be \n",
       "written to minimize litigation; and\n",
       "    (b) Meets the criteria of section 3(b)(2) requiring that all \n",
       "regulations be written in clear language and contain clear legal \n",
       "standards.\n",
       "\n",
       "Consultation With Indian Tribes (Executive Order 13175 and Department \n",
       "Policy)\n",
       "\n",
       "    The Department of the Interior strives to strengthen its \n",
       "government-to-government relationship with Indian tribes through a \n",
       "commitment to consultation with Indian tribes and recognition of their \n",
       "right to self-governance and tribal sovereignty. We have evaluated this \n",
       "rule under the criteria in Executive Order 13175 and under the \n",
       "Department's tribal consultation policy and have determined that tribal \n",
       "consultation is not required because the rule will have no substantial \n",
       "direct effect on federally recognized Indian tribes.\n",
       "\n",
       "Paperwork Reduction Act (PRA) (44 U.S.C. 3501 et seq.)\n",
       "\n",
       "    This rule does not contain any new collection of information that \n",
       "requires approval by OMB under the PRA of 1995. OMB has approved the \n",
       "information collection requirements associated with NPS special use \n",
       "permits and has assigned OMB control number 1024-0026 (expires 08/31/\n",
       "2016). An agency may not conduct or sponsor, and a person is not \n",
       "required to respond to, a collection of information unless it displays \n",
       "a currently valid OMB control number.\n",
       "\n",
       "National Environmental Policy Act (NEPA)\n",
       "\n",
       "    This rule constitutes a major federal action with the potential to \n",
       "significantly affect the quality of the human environment. In \n",
       "accordance with NEPA, the NPS prepared the plan/DEIS, which was \n",
       "released for public comment on May 23, 2014, with the public comment \n",
       "period extended to September 4, 2014, then again to September 19, 2014. \n",
       "A full description of the alternatives that were considered, the \n",
       "environmental impacts associated with the project, and public \n",
       "involvement, and other supporting documentation, can be found online at \n",
       "<a href=\"http://www.parkplanning.nps.gov/calo\">http://www.parkplanning.nps.gov/calo</a>. The NPS considered public \n",
       "comments made on the plan/DEIS in drafting this proposed rule. The NPS \n",
       "will evaluate substantive comments received on the proposed rule in \n",
       "developing the Final ORV Management Plan/Final Environmental Impact \n",
       "Statement, Record of Decision, and Final Rule.\n",
       "\n",
       "Effects on the Energy Supply (Executive Order 13211)\n",
       "\n",
       "    This rule is not a significant energy action under the definition \n",
       "in E.O. 13211. A Statement of Energy Effects is not required.\n",
       "\n",
       "Clarity of This Rule\n",
       "\n",
       "    We are required by Executive Orders 12866 and 12988, and by the \n",
       "Presidential Memorandum of June 1, 1998, to write all rules in plain \n",
       "language. This means that each rule we publish must:\n",
       "    (a) Be logically organized;\n",
       "    (b) Use the active voice to address readers directly;\n",
       "    (c) Use clear language rather than jargon;\n",
       "    (d) Be divided into short sections and sentences; and\n",
       "    (e) Use lists and tables wherever possible.\n",
       "    If you feel that we have not met these requirements, send us \n",
       "comments by one of the methods listed in the ADDRESSES section. To \n",
       "better help us revise the rule, your comments should be as specific as \n",
       "possible. For example, you should tell us the numbers of the sections \n",
       "or paragraphs that are unclearly written, which sections or sentences \n",
       "are too long, the sections where you feel lists or tables would be \n",
       "useful, etc.\n",
       "\n",
       "Public Participation\n",
       "\n",
       "    All submissions received must include the agency name and RIN for \n",
       "this rulemaking: 1024-AE24. All comments received through the Federal \n",
       "eRulemaking portal at <a href=\"http://www.regulations.gov\">http://www.regulations.gov</a> will be available \n",
       "without change. Before including your address, phone number, email \n",
       "address, or other personal identifying information in your comment, you \n",
       "should be aware that your entire comment including your personal \n",
       "identifying information may be made publicly available at any time. \n",
       "While you can ask us in your comment to withhold your personal \n",
       "identifying information, we cannot guarantee that we will be able to do \n",
       "so. To view comments received through the Federal eRulemaking portal, \n",
       "go to <a href=\"http://www.regulations.gov\">http://www.regulations.gov</a> and enter 1024- AE24 in the search \n",
       "box.\n",
       "    Drafting Information: The primary authors of this regulation are: \n",
       "Russel J. Wilson, Chief, Regulations, Jurisdiction, and Special Park \n",
       "Uses, and A.J. North, Regulations Coordinator, National Park Service, \n",
       "Washington, DC.\n",
       "\n",
       "List of Subjects in 36 CFR Part 7\n",
       "\n",
       "    District of Columbia, National Parks, Reporting and recordkeeping \n",
       "requirements.\n",
       "\n",
       "    In consideration of the foregoing, the National Park Service \n",
       "proposes to amend 36 CFR part 7 as follows:\n",
       "\n",
       "PART 7--SPECIAL REGULATIONS, AREAS OF THE NATIONAL PARK SYSTEM\n",
       "\n",
       "0\n",
       "1. The authority citation for part 7 continues to read as follows:\n",
       "\n",
       "    Authority:  54 U.S.C. 100101, 100751, 320102; Sec. 7.96 also \n",
       "issued under DC Code 10-137 and DC Code 50-2201.07.\n",
       "\n",
       "0\n",
       "2. In Sec.  7.49, add paragraph (c) to read as follows:\n",
       "\n",
       "\n",
       "Sec.  7.49  Cape Lookout National Seashore.\n",
       "\n",
       "* * * * *\n",
       "    (c) Off-road motor vehicle use. (1) Definitions. In addition to the \n",
       "definitions found in Sec.  1.4 of this chapter, the following terms \n",
       "apply in this paragraph (c):\n",
       "    All-terrain vehicle (ATV)--A motorized off-highway vehicle designed \n",
       "to travel on four low-pressure tires, having a seat designed to be \n",
       "straddled by the operator and handlebars for steering control.\n",
       "    Back Route--A marked and maintained ORV corridor located behind the \n",
       "dunes running parallel to the beach, known locally as the back road.\n",
       "    ORV means a motor vehicle used off of Seashore roads (off-road).\n",
       "    Utility Terrain Vehicle (UTV)--A motorized off-highway vehicle \n",
       "designed to travel on four low-pressure tires. Differs from ATVs in \n",
       "that UTVs\n",
       "\n",
       "[[Page 79018]]\n",
       "\n",
       "typically have a side-by-side seating arrangement, many have seat belts \n",
       "and roll-over protection, and most have a cargo box at the rear of the \n",
       "vehicle. UTVs generally have a higher payload capability and are longer \n",
       "and wider than ATVs.\n",
       "    (2) ORV permits. The Superintendent administers the NPS special \n",
       "park use permit system at the Seashore, including permits for ORV use, \n",
       "and charges fees to recover ORV program administration costs.\n",
       "    (i) A permit issued by the Superintendent is required to operate a \n",
       "vehicle on designated ORV routes at the Seashore. The Superintendent is \n",
       "responsible for determining or resolving the eligibility of a vehicle \n",
       "for an ORV permit. The cost of the permits will be set by the \n",
       "Superintendent.\n",
       "    (ii) Operation of a motor vehicle authorized under an ORV permit is \n",
       "limited to those routes designated in the table in paragraph (c)(7) of \n",
       "this section.\n",
       "    (iii) The Superintendent will establish an annual limit on the \n",
       "number of ORV permits that may be issued, not to exceed 5,500, and will \n",
       "notify the public under Sec.  1.7 of this chapter.\n",
       "    (iv) In order to obtain a permit, an applicant must comply with \n",
       "vehicle and equipment requirements, obtain an education certificate, \n",
       "acknowledge in writing an understanding of the rules governing ORV use \n",
       "at the Seashore, and pay the permit fee.\n",
       "    (v) Each permit holder must affix the permit in a manner and \n",
       "location specified by the Superintendent to the vehicle authorized for \n",
       "off-road use.\n",
       "    (3) Vehicle, operator and equipment requirements. The following \n",
       "requirements apply for driving a motor vehicle off-road:\n",
       "    (i) The ORV (except ATVs and UTVs) must be registered, licensed, \n",
       "and insured for highway use and must comply with inspection regulations \n",
       "within the state, province, or country where the vehicle is registered.\n",
       "    (ii) The ORV (except ATVs and UTVs) must have no more than two \n",
       "axles, no more than six wheels and no less than four wheels. ATVs and \n",
       "UTVs may have no more than three axles, no more than six wheels and no \n",
       "less than four wheels. An ORV's wheelbase must not exceed 180 inches; \n",
       "this requirement will take effect one year after the effective date of \n",
       "the final rule.\n",
       "    (iii) A towed trailer must have no more than two axles and must not \n",
       "exceed 30 feet in length. This paragraph will take effect one year \n",
       "after the effective date of the Final Rule.\n",
       "    (iv) The ORV (except ATVs and UTVs) must carry a low-pressure tire \n",
       "gauge, shovel, jack, and jack stand or support board.\n",
       "    (v) ORV operators must possess a valid driver's license and an \n",
       "education certificate in addition to the vehicle being permitted.\n",
       "    (vi) ATV operators must wear a U.S. Department of Transportation \n",
       "approved helmet and eye protection.\n",
       "    (vii) Riding on the tailgate or roof or hanging outside of moving \n",
       "ORVs is prohibited.\n",
       "    (viii) While riding in a truck bed a person must be seated on the \n",
       "floor. Children under 16 years of age riding in truck beds must also be \n",
       "accompanied by an adult riding in the bed.\n",
       "    (ix) Passengers are prohibited from riding in or on a trailer, \n",
       "unless specifically authorized by the NPS in a permit, commercial use \n",
       "authorization or concession contract.\n",
       "    (4) Vehicle inspection. An authorized person may inspect the \n",
       "vehicle to determine compliance with the requirements of this \n",
       "regulation.\n",
       "    (5) Prohibited vehicles. Use of a motorcycle, tracked vehicle, farm \n",
       "vehicle, a vehicle with a two-stroke engine, and combination vehicles \n",
       "(i.e. amphibious ATVs) are prohibited in the Seashore. The use of high \n",
       "performance/sport model ATVs and UTVs will be prohibited one year after \n",
       "the effective date of the final rule.\n",
       "    (6) Special use permits for off-road driving, temporary use. The \n",
       "Superintendent may issue a special use permit for temporary off-road \n",
       "vehicle use to allow vehicular transport of mobility impaired \n",
       "individuals, subject to the conditions of the permit.\n",
       "    (7) ORV routes. (i) The Seashore is closed to ORV use from December \n",
       "16 through March 15.\n",
       "    (ii) The Seashore is open to ORV use from March 16 through December \n",
       "15, with seasonal closures for species management from March 16 through \n",
       "December15 and pedestrian-only areas from May 1-September 14.\n",
       "    (iii) The following tables designate ORV routes, ORV ramps, areas \n",
       "closed to ORVs, and pedestrian only areas. Maps depicting the \n",
       "designated routes and ramps, areas closed and pedestrian only areas are \n",
       "available in the Office of the Superintendent and for review on the \n",
       "Seashore Web site.\n",
       "\n",
       "------------------------------------------------------------------------\n",
       " \n",
       "-------------------------------------------------------------------------\n",
       "                       Designated ORV Beach Route\n",
       "------------------------------------------------------------------------\n",
       "The oceanside beach of North and South Core Banks from the toe of the\n",
       " primary dune line or vegetation line to the water line on the seaward\n",
       " side, that is accessible and otherwise open (not closed due to\n",
       " pedestrian, safety or species resource protection closures). The ORV\n",
       " corridor would be marked on the landward side.\n",
       "------------------------------------------------------------------------\n",
       "\n",
       "\n",
       "------------------------------------------------------------------------\n",
       " \n",
       "                   Designated ORV Back Route and Ramps\n",
       "------------------------------------------------------------------------\n",
       "The back route is a marked route parallel to the ocean beach behind the\n",
       " primary dune on North Core and South Core Banks. Marked crossover\n",
       " routes or ``ramps'' between the ocean beach and back route are further\n",
       " described below..\n",
       "------------------------------------------------------------------------\n",
       "North Core Banks..................  Route would parallel the beach\n",
       "                                     behind the primary dune line from\n",
       "                                     approximately 0.5 miles south of\n",
       "                                     Portsmouth Flats at mile marker 4\n",
       "                                     to 0.5 mile north of Old Drum Inlet\n",
       "                                     at mile marker 18 with the\n",
       "                                     exception there is no back route at\n",
       "                                     Kathryn-Jane Flats from mile 6 to\n",
       "                                     mile 7. South of Old Drum Inlet the\n",
       "                                     back route would continue from\n",
       "                                     approximately mile maker 19.5 to\n",
       "                                     mile marker 21 immediately north of\n",
       "                                     the Ophelia Inlet.\n",
       "                                    Ramps: 4a, 4b, 5a, 5b, 6, 7a, 7b,\n",
       "                                     8a, 8b, 9, 10a, 10b, 11a, 11b, 12,\n",
       "                                     13a, 13b, 14, 15a, 15b, 16, 17a,\n",
       "                                     17b, 18a, 18b, 19, 20, 21.\n",
       "South Core Banks..................  Route would parallel the beach\n",
       "                                     behind the primary dune line\n",
       "                                     approximately 1 mile south of\n",
       "                                     Ophelia Inlet at mile marker 24 to\n",
       "                                     the point of Cape Lookout at mile\n",
       "                                     marker 44. Additionally, the back\n",
       "                                     route on South Core Banks would be\n",
       "                                     extended from mile marker 44a to\n",
       "                                     mile marker 44b; this by-pass\n",
       "                                     section of back route would only be\n",
       "                                     open when the area of the Cape\n",
       "                                     closes to through traffic.\n",
       "                                    Ramps 24, 25, 26a, 26b, 27, 28, 29,\n",
       "                                     29a, 29b, 30, 31, 32, 33, 34, 35a,\n",
       "                                     35b, 36, 37a, 37b, 38, 39, 40, 41a,\n",
       "                                     41b, 42a, 42b, 43, 44a, 44b, 45.\n",
       "------------------------------------------------------------------------\n",
       "\n",
       "\n",
       "[[Page 79019]]\n",
       "\n",
       "\n",
       "------------------------------------------------------------------------\n",
       " \n",
       "                   Designated Miscellaneous ORV Routes\n",
       "------------------------------------------------------------------------\n",
       "A marked route from the beach to the Portsmouth Village parking area\n",
       " through Portsmouth Flats.\n",
       "Marked routes within the Long Point and Great Island Cabin Camp areas\n",
       " including vehicle ferry landings.\n",
       "Designated parking areas at Long Point, Great Island, and the Cape\n",
       " Lookout Light Station area, and the marked routes to these areas.\n",
       "Marked Routes within the Cape Lookout Village Historic District.\n",
       "Marked sound-side access routes:\n",
       "    North Core Banks--mile marker 9, mile marker 12, mile marker 18.\n",
       "    South Core Banks--mile marker 26.5, mile marker 31.5, mile marker\n",
       "     35.5 (Codds Creek), mile marker 45.\n",
       "------------------------------------------------------------------------\n",
       "\n",
       "\n",
       "------------------------------------------------------------------------\n",
       " \n",
       "                          Areas Closed to ORVs\n",
       "------------------------------------------------------------------------\n",
       " Sections of the islands isolated from designated ferry landings through\n",
       " the formation of new inlets, such as Middle Core Banks.\n",
       "All areas of the park not designated for ORV use.\n",
       "------------------------------------------------------------------------\n",
       "\n",
       "\n",
       "------------------------------------------------------------------------\n",
       " \n",
       "                          Pedestrian-Only Areas\n",
       "------------------------------------------------------------------------\n",
       "All other areas of the Seashore     Entire area, year round closure.\n",
       " not designated for ORVs above,\n",
       " including Shackleford Banks.\n",
       "Long Point........................  Seasonal pedestrian only area May 1\n",
       "                                     to September 14, on the ocean beach\n",
       "                                     at Long Point Cabin Camp from ramp\n",
       "                                     16 to ramp 17a (approximately 0.50\n",
       "                                     miles). An ORV route will be\n",
       "                                     established through this area.\n",
       "Great Island......................  Seasonal pedestrian only area May 1\n",
       "                                     to September 14, on the ocean beach\n",
       "                                     at the Great Island Cabin Camp\n",
       "                                     pedestrian-only area from ramp 29\n",
       "                                     to ramp 30 (approximately 1.9\n",
       "                                     miles), to be determined by beach\n",
       "                                     profile. An ORV route will be\n",
       "                                     established through this area.\n",
       "Codds Creek.......................  Seasonal pedestrian only area May 1\n",
       "                                     to September 14, on the ocean beach\n",
       "                                     near Codds Creek for a total\n",
       "                                     closure of 0.8 miles between mile\n",
       "                                     marker/ramps 35a and 35b for\n",
       "                                     pedestrians only.\n",
       "Light Station.....................  A year-round pedestrian-only area at\n",
       "                                     the lighthouse would run from ramp\n",
       "                                     41a to ramp 41b for a total closure\n",
       "                                     of approximately 0.7 miles.\n",
       "Power Squadron Spit...............  A year-round pedestrian-only area at\n",
       "                                     the lighthouse would run from Mile\n",
       "                                     Marker 46.2 to the end of the Spit.\n",
       "------------------------------------------------------------------------\n",
       "\n",
       "    (8) Superintendent's closures. (i) The Superintendent may \n",
       "temporarily limit, restrict, or terminate access to routes or areas \n",
       "designated for off-road use after taking into consideration public \n",
       "health and safety, natural and cultural resource protection, and other \n",
       "management activities and objectives.\n",
       "    (ii) The public will be notified of such closures through one or \n",
       "more of the methods listed in Sec.  1.7 of this chapter.\n",
       "    (iii) Violation of any closure or restriction is prohibited.\n",
       "    (9) Rules for vehicle operation. (i) Notwithstanding North Carolina \n",
       "law, including the definition of ``Public Vehicular Area,'' the \n",
       "operator of an ORV anywhere in the Seashore, whether in motion or \n",
       "parked, must comply with all North Carolina traffic laws that would \n",
       "apply if the operator were operating the vehicle on a North Carolina \n",
       "highway. ATVs/UTVs are not allowed on highways under North Carolina \n",
       "traffic law. However, this special regulation would allow ATVs/UTVs use \n",
       "within the Seashore, and would require ATVs/UTVs operators to comply \n",
       "with this and other applicable regulations, and non-conflicting North \n",
       "Carolina law in the same manner as all other vehicles operating off-\n",
       "road.\n",
       "    (ii) In addition to the requirements of part 4 of this chapter, the \n",
       "following restrictions apply:\n",
       "    (A) A vehicle operator must yield to pedestrians on all designated \n",
       "ORV routes.\n",
       "    (B) The speed limit for off road driving is 25 mph, unless \n",
       "otherwise designated.\n",
       "    (C) A vehicle operator must slow to 15 mph when traveling within \n",
       "100 feet or less of any person, vehicle, campsite, structure or while \n",
       "pulling a trailer.\n",
       "    (D) When driving on a designated route, an operator must lower the \n",
       "vehicle's tire pressure sufficiently to maintain adequate traction.\n",
       "    (E) Vehicles may only be transported to the Seashore by NPS \n",
       "authorized ferries.\n",
       "    (10) Night driving restrictions. (i) Hours of operation and night \n",
       "driving restrictions are listed in the following table:\n",
       "\n",
       "------------------------------------------------------------------------\n",
       " \n",
       "------------------------------------------------------------------------\n",
       "              Hours of Operation/Night Driving Restrictions\n",
       "------------------------------------------------------------------------\n",
       "December 16 through March 15..............  Seashore closed to\n",
       "                                             recreational ORV use.\n",
       "March 16 through April 30 and September 15  All designated ORV routes\n",
       " through December 15.                        are open 24 hours a day.\n",
       "May 1 through September 14................  Designated ORV routes are\n",
       "                                             closed from 9 p.m. to 6\n",
       "                                             a.m. to reduce potential\n",
       "                                             impacts to wildlife. The\n",
       "                                             back route is open 5 a.m.\n",
       "                                             to 10 p.m.\n",
       "March 16 through December 15..............  Designated ORV routes within\n",
       "                                             cabin areas at Long Point\n",
       "                                             and Great Island are open\n",
       "                                             24 hours a day.\n",
       "------------------------------------------------------------------------\n",
       "\n",
       "    (ii) Maps are available in the office of the Superintendent and on \n",
       "the Seashore's Web site that show routes closed due to resource and \n",
       "visitor protection.\n",
       "    (11) Parking permit. (i) A parking permit issued by the \n",
       "Superintendent is required for long-term vehicle parking at designated \n",
       "locations.\n",
       "    (ii) Long-term vehicle lots would be closed from December 16 \n",
       "through March 15.\n",
       "    (12) Park violations. (i) Violating any of the provisions of this \n",
       "paragraph, or the terms, conditions, or requirements of an ORV or other \n",
       "permit authorizing ORV use is prohibited.\n",
       "    (ii) A violation may also result in the suspension or revocation of \n",
       "a NPS permit by the Superintendent.\n",
       "    (13) Information collection. As required by 44 U.S.C. 3501 et seq. \n",
       "The\n",
       "\n",
       "[[Page 79020]]\n",
       "\n",
       "Office of Management and Budget has approved the information collection \n",
       "requirements contained in this paragraph. The OMB approval number is \n",
       "1024-0026. The NPS is collecting this information to provide the \n",
       "Superintendent data necessary to issue ORV special use permits. The \n",
       "information will be used to grant a benefit. The obligation to respond \n",
       "is required to order to obtain the benefit in the form of the ORV \n",
       "permit.\n",
       "\n",
       "    Dated: December 9, 2015.\n",
       "Karen Hyun,\n",
       "Acting Principal Deputy Assistant Secretary for Fish and Wildlife and \n",
       "Parks.\n",
       "[FR Doc. 2015-31793 Filed 12-17-15; 8:45 am]\n",
       " BILLING CODE 4310-EJ-P\n",
       "\n",
       "\n",
       "</bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></bullet></pre>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "doc_list2[2]['text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "doc_list =load(open(\"data/Master_doc_content\",'rb'))\n",
    "document = doc_list[3]\n",
    "document_text = str(document['text'][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "#clean_text, clean_summary = get_document_text(document_text)\n",
    "text = \"\"\"In addition[[Page 123]] to -----causing multiple  diseases and cancers, tobacco smoking has many other \n",
    "adverse effects on  the body, including inflammation and impairment to the immune  system.\n",
    "\\\\1\\\\ ---------------------------------------------------------------------------      \n",
    "\\\\1\\\\ Office of the Surgeon General, ``The Health Consequences of  Smoking--50 Years of Progress,'' (2014), \n",
    "available at http://www.surgeongeneral.gov/library/reports/50-years-of-progress/full-report.pdf.\"\n",
    "#tokens = tokenize_text(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"In addition[[Page 123]] to -----causing multiple  diseases and cancers, tobacco smoking has many other adverse effects on  the body, including inflammation and impairment to the immune  system.\\\\1\\\\ ---------------------------------------------------------------------------      \\\\1\\\\ Office of the Surgeon General, ``The Health Consequences of  Smoking--50 Years of Progress,'' (2014), available at http://www.surgeongeneral.gov/library/reports/50-years-of-progress/full-report.pdf.\""
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'abc[[Page 123]]\\\\59\\\\ce'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# |\\/\\d+\\/       # words of form /num/ -- references in our doc\n",
    "#     |\\[\\[Page \\d+\\]\\] # for page number markings in our doc of form  [[Page num]]\n",
    "#     | -*\n",
    "#     | .*\n",
    "#text = text.replace('\\\\','|')\n",
    "test = \"abc[[Page 123]]\\\\59\\\\c---e\"\n",
    "# cr = re.compile('''\\[\\[Page \\d+\\]\\]\n",
    "#                 |\\\\+\\d+\\\\+\n",
    "#                 |-*''')\n",
    "\n",
    "cr = re.compile(r'''\\\\\\\\\\d+\\\\\n",
    "                |-+''')\n",
    "#cs = re.compile('\\\\\\d+\\\\ ')\n",
    "ct = re.sub(cr, r'', test)\n",
    "#ct = re.sub(cs, '', ct)\n",
    "ct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "abc[[Page 123]]\n",
      "\\ce\n"
     ]
    }
   ],
   "source": [
    "print(ct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clean_text, clean_summary = get_document_text(document_text)\n",
    "#clean_text = clean_text.replace('\\\\','|')\n",
    "sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "raw_sents = sent_tokenizer.tokenize(clean_text)\n",
    "cr = re.compile('\\[\\[Page \\d+\\]\\]|-*|\\\\+[0-9]+\\\\+')\n",
    "clean_sent =[]\n",
    "for sent in raw_sents:\n",
    "    clean_sent.append(re.sub(cr, '', sent))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['  I.',\n",
       " 'Executive Summary  A.',\n",
       " 'Purpose of the Proposed Rule      The purpose of the proposed rule is to require PHAs to, within 18  months of the final rule, establish a policy prohibiting lit tobacco  products, as such term is proposed to be defined in Sec.',\n",
       " '965.653(c).',\n",
       " \"inside all indoor areas of public housing, including but not limited to  living units, indoor common areas, electrical closets, storage units,  and PHA administrative office buildings and in all outdoor areas within  25 feet of the housing and administrative office buildings  (collectively, ``restricted areas'').\",\n",
       " 'As further discussed in this  rule, such a policy is expected to improve indoor air quality in public  housing, benefit the health of public housing residents and PHA staff,  reduce the risk of catastrophic fires, and lower overall maintenance  costs.',\n",
       " 'B.',\n",
       " 'Summary of Major Provisions of the Proposed Rule      This proposed rule would apply to all public housing, other than  dwelling units in mixedfinance buildings.',\n",
       " 'PHAs would be required,  within 18 months of the effective date of the final rule, to establish  policies prohibiting lit tobacco products in all restricted areas.',\n",
       " 'PHAs  may, but would not be required to, further restrict smoking to outdoor  dedicated smoking areas outside the restricted areas, create additional  restricted areas in which smoking is    prohibited (e.g., near a playground), or, alternatively, make their  entire grounds smokefree.',\n",
       " 'PHAs would also be required to document their smokefree policies  in their PHA plans, a process that requires resident engagement and  public meetings.',\n",
       " \"The prohibition on lit tobacco would also be included  in a tenant's lease, which may be done either through an amendment  process or as tenants renew their leases annually.\",\n",
       " 'C. Costs and Benefits of This Proposed Rule      The costs to PHAs of implementing smokefree policies may include  training, administrative, legal, and enforcement costs.',\n",
       " 'Of these costs,  HUD expects that the expense of additional enforcement efforts may be  the highest.',\n",
       " 'The costs of implementing a smokefree policy as proposed  by this rule are minimized by the fact that HUD guidance already exists  on many of the topics covered by the smokefree policy proposed to be  required by this rule; that hundreds of PHAs have already voluntarily  implemented smokefree policies; and that infrastructure already exists  for enforcement of lease violations, and violation of the smokefree  policy would be a lease violation.',\n",
       " 'In addition, time spent by PHA staff  on implementing and enforcing the smokefree policy will be partially  offset by the time that staff no longer have to spend mediating  disputes among residents over smoking in secondhand smoke infiltration  within living units.',\n",
       " 'Given the existing HUD guidance, initial learning  costs associated with implementation of a smokefree policy may not be  significant.',\n",
       " 'For the hundreds of PHAs that are already implementing  voluntary smokefree policies, there will be minimal costs for these  PHAs, and, generally, only if their existing policies are not  consistent with the minimum requirements for smokefree policies  proposed by this rule.',\n",
       " 'The benefits of smoke free policies, however, could be  considerable.',\n",
       " 'Over 700,000 units would be affected by this rule  (including over 500,000 units inhabited by elderly households or  households with a nonelderly person with disabilities), and their non smoking residents would have the potential to experience health  benefits from a reduction of exposure to secondhand smoke.',\n",
       " 'PHAs will  also benefit from a reduction of damage caused by smoking, and  residents and PHAs both gain from seeing a reduction in injuries,  deaths, and property damage caused by fires.',\n",
       " 'Estimates of these and  other ruleinduced impacts are summarized in the following table:                                                         Amount (discount            Impact                    Source               rates in                                                         parentheses)  Cost (potentially recurring   PHA Compliance......  $3.2 million.',\n",
       " \"but concentrated during  first few years of the  rule's implementation).\",\n",
       " 'Cost (recurring)............  Smoker Inconvenience  $209 million.',\n",
       " 'Cost (recurring)............  Enforcement.........  Not quantified.',\n",
       " 'Benefit (recurring).........  PHA Reduced           $16 to $38 million.',\n",
       " 'Maintenance.',\n",
       " 'Benefit (recurring).........  PHA Reduced Fire      $32 million.',\n",
       " 'Risk.',\n",
       " 'Benefit (annualized over 10   NonSmoker Health...  Less than:  to 50 years).',\n",
       " '$148 to $447 million                                                      (3%)                                                     $70 to $137 million                                                      (7%).',\n",
       " 'Benefit (recurring).........  NonSmoker Well      $96 to $275 million.',\n",
       " 'Being (PHA                                residents who do                                not live in units                                with smokers).',\n",
       " 'Benefit (recurring).........  Smoker Health.......  Not quantified.',\n",
       " 'Partially Quantified Net      See above...........  Less than: $19 to  Benefits (recurring).',\n",
       " '$302 million (3%)                                                       $97 to $8 million                                                      (7%)       For additional details on the costs and benefits of this rule,  please see the Regulatory Impact Analysis (RIA) for this rule, which  can be found at www.regulations.gov, under the docket number for this  rule.',\n",
       " 'Information on how to view the RIA is included below.',\n",
       " 'II.',\n",
       " 'Background  A.',\n",
       " 'The Effects of Smoking on Health      Tobacco smoking has been determined to be a cause of diseases of  nearly all organs in the body, and research continues to newly identify  diseases caused by smoking, including diabetes mellitus, rheumatoid  arthritis, and colorectal cancer.',\n",
       " \"In addition to causing multiple  diseases and cancers, tobacco smoking has many other adverse effects on  the body, including inflammation and impairment to the immune  system.\\\\1\\\\       \\\\1\\\\ Office of the Surgeon General, ``The Health Consequences of  Smoking50 Years of Progress,'' (2014), available at http://www.surgeongeneral.gov/library/reports/50yearsofprogress/fullreport.pdf.\",\n",
       " '      Adverse effects of tobacco use are not limited to the smoker.',\n",
       " 'The  U.S.',\n",
       " 'Surgeon General estimates that exposure to secondhand tobacco  smoke (i.e., the smoke that comes from burning tobacco products and is  exhaled by smokers) is responsible for the death of 41,000 adults non smokers in the United States each year from lung cancer and heart  disease.\\\\2\\\\ Secondhand smoke (SHS) contains hundreds of toxic chemicals  and is designated as a known human carcinogen by the U.S. Environmental  Protection Agency, the U.S. National Toxicology Program, and the  International Agency for Research on Cancer.\\\\3\\\\ Exposure to SHS can  also cause sudden infant death syndrome and respiratory symptoms such  as cough and wheeze, middle ear infections, and slowed lung growth and  reduced lung function in children, and increased risk of stroke in  adults.\\\\4\\\\ The Surgeon General has concluded that there is no riskfree  level of exposure to SHS, and that eliminating smoking in indoor spaces  fully protects nonsmokers from exposure to secondhand smoke.',\n",
       " 'Separating  smokers from nonsmokers, cleaning the air, and ventilating buildings  cannot eliminate exposures of nonsmokers to secondhand smoke.\\\\5\\\\       \\\\2\\\\ Id.',\n",
       " \"\\\\3\\\\ American Cancer Society, ``Secondhand Smoke,'' http://www.cancer.org/cancer/cancercauses/tobaccocancer/secondhandsmoke.\",\n",
       " \"\\\\4\\\\ 2014 Surgeon General's Report, footnote 1.\",\n",
       " '\\\\5\\\\ U.S. Dept.',\n",
       " \"of Health and Human Services, ``The Health  Consequences of Involuntary Exposure to Tobacco Smoke: A Report of  the Surgeon General,'' (2006), available at http://www.surgeongeneral.gov/library/reports/secondhandsmoke/fullreport.pdf.\",\n",
       " '      The effects of SHS are especially damaging in children and unborn  fetuses.',\n",
       " \"The Surgeon General estimates that SHS is responsible for the  death of hundreds of newborns from Sudden Infant Death Syndrome (SIDS)  each year.\\\\6\\\\ Lead in SHS is also a significant source of lead in house  dust and children's blood.\",\n",
       " 'The CDC confirmed the association between  SHS exposure    and bloodlead levels in youth and adults, concluding that youth with  SHS exposure had blood lead levels high enough to result in adverse  cognitive outcomes.\\\\7\\\\       \\\\6\\\\ Id.',\n",
       " \"\\\\7\\\\ Patricia Richter et al., ``Trends in Tobacco Smoke Exposure  and Blood Lead Levels Among Youth and Adults in the United States:  The National Health and Nutrition Examination Survey, 19992008,''  Preventing Chronic Disease, (December 19, 2013), available at http://www.cdc.gov/pcd/issues/2013/pdf/13_0056.pdf.\",\n",
       " '      Approximately half of the U.S. population is protected from SHS  exposure through statewide, municipal, and federal laws prohibiting  smoking in indoor areas of public places and worksites, including bars  and restaurants.',\n",
       " 'However, an estimated 58 million Americans remain  exposed to secondhand smoke, including 15 million children ages 3 to  11.',\n",
       " 'The home is the primary source of exposure for children.\\\\8\\\\ Because  SHS moves throughout buildings, individuals living in multiunit housing  can be exposed to SHS even if no one smokes in their households.',\n",
       " \"Surveys of multiunit housing residents indicate that 26 to 64 percent  of residents reported SHS incursions into their units from external  sources (e.g., hallways or adjacent apartments), and 65 to 90 percent  of the residents experiencing such incursions were bothered by them.\\\\9\\\\       \\\\8\\\\ 2006 Surgeon General's Report, footnote 5; David M. Homa et  al., ``Vital Signs: Disparities in Nonsmokers' Exposure to  Secondhand SmokeUnited States, 19992012,'' Morbidity and  Mortality Weekly Report (February 6, 2015), available at http://www.cdc.gov/mmwr/preview/mmwrhtml/mm6404a7.htm?s_cid=mm6404a7_w.\",\n",
       " \"\\\\9\\\\ Kimberly Snyder et al., ``Smokefree Multiunit Housing: A  Review of the Scientific Literature,'' Tobacco Control (2015),  available at http://tobaccocontrol.bmj.com/content/early/2015/01/07/tobaccocontrol2014051849.short?rss=1.\",\n",
       " '      The movement of contaminants from SHS within buildings has also  been documented through direct measurements of fine particles (an  environmental marker of SHS) in indoor air.',\n",
       " 'SHS can move both from  external hallways into apartments and between adjacent units.\\\\10\\\\ A  study of public housing documented lower concentrations of SHS  contaminants in buildings covered by smokefree policies (i.e.,  policies prohibiting the smoking of tobacco products in all indoor  spaces) compared to buildings without these policies.\\\\11\\\\ Analysis of  data from the National Health and Nutrition Examination Survey (NHANES)  demonstrated evidence of greater SHS exposure among children (aged 6 to  18) living in multiunit housing through measurements of cotinine (a  metabolite of nicotine) in their blood.\\\\12\\\\ The study demonstrated that  children living in nonsmoking households in apartments had 45 percent  higher levels of cotinine in their blood compared to children living in  nonsmoking households in detached homes.',\n",
       " 'CDC researchers analyzed  NHANES data over the period from 19992012 and reported that one of  four nonsmokers (approximately 58 million people) continue to be  exposed to SHS, with the highest exposures among children, nonHispanic  blacks, renters, and those living in poverty.\\\\13\\\\       \\\\10\\\\ Brian A.',\n",
       " \"King et al., ``Secondhand Smoke Transfer in  Multiunit Housing,'' 12 Nicotine and Tobacco Research 1133 (2010),  available at http://ntr.oxfordjournals.org/content/12/11/1133.\",\n",
       " \"\\\\11\\\\ Elizabeth T. Russo, et al., ``Comparison of Indoor Air  Quality in SmokePermitted and SmokeFree Multiunit Housing:  Findings from the Boston Housing Authority,'' 10 Nicotine and  Tobacco Research 1093 (2014), available at http://ntr.oxfordjournals.org/content/early/2014/08/25/ntr.ntu146.abstract?utm_source=rss&amp;utm_medium=rss&amp;utm_campaign=comparisonofindoorairqualityinsmokepermittedandsmokefreemultiunithousingfindingsfromthebostonhousingauthority.\",\n",
       " \"\\\\12\\\\ Karen M. Wilson et al., ``TobaccoSmoke Exposure in  Children Who Live in Multiunit Housing,'' 127 Pediatrics 85 (2011),  available at http://pediatrics.aappublications.org/content/127/1/85.full.pdf+html.\",\n",
       " \"\\\\13\\\\ David M. Homa et al., ``Disparities in Nonsmokers Exposure  to Secondhand Smoke in the United States, 19992012,'' Mortality and  Morbidity Weekly Report, Early Release, 64 (February 3, 2015),  available at http://www.cdc.gov/mmwr/pdf/wk/mm64e0203a1.pdf.\",\n",
       " '      The Surgeon General concluded in 2006 that separating smokers and  nonsmokers, building ventilation, and cleaning the air cannot eliminate  exposure to SHS; that can only be accomplished by eliminating smoking  from indoor spaces.\\\\14\\\\       \\\\14\\\\ U.S. Dept.',\n",
       " 'of Health and Human Services.',\n",
       " 'See footnote note  2.',\n",
       " '  B.',\n",
       " 'The Financial Costs of Smoking      Beyond the increased costs associated with higher healthcare  expenses, tobacco smoking can have profound financial impacts on PHAs  and owners of other multiunit properties.',\n",
       " 'Smoking is the leading cause  of fire deaths in multiunit properties.\\\\15\\\\ In 2011, smoking caused  17,600 residential fires resulting in 490 civilian deaths, 1,370  injuries, and $516 million in direct property damage.\\\\16\\\\ Smoking is  especially dangerous in units where a household member is receiving  oxygen for medical purposes.',\n",
       " 'Research conducted by the U.S. Fire  Protection Association found that for fire deaths during the period  from 20072011 in which oxygen administration equipment was cited as  being involved in the ignition, 82 percent involved smoking materials  as the heat source.\\\\17\\\\       \\\\15\\\\ U.S. Fire Administration, Residential Structure and  Building Fires, http://www.usfa.fema.gov/downloads/pdf/publications/residential_structure_and_building_fires.pdf.',\n",
       " '\\\\16\\\\ Marty Ahrens, Ntl.',\n",
       " \"Fire Protection Assn., ``Home Structure  Fires,'' (April 2013), available at http://www.nfpa.org/~/media/ Files/Research/NFPA%20reports/Occupancies/oshomes.pdf.\",\n",
       " '\\\\17\\\\ John R. Hall, Jr., Ntl.',\n",
       " \"Fire Protection Assn., ``The  SmokingMaterial Fire Problem,'' (July 2013), available at http:// www.nfpa.org/~/media/Files/Research/NFPA%20reports/Major%20Causes/ ossmoking.pdf.\",\n",
       " '      Smoking is also associated with higher maintenance costs for  landlords of multiunit housing.',\n",
       " 'Smoking indoors increases the cost of  rehabilitating a housing unit because of the need for additional  cleaning, painting, and repair of damaged items at unit turnover  compared to nonsmoking units.',\n",
       " 'The cost of cleaning and renovating a  smoking unit adds up quickly, and smaller properties generally pay more  per unit than larger properties when repairing smoking damage.',\n",
       " 'A survey  of public and subsidized housing managers found that the additional  cost of rehabilitating the units of smokers averaged $1,250 to $2,955  per unit, depending on the intensity of smoking.\\\\18\\\\ A study conducted  in California found that the owners of multiunit housing could save  over $18 million per year if the operators of all multiunit housing in  the state adopted smokefree building policies.\\\\19\\\\ Researchers from  the CDC estimated that a nationwide smokefree public housing policy  would result in an estimated annual cost savings of $152.91 million,  including $42.99 million in reduced renovation costs and $15.92 million  in averted fire losses.\\\\20\\\\       \\\\18\\\\ Ntl.',\n",
       " 'Ctr.',\n",
       " \"For Healthy Hsg., ``Reasons to Explore SmokeFree  Housing,'' (Early Fall 2009), available at http://www.nchh.org/portals/0/contents/nchh_green_factsheet_smokefree.pdf.\",\n",
       " \"\\\\19\\\\ Michael K. Ong et al, ``Estimates of SmokingRelated  Properties Costs in California Multiunit Housing,'' 102 Am J Public  Health 490 (2012), available at http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3487653/.\",\n",
       " \"\\\\20\\\\ Brian King et al., ``National and State Cost Savings  Associated With Prohibiting Smoking in Subsidized and Public Housing  in the United States,'' Preventing Chronic Disease (October 2014),  available at http://www.cdc.gov/pcd/issues/2014/pdf/14_0222.pdf.\",\n",
       " '      Selfimposed rules prohibiting smoking in individual households  (referred to as smokefree home rules) are becoming increasingly common  in the United States.',\n",
       " 'CDC researchers found that the prevalence of  smokefree home rules among U.S. households increased from 43 percent  in 19921993 to 83 percent in 20102011, including an increase among  households with at least one adult smoker, implying that the smokers in  these households agree to smoke outside of the home.\\\\21\\\\ Two    national surveys discussed by the CDC researchers identified voluntary  smokefree home rules among residents of multiunit housing in over 70  percent of those surveyed.',\n",
       " 'Additionally, CDC researchers, reviewing  published studies, found that the majority of residents in multiunit  housing expressed support for a complete smokefree building policy in  six of eight reviewed studies.\\\\22\\\\ The findings from these national and  local surveys suggest that a smokefree rule will be supported by a  majority of public housing residents and will help those residents who  already have a smokefree home rule in place achieve the desired goal  of eliminating the presence of SHS in their homes.',\n",
       " '      \\\\21\\\\ Brian A.',\n",
       " \"King et al., ``Prevalence of Smokefree Home  RulesUnited States, 19921993 and 20102011,'' Morbidity and  Mortality Weekly Report (Sept. 5, 2014), available at http://www.cdc.gov/mmwr/preview/mmwrhtml/mm6335a1.htm.\",\n",
       " '\\\\22\\\\ Kimberly Snyder et al., supra note 9.',\n",
       " '  C. Moving to SmokeFree Public Housing Units      HUD determined that the advantages of smokefree housing policies  were sufficient to warrant action by HUD to promote the voluntary  adoption of smokefree policies by PHAs and the owners/operators of  federally subsidized multifamily properties.',\n",
       " \"In 2009, HUD's Office of  Public and Indian Housing published a notice that strongly encouraged  PHAs to adopt smokefree policies in at least some of the properties  that they managed (this notice was reissued in 2012).\\\\23\\\\ HUD's Office  of Housing issued a similar program notice in 2010 that encouraged  owners/operators of subsidized multifamily properties to adopt smoke free policies (also reissued in 2012).\\\\24\\\\ The notices describe the  advantages of smokefree policies, identify required and recommended  actions in implementing smokefree policies, and provide links to  resources (e.g., smoking cessation assistance for residents).\",\n",
       " \"In June  2012, HUD published more detailed information on smokefree housing  policies for residents and the providers of subsidized housing,  referred to as ``smokefree toolkits.''\",\n",
       " \"\\\\25\\\\       \\\\23\\\\ PIH Notices 200921, ``NonSmoking Policies in Public  Housing'' and 201225, ``SmokeFree Policies in Public Housing'',  available at http://portal.hud.gov/hudportal/HUD?src=/program_offices/administration/hudclips/notices/pih.\",\n",
       " \"\\\\24\\\\ Housing Notices 201021, ``Optional SmokeFree Housing  Policy Implementation'' and 201222, ``Further Encouragement for O/ As to Adopt Optional SmokeFree Housing Policies,'' available at  http://portal.hud.gov/hudportal/HUD?src=/program_offices/administration/hudclips/notices/hsg.\",\n",
       " '\\\\25\\\\ See: http://portal.hud.gov/hudportal/HUD?src=/smokefreetoolkits1.',\n",
       " \"      In October 2012, HUD also published a Federal Register notice that  solicited feedback on the HUD's smokefree housing initiative,  specifically seeking information on topics such as best practices and  practical strategies from housing providers who have implemented smoke free policies, potential obstacles to policy implementation and how  these could be overcome, suggestions for supporting housing providers  and residents to facilitate policy implementation, and feedback from  housing providers who have decided not to implement smokefree  policies.\\\\26\\\\ HUD received many comments in response to this  solicitation, largely from public health organizations and State and  local health departments, expressing support for the concept and citing  the great health risks posed by smoking and SHS.\\\\27\\\\       \\\\26\\\\ 77 FR 60712, ``Request for Information on Adopting Smoke Free Policies in PHAs and Multifamily Housing'' (October 4, 2012).\",\n",
       " '\\\\27\\\\ All public comments submitted on the October 4, 2012,  notice can be found under docket 5597N01 in the  www.regulations.gov portal at http://www.regulations.gov/#!docketDetail;D=HUD20120103.',\n",
       " '      In 2014, HUD released additional guidance for PHAs and owners/ agents of subsidized multifamily properties on implementing smokefree  policies.',\n",
       " 'This guidance incorporates some of the feedback that HUD  received from the 2012 Federal Register notice and includes summaries  of interviews with nine early implementers of smokefree housing  policies, including administrators of public housing, subsidized  multifamily housing, and market rate housing.\\\\28\\\\ The guidance includes  best practices around enforcement, especially graduated enforcement to  assist residents with compliance and prevent evictions.',\n",
       " '      \\\\28\\\\ See: http://portal.hud.gov/hudportal/documents/huddoc?id=SFGuidanceManual.pdf.',\n",
       " '      As a result of these combined actions, over 500 PHAs have  implemented smokefree policies in at least one of their buildings.',\n",
       " 'While this voluntary effort has been highly successful, it has also  resulted in a scattered distribution of smokefree policies, with the  greatest concentration in the Northeast, West, and Northwest, which  also results in unequal protection from SHS for public housing  residents.',\n",
       " 'HUD recognizes that additional action is necessary to truly  eliminate the risk of SHS exposure to public housing residents, reduce  the risk of catastrophic fires, lower overall maintenance costs, and  implement uniform requirements to ensure that all public housing  residents are equally protected.',\n",
       " 'Therefore, HUD is proposing to require PHAs to implement smokefree  policies within public housing except for dwelling units in a mixed finance project.',\n",
       " 'Public housing is defined as lowincome housing, and  all necessary appurtenances (e.g., community facilities, public housing  offices, day care centers, and laundry rooms) thereto, assisted under  the U.S. Housing Act of 1937 (the 1937 Act), other than assistance  under section 8 of the 1937 Act.',\n",
       " 'While the smokefree policy will also apply to scattered sites and  single family properties, this requirement would not extend to public  housing units that are part of a mixedfinance project because the PHA  may not be the primary owner, and nonpublic housing units may be  contained within the building.',\n",
       " 'While smoking in single family units  does not lead to smoke intrusion to adjacent units, the risk of fire  and the increased unit turnover costs remain.',\n",
       " 'Further, including all  public housing units covered by this proposed rule means that all  tenants will be treated equally and be subject to the same lease  requirements.',\n",
       " 'This prohibition on smoking would cover all types of lit  tobacco products, including but not limited to cigarettes, cigars, and  pipes.',\n",
       " 'While the prohibition does not specifically cover waterpipe  tobacco smoking (referred to as hookahs), such smoking involves lit  charcoal and results in heating tobacco to temperatures high enough to  produce secondhand smoke that contains harmful toxins.\\\\29\\\\ For this  reason, HUD is seeking comment on whether to include a prohibition on  waterpipe tobacco in the final rule.',\n",
       " '      \\\\29\\\\ See World Health Organization.',\n",
       " 'Advisory note: waterpipe  tobacco smoking: 2nd edition (2015), available at http://www.who.int/tobacco/publications/prod_regulation/waterpipesecondedition/en/.',\n",
       " \"      The prohibition on the use of lit tobacco products in this proposal  does not include electronic nicotine delivery systems (ENDS), including  electronic cigarettes (``ecigarettes'').\",\n",
       " 'The absence of a prohibition  on the use of ecigarettes in this rule should not be read as an  endorsement of ecigarettes as an acceptable health alternative to  cigarettes.',\n",
       " 'The aerosol from ENDS typically contains nicotine derived  from tobacco plants, and may contain other hazardous and potentially  hazardous constituents such as formaldehyde and lead.\\\\30\\\\ Accidental  ingestion of nicotine liquid used in ENDS can cause acute nicotine  toxicity in children, accounting    for an increasing proportion of exposure calls to poison control  centers.\\\\31\\\\ ENDS may also present an additional enforcement challenge  for PHAs that are implementing smokefree policies because the user may  appear to be smoking a conventional cigarette.',\n",
       " 'In light of growing  health concerns regarding exposure to the aerosol of these products  among nonusers, especially children and pregnant women, HUD is seeking  additional comments on the issue of ENDS, and may prohibit the use of  these products in public housing in the final rule.',\n",
       " 'HUD encourages PHAs  that already have smokefree policies to consider whether ENDS should  be included in their smokefree policies.',\n",
       " '      \\\\30\\\\ See Offerman, F.J.',\n",
       " 'The hazards of ecigarettes.',\n",
       " 'June, 2014.',\n",
       " 'ASHRAE Journal.',\n",
       " \"See also National Institute for Occupational Safety  and Health, ``Promoting Health and Preventing Disease and Injury  Through Workplace Tobacco Policies,'' Current Intelligence Bulletin  67 (2015), available at http://www.cdc.gov/niosh/docs/2015113/pdfs/fy15_cib67_2015113_v3.pdf.\",\n",
       " '\\\\31\\\\ CDC.',\n",
       " 'Notes from the field: Calls to Poison Centers for  Exposures to Electronic CigarettesUnited States, September 2010 February 2014.',\n",
       " 'MMWR 2014;63:29293.',\n",
       " \"      In proposing this policy, it is important for HUD to clarify that  HUD's proposal does not prohibit individual PHA residents from smoking.\",\n",
       " 'PHAs should continue leasing to persons who smoke.',\n",
       " \"This rule is not  intended to contradict HUD's goals to end homelessness and help all  Americans secure quality housing.\",\n",
       " \"Rather, HUD is proposing a  prohibition on smoking inside public housing living units and indoor  common areas, public housing administrative office buildings, public  housing community rooms or community facilities, public housing day  care centers and laundry rooms, in outdoor areas within 25 feet of the  housing and administrative office buildings, and in other areas  designated by a PHA as smokefree (collectively, ``restricted areas'').\",\n",
       " 'PHAs will have the discretion to establish outside designated smoking  locations outside of the required 25 feet perimeter, which may include  partially enclosed structures, to accommodate smoking residents, to  establish additional smokefree areas (such as around a playground),  or, alternatively, to make their entire grounds smokefree.',\n",
       " 'In  addition, section 504 of the Rehabilitation Act of 1973 and the  Americans with Disabilities Act provides the participant the right to  seek a reasonable accommodation, including requests from residents with  mobilityimpairment or mental disability.',\n",
       " 'A request for a reasonable  accommodation from an eligible participant must at least be considered,  and granted in appropriate circumstances.',\n",
       " 'To assist PHAs, HUD will work  with its Office of Fair Housing and Equal Opportunity to develop  guidance on accommodating persons with a disability related to smoke free policies.',\n",
       " 'The guidance will be informed by comments on the  proposed rule and issued in advance of the final rule.',\n",
       " 'The benefits of this proposed regulatory action may be substantial,  and beneficiaries include both PHAs and residents of public housing.',\n",
       " 'Over 700,000 units would be affected by this rule (including over  500,000 units inhabited by elderly households or households with a non elderly person with disabilities), and their residents would have the  potential to experience health benefits from a reduction of exposure to  secondhand smoke.',\n",
       " 'There are also over 775,000 children in these units.',\n",
       " 'PHAs will benefit from a reduction of damage and renovation costs  caused by smoking.',\n",
       " 'Both residents and PHAs will gain from reducing  deaths, injuries, and property damage caused by fires.',\n",
       " 'The costs to  PHAs of implementing the smokefree policy proposed by this rule may  include training, administrative, legal, and enforcement costs.',\n",
       " 'Of  these costs to PHAs, HUD expects that the expense of additional  enforcement efforts may be the highest.',\n",
       " 'The costs of implementing the  smokefree policy proposed by this rule are minimized by the fact that  HUD guidance already exists on many of the topics covered by the  proposed regulatory changes, and that over 500 PHAs have already  implemented smokefree policies.',\n",
       " 'Given the existence of this HUD  guidance, initial learning costs associated with implementation of a  smokefree policy as proposed by this rule may not be significant.',\n",
       " 'There may be costs to residents as a result of eviction,  particularly for persons with disabilities, and especially those with  mobility impairments.',\n",
       " 'HUD recognizes that this rule could adversely  impact those with mobility impairment or particular frailties that  prevent them from smoking in designated areas.',\n",
       " 'As mentioned above, HUD  will develop guidance on reasonable accommodation, and HUD solicits  public comment on how to mitigate these potential adverse impacts.',\n",
       " 'HUD recognizes that PHAs developing smokefree housing policies may  need technical assistance in writing the policies, engaging residents,  and assisting residents who want to stop smoking.',\n",
       " 'HUD will continue to  provide free webinars and training sessions addressing these and  related topics.',\n",
       " \"PHAs are encouraged to work with their State HUD  office, State and local tobacco prevention and cessation programs,  state and community health organizations, and the Environmental  Protection Agency's communitybased asthma program network  (www.asthmacommunitynetwork.org).\",\n",
       " 'CDC provides funding and technical  assistance to State tobacco prevention and control programs and  prevention and smoking cessation programs in every state and the  District of Columbia (see http://www.cdc.gov/tobacco/stateandcommunity/tobacco_control_programs/ntcp/index.htm).',\n",
       " \"Contact information for local  organizations will be provided through HUD's Web site on a page  dedicated to smokefree resources that is under development.\",\n",
       " 'D. Discussions With Stakeholders      In addition to the October 2012 Federal Register notice soliciting  information on adopting smokefree policies in HUD subsidized housing,  in March 2015, HUD reached out to organizations representative of the  interests and concerns of PHAs to solicit feedback on moving forward  with smokefree policies in public housing.',\n",
       " 'The organizations expressed  support for smokefree policies but also requested that any regulations  requiring smokefree policies allow sufficient flexibility for PHAs to  tailor such policies to their local conditions.',\n",
       " 'In this rule, HUD has  strived to provide such flexibility.',\n",
       " 'III.',\n",
       " 'This Proposed RuleSummary of Changes  Applicability (Sec.',\n",
       " '965.651)      As stated above, this proposal would apply to all PHAs of any size  and MovingtoWork (MTW) agencies, but it would only apply to public  housing, and would not apply to dwelling units in a mixedfinance  project.',\n",
       " 'Public housing is defined as lowincome housing, and all  necessary appurtenances (e.g., community facilities, public housing  offices, day care centers and laundry rooms) assisted under the U.S.  Housing Act of 1937 (the 1937 Act), other than assistance under section  8 of the 1937 Act.',\n",
       " 'Requirements (Sec.',\n",
       " '965.653)      In Sec.',\n",
       " \"965.653, HUD provides that a PHA's smokefree policy must  prohibit all ``lit tobacco products.''\",\n",
       " \"HUD proposes to define ``lit  tobacco products'' as all lit tobacco products that involve the  ignition and burning of tobacco leaves such as cigarettes, cigars, and  pipes.\",\n",
       " \"HUD is proposing to require that PHAs prohibit all lit tobacco  products not only in dwelling units, but also within indoor common  areas and in outdoor areas within 25 feet of the housing and any PHA  administrative office buildings (the ``restricted areas'').\",\n",
       " 'Outside of  these areas, PHAs would be permitted to limit smoking to outdoor    designated smoking areas, which may include partially enclosed  structures to accommodate residents who smoke, or, alternatively, to  make their entire grounds smokefree.',\n",
       " \"PHAs that are not making the  entire grounds smokefree are encouraged to work with their residents  to identify outdoor designated smoking areas that are accessible within  the grounds of the public housing or administrative office buildings,  that are not frequented by children (e.g., not a playground), and that  are situated in a way that minimizes nonsmoking residents' exposure to  secondhand smoke.\",\n",
       " 'While not required, a designated smoking area with  shade and benches may assist residents with compliance.',\n",
       " 'Implementation (Sec.',\n",
       " '965.653)      HUD is proposing to provide PHAs 18 months from the effective date  of the final rule to implement smokefree public housing, as proposed  by this rule.',\n",
       " \"HUD believes that 18 months will provide PHAs sufficient  time to conduct resident engagement, to hold any public meetings that  are required to amend their PHA plans, and to incorporate the required  new lease provisions during tenants' recertifications or at a date  before the policy is fully effective.\",\n",
       " 'PHAs that already have a smoke free policy in effect will be required to review their existing  policies for compliance with the requirements of this rule, as  presented in the final rule, and amend their policies as necessary in  the same timeframe of 18 months from the effective date of the final  rule in order to implement smokefree public housing, consistent with  the requirements of the final rule.',\n",
       " 'In addition, HUD is proposing to require PHAs to amend their PHA  plans to incorporate the smokefree policy.',\n",
       " 'If the PHA determines the  imposition of a smokefree policy is a significant amendment to the PHA  plan, the PHA must conduct public meetings in accordance with standard  PHA Plan amendment procedures, and these meetings must be held in  accessible buildings and provided in accessible formats, as necessary,  for persons with disabilities and those who are limited in English  proficiency.',\n",
       " 'HUD would recommend that all PHAs conduct meetings with  residents to fully explain the smokefree building requirements and to  best determine which outside areas, if any, to designate as smoking  areas and to accommodate the needs of all residents.',\n",
       " 'Lease Provisions (Sec.',\n",
       " '966.4)      HUD believes that the best way to implement smokefree policies is  to incorporate the prohibition on indoor smoking in the leases each  tenant must sign.',\n",
       " 'This will allow PHAs to use enforcement mechanisms  already in place and provide an additional notification of the policy  to tenants.',\n",
       " 'HUD expects PHAs to follow the PIH administrative grievance  procedures during enforcement of their smokefree housing policies.',\n",
       " 'Because some tenants may not be recertified before the policy takes  effect, PHAs may require that all remaining leases be amended, or may  establish their own schedule for lease amendments, provided that all  leases are amended by the effective date of the policy.',\n",
       " 'IV.',\n",
       " 'Specific Questions for Comments      While HUD welcomes comments on all aspects of this proposed rule,  HUD is seeking specific comment on the following questions:     1.',\n",
       " 'What barriers that PHAs could encounter in implementing smoke free housing?',\n",
       " 'What costs could PHAs incur?',\n",
       " 'Are there any specific costs  to enforcing such a policy?',\n",
       " '2.',\n",
       " 'Does this proposed rule adequately address the adverse effects  of smoking and secondhand smoke on PHAs and PHA residents?',\n",
       " '3.',\n",
       " 'Does this proposed rule create burdens, costs, or confer  benefits specific to families, children, persons with disabilities,  owners, or the elderly, particularly if any individual or family is  evicted as a result of this policy?',\n",
       " '4.',\n",
       " \"For those PHAs that have already implemented a smokefree  policy, what exceptions to the requirements have been granted based on  tenants' requests?\",\n",
       " '5.',\n",
       " 'For those PHAs that have already implemented a smokefree  policy, what experiences, lessons, or advice would you share based on  your experiences with implementing and enforcing the policy?',\n",
       " '6.',\n",
       " 'For those PHAs that have already implemented a smokefree  policy, what tobacco cessation services were offered to residents to  assist with the change?',\n",
       " 'Did you establish partnerships with external  groups to provide or refer residents to these services?',\n",
       " '7.',\n",
       " 'Are there specific areas of support that HUD could provide PHAs  that would be particularly helpful in the implementation of the  proposed rule?',\n",
       " '8.',\n",
       " 'Should the policy extend to electronic nicotine delivery  systems, such as ecigarettes?',\n",
       " '9.',\n",
       " 'Should the policy extend to waterpipe tobacco smoking?',\n",
       " 'Does such  smoking increase the risk of fire or property damage?',\n",
       " \"V. Findings and Certifications  Executive Order 12866, Regulatory Planning and Review      The Office of Management and Budget (OMB) reviewed this proposed  rule under Executive Order 12866 (entitled ``Regulatory Planning and  Review'').\",\n",
       " 'OMB determined that this rule was economically significant  under the order.',\n",
       " 'The docket file is available for public inspection in  the Regulations Division, Office of General Counsel, U.S. Department of  Housing and Urban Development, 451 7th Street SW., Room 10276,  Washington, DC 204100500.',\n",
       " \"The initial Regulatory Impact Analysis (RIA)  prepared for this rule is also available for public inspection in the  Regulations Division and may be viewed online at www.regulations.gov,  under the docket number above, or on HUD's Web site at http://portal.hud.gov/hudportal/HUD?src=/program_offices/administration/hudclips/ia/.\",\n",
       " 'Due to security measures at the HUD Headquarters  building, an advance appointment to review the public comments must be  scheduled by calling the Regulations Division at (202) 7083055 (this  is not a tollfree number).',\n",
       " 'Individuals with speech or hearing  impairments may access this number via TTY by calling the Federal Relay  Service at (800) 8778339.',\n",
       " 'Information Collection Requirements      The information collection requirements contained in this proposed  rule have been submitted to the Office of Management and Budget (OMB)  under the Paperwork Reduction Act of 1995 (44 U.S.C.',\n",
       " '35013520) and  assigned OMB control number 25770226.',\n",
       " 'In accordance with the Paperwork  Reduction Act, an agency may not conduct or sponsor, and a person is  not required to respond to, a collection of information, unless the  collection displays a currently valid OMB control number.',\n",
       " 'Unfunded Mandates Reform Act      Title II of the Unfunded Mandates Reform Act of 1995 (UMRA)  establishes requirements for federal agencies to assess the effects of  their regulatory actions on state, local, and tribal governments and  the private sector.',\n",
       " 'This rule will not impose any federal mandates on  any state, local, or tribal governments or the private sector within  the meaning of UMRA.',\n",
       " 'Environmental Review      A Finding of No Significant Impact with respect to the environment  has been made in accordance with HUD regulations in 24 CFR part 50 that    implement section 102(2)(C) of the National Environmental Policy Act of  1969 (42 U.S.C.',\n",
       " '4332(2)(C)).',\n",
       " 'The Finding is available for public  inspection during regular business hours in the Regulations Division,  Office of General Counsel, Department of Housing and Urban Development,  451 7th Street SW., Room 10276, Washington, DC 204100500.',\n",
       " 'Due to  security measures at the HUD Headquarters building, please schedule an  appointment to review the Finding by calling the Regulations Division  at 2027083055 (this is not a tollfree number).',\n",
       " 'Individuals with  speech or hearing impairments may access this number via TTY by calling  the Federal Relay Service at 8008778339.',\n",
       " 'Impact on Small Entities      The Regulatory Flexibility Act (RFA) (5 U.S.C.',\n",
       " '601 et seq.',\n",
       " '),  generally requires an agency to conduct a regulatory flexibility  analysis of any rule subject to notice and comment rulemaking  requirements unless the agency certifies that the rule will not have a  significant economic impact on a substantial number of small entities.',\n",
       " 'This rule prohibits smoking of tobacco in all indoor areas of and  within 25 feet of any public housing and administrative office  buildings for all PHAs, regardless of size.',\n",
       " \"There are 2334 ``small'' PHAs (defined as PHAs with fewer than 250  units), which make up 75 percent of the public housing stock across the  country.\",\n",
       " 'Of this number, approximately 378 have already instituted a  voluntary full or partial policy on indoor tobacco smoking.',\n",
       " \"HUD anticipates that implementation of the policy will impose  minimal additional costs, as creation of the smokefree policy only  requires amendment of leases and the PHA plan, both of which may be  done as part of a PHA's normal course of business.\",\n",
       " 'Additionally,  enforcement of the policy will add minimal incremental costs, as PHAs  must already regularly inspect public housing units and enforce lease  provisions.',\n",
       " 'Any costs of this rule are mitigated by the fact that PHAs  have up to 18 months to implement the policy, allowing for costs to be  spread across that time period.',\n",
       " 'While there are significant benefits to the smokefree policy  requirement, the majority of those benefits accrue to the public  housing residents themselves, not to the PHAs.',\n",
       " \"PHAs will realize  monetary benefits due to reduced unit turnover costs and reduced fire  and fire prevention costs, but these benefits are variable according to  the populations of each PHA and the PHA's existing practices.\",\n",
       " 'Finally, this rule does not impose a disproportionate burden on  small PHAs.',\n",
       " 'The rule does not require a fixed expenditure; rather, all  costs should be proportionate to the size of the PHA implementing and  enforcing the smokefree policy.',\n",
       " 'Therefore, the undersigned certifies that this rule will not have a  significant impact on a substantial number of small entities.',\n",
       " \"Notwithstanding HUD's view that this rule will not have a  significant effect on a substantial number of small entities, HUD  specifically invites comments regarding any less burdensome  alternatives to this rule that will meet HUD's objectives as described  in the preamble.\",\n",
       " \"Executive Order 13132, Federalism      Executive Order 13132 (entitled ``Federalism'') prohibits an agency  from publishing any rule that has federalism implications if the rule  either imposes substantial direct compliance costs on state and local  governments or is not required by statute, or the rule preempts state  law, unless the agency meets the consultation and funding requirements  of section 6 of the Executive Order.\",\n",
       " 'This final rule does not have  federalism implications and does not impose substantial direct  compliance costs on state and local governments nor preempt state law  within the meaning of the Executive Order.',\n",
       " 'Catalog of Federal Domestic Assistance      The Catalog of Federal Domestic Assistance number for the Public  Housing program is 14.872.',\n",
       " 'List of Subjects  24 CFR Part 965      Government procurement, Grant programshousing and community  development, Lead poisoning, Loan programshousing and community  development, Public housing, Reporting and recordkeeping requirements,  Utilities.',\n",
       " '24 CFR Part 966      Grant programshousing and community development, Public housing,  Reporting and recordkeeping requirements.',\n",
       " 'Accordingly, for the reasons stated in the preamble, HUD proposes  to amend 24 CFR parts 965 and 966 as follows:  PART 965PHAOWNED OR LEASED PROJECTSGENERAL PROVISIONS  0 1.',\n",
       " 'The authority citation for 24 CFR part 965 continues to read as  follows:      Authority:  42 U.S.C.',\n",
       " '1547, 1437a, 1437d, 1437g, and 3535(d).',\n",
       " 'Subpart H is also issued under 42 U.S.C.',\n",
       " '48214846.',\n",
       " '0 2.',\n",
       " 'Add subpart G to read as follows: Subpart GSmokeFree Public Housing 965.651 Applicability 965.653 Smokefree public housing 965.655 Implementation  Subpart GSmokeFree Public Housing   Sec.',\n",
       " '965.651  Applicability.',\n",
       " 'This subpart applies to public housing units, except for dwelling  units in a mixedfinance project.',\n",
       " 'Public housing is defined as low income housing, and all necessary appurtenances (e.g., community  facilities, public housing offices, day care centers, and laundry  rooms) thereto, assisted under the U.S. Housing Act of 1937 (the 1937  Act), other than assistance under section 8 of the 1937 Act.',\n",
       " 'Sec.',\n",
       " '965.653  Smokefree public housing.',\n",
       " '(a) In general.',\n",
       " \"PHAs must design and implement a policy prohibiting  the use of lit tobacco products in all public housing living units and  interior common areas (including but not limited to hallways, rental  and administrative offices, community centers, day care centers,  laundry centers, and similar structures), as well as in outdoor areas  within 25 feet from public housing and administrative office buildings  (collectively, ``restricted areas'') in which public housing is  located.\",\n",
       " '(b) Designated smoking areas.',\n",
       " 'PHAs may limit smoking to designated  smoking areas on the grounds of the public housing or administrative  office buildings, which may include partially enclosed structures, to  accommodate residents who smoke.',\n",
       " 'These areas must be outside of any  restricted areas, as defined in paragraph (a) of this section.',\n",
       " 'Alternatively, PHAs may choose to create additional smokefree areas  outside the restricted areas or to make their entire grounds smoke free.',\n",
       " '(c) Lit tobacco products.',\n",
       " 'Lit tobacco products are those that  involve the ignition and burning of tobacco leaves, such cigarettes,  cigars, and pipes.',\n",
       " \"A PHA's smokefree policy must, at a minimum,  include a prohibition on the use of all lit tobacco products.\",\n",
       " 'Sec.',\n",
       " '965.655  Implementation.',\n",
       " '(a) Amendments.',\n",
       " 'PHAs are required to implement the requirements of  this subpart by amending each of the following:        (1) All applicable PHA plans, according to the provisions in 24 CFR  part 903.',\n",
       " '(2) Tenant leases, according to the provisions of 24 CFR 966.4.',\n",
       " '(b) Deadline.',\n",
       " 'All PHAs must be in full compliance, with effective  policy amendments, by [INSERT, AT THE FINAL RULE STAGE, THE DATE THAT  IS 540 DAYS AFTER THE EFFECTIVE DATE OF THE FINAL RULE].',\n",
       " 'PART 966PUBLIC HOUSING LEASE AND GRIEVANCE PROCEDURE  0 3.',\n",
       " 'The authority section for 24 CFR part 966 continues to read as  follows:      Authority:  42 U.S.C.',\n",
       " '1437d and 3535(d).',\n",
       " '0 4.',\n",
       " 'In Sec.',\n",
       " '966.4, revise paragraphs (f) (12) (i) and (ii) to read as  follows:   Sec.',\n",
       " '966.4  Lease Requirements.',\n",
       " \"(f)        (12)        (i) To assure that no tenant, member of the tenant's household, or  guest engages in:     (A) Criminal activity.\",\n",
       " '(1) Any criminal activity that threatens the  health, safety or right to peaceful enjoyment of the premises by other  residents;     (2) Any drugrelated criminal activity on or off the premises; or     (B) Civil activity.',\n",
       " 'For any units covered by 24 CFR part 965,  subpart G, any smoking of lit tobacco products in restricted areas, as  defined by 24 CFR 965.653(a), or in other outdoor areas that the PHA  has designated as smokefree.',\n",
       " \"(ii) To assure that no other person under the tenant's control  engages in:     (A) Criminal activity.\",\n",
       " '(1) Any criminal activity that threatens the  health, safety or right to peaceful enjoyment of the premises by other  residents;     (2) Any drugrelated criminal activity on the premises; or     (B) Civil activity.',\n",
       " 'For any units covered by 24 CFR part 965,  subpart G, any smoking of lit tobacco products in restricted areas, as  defined by 24 CFR 965.653(a), or in other outdoor areas that the PHA  has designated as smokefree.',\n",
       " 'Dated: October 22, 2015.',\n",
       " 'Lourdes Castro Ram[iacute]rez, Principal Deputy Assistant Secretary for Public and Indian Housing.',\n",
       " '[FR Doc.',\n",
       " '201529346 Filed 111615; 8:45 am]  BILLING CODE 421067P   ']"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clean_sent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['  I.',\n",
       " 'Executive Summary  A.',\n",
       " 'Purpose of the Proposed Rule      The purpose of the proposed rule is to require PHAs to, within 18  months of the final rule, establish a policy prohibiting lit tobacco  products, as such term is proposed to be defined in Sec.',\n",
       " '965.653(c).',\n",
       " \"inside all indoor areas of public housing, including but not limited to  living units, indoor common areas, electrical closets, storage units,  and PHA administrative office buildings and in all outdoor areas within  25 feet of the housing and administrative office buildings  (collectively, ``restricted areas'').\",\n",
       " 'As further discussed in this  rule, such a policy is expected to improve indoor air quality in public  housing, benefit the health of public housing residents and PHA staff,  reduce the risk of catastrophic fires, and lower overall maintenance  costs.',\n",
       " 'B.',\n",
       " 'Summary of Major Provisions of the Proposed Rule      This proposed rule would apply to all public housing, other than  dwelling units in mixed-finance buildings.',\n",
       " 'PHAs would be required,  within 18 months of the effective date of the final rule, to establish  policies prohibiting lit tobacco products in all restricted areas.',\n",
       " 'PHAs  may, but would not be required to, further restrict smoking to outdoor  dedicated smoking areas outside the restricted areas, create additional  restricted areas in which smoking is  [[Page 71763]]  prohibited (e.g., near a playground), or, alternatively, make their  entire grounds smoke-free.',\n",
       " 'PHAs would also be required to document their smoke-free policies  in their PHA plans, a process that requires resident engagement and  public meetings.',\n",
       " \"The prohibition on lit tobacco would also be included  in a tenant's lease, which may be done either through an amendment  process or as tenants renew their leases annually.\",\n",
       " 'C. Costs and Benefits of This Proposed Rule      The costs to PHAs of implementing smoke-free policies may include  training, administrative, legal, and enforcement costs.',\n",
       " 'Of these costs,  HUD expects that the expense of additional enforcement efforts may be  the highest.',\n",
       " 'The costs of implementing a smoke-free policy as proposed  by this rule are minimized by the fact that HUD guidance already exists  on many of the topics covered by the smoke-free policy proposed to be  required by this rule; that hundreds of PHAs have already voluntarily  implemented smoke-free policies; and that infrastructure already exists  for enforcement of lease violations, and violation of the smoke-free  policy would be a lease violation.',\n",
       " 'In addition, time spent by PHA staff  on implementing and enforcing the smoke-free policy will be partially  offset by the time that staff no longer have to spend mediating  disputes among residents over smoking in secondhand smoke infiltration  within living units.',\n",
       " 'Given the existing HUD guidance, initial learning  costs associated with implementation of a smoke-free policy may not be  significant.',\n",
       " 'For the hundreds of PHAs that are already implementing  voluntary smoke-free policies, there will be minimal costs for these  PHAs, and, generally, only if their existing policies are not  consistent with the minimum requirements for smoke-free policies  proposed by this rule.',\n",
       " 'The benefits of smoke free policies, however, could be  considerable.',\n",
       " 'Over 700,000 units would be affected by this rule  (including over 500,000 units inhabited by elderly households or  households with a non-elderly person with disabilities), and their non- smoking residents would have the potential to experience health  benefits from a reduction of exposure to secondhand smoke.',\n",
       " 'PHAs will  also benefit from a reduction of damage caused by smoking, and  residents and PHAs both gain from seeing a reduction in injuries,  deaths, and property damage caused by fires.',\n",
       " 'Estimates of these and  other rule-induced impacts are summarized in the following table:  ------------------------------------------------------------------------                                                       Amount (discount            Impact                    Source               rates in                                                         parentheses) ------------------------------------------------------------------------ Cost (potentially recurring   PHA Compliance......  $3.2 million.',\n",
       " \"but concentrated during  first few years of the  rule's implementation).\",\n",
       " 'Cost (recurring)............  Smoker Inconvenience  $209 million.',\n",
       " 'Cost (recurring)............  Enforcement.........  Not quantified.',\n",
       " 'Benefit (recurring).........  PHA Reduced           $16 to $38 million.',\n",
       " 'Maintenance.',\n",
       " 'Benefit (recurring).........  PHA Reduced Fire      $32 million.',\n",
       " 'Risk.',\n",
       " 'Benefit (annualized over 10   Non-Smoker Health...  Less than:  to 50 years).',\n",
       " '$148 to $447 million                                                      (3%)                                                     $70 to $137 million                                                      (7%).',\n",
       " 'Benefit (recurring).........  Non-Smoker Well-      $96 to $275 million.',\n",
       " 'Being (PHA                                residents who do                                not live in units                                with smokers).',\n",
       " 'Benefit (recurring).........  Smoker Health.......  Not quantified.',\n",
       " 'Partially Quantified Net      See above...........  Less than: -$19 to  Benefits (recurring).',\n",
       " '$302 million (3%) -                                                      $97 to -$8 million                                                      (7%) ------------------------------------------------------------------------      For additional details on the costs and benefits of this rule,  please see the Regulatory Impact Analysis (RIA) for this rule, which  can be found at www.regulations.gov, under the docket number for this  rule.',\n",
       " 'Information on how to view the RIA is included below.',\n",
       " 'II.',\n",
       " 'Background  A.',\n",
       " 'The Effects of Smoking on Health      Tobacco smoking has been determined to be a cause of diseases of  nearly all organs in the body, and research continues to newly identify  diseases caused by smoking, including diabetes mellitus, rheumatoid  arthritis, and colorectal cancer.',\n",
       " \"In addition to causing multiple  diseases and cancers, tobacco smoking has many other adverse effects on  the body, including inflammation and impairment to the immune  system.\\\\1\\\\ ---------------------------------------------------------------------------      \\\\1\\\\ Office of the Surgeon General, ``The Health Consequences of  Smoking--50 Years of Progress,'' (2014), available at http://www.surgeongeneral.gov/library/reports/50-years-of-progress/full-report.pdf.\",\n",
       " '---------------------------------------------------------------------------      Adverse effects of tobacco use are not limited to the smoker.',\n",
       " 'The  U.S.',\n",
       " 'Surgeon General estimates that exposure to secondhand tobacco  smoke (i.e., the smoke that comes from burning tobacco products and is  exhaled by smokers) is responsible for the death of 41,000 adults non- smokers in the United States each year from lung cancer and heart  disease.\\\\2\\\\ Secondhand smoke (SHS) contains hundreds of toxic chemicals  and is designated as a known human carcinogen by the U.S. Environmental  Protection Agency, the U.S. National Toxicology Program, and the  International Agency for Research on Cancer.\\\\3\\\\ Exposure to SHS can  also cause sudden infant death syndrome and respiratory symptoms such  as cough and wheeze, middle ear infections, and slowed lung growth and  reduced lung function in children, and increased risk of stroke in  adults.\\\\4\\\\ The Surgeon General has concluded that there is no risk-free  level of exposure to SHS, and that eliminating smoking in indoor spaces  fully protects nonsmokers from exposure to secondhand smoke.',\n",
       " 'Separating  smokers from nonsmokers, cleaning the air, and ventilating buildings  cannot eliminate exposures of nonsmokers to secondhand smoke.\\\\5\\\\ ---------------------------------------------------------------------------      \\\\2\\\\ Id.',\n",
       " \"\\\\3\\\\ American Cancer Society, ``Secondhand Smoke,'' http://www.cancer.org/cancer/cancercauses/tobaccocancer/secondhand-smoke.\",\n",
       " \"\\\\4\\\\ 2014 Surgeon General's Report, footnote 1.\",\n",
       " '\\\\5\\\\ U.S. Dept.',\n",
       " \"of Health and Human Services, ``The Health  Consequences of Involuntary Exposure to Tobacco Smoke: A Report of  the Surgeon General,'' (2006), available at http://www.surgeongeneral.gov/library/reports/secondhandsmoke/fullreport.pdf.\",\n",
       " '---------------------------------------------------------------------------      The effects of SHS are especially damaging in children and unborn  fetuses.',\n",
       " \"The Surgeon General estimates that SHS is responsible for the  death of hundreds of newborns from Sudden Infant Death Syndrome (SIDS)  each year.\\\\6\\\\ Lead in SHS is also a significant source of lead in house  dust and children's blood.\",\n",
       " 'The CDC confirmed the association between  SHS exposure  [[Page 71764]]  and blood-lead levels in youth and adults, concluding that youth with  SHS exposure had blood lead levels high enough to result in adverse  cognitive outcomes.\\\\7\\\\ ---------------------------------------------------------------------------      \\\\6\\\\ Id.',\n",
       " \"\\\\7\\\\ Patricia Richter et al., ``Trends in Tobacco Smoke Exposure  and Blood Lead Levels Among Youth and Adults in the United States:  The National Health and Nutrition Examination Survey, 1999-2008,''  Preventing Chronic Disease, (December 19, 2013), available at http://www.cdc.gov/pcd/issues/2013/pdf/13_0056.pdf.\",\n",
       " '---------------------------------------------------------------------------      Approximately half of the U.S. population is protected from SHS  exposure through statewide, municipal, and federal laws prohibiting  smoking in indoor areas of public places and worksites, including bars  and restaurants.',\n",
       " 'However, an estimated 58 million Americans remain  exposed to secondhand smoke, including 15 million children ages 3 to  11.',\n",
       " 'The home is the primary source of exposure for children.\\\\8\\\\ Because  SHS moves throughout buildings, individuals living in multiunit housing  can be exposed to SHS even if no one smokes in their households.',\n",
       " \"Surveys of multiunit housing residents indicate that 26 to 64 percent  of residents reported SHS incursions into their units from external  sources (e.g., hallways or adjacent apartments), and 65 to 90 percent  of the residents experiencing such incursions were bothered by them.\\\\9\\\\ ---------------------------------------------------------------------------      \\\\8\\\\ 2006 Surgeon General's Report, footnote 5; David M. Homa et  al., ``Vital Signs: Disparities in Nonsmokers' Exposure to  Secondhand Smoke--United States, 1999-2012,'' Morbidity and  Mortality Weekly Report (February 6, 2015), available at http://www.cdc.gov/mmwr/preview/mmwrhtml/mm6404a7.htm?s_cid=mm6404a7_w.\",\n",
       " \"\\\\9\\\\ Kimberly Snyder et al., ``Smoke-free Multiunit Housing: A  Review of the Scientific Literature,'' Tobacco Control (2015),  available at http://tobaccocontrol.bmj.com/content/early/2015/01/07/tobaccocontrol-2014-051849.short?rss=1.\",\n",
       " '---------------------------------------------------------------------------      The movement of contaminants from SHS within buildings has also  been documented through direct measurements of fine particles (an  environmental marker of SHS) in indoor air.',\n",
       " 'SHS can move both from  external hallways into apartments and between adjacent units.\\\\10\\\\ A  study of public housing documented lower concentrations of SHS  contaminants in buildings covered by smoke-free policies (i.e.,  policies prohibiting the smoking of tobacco products in all indoor  spaces) compared to buildings without these policies.\\\\11\\\\ Analysis of  data from the National Health and Nutrition Examination Survey (NHANES)  demonstrated evidence of greater SHS exposure among children (aged 6 to  18) living in multiunit housing through measurements of cotinine (a  metabolite of nicotine) in their blood.\\\\12\\\\ The study demonstrated that  children living in non-smoking households in apartments had 45 percent  higher levels of cotinine in their blood compared to children living in  non-smoking households in detached homes.',\n",
       " 'CDC researchers analyzed  NHANES data over the period from 1999-2012 and reported that one of  four nonsmokers (approximately 58 million people) continue to be  exposed to SHS, with the highest exposures among children, non-Hispanic  blacks, renters, and those living in poverty.\\\\13\\\\ ---------------------------------------------------------------------------      \\\\10\\\\ Brian A.',\n",
       " \"King et al., ``Secondhand Smoke Transfer in  Multiunit Housing,'' 12 Nicotine and Tobacco Research 1133 (2010),  available at http://ntr.oxfordjournals.org/content/12/11/1133.\",\n",
       " \"\\\\11\\\\ Elizabeth T. Russo, et al., ``Comparison of Indoor Air  Quality in Smoke-Permitted and Smoke-Free Multiunit Housing:  Findings from the Boston Housing Authority,'' 10 Nicotine and  Tobacco Research 1093 (2014), available at http://ntr.oxfordjournals.org/content/early/2014/08/25/ntr.ntu146.abstract?utm_source=rss&amp;utm_medium=rss&amp;utm_campaign=comparison-of-indoor-air-quality-in-smoke-permitted-and-smoke-free-multiunit-housing-findings-from-the-boston-housing-authority.\",\n",
       " \"\\\\12\\\\ Karen M. Wilson et al., ``Tobacco-Smoke Exposure in  Children Who Live in Multiunit Housing,'' 127 Pediatrics 85 (2011),  available at http://pediatrics.aappublications.org/content/127/1/85.full.pdf+html.\",\n",
       " \"\\\\13\\\\ David M. Homa et al., ``Disparities in Nonsmokers Exposure  to Secondhand Smoke in the United States, 1999-2012,'' Mortality and  Morbidity Weekly Report, Early Release, 64 (February 3, 2015),  available at http://www.cdc.gov/mmwr/pdf/wk/mm64e0203a1.pdf.\",\n",
       " '---------------------------------------------------------------------------      The Surgeon General concluded in 2006 that separating smokers and  nonsmokers, building ventilation, and cleaning the air cannot eliminate  exposure to SHS; that can only be accomplished by eliminating smoking  from indoor spaces.\\\\14\\\\ ---------------------------------------------------------------------------      \\\\14\\\\ U.S. Dept.',\n",
       " 'of Health and Human Services.',\n",
       " 'See footnote note  2.',\n",
       " '---------------------------------------------------------------------------  B.',\n",
       " 'The Financial Costs of Smoking      Beyond the increased costs associated with higher healthcare  expenses, tobacco smoking can have profound financial impacts on PHAs  and owners of other multiunit properties.',\n",
       " 'Smoking is the leading cause  of fire deaths in multiunit properties.\\\\15\\\\ In 2011, smoking caused  17,600 residential fires resulting in 490 civilian deaths, 1,370  injuries, and $516 million in direct property damage.\\\\16\\\\ Smoking is  especially dangerous in units where a household member is receiving  oxygen for medical purposes.',\n",
       " 'Research conducted by the U.S. Fire  Protection Association found that for fire deaths during the period  from 2007-2011 in which oxygen administration equipment was cited as  being involved in the ignition, 82 percent involved smoking materials  as the heat source.\\\\17\\\\ ---------------------------------------------------------------------------      \\\\15\\\\ U.S. Fire Administration, Residential Structure and  Building Fires, http://www.usfa.fema.gov/downloads/pdf/publications/residential_structure_and_building_fires.pdf.',\n",
       " '\\\\16\\\\ Marty Ahrens, Ntl.',\n",
       " \"Fire Protection Assn., ``Home Structure  Fires,'' (April 2013), available at http://www.nfpa.org/~/media/ Files/Research/NFPA%20reports/Occupancies/oshomes.pdf.\",\n",
       " '\\\\17\\\\ John R. Hall, Jr., Ntl.',\n",
       " \"Fire Protection Assn., ``The  Smoking-Material Fire Problem,'' (July 2013), available at http:// www.nfpa.org/~/media/Files/Research/NFPA%20reports/Major%20Causes/ ossmoking.pdf.\",\n",
       " '---------------------------------------------------------------------------      Smoking is also associated with higher maintenance costs for  landlords of multiunit housing.',\n",
       " 'Smoking indoors increases the cost of  rehabilitating a housing unit because of the need for additional  cleaning, painting, and repair of damaged items at unit turnover  compared to non-smoking units.',\n",
       " 'The cost of cleaning and renovating a  smoking unit adds up quickly, and smaller properties generally pay more  per unit than larger properties when repairing smoking damage.',\n",
       " 'A survey  of public and subsidized housing managers found that the additional  cost of rehabilitating the units of smokers averaged $1,250 to $2,955  per unit, depending on the intensity of smoking.\\\\18\\\\ A study conducted  in California found that the owners of multiunit housing could save  over $18 million per year if the operators of all multiunit housing in  the state adopted smoke-free building policies.\\\\19\\\\ Researchers from  the CDC estimated that a nationwide smoke-free public housing policy  would result in an estimated annual cost savings of $152.91 million,  including $42.99 million in reduced renovation costs and $15.92 million  in averted fire losses.\\\\20\\\\ ---------------------------------------------------------------------------      \\\\18\\\\ Ntl.',\n",
       " 'Ctr.',\n",
       " \"For Healthy Hsg., ``Reasons to Explore Smoke-Free  Housing,'' (Early Fall 2009), available at http://www.nchh.org/portals/0/contents/nchh_green_factsheet_smokefree.pdf.\",\n",
       " \"\\\\19\\\\ Michael K. Ong et al, ``Estimates of Smoking-Related  Properties Costs in California Multiunit Housing,'' 102 Am J Public  Health 490 (2012), available at http://www.ncbi.nlm.nih.gov/pmc/articles/PMC3487653/.\",\n",
       " \"\\\\20\\\\ Brian King et al., ``National and State Cost Savings  Associated With Prohibiting Smoking in Subsidized and Public Housing  in the United States,'' Preventing Chronic Disease (October 2014),  available at http://www.cdc.gov/pcd/issues/2014/pdf/14_0222.pdf.\",\n",
       " '---------------------------------------------------------------------------      Self-imposed rules prohibiting smoking in individual households  (referred to as smoke-free home rules) are becoming increasingly common  in the United States.',\n",
       " 'CDC researchers found that the prevalence of  smoke-free home rules among U.S. households increased from 43 percent  in 1992-1993 to 83 percent in 2010-2011, including an increase among  households with at least one adult smoker, implying that the smokers in  these households agree to smoke outside of the home.\\\\21\\\\ Two  [[Page 71765]]  national surveys discussed by the CDC researchers identified voluntary  smoke-free home rules among residents of multiunit housing in over 70  percent of those surveyed.',\n",
       " 'Additionally, CDC researchers, reviewing  published studies, found that the majority of residents in multiunit  housing expressed support for a complete smoke-free building policy in  six of eight reviewed studies.\\\\22\\\\ The findings from these national and  local surveys suggest that a smoke-free rule will be supported by a  majority of public housing residents and will help those residents who  already have a smoke-free home rule in place achieve the desired goal  of eliminating the presence of SHS in their homes.',\n",
       " '---------------------------------------------------------------------------      \\\\21\\\\ Brian A.',\n",
       " \"King et al., ``Prevalence of Smokefree Home  Rules--United States, 1992-1993 and 2010-2011,'' Morbidity and  Mortality Weekly Report (Sept. 5, 2014), available at http://www.cdc.gov/mmwr/preview/mmwrhtml/mm6335a1.htm.\",\n",
       " '\\\\22\\\\ Kimberly Snyder et al., supra note 9.',\n",
       " '---------------------------------------------------------------------------  C. Moving to Smoke-Free Public Housing Units      HUD determined that the advantages of smoke-free housing policies  were sufficient to warrant action by HUD to promote the voluntary  adoption of smoke-free policies by PHAs and the owners/operators of  federally subsidized multifamily properties.',\n",
       " \"In 2009, HUD's Office of  Public and Indian Housing published a notice that strongly encouraged  PHAs to adopt smoke-free policies in at least some of the properties  that they managed (this notice was reissued in 2012).\\\\23\\\\ HUD's Office  of Housing issued a similar program notice in 2010 that encouraged  owners/operators of subsidized multifamily properties to adopt smoke- free policies (also reissued in 2012).\\\\24\\\\ The notices describe the  advantages of smoke-free policies, identify required and recommended  actions in implementing smoke-free policies, and provide links to  resources (e.g., smoking cessation assistance for residents).\",\n",
       " \"In June  2012, HUD published more detailed information on smoke-free housing  policies for residents and the providers of subsidized housing,  referred to as ``smoke-free toolkits.''\",\n",
       " \"\\\\25\\\\ ---------------------------------------------------------------------------      \\\\23\\\\ PIH Notices 2009-21, ``Non-Smoking Policies in Public  Housing'' and 2012-25, ``Smoke-Free Policies in Public Housing'',  available at http://portal.hud.gov/hudportal/HUD?src=/program_offices/administration/hudclips/notices/pih.\",\n",
       " \"\\\\24\\\\ Housing Notices 2010-21, ``Optional Smoke-Free Housing  Policy Implementation'' and 2012-22, ``Further Encouragement for O/ As to Adopt Optional Smoke-Free Housing Policies,'' available at  http://portal.hud.gov/hudportal/HUD?src=/program_offices/administration/hudclips/notices/hsg.\",\n",
       " '\\\\25\\\\ See: http://portal.hud.gov/hudportal/HUD?src=/smokefreetoolkits1.',\n",
       " \"---------------------------------------------------------------------------      In October 2012, HUD also published a Federal Register notice that  solicited feedback on the HUD's smoke-free housing initiative,  specifically seeking information on topics such as best practices and  practical strategies from housing providers who have implemented smoke- free policies, potential obstacles to policy implementation and how  these could be overcome, suggestions for supporting housing providers  and residents to facilitate policy implementation, and feedback from  housing providers who have decided not to implement smoke-free  policies.\\\\26\\\\ HUD received many comments in response to this  solicitation, largely from public health organizations and State and  local health departments, expressing support for the concept and citing  the great health risks posed by smoking and SHS.\\\\27\\\\ ---------------------------------------------------------------------------      \\\\26\\\\ 77 FR 60712, ``Request for Information on Adopting Smoke- Free Policies in PHAs and Multifamily Housing'' (October 4, 2012).\",\n",
       " '\\\\27\\\\ All public comments submitted on the October 4, 2012,  notice can be found under docket 5597-N-01 in the  www.regulations.gov portal at http://www.regulations.gov/#!docketDetail;D=HUD-2012-0103.',\n",
       " '---------------------------------------------------------------------------      In 2014, HUD released additional guidance for PHAs and owners/ agents of subsidized multifamily properties on implementing smoke-free  policies.',\n",
       " 'This guidance incorporates some of the feedback that HUD  received from the 2012 Federal Register notice and includes summaries  of interviews with nine early implementers of smoke-free housing  policies, including administrators of public housing, subsidized  multifamily housing, and market rate housing.\\\\28\\\\ The guidance includes  best practices around enforcement, especially graduated enforcement to  assist residents with compliance and prevent evictions.',\n",
       " '---------------------------------------------------------------------------      \\\\28\\\\ See: http://portal.hud.gov/hudportal/documents/huddoc?id=SFGuidanceManual.pdf.',\n",
       " '---------------------------------------------------------------------------      As a result of these combined actions, over 500 PHAs have  implemented smoke-free policies in at least one of their buildings.',\n",
       " 'While this voluntary effort has been highly successful, it has also  resulted in a scattered distribution of smoke-free policies, with the  greatest concentration in the Northeast, West, and Northwest, which  also results in unequal protection from SHS for public housing  residents.',\n",
       " 'HUD recognizes that additional action is necessary to truly  eliminate the risk of SHS exposure to public housing residents, reduce  the risk of catastrophic fires, lower overall maintenance costs, and  implement uniform requirements to ensure that all public housing  residents are equally protected.',\n",
       " 'Therefore, HUD is proposing to require PHAs to implement smoke-free  policies within public housing except for dwelling units in a mixed- finance project.',\n",
       " 'Public housing is defined as low-income housing, and  all necessary appurtenances (e.g., community facilities, public housing  offices, day care centers, and laundry rooms) thereto, assisted under  the U.S. Housing Act of 1937 (the 1937 Act), other than assistance  under section 8 of the 1937 Act.',\n",
       " 'While the smoke-free policy will also apply to scattered sites and  single family properties, this requirement would not extend to public  housing units that are part of a mixed-finance project because the PHA  may not be the primary owner, and non-public housing units may be  contained within the building.',\n",
       " 'While smoking in single family units  does not lead to smoke intrusion to adjacent units, the risk of fire  and the increased unit turnover costs remain.',\n",
       " 'Further, including all  public housing units covered by this proposed rule means that all  tenants will be treated equally and be subject to the same lease  requirements.',\n",
       " 'This prohibition on smoking would cover all types of lit  tobacco products, including but not limited to cigarettes, cigars, and  pipes.',\n",
       " 'While the prohibition does not specifically cover waterpipe  tobacco smoking (referred to as hookahs), such smoking involves lit  charcoal and results in heating tobacco to temperatures high enough to  produce secondhand smoke that contains harmful toxins.\\\\29\\\\ For this  reason, HUD is seeking comment on whether to include a prohibition on  waterpipe tobacco in the final rule.',\n",
       " '---------------------------------------------------------------------------      \\\\29\\\\ See World Health Organization.',\n",
       " 'Advisory note: waterpipe  tobacco smoking: 2nd edition (2015), available at http://www.who.int/tobacco/publications/prod_regulation/waterpipesecondedition/en/.',\n",
       " \"---------------------------------------------------------------------------      The prohibition on the use of lit tobacco products in this proposal  does not include electronic nicotine delivery systems (ENDS), including  electronic cigarettes (``e-cigarettes'').\",\n",
       " 'The absence of a prohibition  on the use of e-cigarettes in this rule should not be read as an  endorsement of e-cigarettes as an acceptable health alternative to  cigarettes.',\n",
       " 'The aerosol from ENDS typically contains nicotine derived  from tobacco plants, and may contain other hazardous and potentially  hazardous constituents such as formaldehyde and lead.\\\\30\\\\ Accidental  ingestion of nicotine liquid used in ENDS can cause acute nicotine  toxicity in children, accounting  [[Page 71766]]  for an increasing proportion of exposure calls to poison control  centers.\\\\31\\\\ ENDS may also present an additional enforcement challenge  for PHAs that are implementing smoke-free policies because the user may  appear to be smoking a conventional cigarette.',\n",
       " 'In light of growing  health concerns regarding exposure to the aerosol of these products  among non-users, especially children and pregnant women, HUD is seeking  additional comments on the issue of ENDS, and may prohibit the use of  these products in public housing in the final rule.',\n",
       " 'HUD encourages PHAs  that already have smoke-free policies to consider whether ENDS should  be included in their smoke-free policies.',\n",
       " '---------------------------------------------------------------------------      \\\\30\\\\ See Offerman, F.J.',\n",
       " 'The hazards of e-cigarettes.',\n",
       " 'June, 2014.',\n",
       " 'ASHRAE Journal.',\n",
       " \"See also National Institute for Occupational Safety  and Health, ``Promoting Health and Preventing Disease and Injury  Through Workplace Tobacco Policies,'' Current Intelligence Bulletin  67 (2015), available at http://www.cdc.gov/niosh/docs/2015-113/pdfs/fy15_cib-67_2015-113_v3.pdf.\",\n",
       " '\\\\31\\\\ CDC.',\n",
       " 'Notes from the field: Calls to Poison Centers for  Exposures to Electronic Cigarettes--United States, September 2010- February 2014.',\n",
       " 'MMWR 2014;63:292-93.',\n",
       " \"---------------------------------------------------------------------------      In proposing this policy, it is important for HUD to clarify that  HUD's proposal does not prohibit individual PHA residents from smoking.\",\n",
       " 'PHAs should continue leasing to persons who smoke.',\n",
       " \"This rule is not  intended to contradict HUD's goals to end homelessness and help all  Americans secure quality housing.\",\n",
       " \"Rather, HUD is proposing a  prohibition on smoking inside public housing living units and indoor  common areas, public housing administrative office buildings, public  housing community rooms or community facilities, public housing day  care centers and laundry rooms, in outdoor areas within 25 feet of the  housing and administrative office buildings, and in other areas  designated by a PHA as smoke-free (collectively, ``restricted areas'').\",\n",
       " 'PHAs will have the discretion to establish outside designated smoking  locations outside of the required 25 feet perimeter, which may include  partially enclosed structures, to accommodate smoking residents, to  establish additional smoke-free areas (such as around a playground),  or, alternatively, to make their entire grounds smoke-free.',\n",
       " 'In  addition, section 504 of the Rehabilitation Act of 1973 and the  Americans with Disabilities Act provides the participant the right to  seek a reasonable accommodation, including requests from residents with  mobility-impairment or mental disability.',\n",
       " 'A request for a reasonable  accommodation from an eligible participant must at least be considered,  and granted in appropriate circumstances.',\n",
       " 'To assist PHAs, HUD will work  with its Office of Fair Housing and Equal Opportunity to develop  guidance on accommodating persons with a disability related to smoke- free policies.',\n",
       " 'The guidance will be informed by comments on the  proposed rule and issued in advance of the final rule.',\n",
       " 'The benefits of this proposed regulatory action may be substantial,  and beneficiaries include both PHAs and residents of public housing.',\n",
       " 'Over 700,000 units would be affected by this rule (including over  500,000 units inhabited by elderly households or households with a non- elderly person with disabilities), and their residents would have the  potential to experience health benefits from a reduction of exposure to  secondhand smoke.',\n",
       " 'There are also over 775,000 children in these units.',\n",
       " 'PHAs will benefit from a reduction of damage and renovation costs  caused by smoking.',\n",
       " 'Both residents and PHAs will gain from reducing  deaths, injuries, and property damage caused by fires.',\n",
       " 'The costs to  PHAs of implementing the smoke-free policy proposed by this rule may  include training, administrative, legal, and enforcement costs.',\n",
       " 'Of  these costs to PHAs, HUD expects that the expense of additional  enforcement efforts may be the highest.',\n",
       " 'The costs of implementing the  smoke-free policy proposed by this rule are minimized by the fact that  HUD guidance already exists on many of the topics covered by the  proposed regulatory changes, and that over 500 PHAs have already  implemented smoke-free policies.',\n",
       " 'Given the existence of this HUD  guidance, initial learning costs associated with implementation of a  smoke-free policy as proposed by this rule may not be significant.',\n",
       " 'There may be costs to residents as a result of eviction,  particularly for persons with disabilities, and especially those with  mobility impairments.',\n",
       " 'HUD recognizes that this rule could adversely  impact those with mobility impairment or particular frailties that  prevent them from smoking in designated areas.',\n",
       " 'As mentioned above, HUD  will develop guidance on reasonable accommodation, and HUD solicits  public comment on how to mitigate these potential adverse impacts.',\n",
       " 'HUD recognizes that PHAs developing smoke-free housing policies may  need technical assistance in writing the policies, engaging residents,  and assisting residents who want to stop smoking.',\n",
       " 'HUD will continue to  provide free webinars and training sessions addressing these and  related topics.',\n",
       " \"PHAs are encouraged to work with their State HUD  office, State and local tobacco prevention and cessation programs,  state and community health organizations, and the Environmental  Protection Agency's community-based asthma program network  (www.asthmacommunitynetwork.org).\",\n",
       " 'CDC provides funding and technical  assistance to State tobacco prevention and control programs and  prevention and smoking cessation programs in every state and the  District of Columbia (see http://www.cdc.gov/tobacco/stateandcommunity/tobacco_control_programs/ntcp/index.htm).',\n",
       " \"Contact information for local  organizations will be provided through HUD's Web site on a page  dedicated to smoke-free resources that is under development.\",\n",
       " 'D. Discussions With Stakeholders      In addition to the October 2012 Federal Register notice soliciting  information on adopting smoke-free policies in HUD subsidized housing,  in March 2015, HUD reached out to organizations representative of the  interests and concerns of PHAs to solicit feedback on moving forward  with smoke-free policies in public housing.',\n",
       " 'The organizations expressed  support for smoke-free policies but also requested that any regulations  requiring smoke-free policies allow sufficient flexibility for PHAs to  tailor such policies to their local conditions.',\n",
       " 'In this rule, HUD has  strived to provide such flexibility.',\n",
       " 'III.',\n",
       " 'This Proposed Rule--Summary of Changes  Applicability (Sec.',\n",
       " '965.651)      As stated above, this proposal would apply to all PHAs of any size  and Moving-to-Work (MTW) agencies, but it would only apply to public  housing, and would not apply to dwelling units in a mixed-finance  project.',\n",
       " 'Public housing is defined as low-income housing, and all  necessary appurtenances (e.g., community facilities, public housing  offices, day care centers and laundry rooms) assisted under the U.S.  Housing Act of 1937 (the 1937 Act), other than assistance under section  8 of the 1937 Act.',\n",
       " 'Requirements (Sec.',\n",
       " '965.653)      In Sec.',\n",
       " \"965.653, HUD provides that a PHA's smoke-free policy must  prohibit all ``lit tobacco products.''\",\n",
       " \"HUD proposes to define ``lit  tobacco products'' as all lit tobacco products that involve the  ignition and burning of tobacco leaves such as cigarettes, cigars, and  pipes.\",\n",
       " \"HUD is proposing to require that PHAs prohibit all lit tobacco  products not only in dwelling units, but also within indoor common  areas and in outdoor areas within 25 feet of the housing and any PHA  administrative office buildings (the ``restricted areas'').\",\n",
       " 'Outside of  these areas, PHAs would be permitted to limit smoking to outdoor  [[Page 71767]]  designated smoking areas, which may include partially enclosed  structures to accommodate residents who smoke, or, alternatively, to  make their entire grounds smoke-free.',\n",
       " \"PHAs that are not making the  entire grounds smoke-free are encouraged to work with their residents  to identify outdoor designated smoking areas that are accessible within  the grounds of the public housing or administrative office buildings,  that are not frequented by children (e.g., not a playground), and that  are situated in a way that minimizes nonsmoking residents' exposure to  secondhand smoke.\",\n",
       " 'While not required, a designated smoking area with  shade and benches may assist residents with compliance.',\n",
       " 'Implementation (Sec.',\n",
       " '965.653)      HUD is proposing to provide PHAs 18 months from the effective date  of the final rule to implement smoke-free public housing, as proposed  by this rule.',\n",
       " \"HUD believes that 18 months will provide PHAs sufficient  time to conduct resident engagement, to hold any public meetings that  are required to amend their PHA plans, and to incorporate the required  new lease provisions during tenants' recertifications or at a date  before the policy is fully effective.\",\n",
       " 'PHAs that already have a smoke- free policy in effect will be required to review their existing  policies for compliance with the requirements of this rule, as  presented in the final rule, and amend their policies as necessary in  the same timeframe of 18 months from the effective date of the final  rule in order to implement smoke-free public housing, consistent with  the requirements of the final rule.',\n",
       " 'In addition, HUD is proposing to require PHAs to amend their PHA  plans to incorporate the smoke-free policy.',\n",
       " 'If the PHA determines the  imposition of a smoke-free policy is a significant amendment to the PHA  plan, the PHA must conduct public meetings in accordance with standard  PHA Plan amendment procedures, and these meetings must be held in  accessible buildings and provided in accessible formats, as necessary,  for persons with disabilities and those who are limited in English  proficiency.',\n",
       " 'HUD would recommend that all PHAs conduct meetings with  residents to fully explain the smoke-free building requirements and to  best determine which outside areas, if any, to designate as smoking  areas and to accommodate the needs of all residents.',\n",
       " 'Lease Provisions (Sec.',\n",
       " '966.4)      HUD believes that the best way to implement smoke-free policies is  to incorporate the prohibition on indoor smoking in the leases each  tenant must sign.',\n",
       " 'This will allow PHAs to use enforcement mechanisms  already in place and provide an additional notification of the policy  to tenants.',\n",
       " 'HUD expects PHAs to follow the PIH administrative grievance  procedures during enforcement of their smoke-free housing policies.',\n",
       " 'Because some tenants may not be recertified before the policy takes  effect, PHAs may require that all remaining leases be amended, or may  establish their own schedule for lease amendments, provided that all  leases are amended by the effective date of the policy.',\n",
       " 'IV.',\n",
       " 'Specific Questions for Comments      While HUD welcomes comments on all aspects of this proposed rule,  HUD is seeking specific comment on the following questions:     1.',\n",
       " 'What barriers that PHAs could encounter in implementing smoke- free housing?',\n",
       " 'What costs could PHAs incur?',\n",
       " 'Are there any specific costs  to enforcing such a policy?',\n",
       " '2.',\n",
       " 'Does this proposed rule adequately address the adverse effects  of smoking and secondhand smoke on PHAs and PHA residents?',\n",
       " '3.',\n",
       " 'Does this proposed rule create burdens, costs, or confer  benefits specific to families, children, persons with disabilities,  owners, or the elderly, particularly if any individual or family is  evicted as a result of this policy?',\n",
       " '4.',\n",
       " \"For those PHAs that have already implemented a smoke-free  policy, what exceptions to the requirements have been granted based on  tenants' requests?\",\n",
       " '5.',\n",
       " 'For those PHAs that have already implemented a smoke-free  policy, what experiences, lessons, or advice would you share based on  your experiences with implementing and enforcing the policy?',\n",
       " '6.',\n",
       " 'For those PHAs that have already implemented a smoke-free  policy, what tobacco cessation services were offered to residents to  assist with the change?',\n",
       " 'Did you establish partnerships with external  groups to provide or refer residents to these services?',\n",
       " '7.',\n",
       " 'Are there specific areas of support that HUD could provide PHAs  that would be particularly helpful in the implementation of the  proposed rule?',\n",
       " '8.',\n",
       " 'Should the policy extend to electronic nicotine delivery  systems, such as e-cigarettes?',\n",
       " '9.',\n",
       " 'Should the policy extend to waterpipe tobacco smoking?',\n",
       " 'Does such  smoking increase the risk of fire or property damage?',\n",
       " \"V. Findings and Certifications  Executive Order 12866, Regulatory Planning and Review      The Office of Management and Budget (OMB) reviewed this proposed  rule under Executive Order 12866 (entitled ``Regulatory Planning and  Review'').\",\n",
       " 'OMB determined that this rule was economically significant  under the order.',\n",
       " 'The docket file is available for public inspection in  the Regulations Division, Office of General Counsel, U.S. Department of  Housing and Urban Development, 451 7th Street SW., Room 10276,  Washington, DC 20410-0500.',\n",
       " \"The initial Regulatory Impact Analysis (RIA)  prepared for this rule is also available for public inspection in the  Regulations Division and may be viewed online at www.regulations.gov,  under the docket number above, or on HUD's Web site at http://portal.hud.gov/hudportal/HUD?src=/program_offices/administration/hudclips/ia/.\",\n",
       " 'Due to security measures at the HUD Headquarters  building, an advance appointment to review the public comments must be  scheduled by calling the Regulations Division at (202) 708-3055 (this  is not a toll-free number).',\n",
       " 'Individuals with speech or hearing  impairments may access this number via TTY by calling the Federal Relay  Service at (800) 877-8339.',\n",
       " 'Information Collection Requirements      The information collection requirements contained in this proposed  rule have been submitted to the Office of Management and Budget (OMB)  under the Paperwork Reduction Act of 1995 (44 U.S.C.',\n",
       " '3501-3520) and  assigned OMB control number 2577-0226.',\n",
       " 'In accordance with the Paperwork  Reduction Act, an agency may not conduct or sponsor, and a person is  not required to respond to, a collection of information, unless the  collection displays a currently valid OMB control number.',\n",
       " 'Unfunded Mandates Reform Act      Title II of the Unfunded Mandates Reform Act of 1995 (UMRA)  establishes requirements for federal agencies to assess the effects of  their regulatory actions on state, local, and tribal governments and  the private sector.',\n",
       " 'This rule will not impose any federal mandates on  any state, local, or tribal governments or the private sector within  the meaning of UMRA.',\n",
       " 'Environmental Review      A Finding of No Significant Impact with respect to the environment  has been made in accordance with HUD regulations in 24 CFR part 50 that  [[Page 71768]]  implement section 102(2)(C) of the National Environmental Policy Act of  1969 (42 U.S.C.',\n",
       " '4332(2)(C)).',\n",
       " 'The Finding is available for public  inspection during regular business hours in the Regulations Division,  Office of General Counsel, Department of Housing and Urban Development,  451 7th Street SW., Room 10276, Washington, DC 20410-0500.',\n",
       " 'Due to  security measures at the HUD Headquarters building, please schedule an  appointment to review the Finding by calling the Regulations Division  at 202-708-3055 (this is not a toll-free number).',\n",
       " 'Individuals with  speech or hearing impairments may access this number via TTY by calling  the Federal Relay Service at 800-877-8339.',\n",
       " 'Impact on Small Entities      The Regulatory Flexibility Act (RFA) (5 U.S.C.',\n",
       " '601 et seq.',\n",
       " '),  generally requires an agency to conduct a regulatory flexibility  analysis of any rule subject to notice and comment rulemaking  requirements unless the agency certifies that the rule will not have a  significant economic impact on a substantial number of small entities.',\n",
       " 'This rule prohibits smoking of tobacco in all indoor areas of and  within 25 feet of any public housing and administrative office  buildings for all PHAs, regardless of size.',\n",
       " \"There are 2334 ``small'' PHAs (defined as PHAs with fewer than 250  units), which make up 75 percent of the public housing stock across the  country.\",\n",
       " 'Of this number, approximately 378 have already instituted a  voluntary full or partial policy on indoor tobacco smoking.',\n",
       " \"HUD anticipates that implementation of the policy will impose  minimal additional costs, as creation of the smoke-free policy only  requires amendment of leases and the PHA plan, both of which may be  done as part of a PHA's normal course of business.\",\n",
       " 'Additionally,  enforcement of the policy will add minimal incremental costs, as PHAs  must already regularly inspect public housing units and enforce lease  provisions.',\n",
       " 'Any costs of this rule are mitigated by the fact that PHAs  have up to 18 months to implement the policy, allowing for costs to be  spread across that time period.',\n",
       " 'While there are significant benefits to the smoke-free policy  requirement, the majority of those benefits accrue to the public  housing residents themselves, not to the PHAs.',\n",
       " \"PHAs will realize  monetary benefits due to reduced unit turnover costs and reduced fire  and fire prevention costs, but these benefits are variable according to  the populations of each PHA and the PHA's existing practices.\",\n",
       " 'Finally, this rule does not impose a disproportionate burden on  small PHAs.',\n",
       " 'The rule does not require a fixed expenditure; rather, all  costs should be proportionate to the size of the PHA implementing and  enforcing the smoke-free policy.',\n",
       " 'Therefore, the undersigned certifies that this rule will not have a  significant impact on a substantial number of small entities.',\n",
       " \"Notwithstanding HUD's view that this rule will not have a  significant effect on a substantial number of small entities, HUD  specifically invites comments regarding any less burdensome  alternatives to this rule that will meet HUD's objectives as described  in the preamble.\",\n",
       " \"Executive Order 13132, Federalism      Executive Order 13132 (entitled ``Federalism'') prohibits an agency  from publishing any rule that has federalism implications if the rule  either imposes substantial direct compliance costs on state and local  governments or is not required by statute, or the rule preempts state  law, unless the agency meets the consultation and funding requirements  of section 6 of the Executive Order.\",\n",
       " 'This final rule does not have  federalism implications and does not impose substantial direct  compliance costs on state and local governments nor preempt state law  within the meaning of the Executive Order.',\n",
       " 'Catalog of Federal Domestic Assistance      The Catalog of Federal Domestic Assistance number for the Public  Housing program is 14.872.',\n",
       " 'List of Subjects  24 CFR Part 965      Government procurement, Grant programs-housing and community  development, Lead poisoning, Loan programs-housing and community  development, Public housing, Reporting and recordkeeping requirements,  Utilities.',\n",
       " '24 CFR Part 966      Grant programs-housing and community development, Public housing,  Reporting and recordkeeping requirements.',\n",
       " 'Accordingly, for the reasons stated in the preamble, HUD proposes  to amend 24 CFR parts 965 and 966 as follows:  PART 965--PHA-OWNED OR LEASED PROJECTS--GENERAL PROVISIONS  0 1.',\n",
       " 'The authority citation for 24 CFR part 965 continues to read as  follows:      Authority:  42 U.S.C.',\n",
       " '1547, 1437a, 1437d, 1437g, and 3535(d).',\n",
       " 'Subpart H is also issued under 42 U.S.C.',\n",
       " '4821-4846.',\n",
       " '0 2.',\n",
       " 'Add subpart G to read as follows: Subpart G--Smoke-Free Public Housing 965.651 Applicability 965.653 Smoke-free public housing 965.655 Implementation  Subpart G--Smoke-Free Public Housing   Sec.',\n",
       " '965.651  Applicability.',\n",
       " 'This subpart applies to public housing units, except for dwelling  units in a mixed-finance project.',\n",
       " 'Public housing is defined as low- income housing, and all necessary appurtenances (e.g., community  facilities, public housing offices, day care centers, and laundry  rooms) thereto, assisted under the U.S. Housing Act of 1937 (the 1937  Act), other than assistance under section 8 of the 1937 Act.',\n",
       " 'Sec.',\n",
       " '965.653  Smoke-free public housing.',\n",
       " '(a) In general.',\n",
       " \"PHAs must design and implement a policy prohibiting  the use of lit tobacco products in all public housing living units and  interior common areas (including but not limited to hallways, rental  and administrative offices, community centers, day care centers,  laundry centers, and similar structures), as well as in outdoor areas  within 25 feet from public housing and administrative office buildings  (collectively, ``restricted areas'') in which public housing is  located.\",\n",
       " '(b) Designated smoking areas.',\n",
       " 'PHAs may limit smoking to designated  smoking areas on the grounds of the public housing or administrative  office buildings, which may include partially enclosed structures, to  accommodate residents who smoke.',\n",
       " 'These areas must be outside of any  restricted areas, as defined in paragraph (a) of this section.',\n",
       " 'Alternatively, PHAs may choose to create additional smoke-free areas  outside the restricted areas or to make their entire grounds smoke- free.',\n",
       " '(c) Lit tobacco products.',\n",
       " 'Lit tobacco products are those that  involve the ignition and burning of tobacco leaves, such cigarettes,  cigars, and pipes.',\n",
       " \"A PHA's smoke-free policy must, at a minimum,  include a prohibition on the use of all lit tobacco products.\",\n",
       " 'Sec.',\n",
       " '965.655  Implementation.',\n",
       " '(a) Amendments.',\n",
       " 'PHAs are required to implement the requirements of  this subpart by amending each of the following:  [[Page 71769]]      (1) All applicable PHA plans, according to the provisions in 24 CFR  part 903.',\n",
       " '(2) Tenant leases, according to the provisions of 24 CFR 966.4.',\n",
       " '(b) Deadline.',\n",
       " 'All PHAs must be in full compliance, with effective  policy amendments, by [INSERT, AT THE FINAL RULE STAGE, THE DATE THAT  IS 540 DAYS AFTER THE EFFECTIVE DATE OF THE FINAL RULE].',\n",
       " 'PART 966--PUBLIC HOUSING LEASE AND GRIEVANCE PROCEDURE  0 3.',\n",
       " 'The authority section for 24 CFR part 966 continues to read as  follows:      Authority:  42 U.S.C.',\n",
       " '1437d and 3535(d).',\n",
       " '0 4.',\n",
       " 'In Sec.',\n",
       " '966.4, revise paragraphs (f) (12) (i) and (ii) to read as  follows:   Sec.',\n",
       " '966.4  Lease Requirements.',\n",
       " \"(f)        (12)        (i) To assure that no tenant, member of the tenant's household, or  guest engages in:     (A) Criminal activity.\",\n",
       " '(1) Any criminal activity that threatens the  health, safety or right to peaceful enjoyment of the premises by other  residents;     (2) Any drug-related criminal activity on or off the premises; or     (B) Civil activity.',\n",
       " 'For any units covered by 24 CFR part 965,  subpart G, any smoking of lit tobacco products in restricted areas, as  defined by 24 CFR 965.653(a), or in other outdoor areas that the PHA  has designated as smoke-free.',\n",
       " \"(ii) To assure that no other person under the tenant's control  engages in:     (A) Criminal activity.\",\n",
       " '(1) Any criminal activity that threatens the  health, safety or right to peaceful enjoyment of the premises by other  residents;     (2) Any drug-related criminal activity on the premises; or     (B) Civil activity.',\n",
       " 'For any units covered by 24 CFR part 965,  subpart G, any smoking of lit tobacco products in restricted areas, as  defined by 24 CFR 965.653(a), or in other outdoor areas that the PHA  has designated as smoke-free.',\n",
       " 'Dated: October 22, 2015.',\n",
       " 'Lourdes Castro Ram[iacute]rez, Principal Deputy Assistant Secretary for Public and Indian Housing.',\n",
       " '[FR Doc.',\n",
       " '2015-29346 Filed 11-16-15; 8:45 am]  BILLING CODE 4210-67-P   ']"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "raw_sents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"In addition to causing multiple  diseases and cancers, tobacco smoking has many other adverse effects on  the body, including inflammation and impairment to the immune  system.        Office of the Surgeon General, ``The Health Consequences of  Smoking50 Years of Progress,'' (2014), available at http://www.surgeongeneral.gov/library/reports/50yearsofprogress/fullreport.pdf.\""
      ]
     },
     "execution_count": 194,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "re.sub(r\"\\[+Page\\s*\\d+\\]+|\\\\+\\d+\\\\+|-+\",'',text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sl = get_sentences(clean_text)\n",
    "s = handle_tables(sl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 0\n",
      "1 0\n",
      "2 0\n",
      "3 0\n",
      "4 -20\n",
      "5 -20\n",
      "6 0\n",
      "7 -20\n",
      "8 0\n",
      "9 0\n",
      "10 0\n",
      "11 0\n",
      "12 -20\n",
      "13 -20\n",
      "14 0\n",
      "15 0\n",
      "16 0\n",
      "17 0\n",
      "18 0\n",
      "19 0\n",
      "20 0\n",
      "21 0\n",
      "22 -20\n",
      "23 0\n",
      "24 0\n",
      "25 0\n",
      "26 0\n",
      "27 0\n",
      "28 0\n",
      "29 0\n",
      "30 0\n",
      "31 0\n",
      "32 0\n",
      "33 0\n",
      "34 0\n",
      "35 0\n",
      "36 0\n",
      "37 0\n",
      "38 0\n",
      "39 0\n",
      "40 0\n",
      "41 0\n",
      "42 0\n",
      "43 0\n",
      "44 -20\n",
      "45 -20\n",
      "46 0\n",
      "47 -20\n",
      "48 -20\n",
      "49 -20\n",
      "50 -20\n",
      "51 -20\n",
      "52 -20\n",
      "53 0\n",
      "54 0\n",
      "55 -20\n",
      "56 -20\n",
      "57 -20\n",
      "58 -20\n",
      "59 0\n",
      "60 -20\n",
      "61 -20\n",
      "62 -20\n",
      "63 -20\n",
      "64 -20\n",
      "65 -20\n",
      "66 -20\n",
      "67 -20\n",
      "68 -20\n",
      "69 0\n",
      "70 0\n",
      "71 -20\n",
      "72 -20\n",
      "73 -20\n",
      "74 -20\n",
      "75 -20\n",
      "76 -20\n",
      "77 0\n",
      "78 0\n",
      "79 -20\n",
      "80 -20\n",
      "81 -20\n",
      "82 0\n",
      "83 -20\n",
      "84 -20\n",
      "85 0\n",
      "86 0\n",
      "87 0\n",
      "88 -20\n",
      "89 0\n",
      "90 -20\n",
      "91 -20\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(s)):\n",
    "    print(str(i)+\" \"+str(s[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0  A discussion of the changes we are making in   this rule follows.\n",
      "1 Research Room Hours in DC Area Facilities        Our research center and Central Research Room in the National   Archives Building and the research rooms at the National Archives at   College Park facility are currently open for research Monday through   Friday from 8:45 a.m. to 5 p.m.; on Tuesday, Thursday and Friday   evenings from 5 p.m. to 9 p.m.; and Saturdays from 8:45 a.m. to 4:45   p.m.\n",
      "2 This interim final rule would eliminate Saturday hours and change   the research room hours to 9 a.m. to 5 p.m. on weekdays, more closely   reflecting NARA official business hours in those facilities.\n",
      "3 The new   research room hours are specified in Sec.\n",
      "4 Sec.\n",
      "5 1253.1(a) and   1253.2(b).\n",
      "6 We are also amending Sec.\n",
      "7 1253.8 since we no longer will   have Saturday hours.\n",
      "8 During the evening and Saturday hours we must provide staff to   supervise the seven research rooms and assist researchers.\n",
      "9 We also   require additional security guard presence and incur additional utility   costs because the buildings are open to the public.\n",
      "10 We determined that   carrying out the necessary reduction in hours by eliminating evening   and Saturday hours would inconvenience the fewest researchers.\n",
      "11 Researchers who conduct research in original archival records in the   evening or on Saturday currently must make a reference request in  person before 3:30 on weekdays to have the records identified and   retrieved from the stack areas for their research use; no records are   retrieved during those extended hours.\n",
      "12 NARA had 96,393 researcher   visits in FY 2005 in our DC area research rooms.\n",
      "13 The following charts   show that at both facilities, significantly fewer researchers used the   research rooms during evening and Saturday hours in FY 2005:                       2005 Evening/Saturday Research Room Usage at the National Archives Building                                                  Number of researchers         Researchers as a percentage of total                                           researchers during the quarter                                                                                  National Archives Building only     Evening research                      Evening research                                         room usage from   Saturday research   room usage from   Saturday research                                        5:30 p.m. forward      room usage     5:30 p.m. forward      room usage                                                                                   (percent)         (percent)    JanMar 2005........................                968                691                 14                 10  AprJun 2005........................              1,061                653                 12                  8  JulySep 2005.......................              1,268                823                 13                  8  OctDec 2005........................                966                594                 15                  9                                                     Total Evening/Saturday Use                    4,263              2,761                 13                  9       during 2005....................                       2005 Evening/Saturday Research Room Usage at the National Archives at College Park                                                  Number of researchers         Researchers as a percentage of total                                           researchers during the quarter                                                                                 National Archives at College Park    Evening research                      Evening research                  only                   room usage from   Saturday research   room usage from   Saturday research                                        5:30 p.m. forward      room usage     5:30 p.m. forward      room usage                                                                                  (percent)          (percent)    JanMar 2005........................              1,992                734                 19                  7  AprJun 2005........................              2,314                798                 17                  6  JulySep 2005.......................              2,434                817                 17                  6  OctDec 2005........................              2,004                602                 17                  5                                             Total Evening/Saturday Use                    8,744              2,951                 17                  6       during 2005....................      Research Room Hours in Regional Archives        Our regulations only list the core MondayFriday hours for NARA's   regional archives (TuesdaySaturday hours for the Southeast Region in   Morrow, GA) in 36 CFR 1253.7, which are not changing.\n",
      "14 Currently, most   of our regional archives research rooms also are open one evening per   week and/or certain Saturdays of the month primarily for microfilm   research.\n",
      "15 Researchers who conduct research in original archival records   in the evening or on Saturday must make a reference request inperson   before 3:30 on weekdays to have the records identified and retrieved   from the stack areas for their research use; no records are retrieved   during those extended hours.\n",
      "16 The extended hours, which are subject to   more frequent modification, are listed on the NARA Web site (http://www.archives.gov  ) and posted in the regional research rooms.\n",
      "17 In some of     the regional archives, we are reducing extended hours beginning October   2, 2006.\n",
      "18 We are making one unrelated change to Sec.\n",
      "19 1253.7 to add the   archival research room at the National Personnel Records Center in St.   Louis, MO.\n",
      "20 Museum Hours at the National Archives Building        This interim final rule also modifies the hours the exhibit areas   in the National Archives Building, Washington, DC, are open to the   public.\n",
      "21 Currently the National Archives Experience (our Washington DC   museum) including the Rotunda for the Charters of Freedom (displaying   the Declaration of Independence, Constitution, and Bill of Rights) is   open to the public as follows:       The day after Labor Day through March 31, 10 a.m. to 5:30   p.m. (closed on December 25);       April 1 through the Friday before Memorial Day, 10 a.m. to   7 p.m.;       Memorial Day weekend through Labor Day, 10 a.m. to 9 p.m.    We are revising Sec.\n",
      "22 1280.62 to make the public hours the day after   Labor Day through March 14, 10 a.m. to 5:30 p.m., and March 15 through   Labor Day, 10 a.m. to 7 p.m.\n",
      "23 The building will be closed on   Thanksgiving and December 25.\n",
      "24 In addition, because transit through the   building and through the line to view the Charters of Freedom cannot   usually be completed in less than a 30 minute visit, we are   establishing a policy of ``last admission'' at least 30 minutes before   close.\n",
      "25 This policy, which is followed by many museums, will allow us to   clear the exhibit areas of visitors at the stated closing time and   ensure that visitors coming into the building have at least some   opportunity to see the Charters of Freedom.\n",
      "26 Finally, because the   renovation of the National Archives Building provides direct street  level access to the building entrance on Constitution Avenue, we are   rewording Sec.\n",
      "27 1280.60 to remove directions for disabled and persons   with strollers to use the Pennsylvania Avenue entrance to the building.\n",
      "28 Although we are reducing the hours exhibit areas in the National   Archives Building, Washington, DC, are open during the summer months   (Memorial Day through Labor Day), the new summer hours are consistent   with those at the National Museum of American History (10 a.m.6:30   p.m.).\n",
      "29 We have found that the last two hours of the day in the summer   have the lowest average attendance, so this action will affect the   fewest visitors on those days.\n",
      "30 By changing to a two season calendar   with the high season beginning March 15, we are increasing the hours   the exhibit areas are open during peak demand (school break, Cherry   Blossom Festival) in late March.\n",
      "31 As with the change in research room   hours, the changes in hours for the exhibit areas will significantly   reduce our security guard expenses.\n",
      "32 As we noted in the SUMMARY paragraph of this preamble, NARA must   carry out multiple missioncritical activities.\n",
      "33 There are many actions   that must be carried out in order to provide access to archival records   in NARA research rooms; to assist researchers via online descriptions   in our Archival Research Catalog or via written reference requests; and   to develop public programs and exhibits.\n",
      "34 When archival records are   transferred to NARA's custody from the creating agency or at the end of   a Presidential Administration, NARA staff must organize these records,   assess and address their condition, carry out declassification review   on them as needed, describe them, and otherwise prepare them for safe   and efficient use by researchers.\n",
      "35 The actions that we are        taking in this interim final rule will assist us in conducting all of   our missioncritical programs.\n",
      "36 The issuance of an ``interimfinal rule'' may be followed under the   ``goodcause'' exemption of 5 U.S.C.\n",
      "37 553(b)(3)(B) as ``impracticable''   or ``contrary to the public interest.''\n",
      "38 In this instance, good cause   exists because NARA must institute these changes at the beginning of   the next fiscal year, which does not leave sufficient time for NARA to   issue a final rule following the 45 day comment period under this   notice.\n",
      "39 This rule is not a significant regulatory action for the purposes   of Executive Order 12866 and has not been reviewed by the Office of   Management and Budget.\n",
      "40 As required by the Regulatory Flexibility Act, I   certify that this rule will not have a significant impact on a   substantial number of small entities because it affects individual   researchers and museum visitors.\n",
      "41 This regulation does not have any   federalism implications.\n",
      "42 This rule is not a major rule as defined in 5   U.S.C.\n",
      "43 Chapter 8, Congressional Review of Agency Rulemaking.\n",
      "44 List of Subjects    36 CFR Part 1253        Archives and records.\n",
      "45 36 CFR Part 1280        Federal buildings and facilities.\n",
      "46 0  For the reasons set forth in the preamble, NARA amends chapter XII of   title 36, Code of Federal Regulations, as follows:    PART 1253LOCATION OF NARA FACILITIES AND HOURS OF USE    0  1.\n",
      "47 The authority citation for part 1253 continues to read as follows:        Authority: 44 U.S.C.\n",
      "48 2104(a).\n",
      "49 0  2.\n",
      "50 Amend Sec.\n",
      "51 1253.1 by revising paragraph (a) to read as follows:      Sec.\n",
      "52 1253.1  National Archives Building.\n",
      "53 (a) The National Archives Building is located at 700 Pennsylvania   Avenue, NW., Washington, DC 20408. Business hours are 8:45 a.m. to 5:15   p.m., Monday through Friday, except Federal holidays when the building   is closed.\n",
      "54 Hours for the Research Center and the Central Research room   are 9 a.m. to 5 p.m., Monday through Friday, except Federal holidays.\n",
      "55 0  2.\n",
      "56 Amend Sec.\n",
      "57 1253.2 by revising paragraph (b) to read as follows:      Sec.\n",
      "58 1253.2  National Archives at College Park.\n",
      "59 (b) Research complex hours are 9 a.m. to 5 p.m., Monday through   Friday, except Federal holidays.\n",
      "60 0  3.\n",
      "61 Amend Sec.\n",
      "62 1253.7 by adding a new paragraph (n) to read as follows:      Sec.\n",
      "63 1253.7  Regional Archives.\n",
      "64 (n) National Personnel Records Center archival research room is   located at 9700 Page Ave., St. Louis, MO 631325100.\n",
      "65 The hours are 10   a.m. to 4 p.m., Tuesday through Friday, except Federal holidays.\n",
      "66 0  4.\n",
      "67 Revise Sec.\n",
      "68 1253.8 to read as follows:      Sec.\n",
      "69 1253.8  Are NARA research room facilities closed on Federal   holidays?\n",
      "70 NARA research room facilities are closed on all Federal holidays.\n",
      "71 PART 1280PUBLIC USE OF NARA FACILITIES    0  5.\n",
      "72 The authority citation for part 1280 continues to read as follows:        Authority: 44 U.S.C.\n",
      "73 2102 notes, 2104(a), 2112(a)(1)(A)(iii),   2903.\n",
      "74 0  6.\n",
      "75 Amend Sec.\n",
      "76 1280.60 by revising paragraph (b) to read as follows:      Sec.\n",
      "77 1280.60  Where do I enter the National Archives Building in   Washington, DC?\n",
      "78 (b) To visit the exhibit areas of the National Archives Building,   including the National Archives Experience and Rotunda, you must enter   through the Constitution Avenue entrance.\n",
      "79 0  7.\n",
      "80 Revise Sec.\n",
      "81 1280.62 to read as follows:      Sec.\n",
      "82 1280.62  When are the exhibit areas in the National Archives   Building open?\n",
      "83 The exhibit areas are open to the public from 10 a.m. until 5:30   p.m. from the day after Labor Day through March 14.\n",
      "84 The exhibit areas   are open from 10 a.m. until 7 p.m. from March 15 through Labor Day.\n",
      "85 Last admission to the exhibit areas of the building will be no later   than 30 minutes before the stated closing hour.\n",
      "86 The Archivist of the   United States reserves the authority to close the exhibit areas to the   public at any time for special events or other purposes.\n",
      "87 The building   is closed on Thanksgiving and December 25.\n",
      "88 Dated: July 18, 2006.\n",
      "89 Allen Weinstein,  Archivist of the United States.\n",
      "90 [FR Doc.\n",
      "91 E611763 Filed 72406; 8:45 am]    BILLING CODE 751501P  \n"
     ]
    }
   ],
   "source": [
    "for i in range(len(sl)):\n",
    "    print(str(i)+\" \"+str(sl[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ss = re.sub(r\"\\s+\",' ',sl[13])\n",
    "dots = sl[13].count('.')\n",
    "numbers = sl[13].count('1') +sent.count('2') +sent.count('3')+ sent.count('4') +sent.count('5')+sent.count('6')+sent.count('7')+sent.count('8')+sent.count('9')+sent.count('10')\n",
    "sent_len = len(sl[13])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "275"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dots + numbers\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3294"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sent_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1820"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sl[13].split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Annual Burden Estimate: This proposal would result in a 20year recordkeeping and reporting burden as follows: Summary of time and costs (20year): The following table sums up the costs and time: Total cost Annual cost Total time Annual time Pilotrelated costs: TriggerInitial $4,221,982 $211,099 82,923.34 4,146.17 Registration............... NonTriggerInitial 191,555,276 9,577,764 3,521,734.67 176,086.73 Registration............... NonTriggerRenewal........ 149,053,511 7,452,676 2,740,341.74 137,017.09 Additional/Replacement...... 9,654,806 482,740 363,509.25 18,175.46 Portals: KTC......................... 10,100,840 505,042 655,048.00 32,752.40 DPE......................... 17,545,925 877,296 233,945.67 11,697.28 FAA Contractor.................. 5,328,284 266,414 N/A N/A Total....................... 387,460,624 19,373,031 7,597,502.66 379,875.13 The agency is soliciting comments to (1) Evaluate whether the proposed information requirement is necessary for the proper performance of the functions of the agency, including whether the information will have practical utility; (2) Evaluate the accuracy of the agency's estimate of the burden; (3) Enhance the quality, utility, and clarity of the information to be collected; and (4) Minimize the burden of collecting information on those who are to respond, including by using appropriate automated, electronic, mechanical, or other technological collection techniques or other forms of information technology.\""
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tagged_tokens = tag_my_text(tokenize_text_sent(clean_text))\n",
    "np_list = noun_phrase_finder(tagged_tokens)\n",
    "keywords = get_top_np(np_list)\n",
    "top_np = get_top_unigrams(np_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "table_scores = handle_tables(sl)\n",
    "sentence_length_scores = get_sentence_lengths(sl)\n",
    "keyphrase_scores = get_keyphrase_scores(clean_text,sl, keywords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-20"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table_scores[13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-100]"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "handle_tables([sl[271]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stepped_sentence_length =[]\n",
    "for each_score in sentence_length_scores:\n",
    "        s = each_score//10\n",
    "        if s>10:\n",
    "            s = 10\n",
    "        stepped_sentence_length.append(s)\n",
    "stepped_sentence_length[13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "40"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "keyphrase_scores[13]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "s1 = \"The fortyfive NAICS codes in which WOSBs are underrepresented are: 2213Water, Sewage and Other systems; 2361Residential Building Construction; 2371Utility System Construction; 2381Foundation, Structure, and Building Exterior Contractors; 2382Building Equipment Contractors; 2383Building Finishing Contractors; 2389Other Specialty Trade Contractors; 3149Other Textile Product Mills; 3159 Apparel Accessories and Other Apparel Manufacturing; 3219Other Wood Product Manufacturing; 3222Converted Paper Product Manufacturing; 3321; Forging and Stamping; 3323Architectural and Structural Metals Manufacturing; 3324Boiler, Tank, and Shipping Container Manufacturing; 3333Commercial and Service Industry Machinery Manufacturing; 3342Communications Equipment Manufacturing; 3345 Navigational, Measuring, Electromedical, and Control Instruments Manufacturing; 3346Manufacturing and Reproducing Magnetic and Optical Media; 3353Electrical Equipment Manufacturing; 3359Other Electrical Equipment and Component Manufacturing; 3369Other Transportation Equipment Manufacturing; 4842Specialized Freight Trucking; 4881 Support Activities for Air Transportation; 4884Support Activities for Road Transportation; 4885Freight Transportation Arrangement; 5121 Motion Picture and Video Industries; 5311Lessors of Real Estate; 5413Architectural, Engineering, and Related Services; 5414 Specialized Design Services; 5415Computer Systems Design and Related Services; 5416 Management, Scientific, and Technical Consulting Services; 5419Other Professional, Scientific, and Technical Services; 5611Office Administrative Services; 5612Facilities Support Services; 5614 Business Support Services; 5616Investigation and Security Services; 5617Services to Buildings and Dwellings; 6116Other Schools and Instruction; 6214Outpatient Care Centers; 6219Other Ambulatory Health Care Services; 7115Independent Artists, Writers, and Performers; 7223Special Food Services; 8111Automotive Repair and Maintenance; 8113Commercial and Industrial Machinery and Equipment (except Automotive and Electronic) Repair and Maintenance; and 8114 Personal and Household Goods Repair and Maintenance.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "228"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(s1.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}