beta regression topic relationships.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "── Attaching packages ─────────────────────────────────────── tidyverse 1.2.1 ──\n",
      "✔ ggplot2 3.1.0     ✔ purrr   0.2.5\n",
      "✔ tibble  2.0.1     ✔ dplyr   0.7.8\n",
      "✔ tidyr   0.8.2     ✔ stringr 1.3.1\n",
      "✔ readr   1.3.1     ✔ forcats 0.3.0\n",
      "Warning message:\n",
      "“package ‘tibble’ was built under R version 3.5.2”── Conflicts ────────────────────────────────────────── tidyverse_conflicts() ──\n",
      "✖ dplyr::filter() masks stats::filter()\n",
      "✖ dplyr::lag()    masks stats::lag()\n",
      "\n",
      "Attaching package: ‘magrittr’\n",
      "\n",
      "The following object is masked from ‘package:purrr’:\n",
      "\n",
      "    set_names\n",
      "\n",
      "The following object is masked from ‘package:tidyr’:\n",
      "\n",
      "    extract\n",
      "\n"
     ]
    }
   ],
   "source": [
    "library(tidyverse)\n",
    "library(magrittr)\n",
    "library(betareg)\n",
    "library(broom)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'all_arts'</li>\n",
       "\t<li>'all_levels'</li>\n",
       "\t<li>'all_yrs'</li>\n",
       "\t<li>'betaregs'</li>\n",
       "\t<li>'df'</li>\n",
       "\t<li>'idx'</li>\n",
       "\t<li>'sig_contrasts.demos_no_grp'</li>\n",
       "\t<li>'thetas'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'all\\_arts'\n",
       "\\item 'all\\_levels'\n",
       "\\item 'all\\_yrs'\n",
       "\\item 'betaregs'\n",
       "\\item 'df'\n",
       "\\item 'idx'\n",
       "\\item 'sig\\_contrasts.demos\\_no\\_grp'\n",
       "\\item 'thetas'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'all_arts'\n",
       "2. 'all_levels'\n",
       "3. 'all_yrs'\n",
       "4. 'betaregs'\n",
       "5. 'df'\n",
       "6. 'idx'\n",
       "7. 'sig_contrasts.demos_no_grp'\n",
       "8. 'thetas'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "[1] \"all_arts\"                   \"all_levels\"                \n",
       "[3] \"all_yrs\"                    \"betaregs\"                  \n",
       "[5] \"df\"                         \"idx\"                       \n",
       "[7] \"sig_contrasts.demos_no_grp\" \"thetas\"                    "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "load('../data/artsengagement.Rda')\n",
    "load('../data/open_response_data/labeled_topic_prevs.Rda')\n",
    "source('../scripts/select_helpers.R')\n",
    "ls()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Arts Identity Merging\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'sr_identity_musician'</li>\n",
       "\t<li>'sr_identity_visart'</li>\n",
       "\t<li>'sr_identity_dancer'</li>\n",
       "\t<li>'sr_identity_cwriter'</li>\n",
       "\t<li>'sr_identity_film'</li>\n",
       "\t<li>'sr_identity_gdesigner'</li>\n",
       "\t<li>'sr_identity_architect'</li>\n",
       "\t<li>'sr_identity_actor'</li>\n",
       "\t<li>'sr_identity_designer'</li>\n",
       "\t<li>'sr_identity_artsorg1'</li>\n",
       "\t<li>'sr_identity_artsorg2'</li>\n",
       "\t<li>'sr_identity_communityarts'</li>\n",
       "\t<li>'sr_identity_studyarts'</li>\n",
       "\t<li>'sr_identity_artsoutsidemajor'</li>\n",
       "\t<li>'sr_identity_buy'</li>\n",
       "\t<li>'sr_identity_attendyearly'</li>\n",
       "\t<li>'sr_identity_attendmonthly'</li>\n",
       "\t<li>'sr_identity_attendweekly'</li>\n",
       "\t<li>'sr_identity_makemusic'</li>\n",
       "\t<li>'sr_identity_makevisart'</li>\n",
       "\t<li>'sr_identity_dance'</li>\n",
       "\t<li>'sr_identity_cwrite'</li>\n",
       "\t<li>'sr_identity_makefilms'</li>\n",
       "\t<li>'sr_identity_design'</li>\n",
       "\t<li>'sr_identity_construct'</li>\n",
       "\t<li>'sr_identity_perform'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'sr\\_identity\\_musician'\n",
       "\\item 'sr\\_identity\\_visart'\n",
       "\\item 'sr\\_identity\\_dancer'\n",
       "\\item 'sr\\_identity\\_cwriter'\n",
       "\\item 'sr\\_identity\\_film'\n",
       "\\item 'sr\\_identity\\_gdesigner'\n",
       "\\item 'sr\\_identity\\_architect'\n",
       "\\item 'sr\\_identity\\_actor'\n",
       "\\item 'sr\\_identity\\_designer'\n",
       "\\item 'sr\\_identity\\_artsorg1'\n",
       "\\item 'sr\\_identity\\_artsorg2'\n",
       "\\item 'sr\\_identity\\_communityarts'\n",
       "\\item 'sr\\_identity\\_studyarts'\n",
       "\\item 'sr\\_identity\\_artsoutsidemajor'\n",
       "\\item 'sr\\_identity\\_buy'\n",
       "\\item 'sr\\_identity\\_attendyearly'\n",
       "\\item 'sr\\_identity\\_attendmonthly'\n",
       "\\item 'sr\\_identity\\_attendweekly'\n",
       "\\item 'sr\\_identity\\_makemusic'\n",
       "\\item 'sr\\_identity\\_makevisart'\n",
       "\\item 'sr\\_identity\\_dance'\n",
       "\\item 'sr\\_identity\\_cwrite'\n",
       "\\item 'sr\\_identity\\_makefilms'\n",
       "\\item 'sr\\_identity\\_design'\n",
       "\\item 'sr\\_identity\\_construct'\n",
       "\\item 'sr\\_identity\\_perform'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'sr_identity_musician'\n",
       "2. 'sr_identity_visart'\n",
       "3. 'sr_identity_dancer'\n",
       "4. 'sr_identity_cwriter'\n",
       "5. 'sr_identity_film'\n",
       "6. 'sr_identity_gdesigner'\n",
       "7. 'sr_identity_architect'\n",
       "8. 'sr_identity_actor'\n",
       "9. 'sr_identity_designer'\n",
       "10. 'sr_identity_artsorg1'\n",
       "11. 'sr_identity_artsorg2'\n",
       "12. 'sr_identity_communityarts'\n",
       "13. 'sr_identity_studyarts'\n",
       "14. 'sr_identity_artsoutsidemajor'\n",
       "15. 'sr_identity_buy'\n",
       "16. 'sr_identity_attendyearly'\n",
       "17. 'sr_identity_attendmonthly'\n",
       "18. 'sr_identity_attendweekly'\n",
       "19. 'sr_identity_makemusic'\n",
       "20. 'sr_identity_makevisart'\n",
       "21. 'sr_identity_dance'\n",
       "22. 'sr_identity_cwrite'\n",
       "23. 'sr_identity_makefilms'\n",
       "24. 'sr_identity_design'\n",
       "25. 'sr_identity_construct'\n",
       "26. 'sr_identity_perform'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       " [1] \"sr_identity_musician\"         \"sr_identity_visart\"          \n",
       " [3] \"sr_identity_dancer\"           \"sr_identity_cwriter\"         \n",
       " [5] \"sr_identity_film\"             \"sr_identity_gdesigner\"       \n",
       " [7] \"sr_identity_architect\"        \"sr_identity_actor\"           \n",
       " [9] \"sr_identity_designer\"         \"sr_identity_artsorg1\"        \n",
       "[11] \"sr_identity_artsorg2\"         \"sr_identity_communityarts\"   \n",
       "[13] \"sr_identity_studyarts\"        \"sr_identity_artsoutsidemajor\"\n",
       "[15] \"sr_identity_buy\"              \"sr_identity_attendyearly\"    \n",
       "[17] \"sr_identity_attendmonthly\"    \"sr_identity_attendweekly\"    \n",
       "[19] \"sr_identity_makemusic\"        \"sr_identity_makevisart\"      \n",
       "[21] \"sr_identity_dance\"            \"sr_identity_cwrite\"          \n",
       "[23] \"sr_identity_makefilms\"        \"sr_identity_design\"          \n",
       "[25] \"sr_identity_construct\"        \"sr_identity_perform\"         "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df %>% select(contains('sr_identity')) %>% names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "identity <- df %>% select('sr_identity_musician','sr_identity_visart','sr_identity_dancer','sr_identity_cwriter',\n",
    "              'sr_identity_film','sr_identity_gdesigner','sr_identity_architect','sr_identity_actor',\n",
    "              'sr_identity_designer','sr_identity_studyarts', 'sr_identity_makemusic','sr_identity_makevisart',\n",
    "              'sr_identity_dance','sr_identity_cwrite','sr_identity_makefilms','sr_identity_perform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "identity_agg <- c()\n",
    "for(i in seq(nrow(identity))) {\n",
    "    artist <- NA\n",
    "    for(j in seq(identity)) {\n",
    "        if (!is.na(identity[i,j]) & identity[i,j] == \"Yes\") {\n",
    "            artist <- 1\n",
    "            break\n",
    "        }\n",
    "        if (!is.na(identity[i,j]) & identity[i,j] == \"No\") {\n",
    "            artist <- 2\n",
    "        }\n",
    "    }\n",
    "    identity_agg %<>% c(artist)\n",
    "}\n",
    "\n",
    "df$sr_identity_artist <- factor(identity_agg, labels=c(\"Yes\",\"No\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "motiv_demos <- df %>% select(contains('sr_motivation')) %>% names\n",
    "identity_demos <- df %>% select(contains('sr_identity')) %>% names\n",
    "participation_demos <- df %>% select(all_arts('participation')) %>% select(starts_with('sr')) %>% names\n",
    "real_demos <- c('ethnic_group', 'sex', 'school', 'parented', 'income', 'hstype', 'hssize',\n",
    "           'hslocation', 'hs_arts_freq', 'hs_encouragement', 'hs_required', 'hs_fees',\n",
    "           'so_childhood1', 'so_childhood3', 'artsincollege', 'so_childhood5', 'sr_participated',\n",
    "               'sr_highestdegreeplanned')\n",
    "demo_groups <- c('real_demos','motiv_demos','identity_demos','participation_demos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "demo_groups <- list(real_demos, motiv_demos, identity_demos, participation_demos)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Replace 0 values with small nonzero value bc these categories were manually created"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Played a Role</th><th scope=col>Didn't Play a Role</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>0.07166238</td><td>0.00000000</td></tr>\n",
       "\t<tr><td>0.07834615</td><td>0.00000000</td></tr>\n",
       "\t<tr><td>0.04303464</td><td>0.00000000</td></tr>\n",
       "\t<tr><td>0.00000000</td><td>0.07846073</td></tr>\n",
       "\t<tr><td>0.12374583</td><td>0.00000000</td></tr>\n",
       "\t<tr><td>0.07635667</td><td>0.00000000</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ll}\n",
       " Played a Role & Didn't Play a Role\\\\\n",
       "\\hline\n",
       "\t 0.07166238 & 0.00000000\\\\\n",
       "\t 0.07834615 & 0.00000000\\\\\n",
       "\t 0.04303464 & 0.00000000\\\\\n",
       "\t 0.00000000 & 0.07846073\\\\\n",
       "\t 0.12374583 & 0.00000000\\\\\n",
       "\t 0.07635667 & 0.00000000\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "| Played a Role | Didn't Play a Role |\n",
       "|---|---|\n",
       "| 0.07166238 | 0.00000000 |\n",
       "| 0.07834615 | 0.00000000 |\n",
       "| 0.04303464 | 0.00000000 |\n",
       "| 0.00000000 | 0.07846073 |\n",
       "| 0.12374583 | 0.00000000 |\n",
       "| 0.07635667 | 0.00000000 |\n",
       "\n"
      ],
      "text/plain": [
       "  Played a Role Didn't Play a Role\n",
       "1 0.07166238    0.00000000        \n",
       "2 0.07834615    0.00000000        \n",
       "3 0.04303464    0.00000000        \n",
       "4 0.00000000    0.07846073        \n",
       "5 0.12374583    0.00000000        \n",
       "6 0.07635667    0.00000000        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "thetas[[9]] %>% select(contains('Role')) %>% head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "zeroval <- 0.0001\n",
    "for(i in seq(nrow(thetas[[9]]))) {\n",
    "    if (thetas[[9]][i,2] == 0) {\n",
    "        thetas[[9]][i,2] = zeroval\n",
    "        thetas[[9]][i,11] = thetas[[9]][i,11] - zeroval\n",
    "    }\n",
    "    if (thetas[[9]][i,11] == 0) {\n",
    "        thetas[[9]][i,11] = zeroval\n",
    "        thetas[[9]][i,2] = thetas[[9]][i,2] - zeroval\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>Played a Role</th><th scope=col>Didn't Play a Role</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>0.07156238</td><td>0.00010000</td></tr>\n",
       "\t<tr><td>0.07824615</td><td>0.00010000</td></tr>\n",
       "\t<tr><td>0.04293464</td><td>0.00010000</td></tr>\n",
       "\t<tr><td>0.00010000</td><td>0.07836073</td></tr>\n",
       "\t<tr><td>0.12364583</td><td>0.00010000</td></tr>\n",
       "\t<tr><td>0.07625667</td><td>0.00010000</td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|ll}\n",
       " Played a Role & Didn't Play a Role\\\\\n",
       "\\hline\n",
       "\t 0.07156238 & 0.00010000\\\\\n",
       "\t 0.07824615 & 0.00010000\\\\\n",
       "\t 0.04293464 & 0.00010000\\\\\n",
       "\t 0.00010000 & 0.07836073\\\\\n",
       "\t 0.12364583 & 0.00010000\\\\\n",
       "\t 0.07625667 & 0.00010000\\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "| Played a Role | Didn't Play a Role |\n",
       "|---|---|\n",
       "| 0.07156238 | 0.00010000 |\n",
       "| 0.07824615 | 0.00010000 |\n",
       "| 0.04293464 | 0.00010000 |\n",
       "| 0.00010000 | 0.07836073 |\n",
       "| 0.12364583 | 0.00010000 |\n",
       "| 0.07625667 | 0.00010000 |\n",
       "\n"
      ],
      "text/plain": [
       "  Played a Role Didn't Play a Role\n",
       "1 0.07156238    0.00010000        \n",
       "2 0.07824615    0.00010000        \n",
       "3 0.04293464    0.00010000        \n",
       "4 0.00010000    0.07836073        \n",
       "5 0.12364583    0.00010000        \n",
       "6 0.07625667    0.00010000        "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "thetas[[9]] %>% select(contains('Role')) %>% head"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "----\n",
    "----\n",
    "----\n",
    "## Grouped Demos"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'barriers'</li>\n",
       "\t<li>'change'</li>\n",
       "\t<li>'definition'</li>\n",
       "\t<li>'behavior2'</li>\n",
       "\t<li>'childhood2'</li>\n",
       "\t<li>'feel'</li>\n",
       "\t<li>'sr_aftercollege4'</li>\n",
       "\t<li>'sr_career'</li>\n",
       "\t<li>'sr_development'</li>\n",
       "\t<li>'sr_othergrowth'</li>\n",
       "\t<li>'sr_role'</li>\n",
       "\t<li>'sr_society'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'barriers'\n",
       "\\item 'change'\n",
       "\\item 'definition'\n",
       "\\item 'behavior2'\n",
       "\\item 'childhood2'\n",
       "\\item 'feel'\n",
       "\\item 'sr\\_aftercollege4'\n",
       "\\item 'sr\\_career'\n",
       "\\item 'sr\\_development'\n",
       "\\item 'sr\\_othergrowth'\n",
       "\\item 'sr\\_role'\n",
       "\\item 'sr\\_society'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'barriers'\n",
       "2. 'change'\n",
       "3. 'definition'\n",
       "4. 'behavior2'\n",
       "5. 'childhood2'\n",
       "6. 'feel'\n",
       "7. 'sr_aftercollege4'\n",
       "8. 'sr_career'\n",
       "9. 'sr_development'\n",
       "10. 'sr_othergrowth'\n",
       "11. 'sr_role'\n",
       "12. 'sr_society'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       " [1] \"barriers\"         \"change\"           \"definition\"       \"behavior2\"       \n",
       " [5] \"childhood2\"       \"feel\"             \"sr_aftercollege4\" \"sr_career\"       \n",
       " [9] \"sr_development\"   \"sr_othergrowth\"   \"sr_role\"          \"sr_society\"      "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "thetas %>% names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "sig_contrasts.demos <- c()\n",
    "for(k in c(4,8)) {\n",
    "    for(d in seq(demo_groups)) {\n",
    "        question_name <- names(thetas)[k]\n",
    "        merged <- merge(thetas[[question_name]], select(df, key, demo_groups[[d]]), by = 'key')\n",
    "        merged <- merged[-1]\n",
    "        for(l in seq(demo_groups[[d]])) {\n",
    "            ## NA the lonely eggs\n",
    "            merged[[demo_groups[[d]][l]]][merged[[demo_groups[[d]][l]]] %in% \n",
    "                             (merged[[demo_groups[[d]][l]]] %>% table %>% tidy %>% filter(n < 25) %>% pull(1))] <- NA\n",
    "            merged[[demo_groups[[d]][l]]] %<>% droplevels\n",
    "            if (nlevels(merged[[demo_groups[[d]][l]]]) < 2)\n",
    "                merged %<>% select(-matches(demo_groups[[d]][l]))\n",
    "        }\n",
    "        # now we only have good demos, and all topics\n",
    "        topics <- merged[1:(ncol(thetas[[question_name]])-1)]\n",
    "        demo <- merged[ncol(thetas[[question_name]]):ncol(merged)]\n",
    "\n",
    "        for(i in seq(topics)) {\n",
    "            temp <- bind_cols(topics[i], demo)\n",
    "\n",
    "            names(temp) <- c('topic', names(demo))\n",
    "\n",
    "            beta.out <- betareg(topic ~ ., temp)\n",
    "            df.residual <- beta.out$df.residual\n",
    "            df.null <- beta.out$df.null\n",
    "            r.squared <- beta.out$pseudo.r.squared\n",
    "\n",
    "            options(warn=-1)\n",
    "            beta.out %<>% tidy %>% filter(component == 'mean') %>% \n",
    "                                        select(-component) %>% filter(term != '(Intercept)')\n",
    "            options(warn=0)\n",
    "\n",
    "            beta.out$p.value %<>% p.adjust(method='holm')\n",
    "\n",
    "            beta.out %<>% filter(p.value < 0.05) %>% mutate(topic = names(topics[i]), \n",
    "                                                            question = question_name,\n",
    "                                                            residual_df = df.residual,\n",
    "                                                            null_df = df.null,\n",
    "                                                            r_sq = r.squared)\n",
    "\n",
    "            if(nrow(beta.out) > 0) {\n",
    "                options(warn=-1)\n",
    "                sig_contrasts.demos %<>% bind_rows(beta.out)\n",
    "                options(warn=0)\n",
    "            }\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_list <- gregexpr(\"[A-Z]|[0-9]\", sig_contrasts.demos$term)\n",
    "demo_list <- c()\n",
    "level_list <- c()\n",
    "for(i in seq(sig_contrasts.demos$term)) {\n",
    "    demo_list %<>% c(substr(sig_contrasts.demos$term[i], 1, split_list[[i]] - 1))\n",
    "    level_list %<>% c(substr(sig_contrasts.demos$term[i], split_list[[i]],\n",
    "                             nchar(sig_contrasts.demos$term[i])))\n",
    "}\n",
    "sig_contrasts.demos$demo <- demo_list\n",
    "sig_contrasts.demos$level <- level_list\n",
    "sig_contrasts.demos$term <- NULL\n",
    "\n",
    "empty_list <- sig_contrasts.demos %>% filter(demo == '')\n",
    "split_list <- gregexpr(\"[0-9]\", empty_list$level)\n",
    "demo_list <- c()\n",
    "level_list <- c()\n",
    "for(i in seq(empty_list$level)) {\n",
    "    demo_list %<>% c(substr(empty_list$level[i], 1, split_list[[i]] - 1))\n",
    "    level_list %<>% c(substr(empty_list$level[i], split_list[[i]],\n",
    "                             nchar(empty_list$level[i])))\n",
    "}\n",
    "empty_list$demo <- demo_list\n",
    "empty_list$level <- level_list\n",
    "sig_contrasts.demos %<>% bind_rows(empty_list)\n",
    "sig_contrasts.demos %<>% filter(demo != '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "82"
      ],
      "text/latex": [
       "82"
      ],
      "text/markdown": [
       "82"
      ],
      "text/plain": [
       "[1] 82"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>estimate</th><th scope=col>std.error</th><th scope=col>statistic</th><th scope=col>p.value</th><th scope=col>topic</th><th scope=col>question</th><th scope=col>residual_df</th><th scope=col>null_df</th><th scope=col>r_sq</th><th scope=col>demo</th><th scope=col>level</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td> 0.9933077                                                                         </td><td>0.1917906                                                                          </td><td> 5.179128                                                                          </td><td>6.687779e-06                                                                       </td><td><span style=white-space:pre-wrap>More Confident &amp; Social                </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4515716                                                                          </td><td><span style=white-space:pre-wrap>ethnic_group           </span>                    </td><td><span style=white-space:pre-wrap>Asian                          </span>            </td></tr>\n",
       "\t<tr><td>-0.8221978                                                                         </td><td>0.1416449                                                                          </td><td>-5.804642                                                                          </td><td>2.064112e-07                                                                       </td><td><span style=white-space:pre-wrap>More Confident &amp; Social                </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4515716                                                                          </td><td><span style=white-space:pre-wrap>sex                    </span>                    </td><td><span style=white-space:pre-wrap>Female                         </span>            </td></tr>\n",
       "\t<tr><td>-0.9405522                                                                         </td><td>0.2549704                                                                          </td><td>-3.688869                                                                          </td><td>6.081847e-03                                                                       </td><td><span style=white-space:pre-wrap>More Confident &amp; Social                </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4515716                                                                          </td><td><span style=white-space:pre-wrap>school                 </span>                    </td><td><span style=white-space:pre-wrap>Ross School of Business        </span>            </td></tr>\n",
       "\t<tr><td>-1.7024398                                                                         </td><td>0.4044585                                                                          </td><td>-4.209183                                                                          </td><td>7.432575e-04                                                                       </td><td><span style=white-space:pre-wrap>More Confident &amp; Social                </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4515716                                                                          </td><td><span style=white-space:pre-wrap>hssize                 </span>                    </td><td><span style=white-space:pre-wrap>More than 3001                 </span>            </td></tr>\n",
       "\t<tr><td> 1.3042886                                                                         </td><td>0.3311390                                                                          </td><td> 3.938795                                                                          </td><td>2.292976e-03                                                                       </td><td><span style=white-space:pre-wrap>More Confident &amp; Social                </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4515716                                                                          </td><td><span style=white-space:pre-wrap>hslocation             </span>                    </td><td><span style=white-space:pre-wrap>Rural                          </span>            </td></tr>\n",
       "\t<tr><td>-0.7694124                                                                         </td><td>0.1454879                                                                          </td><td>-5.288497                                                                          </td><td>3.823086e-06                                                                       </td><td><span style=white-space:pre-wrap>More Confident &amp; Social                </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4515716                                                                          </td><td><span style=white-space:pre-wrap>hs_arts_freq           </span>                    </td><td><span style=white-space:pre-wrap>Frequently                     </span>            </td></tr>\n",
       "\t<tr><td> 0.5763670                                                                         </td><td>0.1837866                                                                          </td><td> 3.136067                                                                          </td><td>4.451987e-02                                                                       </td><td><span style=white-space:pre-wrap>More Confident &amp; Social                </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4515716                                                                          </td><td><span style=white-space:pre-wrap>hs_encouragement       </span>                    </td><td><span style=white-space:pre-wrap>Yes                            </span>            </td></tr>\n",
       "\t<tr><td> 0.4439039                                                                         </td><td>0.1422351                                                                          </td><td> 3.120915                                                                          </td><td>4.507247e-02                                                                       </td><td><span style=white-space:pre-wrap>More Confident &amp; Social                </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4515716                                                                          </td><td><span style=white-space:pre-wrap>hs_fees                </span>                    </td><td><span style=white-space:pre-wrap>Yes                            </span>            </td></tr>\n",
       "\t<tr><td>-1.0383347                                                             </td><td>0.2781618                                                              </td><td>-3.732845                                                              </td><td>5.869201e-03                                                           </td><td>Deeper Understanding of Art &amp; The World                            </td><td>behavior2                                                              </td><td>30                                                                     </td><td>62                                                                     </td><td>0.4455261                                                              </td><td><span style=white-space:pre-wrap>school                 </span>        </td><td><span style=white-space:pre-wrap>Ross School of Business        </span></td></tr>\n",
       "\t<tr><td> 0.9149535                                                             </td><td>0.1912546                                                              </td><td> 4.783955                                                              </td><td>5.500143e-05                                                           </td><td>Deeper Understanding of Art &amp; The World                            </td><td>behavior2                                                              </td><td>30                                                                     </td><td>62                                                                     </td><td>0.4455261                                                              </td><td>sr_highestdegreeplanned                                                </td><td><span style=white-space:pre-wrap>Master's degree                </span></td></tr>\n",
       "\t<tr><td> 1.0567834                             </td><td>0.3313881                              </td><td> 3.188960                              </td><td>3.712429e-02                           </td><td>More Critical Thinking                 </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.7115845                              </td><td>ethnic_group                           </td><td>Hispanic                               </td></tr>\n",
       "\t<tr><td> 0.5073364                             </td><td>0.1539249                              </td><td> 3.295999                              </td><td>2.647950e-02                           </td><td>More Critical Thinking                 </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.7115845                              </td><td>school                                 </td><td>Engineering                            </td></tr>\n",
       "\t<tr><td> 0.7546784                             </td><td>0.2089958                              </td><td> 3.610975                              </td><td>8.846397e-03                           </td><td>More Critical Thinking                 </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.7115845                              </td><td>parented                               </td><td>Master's                               </td></tr>\n",
       "\t<tr><td>-0.7369082                             </td><td>0.1770650                              </td><td>-4.161795                              </td><td>9.472684e-04                           </td><td>More Critical Thinking                 </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.7115845                              </td><td>income                                 </td><td>More than $150,000                     </td></tr>\n",
       "\t<tr><td> 0.8007149                             </td><td>0.2399230                              </td><td> 3.337383                              </td><td>2.367999e-02                           </td><td>More Critical Thinking                 </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.7115845                              </td><td>hslocation                             </td><td>Rural                                  </td></tr>\n",
       "\t<tr><td>-0.4789299                             </td><td>0.1084851                              </td><td>-4.414705                              </td><td>3.135581e-04                           </td><td>More Critical Thinking                 </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.7115845                              </td><td>hs_arts_freq                           </td><td>Frequently                             </td></tr>\n",
       "\t<tr><td> 0.5912219                             </td><td>0.1022762                              </td><td> 5.780640                              </td><td>2.381338e-07                           </td><td>More Critical Thinking                 </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.7115845                              </td><td>hs_fees                                </td><td>Yes                                    </td></tr>\n",
       "\t<tr><td> 0.3782069                             </td><td>0.1189064                              </td><td> 3.180710                              </td><td>3.712429e-02                           </td><td>More Critical Thinking                 </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.7115845                              </td><td>sr_participated                        </td><td>Yes                                    </td></tr>\n",
       "\t<tr><td> 0.8000761                             </td><td>0.1800304                              </td><td> 4.444117                              </td><td>2.735863e-04                           </td><td>Attending More Events                  </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.5050804                              </td><td>ethnic_group                           </td><td>Asian                                  </td></tr>\n",
       "\t<tr><td>-0.8180881                             </td><td>0.2122767                              </td><td>-3.853877                              </td><td>3.487865e-03                           </td><td>Attending More Events                  </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.5050804                              </td><td>school                                 </td><td>Engineering                            </td></tr>\n",
       "\t<tr><td> 0.7678724                             </td><td>0.2258567                              </td><td> 3.399820                              </td><td>1.955474e-02                           </td><td>Attending More Events                  </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.5050804                              </td><td>hstype                                 </td><td>Private religious                      </td></tr>\n",
       "\t<tr><td> 1.0608601                             </td><td>0.3134339                              </td><td> 3.384638                              </td><td>1.995623e-02                           </td><td>Attending More Events                  </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.5050804                              </td><td>hslocation                             </td><td>Suburban                               </td></tr>\n",
       "\t<tr><td> 1.6145819                             </td><td>0.3151517                              </td><td> 5.123189                              </td><td>9.613154e-06                           </td><td>Attending More Events                  </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.5050804                              </td><td>hslocation                             </td><td>Rural                                  </td></tr>\n",
       "\t<tr><td> 0.4632224                             </td><td>0.1431172                              </td><td> 3.236665                              </td><td>3.265247e-02                           </td><td>Attending More Events                  </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.5050804                              </td><td>sr_highestdegreeplanned                </td><td>Doctoral or professional degree        </td></tr>\n",
       "\t<tr><td> 1.7307381                                                                     </td><td>0.3704740                                                                      </td><td> 4.671686                                                                      </td><td>9.559614e-05                                                                   </td><td><span style=white-space:pre-wrap>More Aware                             </span></td><td>behavior2                                                                      </td><td>30                                                                             </td><td>62                                                                             </td><td>0.6253664                                                                      </td><td><span style=white-space:pre-wrap>school                 </span>                </td><td><span style=white-space:pre-wrap>Stamps School of Art &amp; Design  </span>    </td></tr>\n",
       "\t<tr><td> 0.4328560                                                                         </td><td>0.1228578                                                                          </td><td> 3.523227                                                                          </td><td>1.236346e-02                                                                       </td><td><span style=white-space:pre-wrap>More Appreciative of Arts &amp; Others     </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4716291                                                                          </td><td><span style=white-space:pre-wrap>hs_required            </span>                    </td><td><span style=white-space:pre-wrap>Yes                            </span>            </td></tr>\n",
       "\t<tr><td>-0.5402647                                                                         </td><td>0.1194513                                                                          </td><td>-4.522888                                                                          </td><td>1.952046e-04                                                                       </td><td><span style=white-space:pre-wrap>More Appreciative of Arts &amp; Others     </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4716291                                                                          </td><td><span style=white-space:pre-wrap>hs_fees                </span>                    </td><td><span style=white-space:pre-wrap>Yes                            </span>            </td></tr>\n",
       "\t<tr><td> 0.4366903                                                                         </td><td>0.1168151                                                                          </td><td> 3.738305                                                                          </td><td>5.557954e-03                                                                       </td><td><span style=white-space:pre-wrap>More Appreciative of Arts &amp; Others     </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4716291                                                                          </td><td><span style=white-space:pre-wrap>so_childhood           </span>                    </td><td><span style=white-space:pre-wrap>5No                            </span>            </td></tr>\n",
       "\t<tr><td> 0.5604396                                                                         </td><td>0.1429007                                                                          </td><td> 3.921881                                                                          </td><td>2.723667e-03                                                                       </td><td><span style=white-space:pre-wrap>More Appreciative of Arts &amp; Others     </span></td><td>behavior2                                                                          </td><td>30                                                                                 </td><td>62                                                                                 </td><td>0.4716291                                                                          </td><td>sr_highestdegreeplanned                                                            </td><td><span style=white-space:pre-wrap>Master's degree                </span>            </td></tr>\n",
       "\t<tr><td> 0.5379562                             </td><td>0.1595643                              </td><td> 3.371406                              </td><td>2.393134e-02                           </td><td>New Outlooks                           </td><td>behavior2                              </td><td>30                                     </td><td>62                                     </td><td>0.4571121                              </td><td>sex                                    </td><td>Female                                 </td></tr>\n",
       "\t<tr><td>⋮</td><td>⋮</td><td>⋮</td><td>⋮</td><td>⋮</td><td>⋮</td><td>⋮</td><td>⋮</td><td>⋮</td><td>⋮</td><td>⋮</td></tr>\n",
       "\t<tr><td> 1.3705931                                            </td><td>0.22732657                                            </td><td> 6.029181                                             </td><td>5.273350e-08                                          </td><td>Misc. Music                                           </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.48368240                                            </td><td>hs_arts_freq                                          </td><td>Frequently                                            </td></tr>\n",
       "\t<tr><td>-0.8999722                                            </td><td>0.27161150                                            </td><td>-3.313454                                             </td><td>2.764538e-02                                          </td><td>Misc. Music                                           </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.48368240                                            </td><td>hs_encouragement                                      </td><td>Yes                                                   </td></tr>\n",
       "\t<tr><td>-0.9287445                                            </td><td>0.26310237                                            </td><td>-3.529974                                             </td><td>1.288362e-02                                          </td><td>Misc. Music                                           </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.48368240                                            </td><td>sr_highestdegreeplanned                               </td><td>Master's degree                                       </td></tr>\n",
       "\t<tr><td>-1.3221420                                            </td><td>0.29429467                                            </td><td>-4.492579                                             </td><td>1.759146e-04                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>parented                                              </td><td>Associate's                                           </td></tr>\n",
       "\t<tr><td>-1.3102288                                            </td><td>0.21656955                                            </td><td>-6.049922                                             </td><td>4.347490e-08                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>parented                                              </td><td>Bachelor's                                            </td></tr>\n",
       "\t<tr><td>-1.4650365                                            </td><td>0.22374275                                            </td><td>-6.547861                                             </td><td>1.809374e-09                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>parented                                              </td><td>Master's                                              </td></tr>\n",
       "\t<tr><td>-1.9673279                                            </td><td>0.29688241                                            </td><td>-6.626623                                             </td><td>1.099049e-09                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>parented                                              </td><td>Ph.D or professional degree                           </td></tr>\n",
       "\t<tr><td> 0.7907064                                            </td><td>0.23937033                                            </td><td> 3.303277                                             </td><td>2.006803e-02                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>income$                                               </td><td>100,001-$150,000                                      </td></tr>\n",
       "\t<tr><td>-1.0172833                                            </td><td>0.20464279                                            </td><td>-4.971019                                             </td><td>1.864851e-05                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>hstype                                                </td><td>Private religious                                     </td></tr>\n",
       "\t<tr><td>-0.9580613                                            </td><td>0.20460643                                            </td><td>-4.682459                                             </td><td>7.369798e-05                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>hssize                                                </td><td>1001-2000                                             </td></tr>\n",
       "\t<tr><td> 0.4528630                                            </td><td>0.14280622                                            </td><td> 3.171171                                             </td><td>3.036512e-02                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>hs_encouragement                                      </td><td>Yes                                                   </td></tr>\n",
       "\t<tr><td>-0.3739433                                            </td><td>0.11043067                                            </td><td>-3.386227                                             </td><td>1.558938e-02                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>hs_fees                                               </td><td>Yes                                                   </td></tr>\n",
       "\t<tr><td>-1.1691721                                            </td><td>0.24299331                                            </td><td>-4.811540                                             </td><td>4.043836e-05                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>so_childhood                                          </td><td>1No                                                   </td></tr>\n",
       "\t<tr><td> 0.9124845                                            </td><td>0.15481523                                            </td><td> 5.894023                                             </td><td>1.093025e-07                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>so_childhood                                          </td><td>3No                                                   </td></tr>\n",
       "\t<tr><td> 0.5186899                                            </td><td>0.14394134                                            </td><td> 3.603481                                             </td><td>7.221616e-03                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>sr_participated                                       </td><td>Yes                                                   </td></tr>\n",
       "\t<tr><td>-0.4386914                                            </td><td>0.11843215                                            </td><td>-3.704158                                             </td><td>5.090266e-03                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td> 30                                                   </td><td> 62                                                   </td><td>0.63488174                                            </td><td>sr_highestdegreeplanned                               </td><td>Doctoral or professional degree                       </td></tr>\n",
       "\t<tr><td>-0.2666217                                            </td><td>0.07734493                                            </td><td>-3.447177                                             </td><td>1.359545e-02                                          </td><td>Seeing From More Perspectives                         </td><td>behavior2                                             </td><td>393                                                   </td><td>417                                                   </td><td>0.09086713                                            </td><td>sr_motivation_problemsolving                          </td><td>Yes                                                   </td></tr>\n",
       "\t<tr><td>-0.3099045                                            </td><td>0.09722342                                            </td><td>-3.187549                                             </td><td>3.730584e-02                                          </td><td>More Aware                                            </td><td>behavior2                                             </td><td>363                                                   </td><td>389                                                   </td><td>0.10735024                                            </td><td>sr_identity_dancer                                    </td><td>No                                                    </td></tr>\n",
       "\t<tr><td> 1.0828003                                            </td><td>0.34424499                                            </td><td> 3.145435                                             </td><td>4.809359e-02                                          </td><td>Career Tradeoff Concerns                              </td><td>sr_career                                             </td><td> 42                                                   </td><td> 71                                                   </td><td>0.35591151                                            </td><td>parented                                              </td><td>Associate's                                           </td></tr>\n",
       "\t<tr><td> 0.5198800                                            </td><td>0.13823582                                            </td><td> 3.760820                                             </td><td>4.911361e-03                                          </td><td>Art as a Career                                       </td><td>sr_career                                             </td><td> 42                                                   </td><td> 71                                                   </td><td>0.37205498                                            </td><td>hs_arts_freq                                          </td><td>Frequently                                            </td></tr>\n",
       "\t<tr><td> 0.4578252                                            </td><td>0.12406376                                            </td><td> 3.690241                                             </td><td>6.273164e-03                                          </td><td>Art as a Career                                       </td><td>sr_career                                             </td><td> 42                                                   </td><td> 71                                                   </td><td>0.37205498                                            </td><td>hs_required                                           </td><td>Yes                                                   </td></tr>\n",
       "\t<tr><td> 1.7238058                                            </td><td>0.33406398                                            </td><td> 5.160107                                             </td><td>7.157458e-06                                          </td><td>Want Art to be Part of Future                         </td><td>sr_career                                             </td><td> 42                                                   </td><td> 71                                                   </td><td>0.34033663                                            </td><td>parented                                              </td><td>Associate's                                           </td></tr>\n",
       "\t<tr><td>-0.4778890                                            </td><td>0.14091776                                            </td><td>-3.391262                                             </td><td>1.948007e-02                                          </td><td>Want Art to be Part of Future                         </td><td>sr_career                                             </td><td> 42                                                   </td><td> 71                                                   </td><td>0.34033663                                            </td><td>hs_encouragement                                      </td><td>Yes                                                   </td></tr>\n",
       "\t<tr><td> 0.6918821                                            </td><td>0.19375677                                            </td><td> 3.570880                                             </td><td>1.031774e-02                                          </td><td>Some Impact                                           </td><td>sr_career                                             </td><td> 42                                                   </td><td> 71                                                   </td><td>0.38985069                                            </td><td>hssize                                                </td><td>500-1000                                              </td></tr>\n",
       "\t<tr><td> 0.9332654                                            </td><td>0.26505612                                            </td><td> 3.521011                                             </td><td>1.246726e-02                                          </td><td>Looking for Creative Expression in Non-Artistic Career</td><td>sr_career                                             </td><td> 42                                                   </td><td> 71                                                   </td><td>0.37730674                                            </td><td>ethnic_group                                          </td><td>Hispanic                                              </td></tr>\n",
       "\t<tr><td>-0.7635134                                            </td><td>0.23356800                                            </td><td>-3.268913                                             </td><td>3.022927e-02                                          </td><td>Looking for Creative Expression in Non-Artistic Career</td><td>sr_career                                             </td><td> 42                                                   </td><td> 71                                                   </td><td>0.37730674                                            </td><td>hslocation                                            </td><td>Rural                                                 </td></tr>\n",
       "\t<tr><td>-0.3671697                                            </td><td>0.11887656                                            </td><td>-3.088663                                             </td><td>4.825417e-02                                          </td><td>No Impact 1                                           </td><td>sr_career                                             </td><td>481                                                   </td><td>505                                                   </td><td>0.25552354                                            </td><td>sr_motivation_creativeabilities                       </td><td>Yes                                                   </td></tr>\n",
       "\t<tr><td>-0.5530415                                            </td><td>0.16434058                                            </td><td>-3.365216                                             </td><td>1.988580e-02                                          </td><td>Art as a Career                                       </td><td>sr_career                                             </td><td>466                                                   </td><td>492                                                   </td><td>0.13824483                                            </td><td>sr_identity_film                                      </td><td>No                                                    </td></tr>\n",
       "\t<tr><td>-0.2704459                                            </td><td>0.07336906                                            </td><td>-3.686104                                             </td><td>5.920557e-03                                          </td><td>Want Art to be Part of Future                         </td><td>sr_career                                             </td><td>466                                                   </td><td>492                                                   </td><td>0.16213946                                            </td><td>sr_identity_studyarts                                 </td><td>No                                                    </td></tr>\n",
       "\t<tr><td> 0.2144737                                            </td><td>0.07453378                                            </td><td> 2.877537                                             </td><td>4.408728e-02                                          </td><td>Art as a Career                                       </td><td>sr_career                                             </td><td>485                                                   </td><td>496                                                   </td><td>0.06310533                                            </td><td>sr_film_participation                                 </td><td>Yes                                                   </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lllllllllll}\n",
       " estimate & std.error & statistic & p.value & topic & question & residual\\_df & null\\_df & r\\_sq & demo & level\\\\\n",
       "\\hline\n",
       "\t  0.9933077                                & 0.1917906                                 &  5.179128                                 & 6.687779e-06                              & More Confident \\& Social                 & behavior2                                 & 30                                        & 62                                        & 0.4515716                                 & ethnic\\_group                            & Asian                                    \\\\\n",
       "\t -0.8221978                                & 0.1416449                                 & -5.804642                                 & 2.064112e-07                              & More Confident \\& Social                 & behavior2                                 & 30                                        & 62                                        & 0.4515716                                 & sex                                       & Female                                   \\\\\n",
       "\t -0.9405522                                & 0.2549704                                 & -3.688869                                 & 6.081847e-03                              & More Confident \\& Social                 & behavior2                                 & 30                                        & 62                                        & 0.4515716                                 & school                                    & Ross School of Business                  \\\\\n",
       "\t -1.7024398                                & 0.4044585                                 & -4.209183                                 & 7.432575e-04                              & More Confident \\& Social                 & behavior2                                 & 30                                        & 62                                        & 0.4515716                                 & hssize                                    & More than 3001                           \\\\\n",
       "\t  1.3042886                                & 0.3311390                                 &  3.938795                                 & 2.292976e-03                              & More Confident \\& Social                 & behavior2                                 & 30                                        & 62                                        & 0.4515716                                 & hslocation                                & Rural                                    \\\\\n",
       "\t -0.7694124                                & 0.1454879                                 & -5.288497                                 & 3.823086e-06                              & More Confident \\& Social                 & behavior2                                 & 30                                        & 62                                        & 0.4515716                                 & hs\\_arts\\_freq                          & Frequently                               \\\\\n",
       "\t  0.5763670                                & 0.1837866                                 &  3.136067                                 & 4.451987e-02                              & More Confident \\& Social                 & behavior2                                 & 30                                        & 62                                        & 0.4515716                                 & hs\\_encouragement                        & Yes                                      \\\\\n",
       "\t  0.4439039                                & 0.1422351                                 &  3.120915                                 & 4.507247e-02                              & More Confident \\& Social                 & behavior2                                 & 30                                        & 62                                        & 0.4515716                                 & hs\\_fees                                 & Yes                                      \\\\\n",
       "\t -1.0383347                                & 0.2781618                                 & -3.732845                                 & 5.869201e-03                              & Deeper Understanding of Art \\& The World & behavior2                                 & 30                                        & 62                                        & 0.4455261                                 & school                                    & Ross School of Business                  \\\\\n",
       "\t  0.9149535                                & 0.1912546                                 &  4.783955                                 & 5.500143e-05                              & Deeper Understanding of Art \\& The World & behavior2                                 & 30                                        & 62                                        & 0.4455261                                 & sr\\_highestdegreeplanned                 & Master's degree                          \\\\\n",
       "\t  1.0567834                              & 0.3313881                               &  3.188960                               & 3.712429e-02                            & More Critical Thinking                  & behavior2                               & 30                                      & 62                                      & 0.7115845                               & ethnic\\_group                          & Hispanic                               \\\\\n",
       "\t  0.5073364                              & 0.1539249                               &  3.295999                               & 2.647950e-02                            & More Critical Thinking                  & behavior2                               & 30                                      & 62                                      & 0.7115845                               & school                                  & Engineering                            \\\\\n",
       "\t  0.7546784                              & 0.2089958                               &  3.610975                               & 8.846397e-03                            & More Critical Thinking                  & behavior2                               & 30                                      & 62                                      & 0.7115845                               & parented                                & Master's                               \\\\\n",
       "\t -0.7369082                              & 0.1770650                               & -4.161795                               & 9.472684e-04                            & More Critical Thinking                  & behavior2                               & 30                                      & 62                                      & 0.7115845                               & income                                  & More than \\$150,000                   \\\\\n",
       "\t  0.8007149                              & 0.2399230                               &  3.337383                               & 2.367999e-02                            & More Critical Thinking                  & behavior2                               & 30                                      & 62                                      & 0.7115845                               & hslocation                              & Rural                                  \\\\\n",
       "\t -0.4789299                              & 0.1084851                               & -4.414705                               & 3.135581e-04                            & More Critical Thinking                  & behavior2                               & 30                                      & 62                                      & 0.7115845                               & hs\\_arts\\_freq                        & Frequently                             \\\\\n",
       "\t  0.5912219                              & 0.1022762                               &  5.780640                               & 2.381338e-07                            & More Critical Thinking                  & behavior2                               & 30                                      & 62                                      & 0.7115845                               & hs\\_fees                               & Yes                                    \\\\\n",
       "\t  0.3782069                              & 0.1189064                               &  3.180710                               & 3.712429e-02                            & More Critical Thinking                  & behavior2                               & 30                                      & 62                                      & 0.7115845                               & sr\\_participated                       & Yes                                    \\\\\n",
       "\t  0.8000761                              & 0.1800304                               &  4.444117                               & 2.735863e-04                            & Attending More Events                   & behavior2                               & 30                                      & 62                                      & 0.5050804                               & ethnic\\_group                          & Asian                                  \\\\\n",
       "\t -0.8180881                              & 0.2122767                               & -3.853877                               & 3.487865e-03                            & Attending More Events                   & behavior2                               & 30                                      & 62                                      & 0.5050804                               & school                                  & Engineering                            \\\\\n",
       "\t  0.7678724                              & 0.2258567                               &  3.399820                               & 1.955474e-02                            & Attending More Events                   & behavior2                               & 30                                      & 62                                      & 0.5050804                               & hstype                                  & Private religious                      \\\\\n",
       "\t  1.0608601                              & 0.3134339                               &  3.384638                               & 1.995623e-02                            & Attending More Events                   & behavior2                               & 30                                      & 62                                      & 0.5050804                               & hslocation                              & Suburban                               \\\\\n",
       "\t  1.6145819                              & 0.3151517                               &  5.123189                               & 9.613154e-06                            & Attending More Events                   & behavior2                               & 30                                      & 62                                      & 0.5050804                               & hslocation                              & Rural                                  \\\\\n",
       "\t  0.4632224                              & 0.1431172                               &  3.236665                               & 3.265247e-02                            & Attending More Events                   & behavior2                               & 30                                      & 62                                      & 0.5050804                               & sr\\_highestdegreeplanned               & Doctoral or professional degree        \\\\\n",
       "\t  1.7307381                              & 0.3704740                               &  4.671686                               & 9.559614e-05                            & More Aware                              & behavior2                               & 30                                      & 62                                      & 0.6253664                               & school                                  & Stamps School of Art \\& Design        \\\\\n",
       "\t  0.4328560                                & 0.1228578                                 &  3.523227                                 & 1.236346e-02                              & More Appreciative of Arts \\& Others      & behavior2                                 & 30                                        & 62                                        & 0.4716291                                 & hs\\_required                             & Yes                                      \\\\\n",
       "\t -0.5402647                                & 0.1194513                                 & -4.522888                                 & 1.952046e-04                              & More Appreciative of Arts \\& Others      & behavior2                                 & 30                                        & 62                                        & 0.4716291                                 & hs\\_fees                                 & Yes                                      \\\\\n",
       "\t  0.4366903                                & 0.1168151                                 &  3.738305                                 & 5.557954e-03                              & More Appreciative of Arts \\& Others      & behavior2                                 & 30                                        & 62                                        & 0.4716291                                 & so\\_childhood                            & 5No                                      \\\\\n",
       "\t  0.5604396                                & 0.1429007                                 &  3.921881                                 & 2.723667e-03                              & More Appreciative of Arts \\& Others      & behavior2                                 & 30                                        & 62                                        & 0.4716291                                 & sr\\_highestdegreeplanned                 & Master's degree                          \\\\\n",
       "\t  0.5379562                              & 0.1595643                               &  3.371406                               & 2.393134e-02                            & New Outlooks                            & behavior2                               & 30                                      & 62                                      & 0.4571121                               & sex                                     & Female                                 \\\\\n",
       "\t ⋮ & ⋮ & ⋮ & ⋮ & ⋮ & ⋮ & ⋮ & ⋮ & ⋮ & ⋮ & ⋮\\\\\n",
       "\t  1.3705931                                             & 0.22732657                                             &  6.029181                                              & 5.273350e-08                                           & Misc. Music                                            & behavior2                                              &  30                                                    &  62                                                    & 0.48368240                                             & hs\\_arts\\_freq                                       & Frequently                                            \\\\\n",
       "\t -0.8999722                                             & 0.27161150                                             & -3.313454                                              & 2.764538e-02                                           & Misc. Music                                            & behavior2                                              &  30                                                    &  62                                                    & 0.48368240                                             & hs\\_encouragement                                     & Yes                                                   \\\\\n",
       "\t -0.9287445                                             & 0.26310237                                             & -3.529974                                              & 1.288362e-02                                           & Misc. Music                                            & behavior2                                              &  30                                                    &  62                                                    & 0.48368240                                             & sr\\_highestdegreeplanned                              & Master's degree                                       \\\\\n",
       "\t -1.3221420                                             & 0.29429467                                             & -4.492579                                              & 1.759146e-04                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & parented                                               & Associate's                                           \\\\\n",
       "\t -1.3102288                                             & 0.21656955                                             & -6.049922                                              & 4.347490e-08                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & parented                                               & Bachelor's                                            \\\\\n",
       "\t -1.4650365                                             & 0.22374275                                             & -6.547861                                              & 1.809374e-09                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & parented                                               & Master's                                              \\\\\n",
       "\t -1.9673279                                             & 0.29688241                                             & -6.626623                                              & 1.099049e-09                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & parented                                               & Ph.D or professional degree                           \\\\\n",
       "\t  0.7907064                                             & 0.23937033                                             &  3.303277                                              & 2.006803e-02                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & income\\$                                              & 100,001-\\$150,000                                    \\\\\n",
       "\t -1.0172833                                             & 0.20464279                                             & -4.971019                                              & 1.864851e-05                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & hstype                                                 & Private religious                                     \\\\\n",
       "\t -0.9580613                                             & 0.20460643                                             & -4.682459                                              & 7.369798e-05                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & hssize                                                 & 1001-2000                                             \\\\\n",
       "\t  0.4528630                                             & 0.14280622                                             &  3.171171                                              & 3.036512e-02                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & hs\\_encouragement                                     & Yes                                                   \\\\\n",
       "\t -0.3739433                                             & 0.11043067                                             & -3.386227                                              & 1.558938e-02                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & hs\\_fees                                              & Yes                                                   \\\\\n",
       "\t -1.1691721                                             & 0.24299331                                             & -4.811540                                              & 4.043836e-05                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & so\\_childhood                                         & 1No                                                   \\\\\n",
       "\t  0.9124845                                             & 0.15481523                                             &  5.894023                                              & 1.093025e-07                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & so\\_childhood                                         & 3No                                                   \\\\\n",
       "\t  0.5186899                                             & 0.14394134                                             &  3.603481                                              & 7.221616e-03                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & sr\\_participated                                      & Yes                                                   \\\\\n",
       "\t -0.4386914                                             & 0.11843215                                             & -3.704158                                              & 5.090266e-03                                           & Seeing From More Perspectives                          & behavior2                                              &  30                                                    &  62                                                    & 0.63488174                                             & sr\\_highestdegreeplanned                              & Doctoral or professional degree                       \\\\\n",
       "\t -0.2666217                                             & 0.07734493                                             & -3.447177                                              & 1.359545e-02                                           & Seeing From More Perspectives                          & behavior2                                              & 393                                                    & 417                                                    & 0.09086713                                             & sr\\_motivation\\_problemsolving                       & Yes                                                   \\\\\n",
       "\t -0.3099045                                             & 0.09722342                                             & -3.187549                                              & 3.730584e-02                                           & More Aware                                             & behavior2                                              & 363                                                    & 389                                                    & 0.10735024                                             & sr\\_identity\\_dancer                                 & No                                                    \\\\\n",
       "\t  1.0828003                                             & 0.34424499                                             &  3.145435                                              & 4.809359e-02                                           & Career Tradeoff Concerns                               & sr\\_career                                            &  42                                                    &  71                                                    & 0.35591151                                             & parented                                               & Associate's                                           \\\\\n",
       "\t  0.5198800                                             & 0.13823582                                             &  3.760820                                              & 4.911361e-03                                           & Art as a Career                                        & sr\\_career                                            &  42                                                    &  71                                                    & 0.37205498                                             & hs\\_arts\\_freq                                       & Frequently                                            \\\\\n",
       "\t  0.4578252                                             & 0.12406376                                             &  3.690241                                              & 6.273164e-03                                           & Art as a Career                                        & sr\\_career                                            &  42                                                    &  71                                                    & 0.37205498                                             & hs\\_required                                          & Yes                                                   \\\\\n",
       "\t  1.7238058                                             & 0.33406398                                             &  5.160107                                              & 7.157458e-06                                           & Want Art to be Part of Future                          & sr\\_career                                            &  42                                                    &  71                                                    & 0.34033663                                             & parented                                               & Associate's                                           \\\\\n",
       "\t -0.4778890                                             & 0.14091776                                             & -3.391262                                              & 1.948007e-02                                           & Want Art to be Part of Future                          & sr\\_career                                            &  42                                                    &  71                                                    & 0.34033663                                             & hs\\_encouragement                                     & Yes                                                   \\\\\n",
       "\t  0.6918821                                             & 0.19375677                                             &  3.570880                                              & 1.031774e-02                                           & Some Impact                                            & sr\\_career                                            &  42                                                    &  71                                                    & 0.38985069                                             & hssize                                                 & 500-1000                                              \\\\\n",
       "\t  0.9332654                                             & 0.26505612                                             &  3.521011                                              & 1.246726e-02                                           & Looking for Creative Expression in Non-Artistic Career & sr\\_career                                            &  42                                                    &  71                                                    & 0.37730674                                             & ethnic\\_group                                         & Hispanic                                              \\\\\n",
       "\t -0.7635134                                             & 0.23356800                                             & -3.268913                                              & 3.022927e-02                                           & Looking for Creative Expression in Non-Artistic Career & sr\\_career                                            &  42                                                    &  71                                                    & 0.37730674                                             & hslocation                                             & Rural                                                 \\\\\n",
       "\t -0.3671697                                             & 0.11887656                                             & -3.088663                                              & 4.825417e-02                                           & No Impact 1                                            & sr\\_career                                            & 481                                                    & 505                                                    & 0.25552354                                             & sr\\_motivation\\_creativeabilities                    & Yes                                                   \\\\\n",
       "\t -0.5530415                                             & 0.16434058                                             & -3.365216                                              & 1.988580e-02                                           & Art as a Career                                        & sr\\_career                                            & 466                                                    & 492                                                    & 0.13824483                                             & sr\\_identity\\_film                                   & No                                                    \\\\\n",
       "\t -0.2704459                                             & 0.07336906                                             & -3.686104                                              & 5.920557e-03                                           & Want Art to be Part of Future                          & sr\\_career                                            & 466                                                    & 492                                                    & 0.16213946                                             & sr\\_identity\\_studyarts                              & No                                                    \\\\\n",
       "\t  0.2144737                                             & 0.07453378                                             &  2.877537                                              & 4.408728e-02                                           & Art as a Career                                        & sr\\_career                                            & 485                                                    & 496                                                    & 0.06310533                                             & sr\\_film\\_participation                              & Yes                                                   \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "| estimate | std.error | statistic | p.value | topic | question | residual_df | null_df | r_sq | demo | level |\n",
       "|---|---|---|---|---|---|---|---|---|---|---|\n",
       "|  0.9933077                              | 0.1917906                               |  5.179128                               | 6.687779e-06                            | More Confident & Social                 | behavior2                               | 30                                      | 62                                      | 0.4515716                               | ethnic_group                            | Asian                                   |\n",
       "| -0.8221978                              | 0.1416449                               | -5.804642                               | 2.064112e-07                            | More Confident & Social                 | behavior2                               | 30                                      | 62                                      | 0.4515716                               | sex                                     | Female                                  |\n",
       "| -0.9405522                              | 0.2549704                               | -3.688869                               | 6.081847e-03                            | More Confident & Social                 | behavior2                               | 30                                      | 62                                      | 0.4515716                               | school                                  | Ross School of Business                 |\n",
       "| -1.7024398                              | 0.4044585                               | -4.209183                               | 7.432575e-04                            | More Confident & Social                 | behavior2                               | 30                                      | 62                                      | 0.4515716                               | hssize                                  | More than 3001                          |\n",
       "|  1.3042886                              | 0.3311390                               |  3.938795                               | 2.292976e-03                            | More Confident & Social                 | behavior2                               | 30                                      | 62                                      | 0.4515716                               | hslocation                              | Rural                                   |\n",
       "| -0.7694124                              | 0.1454879                               | -5.288497                               | 3.823086e-06                            | More Confident & Social                 | behavior2                               | 30                                      | 62                                      | 0.4515716                               | hs_arts_freq                            | Frequently                              |\n",
       "|  0.5763670                              | 0.1837866                               |  3.136067                               | 4.451987e-02                            | More Confident & Social                 | behavior2                               | 30                                      | 62                                      | 0.4515716                               | hs_encouragement                        | Yes                                     |\n",
       "|  0.4439039                              | 0.1422351                               |  3.120915                               | 4.507247e-02                            | More Confident & Social                 | behavior2                               | 30                                      | 62                                      | 0.4515716                               | hs_fees                                 | Yes                                     |\n",
       "| -1.0383347                              | 0.2781618                               | -3.732845                               | 5.869201e-03                            | Deeper Understanding of Art & The World | behavior2                               | 30                                      | 62                                      | 0.4455261                               | school                                  | Ross School of Business                 |\n",
       "|  0.9149535                              | 0.1912546                               |  4.783955                               | 5.500143e-05                            | Deeper Understanding of Art & The World | behavior2                               | 30                                      | 62                                      | 0.4455261                               | sr_highestdegreeplanned                 | Master's degree                         |\n",
       "|  1.0567834                              | 0.3313881                               |  3.188960                               | 3.712429e-02                            | More Critical Thinking                  | behavior2                               | 30                                      | 62                                      | 0.7115845                               | ethnic_group                            | Hispanic                                |\n",
       "|  0.5073364                              | 0.1539249                               |  3.295999                               | 2.647950e-02                            | More Critical Thinking                  | behavior2                               | 30                                      | 62                                      | 0.7115845                               | school                                  | Engineering                             |\n",
       "|  0.7546784                              | 0.2089958                               |  3.610975                               | 8.846397e-03                            | More Critical Thinking                  | behavior2                               | 30                                      | 62                                      | 0.7115845                               | parented                                | Master's                                |\n",
       "| -0.7369082                              | 0.1770650                               | -4.161795                               | 9.472684e-04                            | More Critical Thinking                  | behavior2                               | 30                                      | 62                                      | 0.7115845                               | income                                  | More than $150,000                      |\n",
       "|  0.8007149                              | 0.2399230                               |  3.337383                               | 2.367999e-02                            | More Critical Thinking                  | behavior2                               | 30                                      | 62                                      | 0.7115845                               | hslocation                              | Rural                                   |\n",
       "| -0.4789299                              | 0.1084851                               | -4.414705                               | 3.135581e-04                            | More Critical Thinking                  | behavior2                               | 30                                      | 62                                      | 0.7115845                               | hs_arts_freq                            | Frequently                              |\n",
       "|  0.5912219                              | 0.1022762                               |  5.780640                               | 2.381338e-07                            | More Critical Thinking                  | behavior2                               | 30                                      | 62                                      | 0.7115845                               | hs_fees                                 | Yes                                     |\n",
       "|  0.3782069                              | 0.1189064                               |  3.180710                               | 3.712429e-02                            | More Critical Thinking                  | behavior2                               | 30                                      | 62                                      | 0.7115845                               | sr_participated                         | Yes                                     |\n",
       "|  0.8000761                              | 0.1800304                               |  4.444117                               | 2.735863e-04                            | Attending More Events                   | behavior2                               | 30                                      | 62                                      | 0.5050804                               | ethnic_group                            | Asian                                   |\n",
       "| -0.8180881                              | 0.2122767                               | -3.853877                               | 3.487865e-03                            | Attending More Events                   | behavior2                               | 30                                      | 62                                      | 0.5050804                               | school                                  | Engineering                             |\n",
       "|  0.7678724                              | 0.2258567                               |  3.399820                               | 1.955474e-02                            | Attending More Events                   | behavior2                               | 30                                      | 62                                      | 0.5050804                               | hstype                                  | Private religious                       |\n",
       "|  1.0608601                              | 0.3134339                               |  3.384638                               | 1.995623e-02                            | Attending More Events                   | behavior2                               | 30                                      | 62                                      | 0.5050804                               | hslocation                              | Suburban                                |\n",
       "|  1.6145819                              | 0.3151517                               |  5.123189                               | 9.613154e-06                            | Attending More Events                   | behavior2                               | 30                                      | 62                                      | 0.5050804                               | hslocation                              | Rural                                   |\n",
       "|  0.4632224                              | 0.1431172                               |  3.236665                               | 3.265247e-02                            | Attending More Events                   | behavior2                               | 30                                      | 62                                      | 0.5050804                               | sr_highestdegreeplanned                 | Doctoral or professional degree         |\n",
       "|  1.7307381                              | 0.3704740                               |  4.671686                               | 9.559614e-05                            | More Aware                              | behavior2                               | 30                                      | 62                                      | 0.6253664                               | school                                  | Stamps School of Art & Design           |\n",
       "|  0.4328560                              | 0.1228578                               |  3.523227                               | 1.236346e-02                            | More Appreciative of Arts & Others      | behavior2                               | 30                                      | 62                                      | 0.4716291                               | hs_required                             | Yes                                     |\n",
       "| -0.5402647                              | 0.1194513                               | -4.522888                               | 1.952046e-04                            | More Appreciative of Arts & Others      | behavior2                               | 30                                      | 62                                      | 0.4716291                               | hs_fees                                 | Yes                                     |\n",
       "|  0.4366903                              | 0.1168151                               |  3.738305                               | 5.557954e-03                            | More Appreciative of Arts & Others      | behavior2                               | 30                                      | 62                                      | 0.4716291                               | so_childhood                            | 5No                                     |\n",
       "|  0.5604396                              | 0.1429007                               |  3.921881                               | 2.723667e-03                            | More Appreciative of Arts & Others      | behavior2                               | 30                                      | 62                                      | 0.4716291                               | sr_highestdegreeplanned                 | Master's degree                         |\n",
       "|  0.5379562                              | 0.1595643                               |  3.371406                               | 2.393134e-02                            | New Outlooks                            | behavior2                               | 30                                      | 62                                      | 0.4571121                               | sex                                     | Female                                  |\n",
       "| ⋮ | ⋮ | ⋮ | ⋮ | ⋮ | ⋮ | ⋮ | ⋮ | ⋮ | ⋮ | ⋮ |\n",
       "|  1.3705931                                             | 0.22732657                                             |  6.029181                                              | 5.273350e-08                                           | Misc. Music                                            | behavior2                                              |  30                                                    |  62                                                    | 0.48368240                                             | hs_arts_freq                                           | Frequently                                             |\n",
       "| -0.8999722                                             | 0.27161150                                             | -3.313454                                              | 2.764538e-02                                           | Misc. Music                                            | behavior2                                              |  30                                                    |  62                                                    | 0.48368240                                             | hs_encouragement                                       | Yes                                                    |\n",
       "| -0.9287445                                             | 0.26310237                                             | -3.529974                                              | 1.288362e-02                                           | Misc. Music                                            | behavior2                                              |  30                                                    |  62                                                    | 0.48368240                                             | sr_highestdegreeplanned                                | Master's degree                                        |\n",
       "| -1.3221420                                             | 0.29429467                                             | -4.492579                                              | 1.759146e-04                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | parented                                               | Associate's                                            |\n",
       "| -1.3102288                                             | 0.21656955                                             | -6.049922                                              | 4.347490e-08                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | parented                                               | Bachelor's                                             |\n",
       "| -1.4650365                                             | 0.22374275                                             | -6.547861                                              | 1.809374e-09                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | parented                                               | Master's                                               |\n",
       "| -1.9673279                                             | 0.29688241                                             | -6.626623                                              | 1.099049e-09                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | parented                                               | Ph.D or professional degree                            |\n",
       "|  0.7907064                                             | 0.23937033                                             |  3.303277                                              | 2.006803e-02                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | income$                                                | 100,001-$150,000                                       |\n",
       "| -1.0172833                                             | 0.20464279                                             | -4.971019                                              | 1.864851e-05                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | hstype                                                 | Private religious                                      |\n",
       "| -0.9580613                                             | 0.20460643                                             | -4.682459                                              | 7.369798e-05                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | hssize                                                 | 1001-2000                                              |\n",
       "|  0.4528630                                             | 0.14280622                                             |  3.171171                                              | 3.036512e-02                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | hs_encouragement                                       | Yes                                                    |\n",
       "| -0.3739433                                             | 0.11043067                                             | -3.386227                                              | 1.558938e-02                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | hs_fees                                                | Yes                                                    |\n",
       "| -1.1691721                                             | 0.24299331                                             | -4.811540                                              | 4.043836e-05                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | so_childhood                                           | 1No                                                    |\n",
       "|  0.9124845                                             | 0.15481523                                             |  5.894023                                              | 1.093025e-07                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | so_childhood                                           | 3No                                                    |\n",
       "|  0.5186899                                             | 0.14394134                                             |  3.603481                                              | 7.221616e-03                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | sr_participated                                        | Yes                                                    |\n",
       "| -0.4386914                                             | 0.11843215                                             | -3.704158                                              | 5.090266e-03                                           | Seeing From More Perspectives                          | behavior2                                              |  30                                                    |  62                                                    | 0.63488174                                             | sr_highestdegreeplanned                                | Doctoral or professional degree                        |\n",
       "| -0.2666217                                             | 0.07734493                                             | -3.447177                                              | 1.359545e-02                                           | Seeing From More Perspectives                          | behavior2                                              | 393                                                    | 417                                                    | 0.09086713                                             | sr_motivation_problemsolving                           | Yes                                                    |\n",
       "| -0.3099045                                             | 0.09722342                                             | -3.187549                                              | 3.730584e-02                                           | More Aware                                             | behavior2                                              | 363                                                    | 389                                                    | 0.10735024                                             | sr_identity_dancer                                     | No                                                     |\n",
       "|  1.0828003                                             | 0.34424499                                             |  3.145435                                              | 4.809359e-02                                           | Career Tradeoff Concerns                               | sr_career                                              |  42                                                    |  71                                                    | 0.35591151                                             | parented                                               | Associate's                                            |\n",
       "|  0.5198800                                             | 0.13823582                                             |  3.760820                                              | 4.911361e-03                                           | Art as a Career                                        | sr_career                                              |  42                                                    |  71                                                    | 0.37205498                                             | hs_arts_freq                                           | Frequently                                             |\n",
       "|  0.4578252                                             | 0.12406376                                             |  3.690241                                              | 6.273164e-03                                           | Art as a Career                                        | sr_career                                              |  42                                                    |  71                                                    | 0.37205498                                             | hs_required                                            | Yes                                                    |\n",
       "|  1.7238058                                             | 0.33406398                                             |  5.160107                                              | 7.157458e-06                                           | Want Art to be Part of Future                          | sr_career                                              |  42                                                    |  71                                                    | 0.34033663                                             | parented                                               | Associate's                                            |\n",
       "| -0.4778890                                             | 0.14091776                                             | -3.391262                                              | 1.948007e-02                                           | Want Art to be Part of Future                          | sr_career                                              |  42                                                    |  71                                                    | 0.34033663                                             | hs_encouragement                                       | Yes                                                    |\n",
       "|  0.6918821                                             | 0.19375677                                             |  3.570880                                              | 1.031774e-02                                           | Some Impact                                            | sr_career                                              |  42                                                    |  71                                                    | 0.38985069                                             | hssize                                                 | 500-1000                                               |\n",
       "|  0.9332654                                             | 0.26505612                                             |  3.521011                                              | 1.246726e-02                                           | Looking for Creative Expression in Non-Artistic Career | sr_career                                              |  42                                                    |  71                                                    | 0.37730674                                             | ethnic_group                                           | Hispanic                                               |\n",
       "| -0.7635134                                             | 0.23356800                                             | -3.268913                                              | 3.022927e-02                                           | Looking for Creative Expression in Non-Artistic Career | sr_career                                              |  42                                                    |  71                                                    | 0.37730674                                             | hslocation                                             | Rural                                                  |\n",
       "| -0.3671697                                             | 0.11887656                                             | -3.088663                                              | 4.825417e-02                                           | No Impact 1                                            | sr_career                                              | 481                                                    | 505                                                    | 0.25552354                                             | sr_motivation_creativeabilities                        | Yes                                                    |\n",
       "| -0.5530415                                             | 0.16434058                                             | -3.365216                                              | 1.988580e-02                                           | Art as a Career                                        | sr_career                                              | 466                                                    | 492                                                    | 0.13824483                                             | sr_identity_film                                       | No                                                     |\n",
       "| -0.2704459                                             | 0.07336906                                             | -3.686104                                              | 5.920557e-03                                           | Want Art to be Part of Future                          | sr_career                                              | 466                                                    | 492                                                    | 0.16213946                                             | sr_identity_studyarts                                  | No                                                     |\n",
       "|  0.2144737                                             | 0.07453378                                             |  2.877537                                              | 4.408728e-02                                           | Art as a Career                                        | sr_career                                              | 485                                                    | 496                                                    | 0.06310533                                             | sr_film_participation                                  | Yes                                                    |\n",
       "\n"
      ],
      "text/plain": [
       "   estimate   std.error  statistic p.value     \n",
       "1   0.9933077 0.1917906   5.179128 6.687779e-06\n",
       "2  -0.8221978 0.1416449  -5.804642 2.064112e-07\n",
       "3  -0.9405522 0.2549704  -3.688869 6.081847e-03\n",
       "4  -1.7024398 0.4044585  -4.209183 7.432575e-04\n",
       "5   1.3042886 0.3311390   3.938795 2.292976e-03\n",
       "6  -0.7694124 0.1454879  -5.288497 3.823086e-06\n",
       "7   0.5763670 0.1837866   3.136067 4.451987e-02\n",
       "8   0.4439039 0.1422351   3.120915 4.507247e-02\n",
       "9  -1.0383347 0.2781618  -3.732845 5.869201e-03\n",
       "10  0.9149535 0.1912546   4.783955 5.500143e-05\n",
       "11  1.0567834 0.3313881   3.188960 3.712429e-02\n",
       "12  0.5073364 0.1539249   3.295999 2.647950e-02\n",
       "13  0.7546784 0.2089958   3.610975 8.846397e-03\n",
       "14 -0.7369082 0.1770650  -4.161795 9.472684e-04\n",
       "15  0.8007149 0.2399230   3.337383 2.367999e-02\n",
       "16 -0.4789299 0.1084851  -4.414705 3.135581e-04\n",
       "17  0.5912219 0.1022762   5.780640 2.381338e-07\n",
       "18  0.3782069 0.1189064   3.180710 3.712429e-02\n",
       "19  0.8000761 0.1800304   4.444117 2.735863e-04\n",
       "20 -0.8180881 0.2122767  -3.853877 3.487865e-03\n",
       "21  0.7678724 0.2258567   3.399820 1.955474e-02\n",
       "22  1.0608601 0.3134339   3.384638 1.995623e-02\n",
       "23  1.6145819 0.3151517   5.123189 9.613154e-06\n",
       "24  0.4632224 0.1431172   3.236665 3.265247e-02\n",
       "25  1.7307381 0.3704740   4.671686 9.559614e-05\n",
       "26  0.4328560 0.1228578   3.523227 1.236346e-02\n",
       "27 -0.5402647 0.1194513  -4.522888 1.952046e-04\n",
       "28  0.4366903 0.1168151   3.738305 5.557954e-03\n",
       "29  0.5604396 0.1429007   3.921881 2.723667e-03\n",
       "30  0.5379562 0.1595643   3.371406 2.393134e-02\n",
       "⋮  ⋮          ⋮          ⋮         ⋮           \n",
       "53  1.3705931 0.22732657  6.029181 5.273350e-08\n",
       "54 -0.8999722 0.27161150 -3.313454 2.764538e-02\n",
       "55 -0.9287445 0.26310237 -3.529974 1.288362e-02\n",
       "56 -1.3221420 0.29429467 -4.492579 1.759146e-04\n",
       "57 -1.3102288 0.21656955 -6.049922 4.347490e-08\n",
       "58 -1.4650365 0.22374275 -6.547861 1.809374e-09\n",
       "59 -1.9673279 0.29688241 -6.626623 1.099049e-09\n",
       "60  0.7907064 0.23937033  3.303277 2.006803e-02\n",
       "61 -1.0172833 0.20464279 -4.971019 1.864851e-05\n",
       "62 -0.9580613 0.20460643 -4.682459 7.369798e-05\n",
       "63  0.4528630 0.14280622  3.171171 3.036512e-02\n",
       "64 -0.3739433 0.11043067 -3.386227 1.558938e-02\n",
       "65 -1.1691721 0.24299331 -4.811540 4.043836e-05\n",
       "66  0.9124845 0.15481523  5.894023 1.093025e-07\n",
       "67  0.5186899 0.14394134  3.603481 7.221616e-03\n",
       "68 -0.4386914 0.11843215 -3.704158 5.090266e-03\n",
       "69 -0.2666217 0.07734493 -3.447177 1.359545e-02\n",
       "70 -0.3099045 0.09722342 -3.187549 3.730584e-02\n",
       "71  1.0828003 0.34424499  3.145435 4.809359e-02\n",
       "72  0.5198800 0.13823582  3.760820 4.911361e-03\n",
       "73  0.4578252 0.12406376  3.690241 6.273164e-03\n",
       "74  1.7238058 0.33406398  5.160107 7.157458e-06\n",
       "75 -0.4778890 0.14091776 -3.391262 1.948007e-02\n",
       "76  0.6918821 0.19375677  3.570880 1.031774e-02\n",
       "77  0.9332654 0.26505612  3.521011 1.246726e-02\n",
       "78 -0.7635134 0.23356800 -3.268913 3.022927e-02\n",
       "79 -0.3671697 0.11887656 -3.088663 4.825417e-02\n",
       "80 -0.5530415 0.16434058 -3.365216 1.988580e-02\n",
       "81 -0.2704459 0.07336906 -3.686104 5.920557e-03\n",
       "82  0.2144737 0.07453378  2.877537 4.408728e-02\n",
       "   topic                                                  question  residual_df\n",
       "1  More Confident & Social                                behavior2 30         \n",
       "2  More Confident & Social                                behavior2 30         \n",
       "3  More Confident & Social                                behavior2 30         \n",
       "4  More Confident & Social                                behavior2 30         \n",
       "5  More Confident & Social                                behavior2 30         \n",
       "6  More Confident & Social                                behavior2 30         \n",
       "7  More Confident & Social                                behavior2 30         \n",
       "8  More Confident & Social                                behavior2 30         \n",
       "9  Deeper Understanding of Art & The World                behavior2 30         \n",
       "10 Deeper Understanding of Art & The World                behavior2 30         \n",
       "11 More Critical Thinking                                 behavior2 30         \n",
       "12 More Critical Thinking                                 behavior2 30         \n",
       "13 More Critical Thinking                                 behavior2 30         \n",
       "14 More Critical Thinking                                 behavior2 30         \n",
       "15 More Critical Thinking                                 behavior2 30         \n",
       "16 More Critical Thinking                                 behavior2 30         \n",
       "17 More Critical Thinking                                 behavior2 30         \n",
       "18 More Critical Thinking                                 behavior2 30         \n",
       "19 Attending More Events                                  behavior2 30         \n",
       "20 Attending More Events                                  behavior2 30         \n",
       "21 Attending More Events                                  behavior2 30         \n",
       "22 Attending More Events                                  behavior2 30         \n",
       "23 Attending More Events                                  behavior2 30         \n",
       "24 Attending More Events                                  behavior2 30         \n",
       "25 More Aware                                             behavior2 30         \n",
       "26 More Appreciative of Arts & Others                     behavior2 30         \n",
       "27 More Appreciative of Arts & Others                     behavior2 30         \n",
       "28 More Appreciative of Arts & Others                     behavior2 30         \n",
       "29 More Appreciative of Arts & Others                     behavior2 30         \n",
       "30 New Outlooks                                           behavior2 30         \n",
       "⋮  ⋮                                                      ⋮         ⋮          \n",
       "53 Misc. Music                                            behavior2  30        \n",
       "54 Misc. Music                                            behavior2  30        \n",
       "55 Misc. Music                                            behavior2  30        \n",
       "56 Seeing From More Perspectives                          behavior2  30        \n",
       "57 Seeing From More Perspectives                          behavior2  30        \n",
       "58 Seeing From More Perspectives                          behavior2  30        \n",
       "59 Seeing From More Perspectives                          behavior2  30        \n",
       "60 Seeing From More Perspectives                          behavior2  30        \n",
       "61 Seeing From More Perspectives                          behavior2  30        \n",
       "62 Seeing From More Perspectives                          behavior2  30        \n",
       "63 Seeing From More Perspectives                          behavior2  30        \n",
       "64 Seeing From More Perspectives                          behavior2  30        \n",
       "65 Seeing From More Perspectives                          behavior2  30        \n",
       "66 Seeing From More Perspectives                          behavior2  30        \n",
       "67 Seeing From More Perspectives                          behavior2  30        \n",
       "68 Seeing From More Perspectives                          behavior2  30        \n",
       "69 Seeing From More Perspectives                          behavior2 393        \n",
       "70 More Aware                                             behavior2 363        \n",
       "71 Career Tradeoff Concerns                               sr_career  42        \n",
       "72 Art as a Career                                        sr_career  42        \n",
       "73 Art as a Career                                        sr_career  42        \n",
       "74 Want Art to be Part of Future                          sr_career  42        \n",
       "75 Want Art to be Part of Future                          sr_career  42        \n",
       "76 Some Impact                                            sr_career  42        \n",
       "77 Looking for Creative Expression in Non-Artistic Career sr_career  42        \n",
       "78 Looking for Creative Expression in Non-Artistic Career sr_career  42        \n",
       "79 No Impact 1                                            sr_career 481        \n",
       "80 Art as a Career                                        sr_career 466        \n",
       "81 Want Art to be Part of Future                          sr_career 466        \n",
       "82 Art as a Career                                        sr_career 485        \n",
       "   null_df r_sq       demo                           \n",
       "1  62      0.4515716  ethnic_group                   \n",
       "2  62      0.4515716  sex                            \n",
       "3  62      0.4515716  school                         \n",
       "4  62      0.4515716  hssize                         \n",
       "5  62      0.4515716  hslocation                     \n",
       "6  62      0.4515716  hs_arts_freq                   \n",
       "7  62      0.4515716  hs_encouragement               \n",
       "8  62      0.4515716  hs_fees                        \n",
       "9  62      0.4455261  school                         \n",
       "10 62      0.4455261  sr_highestdegreeplanned        \n",
       "11 62      0.7115845  ethnic_group                   \n",
       "12 62      0.7115845  school                         \n",
       "13 62      0.7115845  parented                       \n",
       "14 62      0.7115845  income                         \n",
       "15 62      0.7115845  hslocation                     \n",
       "16 62      0.7115845  hs_arts_freq                   \n",
       "17 62      0.7115845  hs_fees                        \n",
       "18 62      0.7115845  sr_participated                \n",
       "19 62      0.5050804  ethnic_group                   \n",
       "20 62      0.5050804  school                         \n",
       "21 62      0.5050804  hstype                         \n",
       "22 62      0.5050804  hslocation                     \n",
       "23 62      0.5050804  hslocation                     \n",
       "24 62      0.5050804  sr_highestdegreeplanned        \n",
       "25 62      0.6253664  school                         \n",
       "26 62      0.4716291  hs_required                    \n",
       "27 62      0.4716291  hs_fees                        \n",
       "28 62      0.4716291  so_childhood                   \n",
       "29 62      0.4716291  sr_highestdegreeplanned        \n",
       "30 62      0.4571121  sex                            \n",
       "⋮  ⋮       ⋮          ⋮                              \n",
       "53  62     0.48368240 hs_arts_freq                   \n",
       "54  62     0.48368240 hs_encouragement               \n",
       "55  62     0.48368240 sr_highestdegreeplanned        \n",
       "56  62     0.63488174 parented                       \n",
       "57  62     0.63488174 parented                       \n",
       "58  62     0.63488174 parented                       \n",
       "59  62     0.63488174 parented                       \n",
       "60  62     0.63488174 income$                        \n",
       "61  62     0.63488174 hstype                         \n",
       "62  62     0.63488174 hssize                         \n",
       "63  62     0.63488174 hs_encouragement               \n",
       "64  62     0.63488174 hs_fees                        \n",
       "65  62     0.63488174 so_childhood                   \n",
       "66  62     0.63488174 so_childhood                   \n",
       "67  62     0.63488174 sr_participated                \n",
       "68  62     0.63488174 sr_highestdegreeplanned        \n",
       "69 417     0.09086713 sr_motivation_problemsolving   \n",
       "70 389     0.10735024 sr_identity_dancer             \n",
       "71  71     0.35591151 parented                       \n",
       "72  71     0.37205498 hs_arts_freq                   \n",
       "73  71     0.37205498 hs_required                    \n",
       "74  71     0.34033663 parented                       \n",
       "75  71     0.34033663 hs_encouragement               \n",
       "76  71     0.38985069 hssize                         \n",
       "77  71     0.37730674 ethnic_group                   \n",
       "78  71     0.37730674 hslocation                     \n",
       "79 505     0.25552354 sr_motivation_creativeabilities\n",
       "80 492     0.13824483 sr_identity_film               \n",
       "81 492     0.16213946 sr_identity_studyarts          \n",
       "82 496     0.06310533 sr_film_participation          \n",
       "   level                          \n",
       "1  Asian                          \n",
       "2  Female                         \n",
       "3  Ross School of Business        \n",
       "4  More than 3001                 \n",
       "5  Rural                          \n",
       "6  Frequently                     \n",
       "7  Yes                            \n",
       "8  Yes                            \n",
       "9  Ross School of Business        \n",
       "10 Master's degree                \n",
       "11 Hispanic                       \n",
       "12 Engineering                    \n",
       "13 Master's                       \n",
       "14 More than $150,000             \n",
       "15 Rural                          \n",
       "16 Frequently                     \n",
       "17 Yes                            \n",
       "18 Yes                            \n",
       "19 Asian                          \n",
       "20 Engineering                    \n",
       "21 Private religious              \n",
       "22 Suburban                       \n",
       "23 Rural                          \n",
       "24 Doctoral or professional degree\n",
       "25 Stamps School of Art & Design  \n",
       "26 Yes                            \n",
       "27 Yes                            \n",
       "28 5No                            \n",
       "29 Master's degree                \n",
       "30 Female                         \n",
       "⋮  ⋮                              \n",
       "53 Frequently                     \n",
       "54 Yes                            \n",
       "55 Master's degree                \n",
       "56 Associate's                    \n",
       "57 Bachelor's                     \n",
       "58 Master's                       \n",
       "59 Ph.D or professional degree    \n",
       "60 100,001-$150,000               \n",
       "61 Private religious              \n",
       "62 1001-2000                      \n",
       "63 Yes                            \n",
       "64 Yes                            \n",
       "65 1No                            \n",
       "66 3No                            \n",
       "67 Yes                            \n",
       "68 Doctoral or professional degree\n",
       "69 Yes                            \n",
       "70 No                             \n",
       "71 Associate's                    \n",
       "72 Frequently                     \n",
       "73 Yes                            \n",
       "74 Associate's                    \n",
       "75 Yes                            \n",
       "76 500-1000                       \n",
       "77 Hispanic                       \n",
       "78 Rural                          \n",
       "79 Yes                            \n",
       "80 No                             \n",
       "81 No                             \n",
       "82 Yes                            "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sig_contrasts.demos %>% nrow\n",
    "sig_contrasts.demos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4 & 8 investigation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "ungrouped_demos <- c(demo_groups[[1]], demo_groups[[2]], 'sr_identity_artist', demo_groups[[4]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "na_medians <- c()\n",
    "na_means <- c()\n",
    "for(k in seq(thetas)) {\n",
    "    question_name <- names(thetas)[k]\n",
    "    merged <- merge(thetas[[question_name]], select(df, key, ungrouped_demos), by = 'key')\n",
    "    merged <- merged[-1] #remove key\n",
    "    for(l in seq(ungrouped_demos)) {\n",
    "        ## NA the lonely eggs\n",
    "        merged[[ungrouped_demos[l]]][merged[[ungrouped_demos[l]]] %in% \n",
    "                         (merged[[ungrouped_demos[l]]] %>% table %>% tidy %>% filter(n < 25) %>% pull(1))] <- NA\n",
    "        merged[[ungrouped_demos[l]]] %<>% droplevels\n",
    "        if (nlevels(merged[[ungrouped_demos[l]]]) < 2)\n",
    "            merged %<>% select(-matches(ungrouped_demos[l]))\n",
    "    }\n",
    "    \n",
    "    # now we only have good demos, and all topics\n",
    "    topics <- merged[1:(ncol(thetas[[question_name]])-1)]\n",
    "    demo <- merged[ncol(thetas[[question_name]]):ncol(merged)]\n",
    "    \n",
    "    na_medians %<>% c(demo %>% sapply(function(x)(sum(is.na(x)))/nrow(demo)) %>% as.vector %>% median)\n",
    "    na_means %<>% c(demo %>% sapply(function(x)(sum(is.na(x)))/nrow(demo)) %>% as.vector %>% mean)                                   \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table>\n",
       "<thead><tr><th scope=col>na_means</th><th scope=col>na_medians</th><th scope=col>seq.12.</th></tr></thead>\n",
       "<tbody>\n",
       "\t<tr><td>0.46329111</td><td>0.57259528</td><td> 1        </td></tr>\n",
       "\t<tr><td>0.56369913</td><td>0.72809335</td><td> 2        </td></tr>\n",
       "\t<tr><td>0.57852594</td><td>0.73376335</td><td> 3        </td></tr>\n",
       "\t<tr><td>0.28515532</td><td>0.32419355</td><td> 4        </td></tr>\n",
       "\t<tr><td>0.41644596</td><td>0.52490315</td><td> 5        </td></tr>\n",
       "\t<tr><td>0.27844881</td><td>0.31980116</td><td> 6        </td></tr>\n",
       "\t<tr><td>0.03875380</td><td>0.04395604</td><td> 7        </td></tr>\n",
       "\t<tr><td>0.06760413</td><td>0.04339623</td><td> 8        </td></tr>\n",
       "\t<tr><td>0.06907355</td><td>0.03201970</td><td> 9        </td></tr>\n",
       "\t<tr><td>0.07678826</td><td>0.04016478</td><td>10        </td></tr>\n",
       "\t<tr><td>0.06147959</td><td>0.01428571</td><td>11        </td></tr>\n",
       "\t<tr><td>0.08089980</td><td>0.04153355</td><td>12        </td></tr>\n",
       "</tbody>\n",
       "</table>\n"
      ],
      "text/latex": [
       "\\begin{tabular}{r|lll}\n",
       " na\\_means & na\\_medians & seq.12.\\\\\n",
       "\\hline\n",
       "\t 0.46329111 & 0.57259528 &  1        \\\\\n",
       "\t 0.56369913 & 0.72809335 &  2        \\\\\n",
       "\t 0.57852594 & 0.73376335 &  3        \\\\\n",
       "\t 0.28515532 & 0.32419355 &  4        \\\\\n",
       "\t 0.41644596 & 0.52490315 &  5        \\\\\n",
       "\t 0.27844881 & 0.31980116 &  6        \\\\\n",
       "\t 0.03875380 & 0.04395604 &  7        \\\\\n",
       "\t 0.06760413 & 0.04339623 &  8        \\\\\n",
       "\t 0.06907355 & 0.03201970 &  9        \\\\\n",
       "\t 0.07678826 & 0.04016478 & 10        \\\\\n",
       "\t 0.06147959 & 0.01428571 & 11        \\\\\n",
       "\t 0.08089980 & 0.04153355 & 12        \\\\\n",
       "\\end{tabular}\n"
      ],
      "text/markdown": [
       "\n",
       "| na_means | na_medians | seq.12. |\n",
       "|---|---|---|\n",
       "| 0.46329111 | 0.57259528 |  1         |\n",
       "| 0.56369913 | 0.72809335 |  2         |\n",
       "| 0.57852594 | 0.73376335 |  3         |\n",
       "| 0.28515532 | 0.32419355 |  4         |\n",
       "| 0.41644596 | 0.52490315 |  5         |\n",
       "| 0.27844881 | 0.31980116 |  6         |\n",
       "| 0.03875380 | 0.04395604 |  7         |\n",
       "| 0.06760413 | 0.04339623 |  8         |\n",
       "| 0.06907355 | 0.03201970 |  9         |\n",
       "| 0.07678826 | 0.04016478 | 10         |\n",
       "| 0.06147959 | 0.01428571 | 11         |\n",
       "| 0.08089980 | 0.04153355 | 12         |\n",
       "\n"
      ],
      "text/plain": [
       "   na_means   na_medians seq.12.\n",
       "1  0.46329111 0.57259528  1     \n",
       "2  0.56369913 0.72809335  2     \n",
       "3  0.57852594 0.73376335  3     \n",
       "4  0.28515532 0.32419355  4     \n",
       "5  0.41644596 0.52490315  5     \n",
       "6  0.27844881 0.31980116  6     \n",
       "7  0.03875380 0.04395604  7     \n",
       "8  0.06760413 0.04339623  8     \n",
       "9  0.06907355 0.03201970  9     \n",
       "10 0.07678826 0.04016478 10     \n",
       "11 0.06147959 0.01428571 11     \n",
       "12 0.08089980 0.04153355 12     "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "data.frame(na_means, na_medians, seq(12))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Nothing strange here ^^"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ungrouped Demos for comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning message in betareg.fit(X, Y, Z, weights, offset, link, link.phi, type, control):\n",
      "“no valid starting value for precision parameter found, using 1 instead”"
     ]
    },
    {
     "ename": "ERROR",
     "evalue": "Error in optim(par = start, fn = loglikfun, gr = gradfun, method = method, : non-finite value supplied by optim\n",
     "output_type": "error",
     "traceback": [
      "Error in optim(par = start, fn = loglikfun, gr = gradfun, method = method, : non-finite value supplied by optim\nTraceback:\n",
      "1. betareg(topic ~ ., temp)",
      "2. betareg.fit(X, Y, Z, weights, offset, link, link.phi, type, control)",
      "3. optim(par = start, fn = loglikfun, gr = gradfun, method = method, \n .     hessian = hessian, control = control)"
     ]
    }
   ],
   "source": [
    "sig_contrasts.demos_no_grp <- c()\n",
    "betaregs <- c()\n",
    "# bad_input <- c(4,8,Inf)\n",
    "idx = 1\n",
    "for(k in seq(thetas)) {\n",
    "#     if (k == bad_input[idx]) {\n",
    "#         idx = idx + 1\n",
    "#         next\n",
    "#     }\n",
    "    question_name <- names(thetas)[k]\n",
    "    merged <- merge(thetas[[question_name]], select(df, key, ungrouped_demos), by = 'key')\n",
    "    merged <- merged[-1] #remove key\n",
    "    for(l in seq(ungrouped_demos)) {\n",
    "        ## NA the lonely eggs\n",
    "        merged[[ungrouped_demos[l]]][merged[[ungrouped_demos[l]]] %in% \n",
    "                         (merged[[ungrouped_demos[l]]] %>% table %>% tidy %>% filter(n < 25) %>% pull(1))] <- NA\n",
    "        merged[[ungrouped_demos[l]]] %<>% droplevels\n",
    "        if (nlevels(merged[[ungrouped_demos[l]]]) < 2)\n",
    "            merged %<>% select(-matches(ungrouped_demos[l]))\n",
    "    }\n",
    "    \n",
    "    # now we only have good demos, and all topics\n",
    "    topics <- merged[1:(ncol(thetas[[question_name]])-1)]\n",
    "    demo <- merged[ncol(thetas[[question_name]]):ncol(merged)]\n",
    "    \n",
    "    for(i in seq(topics)) {        \n",
    "        temp <- bind_cols(topics[i], demo)\n",
    "\n",
    "        names(temp) <- c('topic', names(demo))\n",
    "\n",
    "        beta.out <- betareg(topic ~ ., temp)#, na.action = na.exclude)\n",
    "        df.residual <- beta.out$df.residual\n",
    "        df.null <- beta.out$df.null\n",
    "        r.squared <- beta.out$pseudo.r.squared\n",
    "        \n",
    "        options(warn=-1)\n",
    "            beta.out %<>% tidy %>% filter(component == 'mean') %>% \n",
    "                                    select(-component) %>% filter(term != '(Intercept)')\n",
    "        options(warn=0)\n",
    "\n",
    "        beta.out$p.value %<>% p.adjust(method='holm')\n",
    "\n",
    "        beta.out %<>% filter(p.value < 0.05) %>% mutate(topic = names(topics[i]), \n",
    "                                                        question = question_name,\n",
    "                                                        residual_df = df.residual,\n",
    "                                                        null_df = df.null,\n",
    "                                                        r_sq = r.squared)\n",
    "\n",
    "        if(nrow(beta.out) > 0) {\n",
    "            options(warn=-1)\n",
    "                sig_contrasts.demos_no_grp %<>% bind_rows(beta.out)\n",
    "            options(warn=0)\n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "split_list <- gregexpr(\"[A-Z]|[0-9]\", sig_contrasts.demos_no_grp$term)\n",
    "demo_list <- c()\n",
    "level_list <- c()\n",
    "for(i in seq(sig_contrasts.demos_no_grp$term)) {\n",
    "    demo_list %<>% c(substr(sig_contrasts.demos_no_grp$term[i], 1, split_list[[i]] - 1))\n",
    "    level_list %<>% c(substr(sig_contrasts.demos_no_grp$term[i], split_list[[i]],\n",
    "                             nchar(sig_contrasts.demos_no_grp$term[i])))\n",
    "}\n",
    "sig_contrasts.demos_no_grp$demo <- demo_list\n",
    "sig_contrasts.demos_no_grp$level <- level_list\n",
    "sig_contrasts.demos_no_grp$term <- NULL\n",
    "\n",
    "empty_list <- sig_contrasts.demos_no_grp %>% filter(demo == '')\n",
    "split_list <- gregexpr(\"[0-9]\", empty_list$level)\n",
    "demo_list <- c()\n",
    "level_list <- c()\n",
    "for(i in seq(empty_list$level)) {\n",
    "    demo_list %<>% c(substr(empty_list$level[i], 1, split_list[[i]] - 1))\n",
    "    level_list %<>% c(substr(empty_list$level[i], split_list[[i]],\n",
    "                             nchar(empty_list$level[i])))\n",
    "}\n",
    "empty_list$demo <- demo_list\n",
    "empty_list$level <- level_list\n",
    "sig_contrasts.demos_no_grp %<>% bind_rows(empty_list)\n",
    "sig_contrasts.demos_no_grp %<>% filter(demo != '')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "294"
      ],
      "text/latex": [
       "294"
      ],
      "text/markdown": [
       "294"
      ],
      "text/plain": [
       "[1] 294"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       ".\n",
       "        barriers           change       childhood2       definition \n",
       "              11                9               30                6 \n",
       "            feel sr_aftercollege4   sr_development   sr_othergrowth \n",
       "              30               24               31               19 \n",
       "         sr_role       sr_society \n",
       "              69               65 "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sig_contrasts.demos_no_grp %>% nrow\n",
    "sig_contrasts.demos_no_grp$question %>% table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "qns <- sig_contrasts.demos_no_grp %>% pull(question) %>% unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<ol class=list-inline>\n",
       "\t<li>'barriers'</li>\n",
       "\t<li>'change'</li>\n",
       "\t<li>'definition'</li>\n",
       "\t<li>'childhood2'</li>\n",
       "\t<li>'feel'</li>\n",
       "\t<li>'sr_aftercollege4'</li>\n",
       "\t<li>'sr_development'</li>\n",
       "\t<li>'sr_othergrowth'</li>\n",
       "\t<li>'sr_role'</li>\n",
       "\t<li>'sr_society'</li>\n",
       "</ol>\n"
      ],
      "text/latex": [
       "\\begin{enumerate*}\n",
       "\\item 'barriers'\n",
       "\\item 'change'\n",
       "\\item 'definition'\n",
       "\\item 'childhood2'\n",
       "\\item 'feel'\n",
       "\\item 'sr\\_aftercollege4'\n",
       "\\item 'sr\\_development'\n",
       "\\item 'sr\\_othergrowth'\n",
       "\\item 'sr\\_role'\n",
       "\\item 'sr\\_society'\n",
       "\\end{enumerate*}\n"
      ],
      "text/markdown": [
       "1. 'barriers'\n",
       "2. 'change'\n",
       "3. 'definition'\n",
       "4. 'childhood2'\n",
       "5. 'feel'\n",
       "6. 'sr_aftercollege4'\n",
       "7. 'sr_development'\n",
       "8. 'sr_othergrowth'\n",
       "9. 'sr_role'\n",
       "10. 'sr_society'\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       " [1] \"barriers\"         \"change\"           \"definition\"       \"childhood2\"      \n",
       " [5] \"feel\"             \"sr_aftercollege4\" \"sr_development\"   \"sr_othergrowth\"  \n",
       " [9] \"sr_role\"          \"sr_society\"      "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "qns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir <- 'topic_demo_csvs/'\n",
    "dir.create(file.path(dir))\n",
    "for(i in seq(qns)) {\n",
    "    write.csv(x = sig_contrasts.demos_no_grp %>% filter(question == qns[i]) %>% arrange(demo),\n",
    "              file = paste0(dir, qns[i], '_demographics.csv'))\n",
    "}"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "R",
   "language": "R",
   "name": "ir"
  },
  "language_info": {
   "codemirror_mode": "r",
   "file_extension": ".r",
   "mimetype": "text/x-r-source",
   "name": "R",
   "pygments_lexer": "r",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}