PrintSparkSettings.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Print Spark Driver Settings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from pyspark.sql import SparkSession"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Setting default log level to \"WARN\".\n",
      "To adjust logging level use sc.setLogLevel(newLevel). For SparkR, use setLogLevel(newLevel).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22/07/02 00:05:22 WARN NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n"
     ]
    }
   ],
   "source": [
    "spark = SparkSession.builder.appName(\"PrintSettings\").getOrCreate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>key</th>\n",
       "      <th>value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spark.app.id</td>\n",
       "      <td>local-1656745529242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>spark.app.name</td>\n",
       "      <td>PrintSettings</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spark.app.startTime</td>\n",
       "      <td>1656745522606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>spark.app.submitTime</td>\n",
       "      <td>1656745517302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>spark.driver.extraJavaOptions</td>\n",
       "      <td>-XX:+IgnoreUnrecognizedVMOptions --add-opens=j...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>spark.driver.host</td>\n",
       "      <td>192.168.1.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>spark.driver.memory</td>\n",
       "      <td>16G</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>spark.driver.port</td>\n",
       "      <td>54827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>spark.executor.extraJavaOptions</td>\n",
       "      <td>-XX:+IgnoreUnrecognizedVMOptions --add-opens=j...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>spark.executor.id</td>\n",
       "      <td>driver</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>spark.master</td>\n",
       "      <td>local[*]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>spark.rdd.compress</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>spark.serializer.objectStreamReset</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>spark.sql.warehouse.dir</td>\n",
       "      <td>file:/Users/Peter/GitRepositories/mmtf-pyspark...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>spark.submit.deployMode</td>\n",
       "      <td>client</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>spark.submit.pyFiles</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>spark.ui.showConsoleProgress</td>\n",
       "      <td>true</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   key  \\\n",
       "0                         spark.app.id   \n",
       "1                       spark.app.name   \n",
       "2                  spark.app.startTime   \n",
       "3                 spark.app.submitTime   \n",
       "4        spark.driver.extraJavaOptions   \n",
       "5                    spark.driver.host   \n",
       "6                  spark.driver.memory   \n",
       "7                    spark.driver.port   \n",
       "8      spark.executor.extraJavaOptions   \n",
       "9                    spark.executor.id   \n",
       "10                        spark.master   \n",
       "11                  spark.rdd.compress   \n",
       "12  spark.serializer.objectStreamReset   \n",
       "13             spark.sql.warehouse.dir   \n",
       "14             spark.submit.deployMode   \n",
       "15                spark.submit.pyFiles   \n",
       "16        spark.ui.showConsoleProgress   \n",
       "\n",
       "                                                value  \n",
       "0                                 local-1656745529242  \n",
       "1                                       PrintSettings  \n",
       "2                                       1656745522606  \n",
       "3                                       1656745517302  \n",
       "4   -XX:+IgnoreUnrecognizedVMOptions --add-opens=j...  \n",
       "5                                         192.168.1.5  \n",
       "6                                                 16G  \n",
       "7                                               54827  \n",
       "8   -XX:+IgnoreUnrecognizedVMOptions --add-opens=j...  \n",
       "9                                              driver  \n",
       "10                                           local[*]  \n",
       "11                                               True  \n",
       "12                                                100  \n",
       "13  file:/Users/Peter/GitRepositories/mmtf-pyspark...  \n",
       "14                                             client  \n",
       "15                                                     \n",
       "16                                               true  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sql(\"SET\").toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "spark.stop()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Spark launch + shutdown time: 45.501788854599 sec.\n"
     ]
    }
   ],
   "source": [
    "end = time.time()\n",
    "print(\"Spark launch + shutdown time:\", end-start, \"sec.\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}