Spark-HelloWorld program (Loading a Dataset).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://10.11.31.77:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v2.2.1</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>local[2]</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>PySparkShell</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=local[2] appName=PySparkShell>"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Transformation are not executed immediately\n",
    "this read method is and transformation function even on giving wrong file name or path it will not through an error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filepath = \"file:///home/mayank/AAPL.csv\"\n",
    "appl = sc.textFile(filepath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "file:///home/mayank/AAPL.csv MapPartitionsRDD[1] at textFile at NativeMethodAccessorImpl.java:0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "those methods that take an immediate response for the operation are called as actions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Date,Open,High,Low,Close,Adj Close,Volume',\n",
       " '2017-11-27,175.050003,175.080002,173.339996,174.089996,174.089996,20716800',\n",
       " '2017-11-28,174.300003,174.869995,171.860001,173.070007,173.070007,26428800',\n",
       " '2017-11-29,172.630005,172.919998,167.160004,169.479996,169.479996,41666400',\n",
       " '2017-11-30,170.429993,172.139999,168.440002,171.850006,171.850006,41527200',\n",
       " '2017-12-01,169.949997,171.669998,168.500000,171.050003,171.050003,39759300',\n",
       " '2017-12-04,172.479996,172.619995,169.630005,169.800003,169.800003,32542400',\n",
       " '2017-12-05,169.059998,171.520004,168.399994,169.639999,169.639999,27350200',\n",
       " '2017-12-06,167.500000,170.199997,166.460007,169.009995,169.009995,28560000',\n",
       " '2017-12-07,169.029999,170.440002,168.910004,169.320007,169.320007,25673300']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appl.take(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Date,Open,High,Low,Close,Adj Close,Volume',\n",
       " '2017-11-27,175.050003,175.080002,173.339996,174.089996,174.089996,20716800',\n",
       " '2017-11-28,174.300003,174.869995,171.860001,173.070007,173.070007,26428800',\n",
       " '2017-11-29,172.630005,172.919998,167.160004,169.479996,169.479996,41666400',\n",
       " '2017-11-30,170.429993,172.139999,168.440002,171.850006,171.850006,41527200',\n",
       " '2017-12-01,169.949997,171.669998,168.500000,171.050003,171.050003,39759300',\n",
       " '2017-12-04,172.479996,172.619995,169.630005,169.800003,169.800003,32542400',\n",
       " '2017-12-05,169.059998,171.520004,168.399994,169.639999,169.639999,27350200',\n",
       " '2017-12-06,167.500000,170.199997,166.460007,169.009995,169.009995,28560000',\n",
       " '2017-12-07,169.029999,170.440002,168.910004,169.320007,169.320007,25673300',\n",
       " '2017-12-08,170.490005,171.000000,168.820007,169.369995,169.369995,23355200',\n",
       " '2017-12-11,169.199997,172.889999,168.789993,172.669998,172.669998,35273800',\n",
       " '2017-12-12,172.149994,172.389999,171.460007,171.699997,171.699997,19409200',\n",
       " '2017-12-13,172.500000,173.539993,172.000000,172.270004,172.270004,23818400',\n",
       " '2017-12-14,172.399994,173.130005,171.649994,172.220001,172.220001,20476500',\n",
       " '2017-12-15,173.630005,174.169998,172.460007,173.970001,173.970001,40169300',\n",
       " '2017-12-18,174.880005,177.199997,174.860001,176.419998,176.419998,29421100',\n",
       " '2017-12-19,175.029999,175.389999,174.089996,174.539993,174.539993,27436400',\n",
       " '2017-12-20,174.869995,175.419998,173.250000,174.350006,174.350006,23475600',\n",
       " '2017-12-21,174.169998,176.020004,174.100006,175.009995,175.009995,20949900',\n",
       " '2017-12-22,174.679993,175.419998,174.500000,175.009995,175.009995,16349400',\n",
       " '2017-12-26,170.800003,171.470001,169.679993,170.570007,170.570007,33043700']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "appl.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}