Examples_SciScript-Python.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import SciServer\n",
    "from SciServer import Authentication, LoginPortal, Config, CasJobs, SkyQuery, SciDrive, SkyServer, Files, Jobs\n",
    "import os;\n",
    "import pandas;\n",
    "import sys;\n",
    "import json;\n",
    "from io import StringIO\n",
    "from io import BytesIO\n",
    "#from PIL import Image\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Define login Name and password before running these examples\n",
    "Authentication_loginName = '***';\n",
    "Authentication_loginPassword = '***'\n",
    "\n",
    "Authentication_login_sharedWithName = '***'\n",
    "Authentication_login_sharedWithPassword = '***'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(SciServer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *******************************************************************************************************\n",
    "# Authentication section\n",
    "# *******************************************************************************************************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(Authentication)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#logging in and getting current token from different ways\n",
    "\n",
    "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);\n",
    "token2 = Authentication.getToken()\n",
    "token3 = Authentication.getKeystoneToken()\n",
    "token4 = Authentication.token.value\n",
    "print(\"token1=\" + token1)#\n",
    "print(\"token2=\" + token2)#\n",
    "print(\"token3=\" + token3)#\n",
    "print(\"token4=\" + token4)#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#getting curent user info\n",
    "\n",
    "user = Authentication.getKeystoneUserWithToken(token1)\n",
    "print(\"userName=\" + user.userName)\n",
    "print(\"id=\" + user.id)\n",
    "iden = Authentication.identArgIdentifier()\n",
    "print(\"ident=\"+iden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reseting the current token to another value:\n",
    "\n",
    "Authentication.setToken(\"myToken1\")\n",
    "token5 = Authentication.getToken()\n",
    "Authentication.setKeystoneToken(\"myToken2\")\n",
    "token6 = Authentication.getKeystoneToken()\n",
    "\n",
    "print(\"token5=\" + token5)\n",
    "print(\"token6=\" + token6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#logging-in again\n",
    "\n",
    "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *******************************************************************************************************\n",
    "# LoginPortal section:\n",
    "# *******************************************************************************************************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(LoginPortal)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#logging in and getting current token from different ways\n",
    "\n",
    "token1 = LoginPortal.login(Authentication_loginName, Authentication_loginPassword);\n",
    "token2 = LoginPortal.getToken()\n",
    "token3 = LoginPortal.getKeystoneToken()\n",
    "print(\"token1=\" + token1)#\n",
    "print(\"token2=\" + token2)#\n",
    "print(\"token3=\" + token3)#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#getting curent user info\n",
    "\n",
    "user = LoginPortal.getKeystoneUserWithToken(token1)\n",
    "print(\"userName=\" + user.userName)\n",
    "print(\"id=\" + user.id)#\n",
    "iden = LoginPortal.identArgIdentifier()\n",
    "print(\"ident=\"+iden)#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#reseting the current token to another value:\n",
    "\n",
    "LoginPortal.setKeystoneToken(\"myToken2\")\n",
    "token6 = LoginPortal.getKeystoneToken()\n",
    "print(\"token6=\" + token6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#logging-in again\n",
    "\n",
    "token1 = LoginPortal.login(Authentication_loginName, Authentication_loginPassword);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *******************************************************************************************************\n",
    "# CasJobs section:\n",
    "# *******************************************************************************************************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(CasJobs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Defining databse context and query, and other variables\n",
    "\n",
    "CasJobs_TestDatabase = \"MyDB\"\n",
    "CasJobs_TestQuery = \"select 4 as Column1, 5 as Column2 \"\n",
    "CasJobs_TestTableName1 = \"MyNewtable1\"\n",
    "CasJobs_TestTableName2 = \"MyNewtable2\"\n",
    "CasJobs_TestTableCSV = u\"Column1,Column2\\n4,5\\n\"\n",
    "CasJobs_TestFitsFile = \"SciScriptTestFile.fits\"\n",
    "CasJobs_TestCSVFile = \"SciScriptTestFile.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get user schema info\n",
    "\n",
    "casJobsId = CasJobs.getSchemaName()\n",
    "print(casJobsId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get info about tables inside MyDB database context:\n",
    "\n",
    "tables = CasJobs.getTables(context=\"MyDB\")\n",
    "print(tables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#execute a quick SQL query:\n",
    "\n",
    "df = CasJobs.executeQuery(sql=CasJobs_TestQuery, context=CasJobs_TestDatabase, format=\"pandas\")\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#submit a job, which inserts the query results into a table in the MyDB database context. \n",
    "#Wait until the job is done and get its status.\n",
    "\n",
    "jobId = CasJobs.submitJob(sql=CasJobs_TestQuery + \" into MyDB.\" + CasJobs_TestTableName1, context=\"MyDB\")\n",
    "jobDescription = CasJobs.waitForJob(jobId=jobId, verbose=False)\n",
    "print(jobId)\n",
    "print(jobDescription)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop or delete table in MyDB database context\n",
    "\n",
    "df = CasJobs.executeQuery(sql=\"DROP TABLE \" + CasJobs_TestTableName1, context=\"MyDB\", format=\"pandas\")\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get job status\n",
    "\n",
    "jobId = CasJobs.submitJob(sql=CasJobs_TestQuery, context=CasJobs_TestDatabase)\n",
    "jobDescription = CasJobs.getJobStatus(jobId)\n",
    "print(jobId)\n",
    "print(jobDescription)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cancel a job\n",
    "\n",
    "jobId = CasJobs.submitJob(sql=CasJobs_TestQuery, context=CasJobs_TestDatabase)\n",
    "jobDescription = CasJobs.cancelJob(jobId=jobId)\n",
    "print(jobId)\n",
    "print(jobDescription)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#execute a query and write a local Fits file containing the query results:\n",
    "\n",
    "result = CasJobs.writeFitsFileFromQuery(fileName=CasJobs_TestFitsFile, queryString=CasJobs_TestQuery, context=\"MyDB\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#delete local FITS file just created:\n",
    "\n",
    "os.remove(CasJobs_TestFitsFile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get a Pandas dataframe containing the results of a query\n",
    "\n",
    "df = CasJobs.getPandasDataFrameFromQuery(queryString=CasJobs_TestQuery, context=CasJobs_TestDatabase)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get numpy array containing the results of a query\n",
    "\n",
    "array = CasJobs.getNumpyArrayFromQuery(queryString=CasJobs_TestQuery, context=CasJobs_TestDatabase)\n",
    "print(array)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#uploads a Pandas dataframe into a Database table\n",
    "\n",
    "df = pandas.read_csv(StringIO(CasJobs_TestTableCSV), index_col=None)\n",
    "result = CasJobs.uploadPandasDataFrameToTable(dataFrame=df, tableName=CasJobs_TestTableName2, context=\"MyDB\")\n",
    "table = CasJobs.executeQuery(sql=\"select * from \" + CasJobs_TestTableName2, context=\"MyDB\", format=\"pandas\")\n",
    "print(result)\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop or delete table just created:\n",
    "\n",
    "result2 = CasJobs.executeQuery(sql=\"DROP TABLE \" + CasJobs_TestTableName2, context=CasJobs_TestDatabase, format=\"pandas\")\n",
    "print(result2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#upload csv data string into a database table:\n",
    "\n",
    "result3 = CasJobs.uploadCSVDataToTable(csvData=CasJobs_TestTableCSV, tableName=CasJobs_TestTableName2, context=\"MyDB\")\n",
    "df2 = CasJobs.executeQuery(sql=\"select * from \" + CasJobs_TestTableName2, context=\"MyDB\", format=\"pandas\")\n",
    "print(result3)\n",
    "print(df2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop or delete table just created:\n",
    "\n",
    "result4 = CasJobs.executeQuery(sql=\"DROP TABLE \" + CasJobs_TestTableName2, context=\"MyDB\", format=\"pandas\")\n",
    "print(result4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *******************************************************************************************************\n",
    "#  SkyServer section:\n",
    "# *******************************************************************************************************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(SkyServer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#defining sql query and SDSS data relelease:\n",
    "\n",
    "SkyServer_TestQuery = \"select top 1 specobjid, ra, dec from specobj order by specobjid\"\n",
    "SkyServer_DataRelease = \"DR13\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Exectute sql query\n",
    "\n",
    "df = SkyServer.sqlSearch(sql=SkyServer_TestQuery, dataRelease=SkyServer_DataRelease)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get an image cutout\n",
    "\n",
    "img = SkyServer.getJpegImgCutout(ra=197.614455642896, dec=18.438168853724, width=2, height=2, scale=0.4, \n",
    "                                 dataRelease=SkyServer_DataRelease,opt=\"OG\",\n",
    "                                 query=\"SELECT TOP 100 p.objID, p.ra, p.dec, p.r FROM fGetObjFromRectEq(197.6,18.4,197.7,18.5) n, PhotoPrimary p WHERE n.objID=p.objID\")\n",
    "plt.imshow(img)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# do a radial search of objects:\n",
    "\n",
    "df = SkyServer.radialSearch(ra=258.25, dec=64.05, radius=0.1, dataRelease=SkyServer_DataRelease)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#do rectangular search of objects:\n",
    "\n",
    "df = SkyServer.rectangularSearch(min_ra=258.3, max_ra=258.31, min_dec=64,max_dec=64.01, dataRelease=SkyServer_DataRelease)\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#do an object search based on RA,Dec coordinates:\n",
    "\n",
    "object = SkyServer.objectSearch(ra=258.25, dec=64.05, dataRelease=SkyServer_DataRelease)\n",
    "print(object)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *******************************************************************************************************\n",
    "# SciDrive section:\n",
    "# *******************************************************************************************************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(SciDrive)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list content and metadata of top level directory in SciDrive\n",
    "\n",
    "dirList = SciDrive.directoryList(\"\")\n",
    "print(dirList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#define name of directory to be created in SciDrive:\n",
    "SciDrive_Directory = \"SciScriptPython\"\n",
    "#define name, path and content of a file to be first created and then uploaded into SciDrive:\n",
    "SciDrive_FileName = \"TestFile.csv\"\n",
    "SciDrive_FilePath = \"./TestFile.csv\"\n",
    "SciDrive_FileContent = \"Column1,Column2\\n4.5,5.5\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a folder or container in SciDrive\n",
    "\n",
    "responseCreate = SciDrive.createContainer(SciDrive_Directory)\n",
    "print(responseCreate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list content and metadata of directory in SciDrive\n",
    "\n",
    "dirList = SciDrive.directoryList(SciDrive_Directory)\n",
    "print(dirList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get the public url to access the directory content in SciDrive\n",
    "\n",
    "url = SciDrive.publicUrl(SciDrive_Directory)\n",
    "print(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Delete folder or container in SciDrive:\n",
    "\n",
    "responseDelete = SciDrive.delete(SciDrive_Directory)\n",
    "print(responseDelete)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create a local file:\n",
    "\n",
    "file = open(SciDrive_FileName, \"w\")\n",
    "file.write(SciDrive_FileContent)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#uploading a file to SciDrive:\n",
    "\n",
    "responseUpload = SciDrive.upload(path=SciDrive_Directory + \"/\" + SciDrive_FileName, localFilePath=SciDrive_FilePath)\n",
    "print(responseUpload)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#download file:\n",
    "\n",
    "stringio = SciDrive.download(path=SciDrive_Directory + \"/\" + SciDrive_FileName, format=\"StringIO\")\n",
    "fileContent = stringio.read()\n",
    "print(fileContent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#upload string data:\n",
    "\n",
    "responseUpload = SciDrive.upload(path=SciDrive_Directory + \"/\" + SciDrive_FileName, data=SciDrive_FileContent)\n",
    "fileContent = SciDrive.download(path=SciDrive_Directory + \"/\" + SciDrive_FileName, format=\"text\")\n",
    "print(fileContent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#delete folder in SciDrive:\n",
    "\n",
    "responseDelete = SciDrive.delete(SciDrive_Directory)\n",
    "print(responseDelete)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#delete local file:\n",
    "\n",
    "os.remove(SciDrive_FilePath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *******************************************************************************************************\n",
    "# SkyQuery section:\n",
    "# *******************************************************************************************************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(SkyQuery)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list all databses or datasets available\n",
    "\n",
    "datasets = SkyQuery.listAllDatasets()\n",
    "print(datasets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get info about the user's personal database or dataset\n",
    "\n",
    "info = SkyQuery.getDatasetInfo(\"MyDB\")\n",
    "print(info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list tables inside dataset\n",
    "\n",
    "tables = SkyQuery.listDatasetTables(\"MyDB\")\n",
    "print(tables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list available job queues\n",
    "\n",
    "queueList = SkyQuery.listQueues()\n",
    "print(queueList)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list available job queues and related info\n",
    "\n",
    "quick = SkyQuery.getQueueInfo('quick')\n",
    "long= SkyQuery.getQueueInfo('long')\n",
    "print(quick)\n",
    "print(long)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define query\n",
    "\n",
    "SkyQuery_Query = \"select 4.5 as Column1, 5.5 as Column2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#submit a query as a job\n",
    "\n",
    "jobId = SkyQuery.submitJob(query=SkyQuery_Query, queue=\"quick\")\n",
    "print(jobId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get status of a submitted job\n",
    "\n",
    "jobId = SkyQuery.submitJob(query=SkyQuery_Query, queue=\"quick\")\n",
    "jobDescription = SkyQuery.getJobStatus(jobId=jobId)\n",
    "print(jobDescription)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wait for a job to be finished and then get the status\n",
    "\n",
    "jobId = SkyQuery.submitJob(query=SkyQuery_Query, queue=\"quick\")\n",
    "jobDescription = SkyQuery.waitForJob(jobId=jobId, verbose=True)\n",
    "print(\"jobDescription=\")\n",
    "print(jobDescription)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cancel a job that is running, and then get its status\n",
    "\n",
    "jobId = SkyQuery.submitJob(query=SkyQuery_Query, queue=\"long\")\n",
    "isCanceled = SkyQuery.cancelJob(jobId)\n",
    "print(isCanceled)\n",
    "print(\"job status:\")\n",
    "print(SkyQuery.getJobStatus(jobId=jobId))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get list of jobs\n",
    "\n",
    "quickJobsList = SkyQuery.listJobs('quick')\n",
    "longJobsList = SkyQuery.listJobs('long')\n",
    "print(quickJobsList)\n",
    "print(longJobsList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#define csv table to be uploaded to into MyDB in SkyQuery\n",
    "\n",
    "SkyQuery_TestTableName = \"TestTable_SciScript_R\"\n",
    "SkyQuery_TestTableCSV = u\"Column1,Column2\\n4.5,5.5\\n\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#uploading the csv table:\n",
    "\n",
    "result = SkyQuery.uploadTable(uploadData=SkyQuery_TestTableCSV, tableName=SkyQuery_TestTableName, datasetName=\"MyDB\", format=\"csv\")\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#downloading table:\n",
    "\n",
    "table = SkyQuery.getTable(tableName=SkyQuery_TestTableName, datasetName=\"MyDB\", top=10)\n",
    "print(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#list tables inside dataset\n",
    "\n",
    "tables = SkyQuery.listDatasetTables(\"MyDB\")\n",
    "print(tables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get dataset table info:\n",
    "\n",
    "info = SkyQuery.getTableInfo(tableName=\"webuser.\" + SkyQuery_TestTableName, datasetName=\"MyDB\")\n",
    "print(info)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#get dataset table columns info\n",
    "\n",
    "columns = SkyQuery.listTableColumns(tableName=\"webuser.\" + SkyQuery_TestTableName, datasetName=\"MyDB\")\n",
    "print(columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#drop (or delete) table from dataset.\n",
    "\n",
    "result = SkyQuery.dropTable(tableName=SkyQuery_TestTableName, datasetName=\"MyDB\");\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *******************************************************************************************************\n",
    "# Files section:\n",
    "# *******************************************************************************************************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(Files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# defining the FileService name\n",
    "\n",
    "Files_FileServiceName = \"FileServiceJHU\"\n",
    "Files_RootVolumeName1 = \"Storage\"\n",
    "Files_UserVolumeName1 = \"UserVolume555\"\n",
    "Files_RootVolumeName2 = \"Storage\"\n",
    "Files_UserVolumeName2 = \"UserVolume999\"\n",
    "Files_NewDirectoryName1 = \"myNewDirectory555\"\n",
    "Files_NewDirectoryName2 = \"myNewDirectory999\"\n",
    "Files_LocalFileName = \"MyNewFile.txt\"\n",
    "Files_LocalFileContent = \"#ID,Column1,Column2\\n1,4.5,5.5\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get available FileServices\n",
    "\n",
    "fileServices = Files.getFileServices();\n",
    "print(fileServices)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get names of file services available to the user.\n",
    "\n",
    "fileServiceNames = Files.getFileServicesNames();\n",
    "print(fileServiceNames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get FileService from Name\n",
    "fileService = Files.getFileServiceFromName(Files_FileServiceName);\n",
    "fileService"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the API endpoint URL of a FileService\n",
    "\n",
    "fileServiceAPIUrl = Files.__getFileServiceAPIUrl(fileService);\n",
    "print(fileServiceAPIUrl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get root volumes\n",
    "\n",
    "rootVolumes = Files.getRootVolumesInfo(fileService)\n",
    "rootVolumes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get root volumes\n",
    "\n",
    "userVolumes = Files.getUserVolumesInfo(fileService)\n",
    "userVolumes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "str(True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create user volume\n",
    "\n",
    "Files.createUserVolume(fileService,\n",
    "    \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1]))\n",
    "Files.createUserVolume(fileService, \n",
    "    \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a directory in the persistent volume\n",
    "\n",
    "Files.createDir(fileService, \n",
    "    \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1]));\n",
    "Files.createDir(fileService, \n",
    "    \"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2]));"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# upload a text string into a file in the remote directory\n",
    "\n",
    "Files.upload(fileService, \n",
    "    \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]), \n",
    "             data=Files_LocalFileContent);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List content of remote directory\n",
    "\n",
    "dirList = Files.dirList(fileService, \n",
    "    \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1]),\n",
    "    level=4)\n",
    "dirList"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# download a remote text file into a local directory\n",
    "\n",
    "Files.download(fileService, \n",
    "    \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]) \n",
    "    ,localFilePath=Files_LocalFileName);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Delete remote file\n",
    "\n",
    "Files.delete(fileService, \n",
    "\"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# upload a local file into a remote directory\n",
    "\n",
    "Files.upload(fileService, \n",
    "\"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName])\n",
    ", localFilePath=Files_LocalFileName, quiet=False);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# copy remote file into remote directory\n",
    "\n",
    "Files.move(fileService, \n",
    "\"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1,Files_LocalFileName]), \n",
    "           fileService, \n",
    "\"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2 ,Files_LocalFileName])\n",
    ");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sharing user volume with another user\n",
    "\n",
    "Files.shareUserVolume(fileService, \n",
    "\"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2]), \n",
    "    sharedWith=Authentication_login_sharedWithName, type=\"USER\",allowedActions=[\"read\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# let the other user log-in\n",
    "\n",
    "token1 = Authentication.login(Authentication_login_sharedWithName, Authentication_login_sharedWithPassword);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the other user downloads the remote text file (from the shared user volume) into a local string variable\n",
    "\n",
    "string = Files.download(fileService, \n",
    "    \"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2,Files_LocalFileName]), \n",
    "    format=\"txt\");\n",
    "print(string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# the original user logs in again\n",
    "\n",
    "token1 = Authentication.login(Authentication_loginName, Authentication_loginPassword);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# delete folders\n",
    "\n",
    "Files.delete(fileService, \n",
    "    \"/\".join([Files_RootVolumeName1,Authentication_loginName,Files_UserVolumeName1,Files_NewDirectoryName1]))\n",
    "Files.delete(fileService, \n",
    "    \"/\".join([Files_RootVolumeName2,Authentication_loginName,Files_UserVolumeName2,Files_NewDirectoryName2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# delete user volumes\n",
    "\n",
    "Files.deleteUserVolume(fileService, \n",
    "    \"/\".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName1]))\n",
    "Files.deleteUserVolume(fileService, \n",
    "    \"/\".join([Files_RootVolumeName1, Authentication_loginName, Files_UserVolumeName2]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# *******************************************************************************************************\n",
    "# Jobs section:\n",
    "# *******************************************************************************************************"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#help(Jobs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Jobs_DockerComputeDomainName = 'Small Jobs Domain'\n",
    "\n",
    "Jobs_FileServiceName = \"FileServiceJHU\"\n",
    "Jobs_RootVolumeName = \"Storage\"\n",
    "Jobs_UserVolumeName = \"JobsTestVolume\"\n",
    "Jobs_DirectoryName = \"JobsTestDirectory\"\n",
    "Jobs_NotebookName = 'TestNotebook.ipynb'\n",
    "Jobs_ShellCommand = 'ls -laht'\n",
    "Jobs_DockerImageName = 'Python + R'\n",
    "\n",
    "Jobs_UserVolumes = [{'name':Jobs_UserVolumeName, 'rootVolumeName':Jobs_RootVolumeName, 'owner':Authentication_loginName, 'needsWriteAccess':True},\n",
    "                    {'name':'scratch', 'rootVolumeName':'Temporary', 'owner':Authentication_loginName, 'needsWriteAccess':True}]\n",
    "Jobs_DataVolumes = [{'name':'SDSS DAS'}]\n",
    "Jobs_Parameters =  'param1=1\\nparam2=2\\nparam3=3'\n",
    "Jobs_Alias = 'MyNewJob'\n",
    "\n",
    "Jobs_SqlQuery = \"select 1;\"\n",
    "Jobs_RBBComputeDomainName = 'Manga (long)'\n",
    "Jobs_DatabaseContextName = \"manga\"\n",
    "Jobs_QueryResultsFile = 'myQueryResults'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get docker compute domains\n",
    "\n",
    "dockerComputeDomains = Jobs.getDockerComputeDomains();\n",
    "print(dockerComputeDomains)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get names of docker compute domains available to the user\n",
    "\n",
    "dockerComputeDomainsNames = Jobs.getDockerComputeDomainsNames()\n",
    "print(dockerComputeDomainsNames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get docker compute domain from name\n",
    "\n",
    "dockerComputeDomain = Jobs.getDockerComputeDomainFromName(Jobs_DockerComputeDomainName)\n",
    "dockerComputeDomain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uploading Jupyter notebook to remote directory\n",
    "\n",
    "fileService = Files.getFileServiceFromName(Jobs_FileServiceName);\n",
    "Files.createUserVolume(fileService, \n",
    "                        \"/\".join([Jobs_RootVolumeName, Authentication_loginName, Jobs_UserVolumeName]))\n",
    "Files.upload(fileService, \n",
    "    \"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]), \n",
    "    localFilePath=Jobs_NotebookName);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# submit a Jupyter notebook job.\n",
    "\n",
    "jobId = Jobs.submitNotebookJob(notebookPath='/home/idies/workspace/' + \n",
    "\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n",
    "                             dockerComputeDomain=dockerComputeDomain,\n",
    "                             dockerImageName=Jobs_DockerImageName,\n",
    "                             userVolumes=Jobs_UserVolumes,\n",
    "                             dataVolumes=Jobs_DataVolumes,\n",
    "                             resultsFolderPath=\"\",\n",
    "                             parameters=Jobs_Parameters,\n",
    "                             jobAlias=Jobs_Alias)\n",
    "print(jobId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get job status\n",
    "\n",
    "jobStatus = Jobs.getJobStatus(jobId)\n",
    "print(jobStatus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get job description\n",
    "\n",
    "job = Jobs.getJobDescription(jobId)\n",
    "print(job)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wait until job is finished and get job status\n",
    "\n",
    "jobId = Jobs.submitNotebookJob(notebookPath='/home/idies/workspace/' + \n",
    "\"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName,Jobs_DirectoryName,Jobs_NotebookName]),\n",
    "                             dockerComputeDomain=dockerComputeDomain,\n",
    "                             dockerImageName=Jobs_DockerImageName, \n",
    "                             userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n",
    "                             resultsFolderPath=\"\",\n",
    "                             parameters=Jobs_Parameters, jobAlias=Jobs_Alias)\n",
    "jobStatus = Jobs.waitForJob(jobId)\n",
    "print(jobStatus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# cancel job\n",
    "\n",
    "jobId = Jobs.submitNotebookJob('/home/idies/workspace/' + Jobs_UserVolumeName + '/' + Jobs_DirectoryName + '/' + Jobs_NotebookName,\n",
    "                             dockerComputeDomain=dockerComputeDomain,\n",
    "                             dockerImageName=Jobs_DockerImageName, \n",
    "                             userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n",
    "                             resultsFolderPath=\"\",\n",
    "                             parameters=Jobs_Parameters, jobAlias=Jobs_Alias)\n",
    "Jobs.cancelJob(jobId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Files.deleteUserVolume(fileService, \"/\".join([Jobs_RootVolumeName,Authentication_loginName,Jobs_UserVolumeName]), quiet=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# submit shell command job\n",
    "\n",
    "jobId = Jobs.submitShellCommandJob(shellCommand=Jobs_ShellCommand,\n",
    "                                    dockerComputeDomain=dockerComputeDomain,\n",
    "                                    dockerImageName=Jobs_DockerImageName,\n",
    "                                    userVolumes=Jobs_UserVolumes, dataVolumes=Jobs_DataVolumes,\n",
    "                                    resultsFolderPath = \"\",\n",
    "                                    jobAlias=Jobs_Alias)\n",
    "jobStatus = Jobs.waitForJob(jobId)\n",
    "print(jobStatus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get relational database (RDB) compute domains\n",
    "\n",
    "rdbComputeDomains = Jobs.getRDBComputeDomains();\n",
    "print(rdbComputeDomains)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get names of relational database (RDB) compute domains\n",
    "\n",
    "rdbComputeDomainsNames = Jobs.getRDBComputeDomainsNames()\n",
    "print(rdbComputeDomainsNames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get relational database (RDB) compute domain from name\n",
    "\n",
    "rdbComputeDomain = Jobs.getRDBComputeDomainFromName(Jobs_RBBComputeDomainName);\n",
    "print(rdbComputeDomain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# submit RDB (relatinal database) query job\n",
    "\n",
    "jobId = Jobs.submitRDBQueryJob(Jobs_SqlQuery, rdbComputeDomain, Jobs_DatabaseContextName, Jobs_QueryResultsFile, Jobs_Alias)\n",
    "print(jobId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get job description\n",
    "\n",
    "job = Jobs.getJobDescription(jobId)\n",
    "print(job)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}