Plot3D_01_fill_iRODS.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits import mplot3d\n",
    "from shutil import copyfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Axes:\n",
    "    \n",
    "    def __init__(self, filename):\n",
    "        # initialise the axes by reading data from the metadata (.meta.txt) file\n",
    "        self.process = True\n",
    "        with open(filename) as f:\n",
    "            lines = [line.rstrip('\\n') for line in open(filename)]\n",
    "        if lines[0] != '#inner loop':\n",
    "            print(\"problem initialising Axes: inner loop \"+filename)\n",
    "            self.process = False\n",
    "        else:\n",
    "            self.no_inner_loop = int(lines[1])\n",
    "            self.min_inner_loop = lines[2]\n",
    "            self.max_inner_loop = lines[3]\n",
    "            self.label_inner_loop = lines[4]\n",
    "            if lines[5] != '#outer loop':\n",
    "                print(\"problem initialising Axes: outer loop \"+filename)\n",
    "                self.process = False\n",
    "            else:\n",
    "                self.no_outer_loop = int(lines[6])\n",
    "                self.min_outer_loop = lines[7]\n",
    "                self.max_outer_loop = lines[8]\n",
    "                self.label_outer_loop = lines[9]\n",
    "                if lines[10] != '#outer most loop':\n",
    "                    self.no_outer_most_loop = 0\n",
    "                    self.min_outer_most_loop = 0\n",
    "                    self.max_outer_most_loop = 0\n",
    "                    self.label_outer_most_loop = \"\"\n",
    "                else:\n",
    "                    self.no_outer_most_loop = int(lines[11])\n",
    "                    self.min_outer_most_loop = lines[12]\n",
    "                    self.max_outer_most_loop = lines[13]\n",
    "                    self.label_outer_most_loop = lines[14]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Data:\n",
    "    \n",
    "    def __init__(self, filename):\n",
    "        # initialise the experimental data by reading the header from the data (.dat) file\n",
    "        self.process = True # should this experiment be processed?\n",
    "        self.axes = []      # axes labels\n",
    "        self.columns = {}   # column labels and index numbers\n",
    "        with open(filename) as f:\n",
    "            lines = [line.rstrip('\\n') for line in open(filename)]\n",
    "            line = 3\n",
    "            col_no = 1\n",
    "            while lines[line].find(\"Column\") != -1:\n",
    "                if lines[line+3].find(\"type: coordinate\") > 0:\n",
    "                    axis_text = lines[line+1][lines[line+1].find(\": \")+2:]\n",
    "                    self.axes.append(axis_text)\n",
    "                    line += 4\n",
    "                elif lines[line+2].find(\"type: value\") > 0:\n",
    "                    column_text = lines[line+1][lines[line+1].find(\": \")+2:]\n",
    "                    self.columns[column_text]=col_no\n",
    "                    line += 3\n",
    "                col_no += 1\n",
    "        if len(self.axes) < 2: self.process = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class File_set:\n",
    "    \n",
    "    def __init__(self, sourceDir, filename):\n",
    "        # locate the dat sets from the .dat file, do sanity check and copy the complete data sets to their target destination\n",
    "        self.input_files_present = False\n",
    "        if sourceDir.startswith(sourceTree) and filename.endswith(\".dat\"):\n",
    "            no_dat = 0\n",
    "            no_py = 0\n",
    "            for file in os.listdir(sourceDir): # if there is more than 1 .dat file in a directory, create an extra directory\n",
    "                if file.endswith(\".dat\"):\n",
    "                    no_dat +=1\n",
    "                elif file.endswith(\".py\"):\n",
    "                    self.pyfile = file\n",
    "                    no_py +=1\n",
    "            if no_dat > 1:\n",
    "                self.extra_dir = filename[:-4]\n",
    "            else:\n",
    "                self.extra_dir = \"\"\n",
    "            \n",
    "            self.datfile = filename\n",
    "            file_base = self.datfile[:-4]\n",
    "            path_base = os.path.join(sourceDir,filename)[:-4]\n",
    "            if os.path.exists(path_base+\".meta.txt\"):\n",
    "                self.metafile = file_base+\".meta.txt\"\n",
    "                if os.path.exists(path_base+\".set\"):\n",
    "                    self.setfile = file_base+\".set\"\n",
    "                    self.input_files_present = True\n",
    "\n",
    "                    if os.path.exists(path_base+\".py\"):\n",
    "                        self.pyfile = file_base+\".py\"\n",
    "                    elif no_py != 1:\n",
    "                        self.pyfile = \"\"\n",
    "        else:\n",
    "            print(\"problem initialising the input files: invalid calling parameters\")        \n",
    "        \n",
    "    def copy_file_set(self, sourceTree, destTree):\n",
    "        success = False\n",
    "        if sourceDir.startswith(sourceTree):\n",
    "            success = True\n",
    "            destDir = destTree+sourceDir[len(sourceTree):]\n",
    "            if not os.path.exists(destDir): os.makedirs(destDir)\n",
    "            if self.extra_dir != \"\":\n",
    "                destDir = os.path.join(destDir,self.extra_dir)\n",
    "                if not os.path.exists(destDir): os.makedirs(destDir)\n",
    "            self.destDir = destDir\n",
    "            if not os.path.isfile(os.path.join(destDir,self.datfile)):\n",
    "                try:\n",
    "                    copyfile(os.path.join(sourceDir,self.datfile), os.path.join(destDir,self.datfile))\n",
    "                except:\n",
    "                    success = False\n",
    "            if not os.path.isfile(os.path.join(destDir,self.metafile)):\n",
    "                try:\n",
    "                    copyfile(os.path.join(sourceDir,self.metafile), os.path.join(destDir,self.metafile))\n",
    "                except:\n",
    "                    success = False\n",
    "            if not os.path.isfile(os.path.join(destDir,self.setfile)):\n",
    "                try:\n",
    "                    copyfile(os.path.join(sourceDir,self.setfile), os.path.join(destDir,self.setfile))\n",
    "                except:\n",
    "                    success = False\n",
    "            if self.pyfile != \"\" and not os.path.isfile(os.path.join(destDir,self.pyfile)):\n",
    "                try:\n",
    "                    copyfile(os.path.join(sourceDir,self.pyfile), os.path.join(destDir,self.pyfile))\n",
    "                except:\n",
    "                    success = False\n",
    "        return success   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_plots(base, sourceDir, destDir, axes, data):\n",
    "\n",
    "    no_plots = 0\n",
    "    datfile = os.path.join(sourceDir,base)+\".dat\"\n",
    "    toBase = os.path.join(destDir,base)+\".dat\"\n",
    "    plot3D = len(data.axes) == 3\n",
    "    if plot3D:\n",
    "        xdata = np.genfromtxt(datfile, usecols=(0))\n",
    "        ydata = np.genfromtxt(datfile, usecols=(1))\n",
    "        zdata = np.genfromtxt(datfile, usecols=(2))\n",
    "    for column_text, col in data.columns.items():\n",
    "        no_plots +=1\n",
    "        if not os.path.exists(toBase+'_plot_'+str(col)+'.jpg'): # don't overwrite\n",
    "            data = np.genfromtxt(datfile, usecols=(col-1))\n",
    "            fig = plt.figure()\n",
    "            if plot3D:\n",
    "                ax = fig.add_subplot(111, projection='3d')\n",
    "                extremes = sorted(data)\n",
    "                data_points = len(extremes)\n",
    "                no_to_include = int(data_points/100.)\n",
    "                large_min = extremes[no_to_include]\n",
    "                small_max = extremes[data_points-1-no_to_include]\n",
    "\n",
    "                xplot=[]\n",
    "                yplot=[]\n",
    "                zplot=[]\n",
    "                cplot=[]\n",
    "                for index, value in enumerate(data):\n",
    "                    if value <= large_min or value >= small_max:\n",
    "                        xplot.append(xdata[index])\n",
    "                        yplot.append(ydata[index])\n",
    "                        zplot.append(zdata[index])\n",
    "                        cplot.append(data[index])\n",
    "\n",
    "                im = ax.scatter3D(xplot, yplot, zplot, c=cplot, cmap=None)\n",
    "\n",
    "                #ax.set_xticks([axes.min_inner_loop, axes.max_inner_loop])\n",
    "                #ax.set_yticks([axes.min_outer_loop, axes.max_outer_loop])\n",
    "                \n",
    "                ax.set_zlabel(axes.label_outer_most_loop)\n",
    "                #ax.set_zticks([axes.min_outer_most_loop, axes.max_outer_most_loop])\n",
    "                #ax.set_zticklabels([axes.min_outer_loop, axes.max_outer_most_loop])\n",
    "\n",
    "            else:\n",
    "                heatmap = np.resize(data,(axes.no_outer_loop,axes.no_inner_loop))\n",
    "    \n",
    "                ax = fig.add_subplot(111)\n",
    "                im = ax.imshow(heatmap, interpolation='nearest', cmap=None)\n",
    "                ax.set_xticks([0, axes.no_inner_loop])\n",
    "                ax.set_xticklabels([axes.min_inner_loop, axes.max_inner_loop])\n",
    "                ax.set_yticks([0, axes.no_outer_loop])\n",
    "                ax.set_yticklabels([axes.min_outer_loop, axes.max_outer_loop])\n",
    "\n",
    "            ax.set_xlabel(axes.label_inner_loop)\n",
    "            ax.set_ylabel(axes.label_outer_loop)\n",
    "\n",
    "            ax.set_title(column_text)\n",
    "\n",
    "            # Add a colour bar along the bottom and label it\n",
    "            cbar = fig.colorbar(ax=ax, mappable=im, orientation='horizontal')\n",
    "            cbar.set_label(base) #looks better this way\n",
    "\n",
    "            #plt.show()\n",
    "            plt.savefig(toBase+'_plot_'+str(col)+'.jpg')\n",
    "            plt.close()\n",
    "    return no_plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_metadata(imetafile, setfile, destDir, no_plots):\n",
    "    \n",
    "    with open(setfile) as f:\n",
    "        lines = [line.rstrip('\\n') for line in open(setfile)]\n",
    "    instrument = \"\"\n",
    "    for line in lines:\n",
    "        try:\n",
    "            tag, value = line.split(\": \")\n",
    "        except:\n",
    "            continue\n",
    "        tag.strip()\n",
    "        tag = tag.replace(\"\\t\", \"\")\n",
    "        if tag == \"Instrument\":\n",
    "            instrument = value+\"_\"\n",
    "        else:\n",
    "            tag.strip()\n",
    "            #SEB if value.strip() != \"None\":\n",
    "            imetafile.write(\"imeta add -C \"+str(destDir)+\" \"+instrument+tag+\" '\"+value+\"' '' \\n\") #SEB added '' unit\n",
    "    #SEB imetafile.write(\"imeta add -C \"+str(destDir)+\" number_of_plots '\"+str(no_plots)+\"' '' \\n\") #SEB added '' unit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# main calling loop\n",
    "sourceTree = 'D:' #Source of files and metadata\n",
    "destTree = 'Y:/garys' #iRODS destination - mapped as disc drive with davrods\n",
    "if not os.path.exists(destTree): os.makedirs(destTree)\n",
    "metadata_file = open(os.path.join(destTree,\"metadata.txt\"),\"w\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "todo_list=[]\n",
    "for sourceDir, subdirList, fileList in os.walk(sourceTree):\n",
    "\n",
    "    for fname in fileList:\n",
    "        #print('\\t%s' % fname)\n",
    "        no_plots = 0\n",
    "        if fname.endswith(\".dat\"):\n",
    "            fs = File_set(sourceDir, fname)\n",
    "            if fs.input_files_present:\n",
    "                make_jpgs = fs.copy_file_set(sourceTree, destTree)\n",
    "                try:\n",
    "                    a = Axes(os.path.join(sourceDir,fs.metafile))\n",
    "                    if a.process:\n",
    "                        d = Data(os.path.join(sourceDir,fs.datfile))\n",
    "                except:\n",
    "                    make_jpgs = False\n",
    "                if make_jpgs and a.process and d.process:\n",
    "                    consistent = True\n",
    "                    if d.axes[0] != a.label_inner_loop or d.axes[1] != a.label_outer_loop:\n",
    "                        print(\"problem: loop information inconsistent \"+os.path.join(sourceDir,fs.metafile))\n",
    "                        consistent = False\n",
    "                    if len(d.axes) == 3:\n",
    "                        if d.axes[2] != a.label_outer_most_loop:\n",
    "                            print(\"problem: 3rd axis information inconsistent \"+os.path.join(sourceDir,fs.metafile))\n",
    "                            consistent = False\n",
    "                    if consistent == True:\n",
    "                        todo_list.append(fname)\n",
    "                        no_plots = generate_plots(fname[:-4], sourceDir, fs.destDir, a, d)\n",
    "                add_metadata(metadata_file, os.path.join(sourceDir,fs.setfile), fs.destDir, no_plots)\n",
    "                # add_metadata only possible if copy_file_set has been run first\n",
    "            else:\n",
    "                print(\"problem: missing input files \"+os.path.join(sourceDir,fname))\n",
    "                continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(todo_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata_file.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}