new local annotation format

.dockerignore

@@ -3,3 +3,4 @@ build
 dist
 *.egg-info/
 Dockerfile
+*.csv

.gitignore

@@ -6,4 +6,10 @@ dist
 .ipynb_checkpoints
 *.DS_Store
 .vscode
-*.py.save
+*.py.save
+
+cellimages
+example_json/
+*.png
+*.ipynb
+psf

CONTRIBUTING.md

@@ -0,0 +1,26 @@
+# Allen Institute Contribution Agreement
+
+This document describes the terms under which you may make “Contributions” — 
+which may include without limitation, software additions, revisions, bug fixes, configuration changes,
+documentation, or any other materials — to any of the projects owned or managed by the Allen Institute.
+If you have questions about these terms, please contact us at terms@alleninstitute.org.  
+
+You certify that:
+
+•	Your Contributions are either:
+
+1.	Created in whole or in part by you and you have the right to submit them under the designated license 
+(described below); or
+2.	Based upon previous work that, to the best of your knowledge, is covered under an appropriate 
+open source license and you have the right under that license to submit that work with modifications,
+whether created in whole or in part by you, under the designated license; or
+
+3.	Provided directly to you by some other person who certified (1) or (2) and you have not modified them.
+
+•	You are granting your Contributions to the Allen Institute under the terms of the [2-Clause BSD license](https://opensource.org/licenses/BSD-2-Clause)
+(the “designated license”).
+
+•	You understand and agree that the Allen Institute projects and your Contributions are public and that 
+a record of the Contributions (including all metadata and personal information you submit with them) is 
+maintained indefinitely and may be redistributed consistent with the Allen Institute’s mission and the 
+2-Clause BSD license.  

Dockerfile

@@ -1,4 +1,4 @@
-FROM fcollman/render-python:latest
+FROM fcollman/render-modules:master
 MAINTAINER Forrest Collman (forrest.collman@gmail.com)
 RUN mkdir -p /usr/local/render-python-apps
 WORKDIR /usr/local/render-python-apps
@@ -7,7 +7,8 @@ RUN pip install -r requirements.txt
 RUN pip install setuptools --upgrade --disable-pip-version-check
 RUN pip install argschema --upgrade --disable-pip-version-check
 RUN pip install jupyter
-RUN apt-get install libspatialindex-dev -y
+RUN apt-get update && apt-get install libspatialindex-dev -y
+RUN conda install nomkl
 COPY . /usr/local/render-python-apps
 
 #RUN git clone https://github.com/fcollman/render-python-apps

allen_utils/conda_env_setup.sh

@@ -0,0 +1,3 @@
+conda create --name render-python-apps --prefix $IMAGE_PROCESSING_DEPLOY_PATH python=2.7
+source activate render-python-apps
+pip install -r ../requirements.txt

allen_utils/docker_setup.sh

@@ -1,5 +1,5 @@
-docker pull atbigdawg:5000/fcollman/render-python:latest
-docker tag atbigdawg:5000/fcollman/render-python:latest fcollman/render-python:latest
+#docker pull atbigdawg:5000/fcollman/render-python:latest
+#docker tag atbigdawg:5000/fcollman/render-python:latest fcollman/render-python:latest
 docker build -t fcollman/render-python-apps .
 docker tag fcollman/render-python-apps atbigdawg:5000/fcollman/render-python-apps
 docker push atbigdawg:5000/fcollman/render-python-apps
@@ -10,14 +10,11 @@ docker run -t --name renderapps \
 -v /nas2:/nas2 \
 -v /nas3:/nas3 \
 -v /nas4:/nas4 \
+-v /nas5:/nas5 \
 -v /data:/data \
 -v /pipeline:/pipeline \
 -v /pipeline/render-python-apps:/usr/local/render-python-apps \
 -v /etc/hosts:/etc/hosts \
---dns 10.128.104.10 \
--p 8888:8888 \
+-p 7777:7777 \
 -e "PASSWORD=$JUPYTERPASSWORD" \
--i -t fcollman/render-python-apps \
-/bin/bash -c "/opt/conda/bin/jupyter notebook --config=/root/.jupyter/jupyter_notebook_config.py --notebook-dir=/pipeline/render-python-apps --no-browser --allow-root" 
-
-
+-i -t fcollman/render-python-apps 

allen_utils/reorg_setup.sh

@@ -1,21 +1,20 @@
-docker pull atbigdawg:5000/fcollman/render-python:master
-docker tag atbigdawg:5000/fcollman/render-python:master fcollman/render-python
-docker build -t fcollman/render-python-apps:reorg .
-docker tag fcollman/render-python-apps:reorg fcollman/render-python-apps:reorg_testsharmi
-docker push atbigdawg:5000/fcollman/render-python-apps
-docker kill renderapps_testsharmi
-docker rm renderapps_testsharmi
-docker run -d --name renderapps_testsharmi \
+docker pull localhost:5000/fcollman/render-python:master
+docker tag localhost:5000/fcollman/render-python:master fcollman/render-python
+docker build -t sharmi/render-python-apps:reorg .
+#docker tag fcollman/render-python-apps:reorg fcollman/render-python-apps:reorg_testsharmi
+#docker push atbigdawg:5000/fcollman/render-python-apps
+docker kill renderapps_develop
+docker rm renderapps_develop
+docker run -d --name renderapps_develop \
 -v /nas2:/nas2 \
 -v /nas:/nas \
 -v /nas3:/nas3 \
 -v /nas4:/nas4 \
 -v /data:/data \
 -v /pipeline:/pipeline \
--v /pipeline/sharmi/Sharmi_tools/render-python-apps:/usr/local/render-python-apps \
+-v /pipeline/sharmi/Sharmi_tools/render-python-apps-branches/DEVELOP/render-python-apps:/usr/local/share/render-python-apps \
 -v /etc/hosts:/etc/hosts \
--p 9999:9999 \
+-p 7777:7777 \
 -e "PASSWORD=$JUPYTERPASSWORD" \
--i -t fcollman/render-python-apps:reorg_testsharmi
-
-
+-i -t sharmi/render-python-apps:reorg \
+/bin/bash -c "jupyter notebook --config=/root/.jupyter/jupyter_notebook_config.py  --no-browser --allow-root --notebook-dir=/pipeline/sharmi/Sharmi_tools/render-python-apps-branches/DEVELOP/render-python-apps"

jupyter_notebook_config.py

@@ -16,7 +16,7 @@
 from IPython.lib import passwd
 
 c.NotebookApp.ip = '*'
-c.NotebookApp.port = 8888
+c.NotebookApp.port = 7777
 c.NotebookApp.open_browser = False
 c.MultiKernelManager.default_kernel_name = 'python2'
 

renderapps/TrakEM2/AnnotationJsonSchema.py

@@ -19,6 +19,7 @@ class Area(argschema.schemas.mm.Schema):
         description='Nx2 numpy array of local points')
     global_path = NumpyArray(argschema.fields.List(argschema.fields.Float),
         description='Nx2 numpy array of global coordinates')
+    z = argschema.fields.Float(required=False,description="z value of tileId")
 
 class AreaList(argschema.schemas.mm.Schema):
     oid = argschema.fields.Str()

renderapps/TrakEM2/ImportTrakEM2Annotations.py

@@ -18,14 +18,15 @@
         "project": "M247514_Rorb_1",
         "client_scripts": "/pipeline/render/render-ws-java-client/src/main/scripts"
     },
-    "EMstack": "ALIGNEM_reg2",
-    "trakem2project": "/nas4/data/EM_annotation/annotationFilesForJHU/annotationTrakEMprojects_M247514_Rorb_1/m247514_Site3Annotation_RD.xml",
-    "output_annotation_file": "/nas4/data/EM_annotation/annotationFilesForJHU/m247514_Site3Annotation_RD_local.json",
-    "output_bounding_box_file": "/nas4/data/EM_annotation/annotationFilesForJHU/m247514_Site3Annotation_RD_bb_local.json",
+    "EMstack": "BIGALIGN_LENS_EMclahe_all",
+    "trakem2project": "/nas4/data/EM_annotation/M247514_Rorb_1/m247514_Site3Annotation_MN.xml",
+    "output_annotation_file": "/nas3/data/M247514_Rorb_1/annotation/m247514_Site3Annotation_MN_local.json",
+    "output_bounding_box_file":"/nas3/data/M247514_Rorb_1/annotation/m247514_Site3Annotation_MN_bb_local.json",
     "renderHome": "/pipeline/render"
 }
 
 
+
 class AnnotationConversionError(Exception):
     pass
 
@@ -98,8 +99,10 @@ def convert_path_to_area(path_numpy, layer_tilespecs):
             local_path_numpy[convert_point_mask_ind, :] = local_points
             point_missing[convert_point_mask_ind] = 0
             local_tileIds[convert_point_mask_ind] = rts.tileId
-
-    assert(np.sum(point_missing) == 0)
+
+    if np.sum(point_missing)>0:
+        raise AnnotationConversionError("unable to find all points {} on the tiles given {}".format(path_numpy,[ts.tileId for poly,ts,rts in layer_tilespecs]))
+
 
     d = {}
     d['tileIds'] = local_tileIds
@@ -134,7 +137,10 @@ def parse_area_lists(render_tilespecs, tem2_tilespecs, tem2_polygons, root, area
             for path in paths:
                 # initialize the path's polygon with the entire path
                 path_numpy = convert_path(path, tform)
-                d = convert_path_to_area(path_numpy, layer_tilespecs)
+                try:
+                    d = convert_path_to_area(path_numpy, layer_tilespecs)
+                except AnnotationConversionError as e:
+                    raise AnnotationConversionError("error in converting synapse oid:{} id:{} on layer:{}, {}".format(al.attrib['oid'],thisid,layerid,e.args))
                 area_list_d['areas'].append(d)
 
         json_output['area_lists'].append(area_list_d)
@@ -168,8 +174,13 @@ def run(self):
             pot_render_tilespecs = self.render.run(renderapi.tilespec.get_tile_specs_from_z,
                                                    self.args['EMstack'],
                                                    ts.z)
-            filepath = (os.path.split(ts.ip.get(0)['imageUrl'])[
-                        1]).split('_flip')[0]
+            try:
+                mml=ts.ip.get(0)
+            except KeyError:
+                mml = ts.channels[0].ip.get(0)
+
+
+            filepath = (os.path.split(mml['imageUrl'])[1]).split('_flip')[0]
             pot_filepaths = [(os.path.split(t.ip.get(0)['imageUrl'])[1]).split(
                 '_flip')[0] for t in pot_render_tilespecs]
             render_tilespecs.append(next(t for t, fp in zip(
@@ -204,7 +215,10 @@ def run(self):
             layer_tilespecs = [(poly, ts, t) for poly, ts, t
                                in zip(tem2_polygons, tem2_tilespecs, render_tilespecs)
                                if ts.tileId in patchids]
-            d = convert_path_to_area(corners,layer_tilespecs)
+            try:
+                d = convert_path_to_area(corners,layer_tilespecs)
+            except AnnotationConversionError as e:
+                raise AnnotationConversionError("unable to find bounding box of layer {} z={}, due to corners not being on layers with patches {} ".format(layer.attrib['oid'],layer.attrib['z'],patchids))
             area_list_d = {}
             area_list_d['oid'] = layer.attrib['oid']
             area_list_d['id'] = int(layer_tilespecs[0][2].z)