Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Multchan #42

Open
wants to merge 5 commits into
base: develop
Choose a base branch
from
Open

Multchan #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d8d8aa2
Updaetd create_subvolume_stack.py to use argschema
TotteKarlsson Dec 6, 2018
34a9b18
Updated to use the 'ImagePyramid'
TotteKarlsson Dec 10, 2018
508f9bf
Removed usage of hardcoded 'processed' folder.
TotteKarlsson Dec 20, 2018
c6ee446
Readded a '_' before "Session"
TotteKarlsson Dec 20, 2018
c7fafac
Added EMAligner requirement to requirement.txt
TotteKarlsson Jan 14, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
8 changes: 1 addition & 7 deletions Dockerfile
View file
Open in desktop
Expand Up @@ -3,7 +3,6 @@ MAINTAINER Forrest Collman (forrest.collman@gmail.com)
RUN mkdir -p /usr/local/render-python-apps
WORKDIR /usr/local/render-python-apps
COPY requirements.txt /usr/local/render-python-apps
RUN pip install -r requirements.txt
RUN pip install setuptools --upgrade --disable-pip-version-check
RUN pip install argschema --upgrade --disable-pip-version-check
RUN pip install jupyter
Expand All @@ -15,14 +14,9 @@ WORKDIR /shared/render-modules
RUN pip install .
WORKDIR /usr/local/render-python-apps


#RUN git clone https://github.com/fcollman/render-python-apps
#WORKDIR render-python-apps
#RUN git pull && git checkout newrender
#RUN python setup.py install
COPY jupyter_notebook_config.py /root/.jupyter/


RUN pip install -r requirements.txt
RUN python setup.py install

CMD ["jupyter", "notebook", "--no-browser", "--allow-root"]
15 changes: 10 additions & 5 deletions renderapps/dataimport/create_fast_stacks_multi.py
View file
Open in desktop
Expand Up @@ -20,6 +20,7 @@
},
"statetableFile": "/nas2/data/M335503_Ai139_smallvol/scripts/statetable_ribbon_5_session_1_section_1",
"projectDirectory": "/nas2/data/M335503_Ai139_smallvol",
"dataOutputFolder": "processed",
"outputStackPrefix": "TESTAcquisition",
"pool_size": 20,
"delete_stack": False
Expand All @@ -31,6 +32,9 @@ class CreateFastStacksParameters(RenderParameters):
description='state table file')
projectDirectory = InputDir(required=True,
description='path to project root')
dataOutputFolder = Str(required=False,
default="processed",
description='Subfolder containing processed data')
reference_channel = Str(required=False,
default="DAPI",
description="will take the first channel which contains these letters and make it the reference image")
Expand All @@ -42,7 +46,7 @@ class CreateFastStacksParameters(RenderParameters):
description='flag to decide whether stack should be deleted before new upload')


def make_tilespec_from_statetable(df, rootdir, outputProject, outputOwner, outputStack, reference_channel='dapi', minval=0, maxval=50000):
def make_tilespec_from_statetable(df, rootdir, outputProject, outputOwner, outputStack, dataOutputFolder="processed", reference_channel='dapi', minval=0, maxval=50000):
df = df[df['zstack'] == 0]
#ribbons = df.groupby('ribbon')
# zoffset=0
Expand All @@ -69,12 +73,12 @@ def make_tilespec_from_statetable(df, rootdir, outputProject, outputOwner, outpu
for ind, row in frame_group.iterrows():
filepath = row.full_path
fileparts = filepath.split(os.path.sep)[1:]
tilespecdir = rootdir + "/processed/downsamp_tilespec/" + \
tilespecdir = rootdir + "/" + dataOutputFolder + "/downsamp_tilespec/" + \
fileparts[5]+"/"+fileparts[6]+"/"+fileparts[7]
if not os.path.isdir(tilespecdir):
os.makedirs(tilespecdir)
downdir = rootdir+"/processed/downsamp_images/" + \
fileparts[5]+"/"+fileparts[6]+"/"+fileparts[7]
downdir = rootdir + "/" + dataOutputFolder + "/downsamp_images/" + \
fileparts[5] + "/" + fileparts[6] + "/"+fileparts[7]

if not os.path.exists(downdir):
os.makedirs(downdir)
Expand Down Expand Up @@ -155,6 +159,7 @@ def run(self):
statetablefile = self.args['statetableFile']
rootdir = self.args['projectDirectory']
reference_channel = self.args['reference_channel']
dataOutputFolder = self.args['dataOutputFolder']
print("This is delete stack : ")
print(self.args['delete_stack'])
# exit(0)
Expand All @@ -169,7 +174,7 @@ def run(self):

self.logger.info("creating tilespecs and cmds....")
tilespecpaths, mipmap_args = make_tilespec_from_statetable(
session_df, rootdir, outputProject, outputOwner, outputStack, reference_channel)
session_df, rootdir, outputProject, outputOwner, outputStack, dataOutputFolder, reference_channel)
self.logger.info("importing tilespecs into render....")
self.logger.info("creating downsampled images ...")
with renderapi.client.WithPool(self.args['pool_size']) as pool:
Expand Down
201 changes: 104 additions & 97 deletions renderapps/materialize/make_downsample_image_stack.py
View file
Open in desktop
Expand Up @@ -9,65 +9,70 @@
import glob
import time
import numpy as np
import time
from PIL import Image
from renderapi.transform.leaf import AffineModel
from renderapi.tilespec import TileSpec, Layout
from renderapi.channel import Channel
from renderapi.image_pyramid import MipMap, ImagePyramid
import tifffile

#modified and fixed by Sharmishtaa Seshamani

example_parameters={
"render":{
"host":"ibs-forrestc-ux1",
"port":80,
"owner":"S3_Run1",
"project":"S3_Run1_Igor",
"client_scripts":"/var/www/render/render-ws-java-client/src/main/scripts"



# modified and fixed by Sharmishtaa Seshamani, Leila, tk and Forrest.
example_parameters = {
"render": {
"host": "ibs-forrestc-ux1",
"port": 80,
"owner": "S3_Run1",
"project": "S3_Run1_Igor",
"client_scripts": "/var/www/render/render-ws-java-client/src/main/scripts"
},
'input_stack':'Stitched_DAPI_1_dropped',
'output_stack':'Stitched_DAPI_1_Lowres_62_to_67',
'image_directory':'/nas3/data/S3_Run1_Igor/processed/Low_res',
'pool_size':20,
'scale': 0.05
'input_stack': 'Stitched_DAPI_1_dropped',
'output_stack': 'Stitched_DAPI_1_Lowres_62_to_67',
'image_directory': '/nas3/data/S3_Run1_Igor/processed/Low_res',
'pool_size': 20,
'scale': 0.05
}


class MakeDownsampleSectionStackParameters(RenderParameters):
input_stack = Str(required=True,
description='stack to make a downsample version of')
scale = Float(required=False,default = .01,
description='scale to make images')
description='stack to make a downsample version of')
scale = Float(required=False, default=.01,
description='scale to make images')
image_directory = Str(required=True,
metadata={'decription','path to save section images'})
metadata={'decription', 'path to save section images'})
numsectionsfile = Str(required=True,
metadata={'decription','file to save length of sections'})
metadata={'decription', 'file to save length of sections'})
output_stack = Str(required=True,
metadata={'description':'output stack to name'})
pool_size = Int(required=False,default=20,
metadata={'description':'number of parallel threads to use'})
minZ = Int(required=False,default=0,
metadata={'description':'Minimum Z value'})
maxZ = Int(required=False,default=100000000,
metadata={'description':'Maximum Z value'})
metadata={'description': 'output stack to name'})
pool_size = Int(required=False, default=20,
metadata={'description': 'number of parallel threads to use'})
minZ = Int(required=False, default=0,
metadata={'description': 'Minimum Z value'})
maxZ = Int(required=False, default=100000000,
metadata={'description': 'Maximum Z value'})


def process_z(render,stack,output_dir,scale,project,tagstr,Z):
def process_z(render, stack, output_dir, scale, project, tagstr, Z):

z = Z[0]
newz = Z[1]

args=['--stack', stack,
'--scale',str(scale),
'--rootDirectory',output_dir ,
str(z)]

args = ['--stack', stack,
'--scale', str(scale),
'--rootDirectory', output_dir,
str(z)]

print args
print project

#############render.run(renderapi.client.call_run_ws_client, 'org.janelia.render.client.RenderSectionClient', add_args = args)



stackbounds = renderapi.stack.get_stack_bounds(stack,render=render)
sectionbounds = renderapi.stack.get_bounds_from_z(stack,z,render=render)
stackbounds = renderapi.stack.get_stack_bounds(stack, render=render)
sectionbounds = renderapi.stack.get_bounds_from_z(stack, z, render=render)

cx1 = (stackbounds['minX'] - stackbounds['maxX'])/2
cy1 = (stackbounds['minY'] - stackbounds['maxY'])/2
Expand All @@ -82,57 +87,65 @@ def process_z(render,stack,output_dir,scale,project,tagstr,Z):

print "This is z: "
print z
print "These are stack bounds!"
print "These are stack bounds: "
print stackbounds
print "These are section bounds!"
print "These are section bounds: "
print sectionbounds


tilespecdir = os.path.join(output_dir,project,stack,'sections_at_%s'%str(scale),'tilespecs_%s'%tagstr)
tilespecdir = os.path.join(output_dir, project, stack, 'sections_at_%s' % str(
scale), 'tilespecs_%s' % tagstr)
tilespecdir = os.path.normpath(tilespecdir)
if os.path.exists(tilespecdir):
print "Path Exists!"
print ("Path Exists: " + tilespecdir)
else:
os.makedirs(tilespecdir)
os.makedirs(tilespecdir)

[q,r] = divmod(z,1000)
[q, r] = divmod(z, 1000)
s = int(r/100)

directory = os.path.join(output_dir,project,stack,'sections_at_%s'%str(scale),'%03d'%q,"%d"%s)
directory = os.path.join(output_dir, project, stack,
'sections_at_%s' % str(scale), '%03d' % q, "%d" % s)
directory = os.path.normpath(directory)
print("checking for: " + directory)
print(os.path.exists(directory))
if not os.path.exists(directory):
os.makedirs(directory)
#im = Image.fromarray(bb)
#im.save(filename)
#tifffile.imsave(filename,bb)
os.makedirs(directory)
print(os.path.exists(directory))
else:
print ("did not make directory: " + directory)

filename = os.path.join(output_dir,project,stack,'sections_at_%s'%str(scale),'%03d'%q,"%d"%s,'%s.tif'%str(z))

filename = os.path.join(output_dir, project, stack, 'sections_at_%s' % str(
scale), '%03d' % q, "%d" % s, '%s.tif' % str(z))
filename = os.path.normpath(filename)
print("checking for: " + filename)
print(os.path.isfile(filename))
if not os.path.isfile(filename):
renderapi.client.renderSectionClient(stack, output_dir, [z], scale=str(scale), render=render, format='tif', doFilter=True, fillWithNoise=False)

tilespecs = renderapi.tilespec.get_tile_specs_from_z(stack,z,render=render)
t = tilespecs[0]
d = t.to_dict()
d['mipmapLevels'][0]['imageUrl'] = filename
d['minIntensity'] = 0
d['maxIntensity'] = 255
d['width'] = stackbounds['maxX']/20
d['height'] = stackbounds['maxY']/20
d['z'] = newz
d['transforms']['specList'][0]['dataString'] = "20.0000000000 0.0000000000 0.0000000000 20.0000000000 %d %d"%(0.0,0.0)
#if len(['transforms']['specList']) > 1:
# for i in range(1,len(['transforms']['specList'])):
# d['transforms']['specList'][i]['dataString'] = "1.0000000000 0.0000000000 0.0000000000 1.0000000000 0000.00 0000.00"
#d['transforms']['specList'][2]['dataString'] = "1.0000000000 0.0000000000 0.0000000000 1.0000000000 0000.00 0000.00"
t.from_dict(d)
renderapi.client.renderSectionClient(stack, output_dir, [z], scale=str(
scale), render=render, format='tif', doFilter=True, fillWithNoise=False)
print(os.path.exists(filename))
else:
print("Did not make file")

tilespecs = renderapi.tilespec.get_tile_specs_from_z(
stack, z, render=render)
t = TileSpec(tileId=tilespecs[0].tileId, imageUrl=filename, width=stackbounds['maxX']
* scale, height=stackbounds['maxY']*scale, minint=0, maxint=255, z=newz, layout=tilespecs[0].layout)
a = AffineModel(M00=20.0000000000, M01=0.0000000000,
M10=0.0000000000, M11=20.0000000000, B0=0.0, B1=0.0)
ip = ImagePyramid()
ip[0] = MipMap(imageUrl=filename)
t.ImagePyramid = ip
t.tforms = [a]
allts = [t]
tilespecfilename = os.path.join(output_dir,project,stack,'sections_at_%s'%str(scale),'tilespecs_%s'%tagstr,'tilespec_%04d.json'%z)
tilespecfilename = os.path.join(output_dir, project, stack,'sections_at_%s'%str(scale),'tilespecs_%s'%tagstr,'tilespec_%04d.json'%z)
print tilespecfilename
fp = open(tilespecfilename,'w')
json.dump([ts.to_dict() for ts in allts] ,fp,indent=4)
fp = open(tilespecfilename, 'w')
json.dump([ts.to_dict() for ts in allts] , fp, indent=4)
fp.close()


# @Parameter(names = "--stack", description = "Stack name", required = true)
# @Parameter(names = "--stack", description = "Stack name", required = true)
# private String stack;
#
# @Parameter(names = "--rootDirectory", description = "Root directory for rendered layers (e.g. /tier2/flyTEM/nobackup/rendered_boxes)", required = true)
Expand All @@ -155,60 +168,54 @@ def process_z(render,stack,output_dir,scale,project,tagstr,Z):
#

class MakeDownsampleSectionStack(RenderModule):
def __init__(self,schema_type=None,*args,**kwargs):
def __init__(self, schema_type=None, *args,**kwargs):
if schema_type is None:
schema_type = MakeDownsampleSectionStackParameters
super(MakeDownsampleSectionStack,self).__init__(schema_type=schema_type,*args,**kwargs)
super(MakeDownsampleSectionStack, self).__init__(schema_type=schema_type, *args,**kwargs)

def run(self):
allzvalues = self.render.run(renderapi.stack.get_z_values_for_stack,
self.args['input_stack'])

allzvalues = np.array(allzvalues)
zvalues = allzvalues[(allzvalues >= self.args['minZ']) & (allzvalues <=self.args['maxZ'])]

tagstr = "%s_%s"%(self.args['minZ'],self.args['maxZ'])
zvalues = allzvalues[(allzvalues >= self.args['minZ']) & (
allzvalues <= self.args['maxZ'])]

tagstr = "%s_%s" %(self.args['minZ'], self.args['maxZ'])

directoryname = os.path.dirname(self.args['numsectionsfile'])
if not os.path.isdir(directoryname):
os.makedirs(directoryname)

f = open(self.args['numsectionsfile'],'w')
f.write("%d"%len(zvalues))
f = open(self.args['numsectionsfile'], 'w')
f.write("%d" % len(zvalues))
f.close()


newzvalues = range(0,len(zvalues))
newzvalues = range(0, len(zvalues))
Z = []
for i in range(0,len(zvalues)):
Z.append( [zvalues[i], newzvalues[i]])
for i in range(0, len(zvalues)):
Z.append([zvalues[i], newzvalues[i]])

print self.args['input_stack']
print self.args['pool_size']
print self.args['image_directory']
print self.args['scale']
print newzvalues
#exit(0)

render=self.render
mypartial = partial(process_z,self.render,self.args['input_stack'],
self.args['image_directory'],self.args['scale'],self.args['render']['project'],tagstr)
render = self.render
mypartial = partial(process_z, self.render, self.args['input_stack'],
self.args['image_directory'], self.args['scale'], self.args['render']['project'],tagstr)
with renderapi.client.WithPool(self.args['pool_size']) as pool:
pool.map(mypartial,Z)

pool.map(mypartial, Z)

t = os.path.join(self.args['image_directory'],self.args['render']['project'],self.args['input_stack'],'sections_at_%s'%str(self.args['scale']),'tilespecs_%s'%tagstr)
jsonfiles = glob.glob("%s/*.json"%t)
t = os.path.join(self.args['image_directory'], self.args['render']['project'], self.args['input_stack'],'sections_at_%s'%str(self.args['scale']),'tilespecs_%s'%tagstr)
jsonfiles = glob.glob("%s/*.json" % t)

renderapi.stack.create_stack(self.args['output_stack'],cycleNumber=5,cycleStepNumber=1,stackResolutionX = 1, stackResolutionY = 1, render=self.render)
renderapi.client.import_jsonfiles_parallel(self.args['output_stack'],jsonfiles,render=self.render)
#sv = renderapi.stack.get_stack_metadata(self.args['input_stack'], render=self.render)
#print sv
#renderapi.stack.set_stack_metadata(self.args['output_stack'], sv, render=self.render)
#renderapi.stack.set_stack_state(self.args['output_stack'],state='COMPLETE',render=self.render)
renderapi.stack.create_stack(self.args['output_stack'], cycleNumber=5, cycleStepNumber=1,stackResolutionX = 1, stackResolutionY = 1, render=self.render)
renderapi.client.import_jsonfiles_parallel(self.args['output_stack'], jsonfiles, render=self.render)

if __name__ == "__main__":
#mod = MakeDownsampleSectionStack(input_data=example_parameters)
mod = MakeDownsampleSectionStack(schema_type=MakeDownsampleSectionStackParameters)
mod = MakeDownsampleSectionStack(
schema_type=MakeDownsampleSectionStackParameters)

mod.run()