[vtkusers] Streaming Pipeline
Mark Bolstad
the.render.dude at gmail.com
Wed Mar 18 11:23:29 EDT 2009
Hello list,
Can someone look at this simple pipeline and explain if I'm using the
StreamingPipeline correctly? It's operating on a set of 6k x 6k images, but
any large image set would work. I would expect that the streamer would help
reduce the memory usage, but I'm blowing the memory on my mac no matter how
I set it,
Note: the image reading code is not the most efficient. We were having some
issues reading these large images and were experimenting with different
techniques. Suggestions for a more efficient pipeline is appreciated. Also,
note that in the real application were reading in two different 6k x 6k
image stacks and overlaying them.
Thanks,
Mark
File:
# ------------------------- imports -------------------------
import os
import vtk
# ------------------------- constants -------------------------
graydir = "/Users/bolstadm/Desktop/6k_6k_150"
grayfilepattern = "image.v0.%05d.png"
# sp maps instead:
# graydir = os.path.join(basedir, "superpixel_maps")
# grayfilepattern = "sp_map.v0.%05d.png"
# there are 30 numbered files in above dir
nfiles = 14
def printEvent(a, b):
print 'event called for piece', a.GetExtentTranslator().GetPiece(),
print 'out of', a.GetExtentTranslator().GetNumberOfPieces()
print 'with extent ', a.GetExtentTranslator().GetExtent()
def printExecEvent(a, b):
print 'event called for piece', a
# ------------------------- script starts here -------------------------
if __name__ == "__main__":
# try to reproduce our failure
print "working with %d files" % nfiles
print "creating factory, first reader"
firstfile = os.path.join(graydir, grayfilepattern % 1)
factory = vtk.vtkImageReader2Factory()
reader = factory.CreateImageReader2(firstfile)
# switch between two reader schemes:
if True:
# use a file appender to read multiple images in
# multiple readers
print "using multiple readers and ImageAppend"
# grab the image size:
reader.SetFileName(firstfile)
reader.Update()
extent = reader.GetDataExtent()
print "extent =", extent
# this will be our data structure; axis 2 is z:
grayreader0 = vtk.vtkImageAppend()
the_exec = vtk.vtkStreamingDemandDrivenPipeline()
grayreader0.SetExecutive( the_exec )
grayreader0.AddObserver("EndEvent", printExecEvent)
grayreader0.SetAppendAxis(2)
# add first plane: will end up in 0-th plane
grayreader0.AddInput(reader.GetOutput())
# don't reuse readers; create one per file:
grayreaderlist = {}
tilelist = [os.path.join(graydir, grayfilepattern % i)
for i in range(1, nfiles + 1)]
print "creating readers"
for tilepath in tilelist:
print tilepath
reader = factory.CreateImageReader2(tilepath)
reader.SetFileName(tilepath)
grayreaderlist[tilepath] = vtk.vtkMemoryLimitImageDataStreamer()
grayreaderlist[tilepath].SetInputConnection(reader.GetOutputPort())
grayreaderlist[tilepath].SetNumberOfStreamDivisions(16)
grayreaderlist[tilepath].SetMemoryLimit( 256 * 1024 )
# make sure we get the correct translator.
#self.imageStreamer.UpdateInformation()
grayreaderlist[tilepath].GetExtentTranslator().SetSplitModeToBlock()
grayreaderlist[tilepath].AddObserver("EndEvent", printEvent)
grayreader0.AddInput(grayreaderlist[tilepath].GetOutput())
# data gets read from disk here
print "updating ImageAppend"
grayreader0.Update()
grayreader0.UpdateWholeExtent()
else:
# read in one reader with a file pattern
print "using a single reader"
# hardcode for now--diff. prefix/pattern than above
reader.SetFilePrefix(os.path.join(basedir, graydir) + "/image.v0.")
reader.SetFilePattern("%s%05d.png")
reader.SetFileNameSliceOffset(1)
reader.Update()
extent = reader.GetDataExtent()
print "extent =", extent
# no ImageAppend, so just pass this reader along:
grayreader0 = reader
print 'Setting a streaming executive'
the_exec.SetUpdateNumberOfPieces(grayreader0.GetExecutive().GetOutputInformation(0),
16);
the_exec.GetExtentTranslator(
grayreader0.GetExecutive().GetOutputInformation(0) ).SetSplitModeToBlock()
# debug
# import pdb; pdb.set_trace()
print "creating color tables"
grayreader = vtk.vtkImageMapToColors()
grayreader.SetInputConnection(grayreader0.GetOutputPort())
graylut = vtk.vtkLookupTable()
graylut.SetTableRange(0, 256)
# can replace these values with those from spmap, doesn't matter
graylut.SetHueRange(0, 0)
graylut.SetSaturationRange(0, 0)
graylut.SetValueRange(0, 1)
graylut.Build()
grayreader.SetLookupTable(graylut)
# this will fail for 14+ planes, with imageappend
print "updating ImageMapToColors"
grayreader.Update()
print "done!"
----------------------------------------------------------------------------
Output:
[bolstadm-lm1:~/test/VTK] bolstadm% python stress-v1.py
working with 14 files
creating factory, first reader
using multiple readers and ImageAppend
extent = (0, 6033, 0, 6209, 0, 0)
creating readers
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00001.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00002.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00003.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00004.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00005.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00006.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00007.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00008.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00009.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00010.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00011.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00012.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00013.png
/Users/bolstadm/Desktop/6k_6k_150/image.v0.00014.png
updating ImageAppend
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece 0 out of 1
with extent (0, 6033, 0, 6209, 0, 0)
event called for piece vtkImageAppend (0x47eaf0)
Debug: Off
Modified Time: 4336
Reference Count: 2
Registered Events:
Registered Observers:
vtkObserver (0x481320)
Event: 4
EventName: EndEvent
Command: 0x4812c0
Priority: 0
Tag: 1
Executive: 0x481260
ErrorCode: Unknown error: 0
Information: 0x47e430
AbortExecute: Off
Progress: 1
Progress Text: (None)
NumberOfThreads: 2
AppendAxis: 2
PreserveExtents: 0
Setting a streaming executive
creating color tables
updating ImageMapToColors
Python(33833,0xa079a720) malloc: *** mmap(size=2248269824) failed (error
code=12)
*** error: can't allocate region
*** set a breakpoint in malloc_error_break to debug
ERROR: In
/Users/bolstadm/Projects/SCE/build/src/VTK5.2.0/Common/vtkDataArrayTemplate.txx,
line 137
vtkUnsignedCharArray (0x4c1b40): Unable to allocate 2248268400 elements of
size 1 bytes.
ERROR: In
/Users/bolstadm/Projects/SCE/build/src/VTK5.2.0/Filtering/vtkImageData.cxx,
line 2117
vtkImageData (0x4c0300): Coordinate (0, 0, 0) out side of array (max = -1
ERROR: In
/Users/bolstadm/Projects/SCE/build/src/VTK5.2.0/Filtering/vtkImageData.cxx,
line 2117
vtkImageData (0x4c0300): Coordinate (0, 0, 8) out side of array (max = -1
Bus error
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://www.vtk.org/pipermail/vtkusers/attachments/20090318/574b36f8/attachment.htm>
More information about the vtkusers
mailing list