[vtkusers] VTK-WPF program crash, Bug in VTK?

[Xiaofeng Z]

I've been trying to solve a program crash problem for the past several days and running out of idea.  I would like people has deeper knowledge of VTK and/or Windows OS to offer some suggestions as to where to look for the problem.

Background:

I'm trying to use VTK in a WPF application.  WPF is the tool allows development of UI application that renders controls in a single top level window, much like the windowless control in visual studio 6.
However, WPF does not allow the program to draw using unmanaged code to the window it manages.  Instead, it supplies a HwndHost class allows a native windowed control to be placed on top of the WPF window. To use HwndHost, the minimum one need to do is to derive from it and override two methods (all error checking code are omitted), one to create the native control, one to destroy it.

        protected override HandleRef BuildWindowCore(HandleRef hwndParent)
        {
            IntPtr hwndHost = Import.CreateChildWindow(hwndParent.Handle, hostHeight, hostWidth);
            return new HandleRef(this, hwndHost);

            return new HandleRef(this, IntPtr.Zero);
        }

        protected override void DestroyWindowCore(System.Runtime.InteropServices.HandleRef hwnd)
        {
            Import.DestroyChild(hwnd.Handle, instance);
        }

Notice that native functions CreateChildWindow and DestroyChild are called to create the hosted window.  The native control can then be inserted into WPF application by attaching the MyHwndHost instance to the WPF visual tree.

            MyHwndHost myHwndHost = new MyHwndHost(hostBorder.ActualWidth, hostBorder.ActualHeight);
            hostBorder.Child = myHwndHost;
            myHwndHost.SubClassChild();   //<<= the subclass the control just like vtkInteractor does

This last statement is the above code snippet is used to simulate the subclassing by vtkWin32RenderWindowInteractor, discussed later.  Under the hood, Upon completion of BuildwondowCore and before calling DestroyWindowCore, HwndHost attaches and detaches it's own window procedure to intercept Window messages send to the hosted window, using SetWindowLongPtr API call.

On the other hand, VTK interactive also uses subclassing to intercept the messages.  Which, in my case, is done after WPF attaches it's Window procedure.  So after the control is rendered, the Window procedure chain looks like this (in the reverse order they're called):

vtkWin32OpenGLRenderWindow::WndProc <- HwndHost.SubclassWndProc <- vtkHandleMessage

These all works out nicely, until the time comes to close the control.  WPF first send a registered message to the hosted window which is passed along to HwndHost's SubclassWndProc by vtkHandleMessage.  At this point, WPF tried to detach itself from the hosted window.  It first calls GetWindowLongPtr to check if it is at the beginning of the chain and this is where the problem shows up.  Instead of getting the address of vtkHandleMessage, it gets an invalid address and crashes the program.

To rule out VTK as the source of the problem, I tried to reproduce it without VTK buy create a Windows Procedure Chain similar to that of VTK (see the source code at the end).  And, as you already guessed it.  The test program without VTK, worked flawlessly.

How much a role does vtk played in crashing the program?  If it does, where the problem most likely to be originated?

xz

__declspec(dllexport) HWND WINAPI CreateChildWindow(HWND hWndParent, int width, int height)
{
    HINSTANCE hApp = (HINSTANCE)GetWindowLongPtr(hWndParent, GWLP_HINSTANCE);

    WNDCLASS wndClass;
    wndClass.style = CS_HREDRAW | CS_VREDRAW | CS_OWNDC;
    wndClass.lpfnWndProc = Win32CtrlImpl::DummyWndProc;
    wndClass.cbClsExtra = 0;
    wndClass.hInstance = hApp;
    wndClass.hIcon = LoadIcon(NULL, IDI_APPLICATION);
    wndClass.hCursor = LoadCursor(NULL, IDC_ARROW);
    wndClass.hbrBackground = (HBRUSH)GetStockObject(BLACK_BRUSH);
    wndClass.lpszMenuName = NULL;
    wndClass.lpszClassName = L"Win32Control";
    wndClass.cbWndExtra = 2*sizeof(LONG_PTR);
    RegisterClass(&wndClass);

    HWND hwnd = CreateWindow
        ( L"Win32Control"
        , L"Win32 Control"
        , WS_CHILD|WS_CLIPCHILDREN
        , 0, 0
        , width, height
        , hWndParent
        , NULL, hApp, NULL);

    SetWindowLongPtr(hwnd, sizeof(LONG_PTR), (LONG_PTR)&wndClass);

    wchar_t buffer[1024];
    swprintf(buffer, L"Initial Window Procedure: %0lx", wndClass.lpfnWndProc);
    MessageBox(hWndParent, buffer, L"Window Procedure", MB_OK);

    return hwnd;
}

__declspec(dllexport) void* WINAPI SubClassChild(HWND hwnd)
{
    Instance* instance = new Instance();

    SetLastError(0);
    LONG_PTR rv = SetWindowLongPtr(hwnd, GWLP_WNDPROC, (LONG_PTR)MyWndProc);
    if (!rv || GetLastError())
        MessageBox(hwnd, L"Failed to subclass window.", L"debug", MB_OK);

    wchar_t buffer[1024];
    swprintf(buffer, L"Window Procedure Prior to sub-classing: %0lx", rv);
    MessageBox(hwnd, buffer, L"Window Procedure", MB_OK);

    instance->wndProc = (WNDPROC)rv;

    rv = SetWindowLongPtr(hwnd, GWLP_USERDATA, (LONG_PTR)instance);
    if (!rv && GetLastError())
        MessageBox(hwnd, L"Failed to store user data.", L"debug", MB_OK);

    ShowWindow(hwnd, SW_NORMAL);

    return instance;
}

__declspec(dllexport) void WINAPI DestroyChild(HWND hwnd, void* p)
{
    Instance* instance = (Instance*)p;

    SetLastError(0);
    LONG_PTR rv = SetWindowLongPtr(hwnd, GWLP_WNDPROC, (LONG_PTR)instance->wndProc);
    if (!rv && GetLastError())
        MessageBox(hwnd, L"Failed to restore window.", L"debug", MB_OK);

    if (rv != (LONG_PTR)&MyWndProc)
        MessageBox(hwnd, L"Bad Window Process", L"debug", MB_OK|MB_ICONQUESTION);

    wchar_t buffer[1024];
    swprintf(buffer, L"Window Procedure Prior to restoring: %0lx", rv);
    MessageBox(hwnd, buffer, L"Window Procedure", MB_OK);

    delete instance;
}

LRESULT CALLBACK MyWndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    // do nothing but to pass the message to the original window procedure
    Instance* instance = (Instance*)GetWindowLongPtr(hWnd, GWLP_USERDATA);
    //WNDPROC wndProc = (WNDPROC)GetWindowLongPtr(hWnd, GWLP_WNDPROC);
    if (!instance)
        MessageBox(hWnd, L"Failed to extract previous windows procedure.", L"debug", MB_OK);

    UINT regMessage = RegisterWindowMessage(L"HwndSubclass.DetachMessage");
    if (message==regMessage)
    {
        wchar_t buffer[1024];
        swprintf(buffer, L"Window Procedure Prior to detaching: %0lx", GetWindowLongPtr(hWnd, GWLP_WNDPROC));
        MessageBox(hWnd, buffer, L"Window Procedure", MB_OK);
    }

    return CallWindowProc(instance->wndProc, hWnd, message, wParam, lParam);
}

LRESULT CALLBACK Win32CtrlImpl::DummyWndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam)
{
    return DefWindowProc(hWnd, message, wParam, lParam);
}




 		 	   		  
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://www.vtk.org/pipermail/vtkusers/attachments/20110118/0d41e4e5/attachment.htm>