[vtkusers] Syncing the View and Projection Matrices of a vtkExternalOpenGLCamera in an ExternalVTKWidget with Unity

[Wheeler, Gavin]

We are trying to build a Unity plugin using Native C++ which will allow us to incorporate VTK rendering. We have a plugin which demonstrates the basics of this based and draws a rotating cube. Our problem is that the cube doesn't move with the rest of the world - the vtkExternalOpenGLCamera doesn't appear to be using the View or Projection matrices given to it.

Platform is Windows 10 64bit application.

A few ways we have investigated this:

* The cube can be moved within the view by setting its position
    * Setting X or Y to +/-1 moves it to the edges of the view - indicating that it's position is in screen space, where the limits are +/_ 1
    * Changing Z doesn't change the view of the cube - indicating that the view is orthographic
* Looking at the transforms/matrices in the camera using the debugger they are identity matrices - which is consostent with the above
* To Unity we have added a cebug script which allows the user to move the camera around
    * The elements of the scene rendered by Unity move as expected
    * The elements in the of the scene rendered by VTK do not move - they stay in the same place relative to the camera
        * This indicates that the View matrix is not being used by the vtkExternalOpenGLCamera
* Getting the camera View and Projection matrices from Unity and passing them to VTK
    * This post suggests that we should not need to do this (http://vtk.1045678.n5.nabble.com/Using-vtkExternalOpenGLCamera-td5744015.html) - the camera position should be automatically be updated by the vtk code - "The external renderer is responsible for updating the VTK camera matrices based on the OpenGL state."
    * However, we added functions to the plugin to pass the View and projection matrix values and then set them in the vtkExternalOpenGLCamera
        * No change is observed in the rendering
        * Running through the code in the debugger the matrices do appear to be set in the camera

During the first run the first run these matrices appear to be successfully set...

* UserProvidedViewTransform - false -> true
* ViewTransform - Concatenation - PreMatrix - Identity -> Matches transpose of supplied view matrix
* ModelViewTransform - Concatenation - PreMatrix - Identity -> Matches transpose of supplied view matrix
* UseExplicitProjectionTransformMatrix - false -> true
* ExplicitProjectionTransformMatrix - undefined -> defined as the transpose of the supplied projection matrix

I've also tried setting these to hard coded values, but with no effect.

So at this point we are a bit stuck as to why it isn't working, and so how to make it work.

* Any suggestions to fix this are welcomed

Core code for the plugin (which includes all of the VTK code follows) NB, this was based on an example native C++ Unity Plugin from here (but a lot of the code has been stripped out for clarity): https://bitbucket.org/Unity-Technologies/graphicsdemos/src/43011994ae74/NativeRenderingPlugin/?at=default

#include "RenderAPI.h"
#include "PlatformBase.h"

// OpenGL Core profile (desktop) or OpenGL ES (mobile) implementation of RenderAPI.
// Supports several flavors: Core, ES2, ES3

//vtk headers
#include "vtkAutoInit.h"
VTK_MODULE_INIT(vtkRenderingOpenGL2); // VTK was built with vtkRenderingOpenGL2
#include <ExternalVTKWidget.h>
#include <vtkActor.h>
#include <vtkProperty.h>
#include <vtkCallbackCommand.h>
#include <vtkCamera.h>
#include <vtkCubeSource.h>
#include <vtkExternalOpenGLRenderWindow.h>
#include <vtkExternalOpenGLCamera.h>
#include <vtkLight.h>
#include <vtkNew.h>
#include <vtkPolyDataMapper.h>
#include "vtkWindows.h" // Needed to include OpenGL header on Windows.
#include <vtk_glew.h>


#if SUPPORT_OPENGL_UNIFIED


#include "GLEW/glew.h"


class RenderAPI_OpenGLCoreES : public RenderAPI
{
public:
    RenderAPI_OpenGLCoreES(UnityGfxRenderer apiType);
    virtual ~RenderAPI_OpenGLCoreES() { }

    virtual void ProcessDeviceEvent(UnityGfxDeviceEventType type, IUnityInterfaces* interfaces);

    virtual bool GetUsesReverseZ() { return false; }

    virtual void UpdateVtk(const double viewMatrix[16], const double projectionMatrix[16]); // std::string &debugStr);

private:
    void CreateResources();

private:
    // vtk
    vtkNew<ExternalVTKWidget> m_ExternalVTKWidget;
    vtkNew<vtkExternalOpenGLRenderer> m_renderer;
    vtkNew<vtkActor> m_CubeActor;
    vtkSmartPointer<vtkExternalOpenGLCamera> m_externalVtkCamera;

    UnityGfxRenderer m_APIType;
    GLuint m_VertexShader;
    GLuint m_Program;
    GLuint m_VertexBuffer;
};


RenderAPI* CreateRenderAPI_OpenGLCoreES(UnityGfxRenderer apiType)
{
    return new RenderAPI_OpenGLCoreES(apiType);
}

enum VertexInputs
{
    kVertexInputPosition = 0,
    kVertexInputColor = 1
};


// Simple vertex shader source
#define VERTEX_SHADER_SRC(ver, attr, varying)                       \
    ver                                                             \
    attr " highp vec3 pos;\n"                                       \
    attr " lowp vec4 color;\n"                                      \
    "\n"                                                            \
    varying " lowp vec4 ocolor;\n"                                  \
    "\n"                                                            \
    "uniform highp mat4 worldMatrix;\n"                             \
    "uniform highp mat4 viewMatrix;\n"                              \
    "uniform highp mat4 projMatrix;\n"                              \
    "\n"                                                            \
    "void main()\n"                                                 \
    "{\n"                                                           \
    "   gl_Position = (projMatrix * viewMatrix * worldMatrix) * vec4(pos,1);\n" \
    "   ocolor = color;\n"                                          \
    "}\n"                                                           \

static const char* kGlesVProgTextGLES2 = VERTEX_SHADER_SRC("\n", "attribute", "varying");
static const char* kGlesVProgTextGLES3 = VERTEX_SHADER_SRC("#version 300 es\n", "in", "out");
#if SUPPORT_OPENGL_CORE
static const char* kGlesVProgTextGLCore = VERTEX_SHADER_SRC("#version 150\n", "in", "out");
#endif

#undef VERTEX_SHADER_SRC


static GLuint CreateShader(GLenum type, const char* sourceText)
{
    GLuint ret = glCreateShader(type);
    glShaderSource(ret, 1, &sourceText, NULL);
    glCompileShader(ret);
    return ret;
}


void RenderAPI_OpenGLCoreES::CreateResources()
{
    // Create shaders
    glewExperimental = GL_TRUE;
    glewInit();
    glGetError(); // Clean up error generated by glewInit

    m_VertexShader = CreateShader(GL_VERTEX_SHADER, kGlesVProgTextGLCore);

    // Link shaders into a program and find uniform locations
    m_Program = glCreateProgram();
    glBindAttribLocation(m_Program, kVertexInputPosition, "pos");
    glBindAttribLocation(m_Program, kVertexInputColor, "color");
    glAttachShader(m_Program, m_VertexShader);
    //glAttachShader(m_Program, m_FragmentShader);
#   if SUPPORT_OPENGL_CORE
    if (m_APIType == kUnityGfxRendererOpenGLCore)
    {
        glBindFragDataLocation(m_Program, 0, "fragColor");
    }
#   endif // if SUPPORT_OPENGL_CORE
    glLinkProgram(m_Program);

    GLint status = 0;
    glGetProgramiv(m_Program, GL_LINK_STATUS, &status);
    assert(status == GL_TRUE);

    // Create vertex buffer
    glGenBuffers(1, &m_VertexBuffer);
    glBindBuffer(GL_ARRAY_BUFFER, m_VertexBuffer);
    glBufferData(GL_ARRAY_BUFFER, 1024, NULL, GL_STREAM_DRAW);

    assert(glGetError() == GL_NO_ERROR);

    //InitialiseWorldTriangleTest();
    // Rotating cube ---------------------------------------------------------------
    // initiate vtk content
    // create the VTK scene
    vtkNew<vtkCubeSource> cs;
    vtkNew<vtkPolyDataMapper> mapper;

    mapper->SetInputConnection(cs->GetOutputPort());

    m_CubeActor->SetMapper(mapper.GetPointer());
    m_CubeActor->GetProperty()->SetColor(1, 1, 0.2);
    m_CubeActor->GetProperty()->SetOpacity(0.5);
    m_CubeActor->SetPosition(0.5, 0, 0);

    // create the VTK external renderer
    vtkNew<vtkExternalOpenGLRenderWindow> renWin;
    m_ExternalVTKWidget->SetRenderWindow(renWin.GetPointer());
    m_ExternalVTKWidget->GetRenderWindow()->AddRenderer(m_renderer.GetPointer());
    m_renderer->AddActor(m_CubeActor.GetPointer());
    m_renderer->ResetCamera();
    m_externalVtkCamera = vtkSmartPointer<vtkExternalOpenGLCamera>(
        dynamic_cast<vtkExternalOpenGLCamera*>(m_renderer->GetActiveCamera()));
}

RenderAPI_OpenGLCoreES::RenderAPI_OpenGLCoreES(UnityGfxRenderer apiType)
    : m_APIType(apiType)
{
}


void RenderAPI_OpenGLCoreES::ProcessDeviceEvent(UnityGfxDeviceEventType type, IUnityInterfaces* interfaces)
{
    if (type == kUnityGfxDeviceEventInitialize)
    {
        CreateResources();
    }
    else if (type == kUnityGfxDeviceEventShutdown)
    {
        //@TODO: release resources
    }
}


void RenderAPI_OpenGLCoreES::UpdateVtk(const double viewMatrix[16], const double projectionMatrix[16]) //std::string &debugStr)
{
    m_externalVtkCamera->SetViewTransformMatrix(viewMatrix);
    m_externalVtkCamera->SetProjectionTransformMatrix(projectionMatrix);

    // draw vtk content
    m_CubeActor->RotateX(1.0f);
    m_CubeActor->RotateY(1.0f);
    m_ExternalVTKWidget->GetRenderWindow()->Render();
}


#endif // #if SUPPORT_OPENGL_UNIFIED




-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://vtk.org/pipermail/vtkusers/attachments/20171218/22903259/attachment.html>