How to create a functioning ray picking

A place to discuss everything related to Newton Dynamics.

Moderators: Sascha Willems, walaber

How to create a functioning ray picking

Postby arkdemon » Sat Jan 18, 2014 12:47 pm

Hello everyone. I am new to this forum so I might not know the rules.
Anyway, I am facing a problem with Newton Game Dynamics: I am trying to implement ray picking in my program but it does not work. Either it selects the last object of the NewtonWorld, either it selects nothing at all. So here is my source if you need it (it is nearly the exact function as the demo):

pick.h:
Code: Select all
#include <newton/Newton.h>
#include <glm/glm.hpp>

struct Pick{
static NewtonBody* MousePickByForce (NewtonWorld* const nWorld, const glm::vec3& origin, const glm::vec3& end, dFloat& paramter, glm::vec3& positionOut, glm::vec3& normalOut);
};


pick.cpp:
Code: Select all
/* Copyright (c) <2009> <Newton Game Dynamics>
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
* arising from the use of this software.
*
* Permission is granted to anyone to use this software for any purpose,
* including commercial applications, and to alter it and redistribute it
* freely
*/


#include "pick.h"
#include <glm/ext.hpp>

class dMousePickClass
{
    public:
    dMousePickClass ()
        :m_param (1.f)
        ,m_body(NULL)
    {
    }

    // implement a ray cast pre-filter
    static unsigned RayCastPrefilter (const NewtonBody* body,  const NewtonCollision* const collision, void* const userData)
    {
        // ray cannot pick trigger volumes
        //return NewtonCollisionIsTriggerVolume(collision) ? 0 : 1;
        return (NewtonBodyGetType(body) == NEWTON_DYNAMIC_BODY) ? 1 : 0;
    }

    static dFloat RayCastFilter (const NewtonBody* const body, const NewtonCollision* const collisionHit, const dFloat* const contact, const dFloat* const normal, dLong collisionID, void* const userData, dFloat intersetParam)
    {
        dFloat mass;
        dFloat Ixx;
        dFloat Iyy;
        dFloat Izz;
        dMousePickClass* const data = (dMousePickClass*) userData;
        NewtonBodyGetMassMatrix (body, &mass, &Ixx, &Iyy, &Izz);
        if ((mass > 0.f) || (NewtonBodyGetType(body) == NEWTON_KINEMATIC_BODY)) {
            float p0[3], p1[3];
            NewtonBodyGetAABB(body,p0, p1);
            if(contact[0] < p0[0] || contact[0] > p1[0] ||
               contact[1] < p0[1] || contact[1] > p1[1] ||
               contact[2] < p0[2] || contact[2] > p1[2]){}
            else
                data->m_body = body;
        }


        if (intersetParam < data->m_param) {
            data->m_param = intersetParam;
            data->m_normal = glm::vec3(normal[0], normal[1], normal[2]);
        }
        return intersetParam;
    }

    glm::vec3 m_normal;
    dFloat m_param;
    const NewtonBody* m_body;
};

NewtonBody* Pick::MousePickByForce (NewtonWorld* const nWorld, const glm::vec3& origin, const glm::vec3& end, dFloat& paramterOut, glm::vec3& positionOut, glm::vec3& normalOut)
{
//   float x = dFloat (mouseX);
//   float y = dFloat (mouseY);
//   glm::vec3 p0 (ScreenToWorld(glm::vec3 (x, y, 0.0f, 0.0f)));
//   glm::vec3 p1 (ScreenToWorld(glm::vec3 (x, y, 1.0f, 0.0f)));

    dMousePickClass rayCast;
    NewtonWorldRayCast(nWorld, glm::value_ptr(origin), glm::value_ptr(end), dMousePickClass::RayCastFilter, &rayCast, dMousePickClass::RayCastPrefilter, 0);

    if (rayCast.m_body) {
        positionOut = origin + (end - origin)*(rayCast.m_param);
        normalOut = rayCast.m_normal;
        paramterOut = rayCast.m_param;
    }
    return (NewtonBody*) rayCast.m_body;
}


and scene.cpp (central class):

Code: Select all
glm::vec3 ScreenToWorld (const glm::vec3& screen)
{
    //Where the values will be stored

    GLint viewport[4];

    //Retrieves the viewport and stores it in the variable
    // get a point on the display array of the windows
    glGetIntegerv(GL_VIEWPORT, viewport);

    //Where the 16 doubles of the matrix are to be stored

    //Retrieve the matrix
    GLdouble modelview[16];
    glGetDoublev(GL_MODELVIEW_MATRIX, modelview);

    GLdouble projection[16];
    glGetDoublev(GL_PROJECTION_MATRIX, projection);

    GLdouble winX = 0.0;
    GLdouble winY = 0.0;
    GLdouble winZ = 0.0; //The coordinates to pass in

    winX = screen.x; //Store the x cord
    winY = screen.y; //Store the y cord
    winZ = screen.z; //Store the Z cord

    //Now windows coordinates start with (0,0) being at the top left
    //whereas OpenGL cords start lower left so we have to do this to convert it:
    //Remember we got viewport value before
    winY = (dFloat)viewport[3] - winY;

    GLdouble objx;
    GLdouble objy;
    GLdouble objz;

    // use the real GL view port
    glGetIntegerv(GL_VIEWPORT, viewport);
    gluUnProject (winX, winY, winZ, modelview, projection, viewport, &objx, &objy, &objz);

    return glm::vec3(dFloat(objx), dFloat(objy), dFloat(objz));
}

void Scene::pick(float x, float y){

    glm::vec3 p0 (ScreenToWorld(glm::vec3 (x, y, 0.0f)));
    glm::vec3 p1 (ScreenToWorld(glm::vec3 (x, y, 1.0f)));


    float param;
    glm::vec3 pos_out;
    glm::vec3 norm_out;
    NewtonBody *picked = Pick::MousePickByForce(world, p0, p1, param, pos_out, norm_out);


   if(!picked)
           printf("Nothing\n");
   else
       exit(0);//to see if when i click it works
   picked = 0;
}


If you need more information, ask me and I will give them.

Thank you for your help.
My name is arkdemon and I don't approve this message :D
User avatar
arkdemon
 
Posts: 90
Joined: Sat Jan 18, 2014 12:38 pm

Re: How to create a functioning ray picking

Postby arkdemon » Sat Jan 18, 2014 1:35 pm

I found out that it clicked because the projection matrix and view matrix it loaded with glLoadMatrix was NULL (I use GLSL) so I changed ScreentoWorld by
Code: Select all
    float ray_x = (x/w - 0.5f) * 2.f;
    float ray_y = (y/h - 0.5f) * 2.f;
    glm::vec4 lRayStart_NDC(ray_x, ray_y, -1.f,// The near plane maps to Z=-1 in Normalized Device Coordinates
         1.f);
    glm::vec4 lRayEnd_NDC(ray_x,ray_y, 0.f, 1.f);

    //Get
    glm::mat4 inv_proj = glm::inverse(m_proj);
    glm::mat4 inv_view = glm::inverse(m_cam->view());

    glm::vec4 lRayStart_cam   = inv_proj * lRayStart_NDC;
    lRayStart_cam            /= lRayStart_cam.w;
    glm::vec4 lRayStart_world = inv_view * lRayStart_cam;
    lRayStart_world          /= lRayStart_world.w;
    glm::vec4 lRayEnd_cam     = inv_proj * lRayEnd_NDC;
    lRayEnd_cam              /= lRayEnd_cam.w;
    glm::vec4 lRayEnd_world   = inv_view * lRayEnd_cam;
    lRayEnd_world            /= lRayEnd_world.w;

    glm::vec3 lRayDir_world = glm::vec3(lRayEnd_world - lRayStart_world);
    lRayDir_world = glm::normalize(lRayDir_world);


    glm::vec3 p0 = glm::vec3(lRayStart_world);
    glm::vec3 p1 = glm::vec3(lRayEnd_world);

which I know it works because I used it to pick objects before using NGD. However, I do not know how to use it with NGD because before I had to send the origin and the direction of the ray whilst here I have to send the start and end of the ray.


Now it does not pick anything
My name is arkdemon and I don't approve this message :D
User avatar
arkdemon
 
Posts: 90
Joined: Sat Jan 18, 2014 12:38 pm

Re: How to create a functioning ray picking

Postby JoeJ » Sat Jan 18, 2014 4:20 pm

I'd start with a line with some fixed world space coordinates.
You can visualize this line and easily verify what bodies it should hit without doupts on camera projection stuff.

Also you can try my code below, it's simpler and the code from demos might do things you do not want or know yet.



Code: Select all

struct RayCastInfo
{
   float param;
   sVec3 normal;
   Body* hitBody;
   
   void Reset ()
   {
      param = 1.1f;
      hitBody = 0;
   }

   RayCastInfo () { Reset(); }
};

unsigned nRayCastPrefilter (const NewtonBody* body, const NewtonCollision* collision, void* userData)
{
   return 1; // het everything for now
}

float nRayCastFilter (const NewtonBody* const body, const NewtonCollision* const shapeHit, const float* const hitContact, const float* const normal, long long collisionID,  void* const userData, float intersectParam)
{
   RayCastInfo *rc = (RayCastInfo*) userData;
   if (intersectParam < rc->param) // hit is closer than previous, so update
   {
      rc->param = intersectParam;
      rc->hitBody = (Body*) body;
      // sMat4 m; BodyGetMatrix ((Body*)body, m);
      // rc->normal = m.Unrotate (*((sVec3*)normal));
   }
   return intersectParam;
}


void RayCast (sVec3 &p0, sVec3 &p1, RayCastInfo &rc)
{
      NewtonWorldRayCast (world, &p0[0], &p1[0], (NewtonWorldRayFilterCallback)nRayCastFilter, (void*)&rc, nRayCastPrefilter, 0);
}
User avatar
JoeJ
 
Posts: 1489
Joined: Tue Dec 21, 2010 6:18 pm

Re: How to create a functioning ray picking

Postby arkdemon » Sat Jan 18, 2014 5:06 pm

Hello, thank you for your answer but the program crashes. When I debugged it crashed here:
Code: Select all
NewtonWorldRayCast(world,glm::value_ptr(p0), glm::value_ptr(p1), (NewtonWorldRayFilterCallback)nRayCastFilter, (void*)&rc, nRayCastPrefilter, 0);
My name is arkdemon and I don't approve this message :D
User avatar
arkdemon
 
Posts: 90
Joined: Sat Jan 18, 2014 12:38 pm

Re: How to create a functioning ray picking

Postby arkdemon » Sun Jan 19, 2014 12:17 pm

anybody?
My name is arkdemon and I don't approve this message :D
User avatar
arkdemon
 
Posts: 90
Joined: Sat Jan 18, 2014 12:38 pm

Re: How to create a functioning ray picking

Postby arkdemon » Sun Jan 19, 2014 3:33 pm

I made it not crash! The problem was that the "world" was static from another file so it crashed because it was NULL. Now I put the Pick function in the physics class. It does not crash anymore but it does not work ...
Still it is a great improvement.
If anyone has an idea to make it work, he would be welcomed. Thank you.

PS: this is how I get p0 and p1:
Code: Select all
    float ray_x = (x/w - 0.5f) * 2.f;
    float ray_y = (y/h - 0.5f) * 2.f;
    glm::vec4 lRayStart_NDC(ray_x, ray_y, -1.f,// The near plane maps to Z=-1 in Normalized Device Coordinates
         1.f);
    glm::vec4 lRayEnd_NDC(ray_x,ray_y, 0.f, 1.f);

    //Get
    glm::mat4 inv_proj = glm::inverse(m_proj);
    glm::mat4 inv_view = glm::inverse(m_cam->view());

    glm::vec4 lRayStart_cam   = inv_proj * lRayStart_NDC;
    lRayStart_cam            /= lRayStart_cam.w;
    glm::vec4 lRayStart_world = inv_view * lRayStart_cam;
    lRayStart_world          /= lRayStart_world.w;
    glm::vec4 lRayEnd_cam     = inv_proj * lRayEnd_NDC;
    lRayEnd_cam              /= lRayEnd_cam.w;
    glm::vec4 lRayEnd_world   = inv_view * lRayEnd_cam;
    lRayEnd_world            /= lRayEnd_world.w;

    glm::vec3 lRayDir_world = glm::vec3(lRayEnd_world - lRayStart_world);
    lRayDir_world = glm::normalize(lRayDir_world);


    glm::vec3 p0 = glm::vec3(lRayStart_world);
    glm::vec3 p1 = glm::vec3(lRayEnd_world);


    glm::vec3 pos_out = glm::vec3(0.f);
    RayCastInfo ray = RayCast(glm::value_ptr(p0),glm::value_ptr(p1));
    NewtonBody *picked = ray.hitBody;


which comes from http://www.opengl-tutorial.org/miscella ... _direction
My name is arkdemon and I don't approve this message :D
User avatar
arkdemon
 
Posts: 90
Joined: Sat Jan 18, 2014 12:38 pm

Re: How to create a functioning ray picking

Postby JoeJ » Sun Jan 19, 2014 4:57 pm

What happens if you do something like

vec p0(-10,1,0);
vec p1(10,1,0);
RenderLine (p0, p1); // visualize to see if object is in between
RayCastInfo ray = RayCast(p0, p1);
NewtonBody *picked = ray.hitBody;
if (!picked) printf("Nothing\n");

Still nothing?
Maybe the returned RayCastInfo object is not the one used during raycast, due to a similar mistake like with world pointer?
If yo set breakpoints in nRayCastFilter, is it called, but there's never a hit?
You know how to use debugger? That's how you can get answers to such questions very quickly.

The projection stuff looks complicated, for me it's just:
p0 = camera.matrix[3]; // eye position
p1 = camera.matrix[3] + camera.matrix[2] * 200.0f; // z axis = looking direction
User avatar
JoeJ
 
Posts: 1489
Joined: Tue Dec 21, 2010 6:18 pm

Re: How to create a functioning ray picking

Postby arkdemon » Wed Jan 22, 2014 2:00 pm

Hi, thank you for answering.
When I "debug" (ie I put some printf) I saw that the raycast function does not go through RayPrefilter and RayFilter, is it normal?
Thank you for your answers.
My name is arkdemon and I don't approve this message :D
User avatar
arkdemon
 
Posts: 90
Joined: Sat Jan 18, 2014 12:38 pm

Re: How to create a functioning ray picking

Postby JoeJ » Wed Jan 22, 2014 4:05 pm

arkdemon wrote:When I "debug" (ie I put some printf)

There's a better way to do that. If you use Visual Studio, you set a breakpoint (red dot) by clickeng at the left edge of code window.
Then you start debugging (F5). If code flow hits the beakpoint, your program pauses, code becomes visible with a yellow arrow where the program pauses at the moment. You can than than execute the next statement (F10) and so forth, meaning you can follow the entire code flow of your program to see where's the bug.
You can also - at any point - see what values the scoped variables have. You would see that a world pointer is zero, or a floting point number is NAN etc...
All this works with debug build, but not in release mode.

arkdemon wrote:I saw that the raycast function does not go through RayPrefilter and RayFilter, is it normal?

Yes, if the ray intersects no object it is correct. I assume Newton calls the PreFilter if a bounding box is hit, and the other callback when a real body is hit.
But if you are sure the ray should hit something, than there's something wrong, but i can't tell what it is.
You could setup a small testworld with two bodies, and post code how you do that, maybe the bug is there...
User avatar
JoeJ
 
Posts: 1489
Joined: Tue Dec 21, 2010 6:18 pm

Re: How to create a functioning ray picking

Postby arkdemon » Wed Jan 22, 2014 4:24 pm

Hi, I use Qt Creator.
Thank you for your answer.
For the code, I will post when I can.
My name is arkdemon and I don't approve this message :D
User avatar
arkdemon
 
Posts: 90
Joined: Sat Jan 18, 2014 12:38 pm

Re: How to create a functioning ray picking

Postby arkdemon » Thu Jan 23, 2014 1:13 pm

Hello, sorry I could not access my computer. SO here is the code:
scene.cpp:
Code: Select all
void Scene::pick(float x, float y, float w, float h){
    float ray_x = (x/w - 0.5f) * 2.f;
    float ray_y = (y/h - 0.5f) * 2.f;
    glm::vec4 lRayStart_NDC(ray_x, ray_y, -1.f,// The near plane maps to Z=-1 in Normalized Device Coordinates
         1.f);
    glm::vec4 lRayEnd_NDC(ray_x,ray_y, 0.f, 1.f);

    //Get
    glm::mat4 inv_proj = glm::inverse(m_proj);
    glm::mat4 inv_view = glm::inverse(m_cam->view());

    glm::vec4 lRayStart_cam   = inv_proj * lRayStart_NDC;
    lRayStart_cam            /= lRayStart_cam.w;
    glm::vec4 lRayStart_world = inv_view * lRayStart_cam;
    lRayStart_world          /= lRayStart_world.w;
    glm::vec4 lRayEnd_cam     = inv_proj * lRayEnd_NDC;
    lRayEnd_cam              /= lRayEnd_cam.w;
    glm::vec4 lRayEnd_world   = inv_view * lRayEnd_cam;
    lRayEnd_world            /= lRayEnd_world.w;

    glm::vec3 lRayDir_world = glm::vec3(lRayEnd_world - lRayStart_world);
    lRayDir_world = glm::normalize(lRayDir_world);


    glm::vec3 p0 = glm::vec3(lRayStart_world);
    glm::vec3 p1 = glm::vec3(lRayEnd_world);

    RayCastInfo ray;
    RayCast(glm::value_ptr(p0),glm::value_ptr(p1), ray);
    NewtonBody *picked = ray.hitBody;
    if (!picked)
        printf("Nothing\n");
    else
        exit(0);
}


and pick.h:
Code: Select all
struct RayCastInfo
{
   float param;
   glm::vec3 normal;
   NewtonBody* hitBody;

   void Reset (){
      normal = glm::vec3(0.f);
      param = 1.1f;//1.1f
      hitBody = 0;
   }

   RayCastInfo () : param(1.1f), hitBody(), normal(0.f){ Reset(); }
};

static unsigned nRayCastPrefilter (const NewtonBody* body, const NewtonCollision* collision, void* userData){
   printf("a");
   return 1; // hit everything for now
}

static float nRayCastFilter(const NewtonBody* const body, const NewtonCollision* const shapeHit,
                      const float* const hitContact, const float* const normal, long long collisionID,
                      void* const userData, float intersectParam){
   RayCastInfo *rc = (RayCastInfo*) userData;
   printf("%.3f %.3f\n", rc->param, intersectParam);
   if (intersectParam < rc->param){ // hit is closer than previous, so update
      rc->param = intersectParam;
      rc->hitBody = (NewtonBody*)(body);
   }

   return intersectParam;
}


static void RayCast (float *p0, float *p1, RayCastInfo &rc){
    rc.Reset();
    NewtonWorldRayCast(world, p0, p1, nRayCastFilter,&rc, nRayCastPrefilter, 0);
}


I think the problem comes from the lRayStart_world and lRayEnd_world or p0 and p1.

Thank you for your answers.
My name is arkdemon and I don't approve this message :D
User avatar
arkdemon
 
Posts: 90
Joined: Sat Jan 18, 2014 12:38 pm

Re: How to create a functioning ray picking

Postby JoeJ » Thu Jan 23, 2014 2:14 pm

arkdemon wrote:I use Qt Creator

I'm sure there are very similar debugging features... check it out. I don't wanna know how much time i've wasted the first years ignoring the existance of a debugger ;)

The code i wanted to see is not the picking code again, but the world creation code, where you create your scene.

However, you have doupts about ray points calculation, so you need to get sure.
Some suggestions:
* printf the values for p0 and p1 and see if those are reasonable. (Or better, set a breakpoint and simply look without the need to use printf all the time)
* Ensure your physics match up with your visuals (do falling boxes collide correctly with ground / other boxes?...)
* Build your test case in worldspace, like i said earlier. Pseudocode again:

NewtonBody *box = Create a 3x3x3 box centered at (0,1,0), so following raycast should hit it:
vec p0(-10,1,0);
vec p1(10,1,0);
//RenderLine (p0, p1); // visualize to see if object is in between
RayCastInfo ray; RayCast(p0, p1, ray);
NewtonBody *picked = ray.hitBody;
if (!picked) printf("Nothing\n");
User avatar
JoeJ
 
Posts: 1489
Joined: Tue Dec 21, 2010 6:18 pm

Re: How to create a functioning ray picking

Postby arkdemon » Thu Jan 23, 2014 3:34 pm

Hello everyone. It works!
The problem was with the unproject function: mine did not work. so here is the code for those who need it:

scene.cpp:
Code: Select all
glm::vec3 unproject(
        float x, float y,
        float w, float h,
        glm::mat4 proj,
        glm::mat4 modview,
        float depth){
    // transformation of normalized coordinates
    glm::vec4 vec1;
    vec1.x = (2.f * x) / w - 1.0f;
    vec1.y = (2.f * y) / h - 1.0f;
    vec1.z = 2.f * depth - 1.f;
    vec1.w = 1.f;

    // multiply inverted matrix with vector
    glm::vec4 rayWorld = glm::inverse(proj * modview) * vec1;

    glm::vec3 result;
    result.x = rayWorld.x / rayWorld.w;
    result.y = rayWorld.y / rayWorld.w;
    result.z = rayWorld.z / rayWorld.w;

    return result;
}

void Scene::pick(float x, float y, float w, float h){
    glm::vec3 p0 = unproject(x, y, w, h, m_proj, m_cam->view(), 0.f);
    glm::vec3 p1 = unproject(x, y, w, h, m_proj, m_cam->view(), 1.f);

    RayCastInfo ray;
    RayCast(glm::value_ptr(p0),glm::value_ptr(p1), ray);
    NewtonBody *picked = ray.hitBody;
    if (!picked)
        printf("Nothing\n");
    else
        exit(0);


it comes from here: http://stackoverflow.com/questions/2084 ... ithout-glm

and pick.h:
Code: Select all
struct RayCastInfo
{
   float param;
   glm::vec3 normal;
   NewtonBody* hitBody;

   void Reset (){
      normal = glm::vec3(0.f);
      param = 1.1f;//1.1f
      hitBody = 0;
   }

   RayCastInfo () : param(1.1f), hitBody(), normal(0.f){ Reset(); }
};

static unsigned nRayCastPrefilter (const NewtonBody* body, const NewtonCollision* collision, void* userData){
   return 1; // hit everything for now
}

static float nRayCastFilter(const NewtonBody* const body, const NewtonCollision* const shapeHit,
                      const float* const hitContact, const float* const normal, long long collisionID,
                      void* const userData, float intersectParam){
   RayCastInfo *rc = (RayCastInfo*) userData;
   if (intersectParam < rc->param){ // hit is closer than previous, so update
      rc->param = intersectParam;
      rc->hitBody = (NewtonBody*)(body);
   }

   return intersectParam;
}


static void RayCast (float *p0, float *p1, RayCastInfo &rc){
    rc.Reset();
    NewtonWorldRayCast(world, p0, p1, nRayCastFilter,&rc, nRayCastPrefilter, 0);
}
My name is arkdemon and I don't approve this message :D
User avatar
arkdemon
 
Posts: 90
Joined: Sat Jan 18, 2014 12:38 pm


Return to General Discussion

Who is online

Users browsing this forum: No registered users and 2 guests