How to add Raycaster support for THREE.js objects?

[JunFang-NWPU]

Hi, thanks for this great work. Currently, it support integrating Three.js scenes into the 3dGS scene. How to add raycaster support for the integrated three.js objects?

Thanks.

[Mark Kellogg]

I admit there is some work to be done in this area :) If you are using a standalone instance of Viewer and passing in a three.js scene via the threeScene parameter, then you can do something like this to raycast against objects in that scene:

const raycaster = new THREE.Raycaster();
const pointer = new THREE.Vector2();
...

<call some function to get mouse screen coordinates and store in 'pointer'>

// update the picking ray with the camera and pointer position
raycaster.setFromCamera(pointer, viewer.camera);

// calculate objects intersecting the picking ray
const intersects = raycaster.intersectObjects(threeScene.children);

I really need to update Viewer so that its built-in raycaster will work on three.js objects as well as splats. That is on my to-do list.

If you are using the DropInViewer, there's actually not a good option right now because I haven't properly implemented a raycast function on SplatMesh so that the standard three.js raycaster can be used on it. That is also on my to-do list :)

[JunFang-NWPU]

Thanks for your comments.

I wrote codes following your suggestions:

` <script type="module">
import * as GaussianSplats3D from '@mkkellogg/gaussian-splats-3d';
import * as THREE from 'three';

const urlParams = new URLSearchParams(window.location.search);
const mode = parseInt(urlParams.get('mode')) || 0;

const threeScene = new THREE.Scene();
const sphereColor = 0xffffff;
const sphereGeometry = new THREE.SphereGeometry(0.02); 
const sphere = new THREE.Mesh(sphereGeometry, new THREE.MeshLambertMaterial({ 'color': sphereColor}));
sphere.position.set(0.02218, -0.07569, 0.03335);
threeScene.add(sphere);

const raycaster = new THREE.Raycaster();
const pointer = new THREE.Vector2();

function onPointerMove(event) {
  pointer.x = (event.clientX / window.innerWidth) * 2 - 1;
  pointer.y = - (event.clientY / window.innerHeight) * 2 + 1;
}

const viewer = new GaussianSplats3D.Viewer({
  'threeScene': threeScene,
  'cameraUp': [0.68502, -0.17084, -0.70821],
  'initialCameraPosition': [0.70281, -0.05095, -0.87661],
  'initialCameraLookAt': [-0.02508, 0.01483, -0.12609],
  'sphericalHarmonicsDegree': 2
});

raycaster.setFromCamera(pointer,viewer.camera);
const intersects = raycaster.intersectObjects( threeScene.children );
for ( let i = 0; i < intersects.length; i ++ ) {
  console.log(intersects[i].object.material.color);
  console.log(pointer);
  intersects[ i ].object.material.color.set( 0xff0000 );
  console.log(intersects[i].object.material.color);
}

window.addEventListener( 'mousemove', onPointerMove );

let path = 'assets/data/dog/dog' + (mode ? '_high' : '') + '.ksplat';
viewer.addSplatScene(path, {
  'streamView': true
})
  .then(() => {
    viewer.start();
  });

</script>`

It can run without error, and output four intesected objects on the [0,0]. However, it can not detect the move of the mouse. I think it should add codes like window.requestAnimationFrame(render); to support update. How can I add such fragments?

Thanks.

[Mark Kellogg]

You could just move the raycasting code to onPointerMove(). This seems to work on my end:

function onPointerMove(event) {
  pointer.x = (event.clientX / window.innerWidth) * 2 - 1;
  pointer.y = - (event.clientY / window.innerHeight) * 2 + 1;
  raycaster.setFromCamera(pointer,viewer.camera);
  const intersects = raycaster.intersectObjects( threeScene.children );
  for ( let i = 0; i < intersects.length; i ++ ) {
    console.log(intersects[i].object.material.color);
    console.log(pointer);
    intersects[ i ].object.material.color.set( 0xff0000 );
    console.log(intersects[i].object.material.color);
  }
}

Or you could add an update function:

let path = 'assets/data/dog/dog' + (mode ? '_high' : '') + '.ksplat';
viewer.addSplatScene(path, {
  'streamView': true
})
.then(() => {
  viewer.start();
  window.requestAnimationFrame(update);
});

function update() {
    window.requestAnimationFrame(update);
    raycaster.setFromCamera(pointer, viewer.camera);
    const intersects = raycaster.intersectObjects( threeScene.children );
    for ( let i = 0; i < intersects.length; i ++ ) {
      console.log(intersects[i].object.material.color);
      console.log(pointer);
      intersects[ i ].object.material.color.set( 0xff0000 );
      console.log(intersects[i].object.material.color);
    }
}

The problem with the second approach is that you will be raycasting every frame, even if the mouse didn't move (but maybe that's OK).

[JunFang-NWPU]

Yes, the raycast works now. However, the color of the object in the threeScene doesn't change when I move the mouse on it. In the following code, color value of the sphere is changed indeed, however, it is not updated in the window. Doesn't I miss something in the render process.

'<script type="module">
import * as GaussianSplats3D from '@mkkellogg/gaussian-splats-3d';
import * as THREE from 'three';

const urlParams = new URLSearchParams(window.location.search);
const mode = parseInt(urlParams.get('mode')) || 0;

const threeScene = new THREE.Scene();
const sphereColor = 0xffffff;
const sphereGeometry = new THREE.SphereGeometry(0.02); 
const sphere = new THREE.Mesh(sphereGeometry, new THREE.MeshLambertMaterial({ 'color': sphereColor}));
sphere.position.set(0.02218, -0.07569, 0.03335);
threeScene.add(sphere);

const raycaster = new THREE.Raycaster();
const pointer = new THREE.Vector2();

function onPointerMove(event) {

  pointer.x = (event.clientX / window.innerWidth) * 2 - 1;
  pointer.y = - (event.clientY / window.innerHeight) * 2 + 1;

  raycaster.setFromCamera(pointer,viewer.camera);
  const intersects = raycaster.intersectObjects(threeScene.children,false);
  if (intersects.length>0) {
    console.log(intersects[0].object.material.color);
    console.log(pointer);
    intersects[0].object.material.color.set( Math.random() * 0xffffff);
    console.log(intersects[0].object.material.color);
  }
}

const viewer = new GaussianSplats3D.Viewer({
  'threeScene': threeScene,
  'cameraUp': [0.68502, -0.17084, -0.70821],
  'initialCameraPosition': [0.70281, -0.05095, -0.87661],
  'initialCameraLookAt': [-0.02508, 0.01483, -0.12609],
  'sphericalHarmonicsDegree': 2
});

window.addEventListener( 'mousemove', onPointerMove );

let path = 'assets/data/dog/dog' + (mode ? '_high' : '') + '.ksplat';
viewer.addSplatScene(path, {
  'streamView': true
})
  .then(() => {
    viewer.start();
  });

</script>'

Thanks.

[Mark Kellogg]

Because you're using THREE.MeshLambertMaterial, you need lights in your scene to properly illuminate the three.js objects. If you change the material to THREE.MeshBasicMaterial, you should see the color change.

[JunFang-NWPU]

Yes, It works now :)