feat: dispatch clicks on the 3d models to the canvases

app/components/NDS.vue

@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
-import { useLoop } from "@tresjs/core";
+import { useLoop, useTresContext } from "@tresjs/core";
 import * as THREE from "three";
 
 const props = defineProps<{
@@ -8,6 +8,11 @@ const props = defineProps<{
   bottomScreenCanvas: HTMLCanvasElement | null;
 }>();
 
+const emit = defineEmits<{
+  topScreenClick: [x: number, y: number];
+  bottomScreenClick: [x: number, y: number];
+}>();
+
 const { state: model } = useLoader(
   GLTFLoader,
   "/models/nintendo-ds/scene.gltf",
@@ -17,6 +22,10 @@ const scene = computed(() => model.value?.scene);
 
 let topScreenTexture: THREE.CanvasTexture | null = null;
 let bottomScreenTexture: THREE.CanvasTexture | null = null;
+let topScreenMesh: THREE.Mesh | null = null;
+let bottomScreenMesh: THREE.Mesh | null = null;
+
+const { camera, renderer } = useTresContext();
 
 watch(
   () => [props.topScreenCanvas, props.bottomScreenCanvas],
@@ -51,6 +60,7 @@ watch(scene, () => {
           emissive: new THREE.Color(0x222222),
           emissiveIntensity: 0.5,
         });
+        topScreenMesh = child;
       } else if (
         material.name?.includes("screen_down") &&
         bottomScreenTexture
@@ -62,6 +72,7 @@ watch(scene, () => {
           emissive: new THREE.Color(0x222222),
           emissiveIntensity: 0.5,
         });
+        bottomScreenMesh = child;
       }
     }
   });
@@ -74,7 +85,55 @@ onRender(() => {
   if (bottomScreenTexture) bottomScreenTexture.needsUpdate = true;
 });
 
+const handleClick = (event: MouseEvent) => {
+  const domElement = renderer.instance.domElement;
+  const rect = domElement.getBoundingClientRect();
+
+  const raycaster = new THREE.Raycaster();
+  raycaster.setFromCamera(
+    new THREE.Vector2(
+      ((event.clientX - rect.left) / rect.width) * 2 - 1,
+      -((event.clientY - rect.top) / rect.height) * 2 + 1,
+    ),
+    camera.activeCamera.value,
+  );
+
+  if (topScreenMesh) {
+    const intersects = raycaster.intersectObject(topScreenMesh);
+    if (intersects[0]) {
+      const uv = intersects[0].uv;
+      if (uv) {
+        const x = Math.floor(uv.x * 256);
+        const y = Math.floor(uv.y * (1024 / 404) * 192);
+        emit("topScreenClick", x, y);
+        return;
+      }
+    }
+  }
+
+  if (bottomScreenMesh) {
+    const intersects = raycaster.intersectObject(bottomScreenMesh);
+    if (intersects[0]) {
+      const uv = intersects[0].uv;
+      if (uv) {
+        const x = Math.floor(uv.x * 256);
+        const y = Math.floor(192 - (1 - uv.y) * (1024 / 532) * 192);
+        emit("bottomScreenClick", x, y);
+      }
+    }
+  }
+};
+
+onMounted(() => {
+  if (renderer) {
+    renderer.instance.domElement.addEventListener("click", handleClick);
+  }
+});
+
 onUnmounted(() => {
+  if (renderer) {
+    renderer.instance.domElement.removeEventListener("click", handleClick);
+  }
   topScreenTexture?.dispose();
   bottomScreenTexture?.dispose();
 });