From e720f5a6c7f0896701dbe8ca9b60d686f82c2cd7 Mon Sep 17 00:00:00 2001
From: Pihkaal <hello@pihkaal.me>
Date: Sun, 14 Dec 2025 14:36:30 +0100
Subject: [PATCH] feat: dispatch clicks on the 3d models to the canvases

---
 app/components/NDS.vue    | 61 ++++++++++++++++++++++++++++++++++++++-
 app/components/Screen.vue | 10 +++----
 app/pages/index.vue       | 30 +++++++++++++++++--
 3 files changed, 92 insertions(+), 9 deletions(-)

diff --git a/app/components/NDS.vue b/app/components/NDS.vue
index f64cc88..418e82e 100644
--- a/app/components/NDS.vue
+++ b/app/components/NDS.vue
@@ -1,6 +1,6 @@
 <script setup lang="ts">
 import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
-import { useLoop } from "@tresjs/core";
+import { useLoop, useTresContext } from "@tresjs/core";
 import * as THREE from "three";
 
 const props = defineProps<{
@@ -8,6 +8,11 @@ const props = defineProps<{
   bottomScreenCanvas: HTMLCanvasElement | null;
 }>();
 
+const emit = defineEmits<{
+  topScreenClick: [x: number, y: number];
+  bottomScreenClick: [x: number, y: number];
+}>();
+
 const { state: model } = useLoader(
   GLTFLoader,
   "/models/nintendo-ds/scene.gltf",
@@ -17,6 +22,10 @@ const scene = computed(() => model.value?.scene);
 
 let topScreenTexture: THREE.CanvasTexture | null = null;
 let bottomScreenTexture: THREE.CanvasTexture | null = null;
+let topScreenMesh: THREE.Mesh | null = null;
+let bottomScreenMesh: THREE.Mesh | null = null;
+
+const { camera, renderer } = useTresContext();
 
 watch(
   () => [props.topScreenCanvas, props.bottomScreenCanvas],
@@ -51,6 +60,7 @@ watch(scene, () => {
           emissive: new THREE.Color(0x222222),
           emissiveIntensity: 0.5,
         });
+        topScreenMesh = child;
       } else if (
         material.name?.includes("screen_down") &&
         bottomScreenTexture
@@ -62,6 +72,7 @@ watch(scene, () => {
           emissive: new THREE.Color(0x222222),
           emissiveIntensity: 0.5,
         });
+        bottomScreenMesh = child;
       }
     }
   });
@@ -74,7 +85,55 @@ onRender(() => {
   if (bottomScreenTexture) bottomScreenTexture.needsUpdate = true;
 });
 
+const handleClick = (event: MouseEvent) => {
+  const domElement = renderer.instance.domElement;
+  const rect = domElement.getBoundingClientRect();
+
+  const raycaster = new THREE.Raycaster();
+  raycaster.setFromCamera(
+    new THREE.Vector2(
+      ((event.clientX - rect.left) / rect.width) * 2 - 1,
+      -((event.clientY - rect.top) / rect.height) * 2 + 1,
+    ),
+    camera.activeCamera.value,
+  );
+
+  if (topScreenMesh) {
+    const intersects = raycaster.intersectObject(topScreenMesh);
+    if (intersects[0]) {
+      const uv = intersects[0].uv;
+      if (uv) {
+        const x = Math.floor(uv.x * 256);
+        const y = Math.floor(uv.y * (1024 / 404) * 192);
+        emit("topScreenClick", x, y);
+        return;
+      }
+    }
+  }
+
+  if (bottomScreenMesh) {
+    const intersects = raycaster.intersectObject(bottomScreenMesh);
+    if (intersects[0]) {
+      const uv = intersects[0].uv;
+      if (uv) {
+        const x = Math.floor(uv.x * 256);
+        const y = Math.floor(192 - (1 - uv.y) * (1024 / 532) * 192);
+        emit("bottomScreenClick", x, y);
+      }
+    }
+  }
+};
+
+onMounted(() => {
+  if (renderer) {
+    renderer.instance.domElement.addEventListener("click", handleClick);
+  }
+});
+
 onUnmounted(() => {
+  if (renderer) {
+    renderer.instance.domElement.removeEventListener("click", handleClick);
+  }
   topScreenTexture?.dispose();
   bottomScreenTexture?.dispose();
 });
diff --git a/app/components/Screen.vue b/app/components/Screen.vue
index 1bceeb2..dd23cda 100644
--- a/app/components/Screen.vue
+++ b/app/components/Screen.vue
@@ -83,17 +83,17 @@ const renderFrame = (timestamp: number) => {
   animationFrameId = requestAnimationFrame(renderFrame);
 };
 
+provide("registerUpdateCallback", registerUpdateCallback);
+provide("registerRenderCallback", registerRenderCallback);
+provide("registerScreenClickCallback", registerScreenClickCallback);
+provide("registerScreenMouseWheelCallback", registerScreenMouseWheelCallback);
+
 onMounted(() => {
   if (!canvas.value) throw new Error("Missing canvas");
 
   ctx = canvas.value.getContext("2d");
   if (!ctx) throw new Error("Missing 2d context");
 
-  provide("registerUpdateCallback", registerUpdateCallback);
-  provide("registerRenderCallback", registerRenderCallback);
-  provide("registerScreenClickCallback", registerScreenClickCallback);
-  provide("registerScreenMouseWheelCallback", registerScreenMouseWheelCallback);
-
   canvas.value.addEventListener("click", handleCanvasClick);
   canvas.value.addEventListener("wheel", handleCanvasWheel, { passive: true });
 
diff --git a/app/pages/index.vue b/app/pages/index.vue
index 06ddc02..988a908 100644
--- a/app/pages/index.vue
+++ b/app/pages/index.vue
@@ -6,6 +6,26 @@ const screen = computed(() => route.query.screen as string | undefined);
 
 const topScreen = useTemplateRef("topScreen");
 const bottomScreen = useTemplateRef("bottomScreen");
+
+const topScreenCanvas = computed(() => topScreen.value?.canvas ?? null);
+const bottomScreenCanvas = computed(() => bottomScreen.value?.canvas ?? null);
+
+const handleScreenClick = (
+  canvas: HTMLCanvasElement | null,
+  x: number,
+  y: number,
+) => {
+  if (!canvas) return;
+
+  const rect = canvas.getBoundingClientRect();
+  const clickEvent = new MouseEvent("click", {
+    bubbles: true,
+    cancelable: true,
+    clientX: (x / SCREEN_WIDTH) * rect.width + rect.left,
+    clientY: (y / SCREEN_HEIGHT) * rect.height + rect.top,
+  });
+  canvas.dispatchEvent(clickEvent);
+};
 </script>
 
 <template>
@@ -18,9 +38,13 @@ const bottomScreen = useTemplateRef("bottomScreen");
       <TresDirectionalLight />
 
       <NDS
-        v-if="topScreen && bottomScreen"
-        :top-screen-canvas="topScreen.canvas"
-        :bottom-screen-canvas="bottomScreen.canvas"
+        v-if="topScreenCanvas && bottomScreenCanvas"
+        :top-screen-canvas="topScreenCanvas"
+        :bottom-screen-canvas="bottomScreenCanvas"
+        @top-screen-click="(x, y) => handleScreenClick(topScreenCanvas, x, y)"
+        @bottom-screen-click="
+          (x, y) => handleScreenClick(bottomScreenCanvas, x, y)
+        "
       />
     </TresCanvas>