Add input handling functionality to the agent. Implement API endpoints for mouse click and text input, and update the web interface to support video capture and interaction features.

lib/agent/web/app.js

@@ -11,8 +11,14 @@
   const startupState = document.getElementById("startup-state");
   const startupAdd = document.getElementById("startup-add");
   const startupRemove = document.getElementById("startup-remove");
+  const videoCanvas = document.getElementById("video-canvas");
+  const videoMeta = document.getElementById("video-meta");
+  const videoStart = document.getElementById("video-start");
+  const videoStop = document.getElementById("video-stop");
 
   let objectUrls = [];
+  let videoRunning = false;
+  let lastMonitor = { left: 0, top: 0, width: 0, height: 0 };
 
   function showError(message) {
     errorEl.textContent = message || "";
@@ -198,6 +204,92 @@
     loadStatus().catch((err) => showError(err.message));
   }
 
+  function clamp(value, min, max) {
+    const n = Number(value);
+    if (!Number.isFinite(n)) return min;
+    return Math.min(max, Math.max(min, Math.round(n)));
+  }
+
+  function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  async function startVideo() {
+    if (videoRunning) return;
+    videoRunning = true;
+    videoStart.disabled = true;
+    videoStop.disabled = false;
+    while (videoRunning) {
+      const started = Date.now();
+      try {
+        await pullVideoFrame();
+      } catch (err) {
+        showError(err.message);
+      }
+      if (!videoRunning) break;
+      const fps = clamp(document.getElementById("video-fps").value, 1, 15);
+      await sleep(Math.max(0, 1000 / fps - (Date.now() - started)));
+    }
+  }
+
+  function stopVideo() {
+    videoRunning = false;
+    videoStart.disabled = false;
+    videoStop.disabled = true;
+  }
+
+  async function pullVideoFrame() {
+    const quality = clamp(document.getElementById("video-quality").value, 1, 100);
+    const monitor = clamp(document.getElementById("video-monitor").value, 0, 64);
+    const res = await api(`/api/v1/screenshot?format=jpeg&quality=${quality}&monitor=${monitor}`);
+    lastMonitor = {
+      left: Number(res.headers.get("X-Monitor-Left") || 0),
+      top: Number(res.headers.get("X-Monitor-Top") || 0),
+      width: Number(res.headers.get("X-Monitor-Width") || 0),
+      height: Number(res.headers.get("X-Monitor-Height") || 0),
+    };
+    const blob = await res.blob();
+    const bitmap = await createImageBitmap(blob);
+    if (videoCanvas.width !== bitmap.width || videoCanvas.height !== bitmap.height) {
+      videoCanvas.width = bitmap.width;
+      videoCanvas.height = bitmap.height;
+    }
+    const ctx = videoCanvas.getContext("2d");
+    ctx.drawImage(bitmap, 0, 0);
+    const width = bitmap.width;
+    const height = bitmap.height;
+    bitmap.close();
+    videoMeta.textContent = `${width}×${height} · monitor ${monitor} @ ${lastMonitor.left},${lastMonitor.top}`;
+  }
+
+  async function sendClick(event, button) {
+    if (!lastMonitor.width || !lastMonitor.height) return;
+    const rect = videoCanvas.getBoundingClientRect();
+    if (!rect.width || !rect.height) return;
+    const bitmapX = Math.floor((event.clientX - rect.left) * (videoCanvas.width / rect.width));
+    const bitmapY = Math.floor((event.clientY - rect.top) * (videoCanvas.height / rect.height));
+    const x = lastMonitor.left + bitmapX;
+    const y = lastMonitor.top + bitmapY;
+    const monitor = clamp(document.getElementById("video-monitor").value, 0, 64);
+    await api("/api/v1/input/click", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ x, y, button, monitor }),
+    });
+  }
+
+  async function sendVideoText() {
+    const field = document.getElementById("video-text");
+    const text = field.value;
+    if (!text) return;
+    await api("/api/v1/input/text", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ text }),
+    });
+    field.value = "";
+  }
+
   async function runCommand() {
     const command = document.getElementById("exec-command").value.trim();
     const timeout = Number(document.getElementById("exec-timeout").value);
@@ -219,6 +311,9 @@
       document.querySelectorAll(".panel").forEach((panel) => panel.classList.remove("active"));
       button.classList.add("active");
       document.getElementById(button.dataset.tab).classList.add("active");
+      if (button.dataset.tab !== "video") {
+        stopVideo();
+      }
     });
   });
 
@@ -240,6 +335,26 @@
   document.getElementById("shot-capture").addEventListener("click", () => {
     captureScreen().catch((err) => showError(err.message));
   });
+  videoStart.addEventListener("click", () => {
+    startVideo().catch((err) => showError(err.message));
+  });
+  videoStop.addEventListener("click", () => stopVideo());
+  videoCanvas.addEventListener("click", (event) => {
+    sendClick(event, "left").catch((err) => showError(err.message));
+  });
+  videoCanvas.addEventListener("contextmenu", (event) => {
+    event.preventDefault();
+    sendClick(event, "right").catch((err) => showError(err.message));
+  });
+  document.getElementById("video-send").addEventListener("click", () => {
+    sendVideoText().catch((err) => showError(err.message));
+  });
+  document.getElementById("video-text").addEventListener("keydown", (event) => {
+    if (event.key === "Enter") {
+      event.preventDefault();
+      sendVideoText().catch((err) => showError(err.message));
+    }
+  });
   document.getElementById("exec-run").addEventListener("click", () => {
     runCommand().catch((err) => showError(err.message));
   });