Add input handling functionality to the agent. Implement API endpoints for mouse click and text input, and update the web interface to support video capture and interaction features.

2026-08-18 19:12:25 +03:00
parent f24e31da27
commit 16d7aa75b3
8 changed files with 447 additions and 26 deletions
@@ -11,8 +11,14 @@
  const startupState = document.getElementById("startup-state");
  const startupAdd = document.getElementById("startup-add");
  const startupRemove = document.getElementById("startup-remove");
+  const videoCanvas = document.getElementById("video-canvas");
+  const videoMeta = document.getElementById("video-meta");
+  const videoStart = document.getElementById("video-start");
+  const videoStop = document.getElementById("video-stop");

  let objectUrls = [];
+  let videoRunning = false;
+  let lastMonitor = { left: 0, top: 0, width: 0, height: 0 };

  function showError(message) {
    errorEl.textContent = message || "";
@@ -198,6 +204,92 @@
    loadStatus().catch((err) => showError(err.message));
  }

+  function clamp(value, min, max) {
+    const n = Number(value);
+    if (!Number.isFinite(n)) return min;
+    return Math.min(max, Math.max(min, Math.round(n)));
+  }
+
+  function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  async function startVideo() {
+    if (videoRunning) return;
+    videoRunning = true;
+    videoStart.disabled = true;
+    videoStop.disabled = false;
+    while (videoRunning) {
+      const started = Date.now();
+      try {
+        await pullVideoFrame();
+      } catch (err) {
+        showError(err.message);
+      }
+      if (!videoRunning) break;
+      const fps = clamp(document.getElementById("video-fps").value, 1, 15);
+      await sleep(Math.max(0, 1000 / fps - (Date.now() - started)));
+    }
+  }
+
+  function stopVideo() {
+    videoRunning = false;
+    videoStart.disabled = false;
+    videoStop.disabled = true;
+  }
+
+  async function pullVideoFrame() {
+    const quality = clamp(document.getElementById("video-quality").value, 1, 100);
+    const monitor = clamp(document.getElementById("video-monitor").value, 0, 64);
+    const res = await api(`/api/v1/screenshot?format=jpeg&quality=${quality}&monitor=${monitor}`);
+    lastMonitor = {
+      left: Number(res.headers.get("X-Monitor-Left") || 0),
+      top: Number(res.headers.get("X-Monitor-Top") || 0),
+      width: Number(res.headers.get("X-Monitor-Width") || 0),
+      height: Number(res.headers.get("X-Monitor-Height") || 0),
+    };
+    const blob = await res.blob();
+    const bitmap = await createImageBitmap(blob);
+    if (videoCanvas.width !== bitmap.width || videoCanvas.height !== bitmap.height) {
+      videoCanvas.width = bitmap.width;
+      videoCanvas.height = bitmap.height;
+    }
+    const ctx = videoCanvas.getContext("2d");
+    ctx.drawImage(bitmap, 0, 0);
+    const width = bitmap.width;
+    const height = bitmap.height;
+    bitmap.close();
+    videoMeta.textContent = `${width}×${height} · monitor ${monitor} @ ${lastMonitor.left},${lastMonitor.top}`;
+  }
+
+  async function sendClick(event, button) {
+    if (!lastMonitor.width || !lastMonitor.height) return;
+    const rect = videoCanvas.getBoundingClientRect();
+    if (!rect.width || !rect.height) return;
+    const bitmapX = Math.floor((event.clientX - rect.left) * (videoCanvas.width / rect.width));
+    const bitmapY = Math.floor((event.clientY - rect.top) * (videoCanvas.height / rect.height));
+    const x = lastMonitor.left + bitmapX;
+    const y = lastMonitor.top + bitmapY;
+    const monitor = clamp(document.getElementById("video-monitor").value, 0, 64);
+    await api("/api/v1/input/click", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ x, y, button, monitor }),
+    });
+  }
+
+  async function sendVideoText() {
+    const field = document.getElementById("video-text");
+    const text = field.value;
+    if (!text) return;
+    await api("/api/v1/input/text", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ text }),
+    });
+    field.value = "";
+  }
+
  async function runCommand() {
    const command = document.getElementById("exec-command").value.trim();
    const timeout = Number(document.getElementById("exec-timeout").value);
@@ -219,6 +311,9 @@
      document.querySelectorAll(".panel").forEach((panel) => panel.classList.remove("active"));
      button.classList.add("active");
      document.getElementById(button.dataset.tab).classList.add("active");
+      if (button.dataset.tab !== "video") {
+        stopVideo();
+      }
    });
  });

@@ -240,6 +335,26 @@
  document.getElementById("shot-capture").addEventListener("click", () => {
    captureScreen().catch((err) => showError(err.message));
  });
+  videoStart.addEventListener("click", () => {
+    startVideo().catch((err) => showError(err.message));
+  });
+  videoStop.addEventListener("click", () => stopVideo());
+  videoCanvas.addEventListener("click", (event) => {
+    sendClick(event, "left").catch((err) => showError(err.message));
+  });
+  videoCanvas.addEventListener("contextmenu", (event) => {
+    event.preventDefault();
+    sendClick(event, "right").catch((err) => showError(err.message));
+  });
+  document.getElementById("video-send").addEventListener("click", () => {
+    sendVideoText().catch((err) => showError(err.message));
+  });
+  document.getElementById("video-text").addEventListener("keydown", (event) => {
+    if (event.key === "Enter") {
+      event.preventDefault();
+      sendVideoText().catch((err) => showError(err.message));
+    }
+  });
  document.getElementById("exec-run").addEventListener("click", () => {
    runCommand().catch((err) => showError(err.message));
  });
@@ -114,6 +114,15 @@
    .shots { display: flex; flex-wrap: wrap; gap: 0.75rem; }
    .shots figure { margin: 0; }
    .shots img { max-width: 100%; height: auto; border: 1px solid var(--line); border-radius: 8px; }
+    #video-canvas {
+      width: 100%;
+      height: auto;
+      display: block;
+      border: 1px solid var(--line);
+      border-radius: 8px;
+      cursor: crosshair;
+      background: #111;
+    }
    .error {
      display: none;
      max-width: 1100px;
@@ -134,6 +143,7 @@
    <button data-tab="status" class="active">Status</button>
    <button data-tab="files">Files</button>
    <button data-tab="screenshot">Screenshot</button>
+    <button data-tab="video">Video</button>
    <button data-tab="exec">Command</button>
  </nav>
  <main>
@@ -178,6 +188,27 @@
      </div>
      <div id="shot-gallery" class="shots"></div>
    </section>
+    <section id="video" class="panel">
+      <div class="row">
+        <button id="video-start" class="primary" type="button">Start</button>
+        <button id="video-stop" type="button" disabled>Stop</button>
+        <label>FPS
+          <input id="video-fps" type="number" min="1" max="15" value="5" style="width:4.5rem">
+        </label>
+        <label>Quality
+          <input id="video-quality" type="number" min="1" max="100" value="40" style="width:5rem">
+        </label>
+        <label>Monitor
+          <input id="video-monitor" type="number" min="0" value="0" style="width:4.5rem">
+        </label>
+      </div>
+      <canvas id="video-canvas" width="1280" height="720"></canvas>
+      <p id="video-meta" class="meta"></p>
+      <div class="row" style="margin-top:0.75rem">
+        <input id="video-text" type="text" placeholder="Type text for the focused field">
+        <button id="video-send" class="primary" type="button">Send</button>
+      </div>
+    </section>
    <section id="exec" class="panel">
      <div class="row">
        <textarea id="exec-command" placeholder="ipconfig /all"></textarea>