Add input handling functionality to the agent. Implement API endpoints for mouse click and text input, and update the web interface to support video capture and interaction features.

lib/agent/agent.go

@@ -14,6 +14,7 @@ import (
 	"tea.chunkbyte.com/kato/go-worm/lib/config"
 	"tea.chunkbyte.com/kato/go-worm/lib/files"
 	"tea.chunkbyte.com/kato/go-worm/lib/helpers"
+	"tea.chunkbyte.com/kato/go-worm/lib/input"
 	"tea.chunkbyte.com/kato/go-worm/lib/instance"
 )
 
@@ -42,6 +43,7 @@ func New() (*Agent, error) {
 		return nil, err
 	}
 	a.guard = guard
+	input.EnableDPIAwareness()
 	return a, nil
 }
 
@@ -63,6 +65,8 @@ func (a *Agent) Serve() error {
 	mux.HandleFunc("/api/v1/screenshot", a.handleScreenshot)
 	mux.HandleFunc("/api/v1/exec", a.handleExec)
 	mux.HandleFunc("/api/v1/startup", a.handleStartup)
+	mux.HandleFunc("/api/v1/input/click", a.handleClick)
+	mux.HandleFunc("/api/v1/input/text", a.handleText)
 
 	a.server = &http.Server{
 		Addr:              a.addr,

lib/agent/handlers.go

@@ -2,7 +2,6 @@ package agent
 
 import (
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"io"
@@ -19,6 +18,7 @@ import (
 	"tea.chunkbyte.com/kato/go-worm/lib/config"
 	"tea.chunkbyte.com/kato/go-worm/lib/files"
 	"tea.chunkbyte.com/kato/go-worm/lib/helpers"
+	"tea.chunkbyte.com/kato/go-worm/lib/input"
 	"tea.chunkbyte.com/kato/go-worm/lib/models"
 	"tea.chunkbyte.com/kato/go-worm/lib/screenshot"
 	"tea.chunkbyte.com/kato/go-worm/lib/startup"
@@ -40,12 +40,14 @@ func (a *Agent) handleOpenAPI(w http.ResponseWriter, r *http.Request) {
 	helpers.WriteJSON(w, http.StatusOK, map[string]any{
 		"openapi": "3.0.3", "info": map[string]string{"title": "Local Management Agent", "version": config.Version},
 		"paths": map[string]any{
-			"/api/v1/status":     map[string]any{"get": map[string]string{"summary": "Agent status"}},
+			"/api/v1/status":      map[string]any{"get": map[string]string{"summary": "Agent status"}},
-			"/api/v1/files":      map[string]any{"get": map[string]string{"summary": "List files"}},
+			"/api/v1/files":       map[string]any{"get": map[string]string{"summary": "List files"}},
-			"/api/v1/download":   map[string]any{"get": map[string]string{"summary": "Download file"}},
+			"/api/v1/download":    map[string]any{"get": map[string]string{"summary": "Download file"}},
-			"/api/v1/screenshot": map[string]any{"get": map[string]string{"summary": "Capture desktop"}},
+			"/api/v1/screenshot":  map[string]any{"get": map[string]string{"summary": "Capture desktop"}},
-			"/api/v1/exec":       map[string]any{"post": map[string]string{"summary": "Run a command"}},
+			"/api/v1/exec":        map[string]any{"post": map[string]string{"summary": "Run a command"}},
-			"/api/v1/startup":    map[string]any{"post": map[string]string{"summary": "Add to Windows startup"}, "delete": map[string]string{"summary": "Remove from Windows startup"}},
+			"/api/v1/startup":     map[string]any{"post": map[string]string{"summary": "Add to Windows startup"}, "delete": map[string]string{"summary": "Remove from Windows startup"}},
+			"/api/v1/input/click": map[string]any{"post": map[string]string{"summary": "Click the desktop"}},
+			"/api/v1/input/text":  map[string]any{"post": map[string]string{"summary": "Type text into the focused field"}},
 		},
 	})
 }
@@ -187,27 +189,101 @@ func (a *Agent) handleScreenshot(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	images, err := screenshot.Capture(format, quality)
+	monitor := 0
+	if raw := r.URL.Query().Get("monitor"); raw != "" {
+		var err error
+		monitor, err = strconv.Atoi(raw)
+		if err != nil || monitor < 0 {
+			helpers.WriteError(w, http.StatusBadRequest, "monitor must be 0 or greater")
+			return
+		}
+	}
+	frame, err := screenshot.CaptureMonitor(monitor, format, quality)
 	if err != nil {
+		if errors.Is(err, screenshot.ErrMonitorNotFound) {
+			helpers.WriteError(w, http.StatusBadRequest, err.Error())
+			return
+		}
 		helpers.Log.Printf("screenshot: %v", err)
 		helpers.WriteError(w, http.StatusServiceUnavailable, "no interactive desktop is available")
 		return
 	}
-	if len(images) == 1 {
+	w.Header().Set("Content-Type", frame.ContentType)
-		w.Header().Set("Content-Type", images[0].ContentType)
+	w.Header().Set("Content-Length", strconv.Itoa(len(frame.Data)))
-		w.Header().Set("Content-Length", strconv.Itoa(len(images[0].Data)))
+	w.Header().Set("X-Monitor-Left", strconv.Itoa(frame.Left))
-		_, _ = w.Write(images[0].Data)
+	w.Header().Set("X-Monitor-Top", strconv.Itoa(frame.Top))
+	w.Header().Set("X-Monitor-Width", strconv.Itoa(frame.Width))
+	w.Header().Set("X-Monitor-Height", strconv.Itoa(frame.Height))
+	_, _ = w.Write(frame.Data)
+}
+
+func (a *Agent) handleClick(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		helpers.WriteError(w, http.StatusMethodNotAllowed, "method not allowed")
 		return
 	}
-	response := models.ScreenshotResponse{Images: make([]models.ScreenshotImage, 0, len(images))}
+	r.Body = http.MaxBytesReader(w, r.Body, config.RequestBodyMax)
-	for i, item := range images {
+	defer r.Body.Close()
-		response.Images = append(response.Images, models.ScreenshotImage{
+	var request models.ClickRequest
-			Monitor:     i,
+	decoder := json.NewDecoder(r.Body)
-			ContentType: item.ContentType,
+	decoder.DisallowUnknownFields()
-			DataBase64:  base64.StdEncoding.EncodeToString(item.Data),
+	if err := decoder.Decode(&request); err != nil {
-		})
+		helpers.WriteError(w, http.StatusBadRequest, "body must contain click coordinates")
+		return
 	}
-	helpers.WriteJSON(w, http.StatusOK, response)
+	left, top, width, height, err := screenshot.MonitorBounds(request.Monitor)
+	if err != nil {
+		if errors.Is(err, screenshot.ErrMonitorNotFound) {
+			helpers.WriteError(w, http.StatusBadRequest, err.Error())
+			return
+		}
+		helpers.WriteError(w, http.StatusBadRequest, "monitor not found")
+		return
+	}
+	if request.X < left || request.Y < top || request.X >= left+width || request.Y >= top+height {
+		helpers.WriteError(w, http.StatusBadRequest, "click is outside the selected monitor")
+		return
+	}
+	if err := input.Click(request.X, request.Y, request.Button); err != nil {
+		if errors.Is(err, input.ErrBadButton) {
+			helpers.WriteError(w, http.StatusBadRequest, err.Error())
+			return
+		}
+		helpers.Log.Printf("click: %v", err)
+		helpers.WriteError(w, http.StatusInternalServerError, "could not click")
+		return
+	}
+	helpers.WriteJSON(w, http.StatusOK, map[string]any{"ok": true, "x": request.X, "y": request.Y})
+}
+
+func (a *Agent) handleText(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		helpers.WriteError(w, http.StatusMethodNotAllowed, "method not allowed")
+		return
+	}
+	r.Body = http.MaxBytesReader(w, r.Body, config.RequestBodyMax)
+	defer r.Body.Close()
+	var request models.TextRequest
+	decoder := json.NewDecoder(r.Body)
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&request); err != nil {
+		helpers.WriteError(w, http.StatusBadRequest, "body must contain text")
+		return
+	}
+	if request.Text == "" {
+		helpers.WriteError(w, http.StatusBadRequest, input.ErrEmptyText.Error())
+		return
+	}
+	if len(request.Text) > config.MaxInputText {
+		helpers.WriteError(w, http.StatusBadRequest, "text is too long")
+		return
+	}
+	if err := input.TypeText(request.Text); err != nil {
+		helpers.Log.Printf("text: %v", err)
+		helpers.WriteError(w, http.StatusInternalServerError, "could not type text")
+		return
+	}
+	helpers.WriteJSON(w, http.StatusOK, map[string]any{"ok": true, "length": len(request.Text)})
 }
 
 func (a *Agent) handleExec(w http.ResponseWriter, r *http.Request) {

lib/agent/web/app.js

@@ -11,8 +11,14 @@
   const startupState = document.getElementById("startup-state");
   const startupAdd = document.getElementById("startup-add");
   const startupRemove = document.getElementById("startup-remove");
+  const videoCanvas = document.getElementById("video-canvas");
+  const videoMeta = document.getElementById("video-meta");
+  const videoStart = document.getElementById("video-start");
+  const videoStop = document.getElementById("video-stop");
 
   let objectUrls = [];
+  let videoRunning = false;
+  let lastMonitor = { left: 0, top: 0, width: 0, height: 0 };
 
   function showError(message) {
     errorEl.textContent = message || "";
@@ -198,6 +204,92 @@
     loadStatus().catch((err) => showError(err.message));
   }
 
+  function clamp(value, min, max) {
+    const n = Number(value);
+    if (!Number.isFinite(n)) return min;
+    return Math.min(max, Math.max(min, Math.round(n)));
+  }
+
+  function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+
+  async function startVideo() {
+    if (videoRunning) return;
+    videoRunning = true;
+    videoStart.disabled = true;
+    videoStop.disabled = false;
+    while (videoRunning) {
+      const started = Date.now();
+      try {
+        await pullVideoFrame();
+      } catch (err) {
+        showError(err.message);
+      }
+      if (!videoRunning) break;
+      const fps = clamp(document.getElementById("video-fps").value, 1, 15);
+      await sleep(Math.max(0, 1000 / fps - (Date.now() - started)));
+    }
+  }
+
+  function stopVideo() {
+    videoRunning = false;
+    videoStart.disabled = false;
+    videoStop.disabled = true;
+  }
+
+  async function pullVideoFrame() {
+    const quality = clamp(document.getElementById("video-quality").value, 1, 100);
+    const monitor = clamp(document.getElementById("video-monitor").value, 0, 64);
+    const res = await api(`/api/v1/screenshot?format=jpeg&quality=${quality}&monitor=${monitor}`);
+    lastMonitor = {
+      left: Number(res.headers.get("X-Monitor-Left") || 0),
+      top: Number(res.headers.get("X-Monitor-Top") || 0),
+      width: Number(res.headers.get("X-Monitor-Width") || 0),
+      height: Number(res.headers.get("X-Monitor-Height") || 0),
+    };
+    const blob = await res.blob();
+    const bitmap = await createImageBitmap(blob);
+    if (videoCanvas.width !== bitmap.width || videoCanvas.height !== bitmap.height) {
+      videoCanvas.width = bitmap.width;
+      videoCanvas.height = bitmap.height;
+    }
+    const ctx = videoCanvas.getContext("2d");
+    ctx.drawImage(bitmap, 0, 0);
+    const width = bitmap.width;
+    const height = bitmap.height;
+    bitmap.close();
+    videoMeta.textContent = `${width}×${height} · monitor ${monitor} @ ${lastMonitor.left},${lastMonitor.top}`;
+  }
+
+  async function sendClick(event, button) {
+    if (!lastMonitor.width || !lastMonitor.height) return;
+    const rect = videoCanvas.getBoundingClientRect();
+    if (!rect.width || !rect.height) return;
+    const bitmapX = Math.floor((event.clientX - rect.left) * (videoCanvas.width / rect.width));
+    const bitmapY = Math.floor((event.clientY - rect.top) * (videoCanvas.height / rect.height));
+    const x = lastMonitor.left + bitmapX;
+    const y = lastMonitor.top + bitmapY;
+    const monitor = clamp(document.getElementById("video-monitor").value, 0, 64);
+    await api("/api/v1/input/click", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ x, y, button, monitor }),
+    });
+  }
+
+  async function sendVideoText() {
+    const field = document.getElementById("video-text");
+    const text = field.value;
+    if (!text) return;
+    await api("/api/v1/input/text", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ text }),
+    });
+    field.value = "";
+  }
+
   async function runCommand() {
     const command = document.getElementById("exec-command").value.trim();
     const timeout = Number(document.getElementById("exec-timeout").value);
@@ -219,6 +311,9 @@
       document.querySelectorAll(".panel").forEach((panel) => panel.classList.remove("active"));
       button.classList.add("active");
       document.getElementById(button.dataset.tab).classList.add("active");
+      if (button.dataset.tab !== "video") {
+        stopVideo();
+      }
     });
   });
 
@@ -240,6 +335,26 @@
   document.getElementById("shot-capture").addEventListener("click", () => {
     captureScreen().catch((err) => showError(err.message));
   });
+  videoStart.addEventListener("click", () => {
+    startVideo().catch((err) => showError(err.message));
+  });
+  videoStop.addEventListener("click", () => stopVideo());
+  videoCanvas.addEventListener("click", (event) => {
+    sendClick(event, "left").catch((err) => showError(err.message));
+  });
+  videoCanvas.addEventListener("contextmenu", (event) => {
+    event.preventDefault();
+    sendClick(event, "right").catch((err) => showError(err.message));
+  });
+  document.getElementById("video-send").addEventListener("click", () => {
+    sendVideoText().catch((err) => showError(err.message));
+  });
+  document.getElementById("video-text").addEventListener("keydown", (event) => {
+    if (event.key === "Enter") {
+      event.preventDefault();
+      sendVideoText().catch((err) => showError(err.message));
+    }
+  });
   document.getElementById("exec-run").addEventListener("click", () => {
     runCommand().catch((err) => showError(err.message));
   });

lib/agent/web/index.html

@@ -114,6 +114,15 @@
     .shots { display: flex; flex-wrap: wrap; gap: 0.75rem; }
     .shots figure { margin: 0; }
     .shots img { max-width: 100%; height: auto; border: 1px solid var(--line); border-radius: 8px; }
+    #video-canvas {
+      width: 100%;
+      height: auto;
+      display: block;
+      border: 1px solid var(--line);
+      border-radius: 8px;
+      cursor: crosshair;
+      background: #111;
+    }
     .error {
       display: none;
       max-width: 1100px;
@@ -134,6 +143,7 @@
     <button data-tab="status" class="active">Status</button>
     <button data-tab="files">Files</button>
     <button data-tab="screenshot">Screenshot</button>
+    <button data-tab="video">Video</button>
     <button data-tab="exec">Command</button>
   </nav>
   <main>
@@ -178,6 +188,27 @@
       </div>
       <div id="shot-gallery" class="shots"></div>
     </section>
+    <section id="video" class="panel">
+      <div class="row">
+        <button id="video-start" class="primary" type="button">Start</button>
+        <button id="video-stop" type="button" disabled>Stop</button>
+        <label>FPS
+          <input id="video-fps" type="number" min="1" max="15" value="5" style="width:4.5rem">
+        </label>
+        <label>Quality
+          <input id="video-quality" type="number" min="1" max="100" value="40" style="width:5rem">
+        </label>
+        <label>Monitor
+          <input id="video-monitor" type="number" min="0" value="0" style="width:4.5rem">
+        </label>
+      </div>
+      <canvas id="video-canvas" width="1280" height="720"></canvas>
+      <p id="video-meta" class="meta"></p>
+      <div class="row" style="margin-top:0.75rem">
+        <input id="video-text" type="text" placeholder="Type text for the focused field">
+        <button id="video-send" class="primary" type="button">Send</button>
+      </div>
+    </section>
     <section id="exec" class="panel">
       <div class="row">
         <textarea id="exec-command" placeholder="ipconfig /all"></textarea>

lib/config/config.go

@@ -16,6 +16,7 @@ const (
 	MaxExecTO        = 120
 	StartupValueName = "LocalManagementAgent"
 	StartupRunKey    = `Software\Microsoft\Windows\CurrentVersion\Run`
+	MaxInputText     = 4096
 )
 
 func EnvOr(name, fallback string) string {

lib/input/input.go

@@ -0,0 +1,116 @@
+package input
+
+import (
+	"errors"
+	"strings"
+	"syscall"
+	"unicode/utf16"
+	"unsafe"
+)
+
+const (
+	inputMouse    = 0
+	inputKeyboard = 1
+
+	mouseLeftDown  = 0x0002
+	mouseLeftUp    = 0x0004
+	mouseRightDown = 0x0008
+	mouseRightUp   = 0x0010
+
+	keyeventfKeyup   = 0x0002
+	keyeventfUnicode = 0x0004
+)
+
+var (
+	ErrBadButton = errors.New("button must be left or right")
+	ErrEmptyText = errors.New("text must not be empty")
+
+	user32                 = syscall.NewLazyDLL("user32.dll")
+	procSetCursorPos       = user32.NewProc("SetCursorPos")
+	procSendInput          = user32.NewProc("SendInput")
+	procSetProcessDPIAware = user32.NewProc("SetProcessDPIAware")
+)
+
+type mouseInput struct {
+	Type      uint32
+	_         uint32
+	Dx        int32
+	Dy        int32
+	MouseData uint32
+	Flags     uint32
+	Time      uint32
+	ExtraInfo uintptr
+}
+
+type keybdInput struct {
+	Type      uint32
+	_         uint32
+	Vk        uint16
+	Scan      uint16
+	Flags     uint32
+	Time      uint32
+	ExtraInfo uintptr
+	_         [8]byte
+}
+
+func EnableDPIAwareness() {
+	_, _, _ = procSetProcessDPIAware.Call()
+}
+
+func Click(x, y int, button string) error {
+	down, up, err := mouseFlags(button)
+	if err != nil {
+		return err
+	}
+	ok, _, callErr := procSetCursorPos.Call(uintptr(x), uintptr(y))
+	if ok == 0 {
+		return callErr
+	}
+	inputs := []mouseInput{
+		{Type: inputMouse, Flags: down},
+		{Type: inputMouse, Flags: up},
+	}
+	return sendMouse(inputs)
+}
+
+func TypeText(text string) error {
+	if text == "" {
+		return ErrEmptyText
+	}
+	units := utf16.Encode([]rune(text))
+	inputs := make([]keybdInput, 0, len(units)*2)
+	for _, unit := range units {
+		inputs = append(inputs,
+			keybdInput{Type: inputKeyboard, Scan: unit, Flags: keyeventfUnicode},
+			keybdInput{Type: inputKeyboard, Scan: unit, Flags: keyeventfUnicode | keyeventfKeyup},
+		)
+	}
+	return sendKeys(inputs)
+}
+
+func mouseFlags(button string) (down, up uint32, err error) {
+	switch strings.ToLower(strings.TrimSpace(button)) {
+	case "", "left":
+		return mouseLeftDown, mouseLeftUp, nil
+	case "right":
+		return mouseRightDown, mouseRightUp, nil
+	default:
+		return 0, 0, ErrBadButton
+	}
+}
+
+func sendMouse(inputs []mouseInput) error {
+	n, _, err := procSendInput.Call(uintptr(len(inputs)), uintptr(unsafe.Pointer(&inputs[0])), uintptr(unsafe.Sizeof(inputs[0])))
+	if n == 0 {
+		return err
+	}
+	return nil
+}
+
+func sendKeys(inputs []keybdInput) error {
+	n, _, err := procSendInput.Call(uintptr(len(inputs)), uintptr(unsafe.Pointer(&inputs[0])), uintptr(unsafe.Sizeof(inputs[0])))
+	if n == 0 {
+		return err
+	}
+	return nil
+}

lib/models/models.go

@@ -38,4 +38,19 @@ type ExecResponse struct {
 type CapturedImage struct {
 	ContentType string
 	Data        []byte
+	Left        int
+	Top         int
+	Width       int
+	Height      int
+}
+
+type ClickRequest struct {
+	X       int    `json:"x"`
+	Y       int    `json:"y"`
+	Button  string `json:"button"`
+	Monitor int    `json:"monitor"`
+}
+
+type TextRequest struct {
+	Text string `json:"text"`
 }

lib/screenshot/screenshot.go

@@ -13,6 +13,8 @@ import (
 	"tea.chunkbyte.com/kato/go-worm/lib/models"
 )
 
+var ErrMonitorNotFound = errors.New("monitor not found")
+
 var (
 	user32                  = syscall.NewLazyDLL("user32.dll")
 	gdi32                   = syscall.NewLazyDLL("gdi32.dll")
@@ -68,19 +70,80 @@ func Capture(format string, quality int) ([]models.CapturedImage, error) {
 	}
 	result := make([]models.CapturedImage, 0, len(monitors))
 	for _, monitor := range monitors {
-		img, err := captureRect(monitor)
+		frame, err := captureAndEncode(monitor, format, quality)
 		if err != nil {
 			return nil, err
 		}
-		data, contentType, err := encodeImage(img, format, quality)
+		result = append(result, frame)
-		if err != nil {
-			return nil, err
-		}
-		result = append(result, models.CapturedImage{ContentType: contentType, Data: data})
 	}
 	return result, nil
 }
 
+func CaptureMonitor(index int, format string, quality int) (models.CapturedImage, error) {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	if err := attachInputDesktop(); err != nil {
+		return models.CapturedImage{}, err
+	}
+	monitors, err := enumerateMonitors()
+	if err != nil {
+		return models.CapturedImage{}, err
+	}
+	if index < 0 || index >= len(monitors) {
+		return models.CapturedImage{}, ErrMonitorNotFound
+	}
+	return captureAndEncode(monitors[index], format, quality)
+}
+
+func MonitorCount() (int, error) {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	if err := attachInputDesktop(); err != nil {
+		return 0, err
+	}
+	monitors, err := enumerateMonitors()
+	if err != nil {
+		return 0, err
+	}
+	return len(monitors), nil
+}
+
+func MonitorBounds(index int) (left, top, width, height int, err error) {
+	runtime.LockOSThread()
+	defer runtime.UnlockOSThread()
+	if err := attachInputDesktop(); err != nil {
+		return 0, 0, 0, 0, err
+	}
+	monitors, err := enumerateMonitors()
+	if err != nil {
+		return 0, 0, 0, 0, err
+	}
+	if index < 0 || index >= len(monitors) {
+		return 0, 0, 0, 0, ErrMonitorNotFound
+	}
+	r := monitors[index]
+	return int(r.Left), int(r.Top), int(r.Right - r.Left), int(r.Bottom - r.Top), nil
+}
+
+func captureAndEncode(monitor rect, format string, quality int) (models.CapturedImage, error) {
+	img, err := captureRect(monitor)
+	if err != nil {
+		return models.CapturedImage{}, err
+	}
+	data, contentType, err := encodeImage(img, format, quality)
+	if err != nil {
+		return models.CapturedImage{}, err
+	}
+	return models.CapturedImage{
+		ContentType: contentType,
+		Data:        data,
+		Left:        int(monitor.Left),
+		Top:         int(monitor.Top),
+		Width:       int(monitor.Right - monitor.Left),
+		Height:      int(monitor.Bottom - monitor.Top),
+	}, nil
+}
+
 func attachInputDesktop() error {
 	h, _, err := procOpenInputDesktop.Call(0, 0, 0x0001|0x0040)
 	if h == 0 {