Add input handling functionality to the agent. Implement API endpoints for mouse click and text input, and update the web interface to support video capture and interaction features.

2026-08-18 19:12:25 +03:00
parent f24e31da27
commit 16d7aa75b3
8 changed files with 447 additions and 26 deletions
@@ -2,7 +2,6 @@ package agent

 import (
 	"context"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"io"
@@ -19,6 +18,7 @@ import (
 	"tea.chunkbyte.com/kato/go-worm/lib/config"
 	"tea.chunkbyte.com/kato/go-worm/lib/files"
 	"tea.chunkbyte.com/kato/go-worm/lib/helpers"
+	"tea.chunkbyte.com/kato/go-worm/lib/input"
 	"tea.chunkbyte.com/kato/go-worm/lib/models"
 	"tea.chunkbyte.com/kato/go-worm/lib/screenshot"
 	"tea.chunkbyte.com/kato/go-worm/lib/startup"
@@ -40,12 +40,14 @@ func (a *Agent) handleOpenAPI(w http.ResponseWriter, r *http.Request) {
 	helpers.WriteJSON(w, http.StatusOK, map[string]any{
 		"openapi": "3.0.3", "info": map[string]string{"title": "Local Management Agent", "version": config.Version},
 		"paths": map[string]any{
-			"/api/v1/status":     map[string]any{"get": map[string]string{"summary": "Agent status"}},
-			"/api/v1/files":      map[string]any{"get": map[string]string{"summary": "List files"}},
-			"/api/v1/download":   map[string]any{"get": map[string]string{"summary": "Download file"}},
-			"/api/v1/screenshot": map[string]any{"get": map[string]string{"summary": "Capture desktop"}},
-			"/api/v1/exec":       map[string]any{"post": map[string]string{"summary": "Run a command"}},
-			"/api/v1/startup":    map[string]any{"post": map[string]string{"summary": "Add to Windows startup"}, "delete": map[string]string{"summary": "Remove from Windows startup"}},
+			"/api/v1/status":      map[string]any{"get": map[string]string{"summary": "Agent status"}},
+			"/api/v1/files":       map[string]any{"get": map[string]string{"summary": "List files"}},
+			"/api/v1/download":    map[string]any{"get": map[string]string{"summary": "Download file"}},
+			"/api/v1/screenshot":  map[string]any{"get": map[string]string{"summary": "Capture desktop"}},
+			"/api/v1/exec":        map[string]any{"post": map[string]string{"summary": "Run a command"}},
+			"/api/v1/startup":     map[string]any{"post": map[string]string{"summary": "Add to Windows startup"}, "delete": map[string]string{"summary": "Remove from Windows startup"}},
+			"/api/v1/input/click": map[string]any{"post": map[string]string{"summary": "Click the desktop"}},
+			"/api/v1/input/text":  map[string]any{"post": map[string]string{"summary": "Type text into the focused field"}},
 		},
 	})
 }
@@ -187,27 +189,101 @@ func (a *Agent) handleScreenshot(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	images, err := screenshot.Capture(format, quality)
+	monitor := 0
+	if raw := r.URL.Query().Get("monitor"); raw != "" {
+		var err error
+		monitor, err = strconv.Atoi(raw)
+		if err != nil || monitor < 0 {
+			helpers.WriteError(w, http.StatusBadRequest, "monitor must be 0 or greater")
+			return
+		}
+	}
+	frame, err := screenshot.CaptureMonitor(monitor, format, quality)
 	if err != nil {
+		if errors.Is(err, screenshot.ErrMonitorNotFound) {
+			helpers.WriteError(w, http.StatusBadRequest, err.Error())
+			return
+		}
 		helpers.Log.Printf("screenshot: %v", err)
 		helpers.WriteError(w, http.StatusServiceUnavailable, "no interactive desktop is available")
 		return
 	}
-	if len(images) == 1 {
-		w.Header().Set("Content-Type", images[0].ContentType)
-		w.Header().Set("Content-Length", strconv.Itoa(len(images[0].Data)))
-		_, _ = w.Write(images[0].Data)
+	w.Header().Set("Content-Type", frame.ContentType)
+	w.Header().Set("Content-Length", strconv.Itoa(len(frame.Data)))
+	w.Header().Set("X-Monitor-Left", strconv.Itoa(frame.Left))
+	w.Header().Set("X-Monitor-Top", strconv.Itoa(frame.Top))
+	w.Header().Set("X-Monitor-Width", strconv.Itoa(frame.Width))
+	w.Header().Set("X-Monitor-Height", strconv.Itoa(frame.Height))
+	_, _ = w.Write(frame.Data)
+}
+
+func (a *Agent) handleClick(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		helpers.WriteError(w, http.StatusMethodNotAllowed, "method not allowed")
 		return
 	}
-	response := models.ScreenshotResponse{Images: make([]models.ScreenshotImage, 0, len(images))}
-	for i, item := range images {
-		response.Images = append(response.Images, models.ScreenshotImage{
-			Monitor:     i,
-			ContentType: item.ContentType,
-			DataBase64:  base64.StdEncoding.EncodeToString(item.Data),
-		})
+	r.Body = http.MaxBytesReader(w, r.Body, config.RequestBodyMax)
+	defer r.Body.Close()
+	var request models.ClickRequest
+	decoder := json.NewDecoder(r.Body)
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&request); err != nil {
+		helpers.WriteError(w, http.StatusBadRequest, "body must contain click coordinates")
+		return
 	}
-	helpers.WriteJSON(w, http.StatusOK, response)
+	left, top, width, height, err := screenshot.MonitorBounds(request.Monitor)
+	if err != nil {
+		if errors.Is(err, screenshot.ErrMonitorNotFound) {
+			helpers.WriteError(w, http.StatusBadRequest, err.Error())
+			return
+		}
+		helpers.WriteError(w, http.StatusBadRequest, "monitor not found")
+		return
+	}
+	if request.X < left || request.Y < top || request.X >= left+width || request.Y >= top+height {
+		helpers.WriteError(w, http.StatusBadRequest, "click is outside the selected monitor")
+		return
+	}
+	if err := input.Click(request.X, request.Y, request.Button); err != nil {
+		if errors.Is(err, input.ErrBadButton) {
+			helpers.WriteError(w, http.StatusBadRequest, err.Error())
+			return
+		}
+		helpers.Log.Printf("click: %v", err)
+		helpers.WriteError(w, http.StatusInternalServerError, "could not click")
+		return
+	}
+	helpers.WriteJSON(w, http.StatusOK, map[string]any{"ok": true, "x": request.X, "y": request.Y})
+}
+
+func (a *Agent) handleText(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		helpers.WriteError(w, http.StatusMethodNotAllowed, "method not allowed")
+		return
+	}
+	r.Body = http.MaxBytesReader(w, r.Body, config.RequestBodyMax)
+	defer r.Body.Close()
+	var request models.TextRequest
+	decoder := json.NewDecoder(r.Body)
+	decoder.DisallowUnknownFields()
+	if err := decoder.Decode(&request); err != nil {
+		helpers.WriteError(w, http.StatusBadRequest, "body must contain text")
+		return
+	}
+	if request.Text == "" {
+		helpers.WriteError(w, http.StatusBadRequest, input.ErrEmptyText.Error())
+		return
+	}
+	if len(request.Text) > config.MaxInputText {
+		helpers.WriteError(w, http.StatusBadRequest, "text is too long")
+		return
+	}
+	if err := input.TypeText(request.Text); err != nil {
+		helpers.Log.Printf("text: %v", err)
+		helpers.WriteError(w, http.StatusInternalServerError, "could not type text")
+		return
+	}
+	helpers.WriteJSON(w, http.StatusOK, map[string]any{"ok": true, "length": len(request.Text)})
 }

 func (a *Agent) handleExec(w http.ResponseWriter, r *http.Request) {