Add input handling functionality to the agent. Implement API endpoints for mouse click and text input, and update the web interface to support video capture and interaction features.

lib/input/input.go

@@ -0,0 +1,116 @@
+package input
+
+import (
+	"errors"
+	"strings"
+	"syscall"
+	"unicode/utf16"
+	"unsafe"
+)
+
+const (
+	inputMouse    = 0
+	inputKeyboard = 1
+
+	mouseLeftDown  = 0x0002
+	mouseLeftUp    = 0x0004
+	mouseRightDown = 0x0008
+	mouseRightUp   = 0x0010
+
+	keyeventfKeyup   = 0x0002
+	keyeventfUnicode = 0x0004
+)
+
+var (
+	ErrBadButton = errors.New("button must be left or right")
+	ErrEmptyText = errors.New("text must not be empty")
+
+	user32                 = syscall.NewLazyDLL("user32.dll")
+	procSetCursorPos       = user32.NewProc("SetCursorPos")
+	procSendInput          = user32.NewProc("SendInput")
+	procSetProcessDPIAware = user32.NewProc("SetProcessDPIAware")
+)
+
+type mouseInput struct {
+	Type      uint32
+	_         uint32
+	Dx        int32
+	Dy        int32
+	MouseData uint32
+	Flags     uint32
+	Time      uint32
+	ExtraInfo uintptr
+}
+
+type keybdInput struct {
+	Type      uint32
+	_         uint32
+	Vk        uint16
+	Scan      uint16
+	Flags     uint32
+	Time      uint32
+	ExtraInfo uintptr
+	_         [8]byte
+}
+
+func EnableDPIAwareness() {
+	_, _, _ = procSetProcessDPIAware.Call()
+}
+
+func Click(x, y int, button string) error {
+	down, up, err := mouseFlags(button)
+	if err != nil {
+		return err
+	}
+	ok, _, callErr := procSetCursorPos.Call(uintptr(x), uintptr(y))
+	if ok == 0 {
+		return callErr
+	}
+	inputs := []mouseInput{
+		{Type: inputMouse, Flags: down},
+		{Type: inputMouse, Flags: up},
+	}
+	return sendMouse(inputs)
+}
+
+func TypeText(text string) error {
+	if text == "" {
+		return ErrEmptyText
+	}
+	units := utf16.Encode([]rune(text))
+	inputs := make([]keybdInput, 0, len(units)*2)
+	for _, unit := range units {
+		inputs = append(inputs,
+			keybdInput{Type: inputKeyboard, Scan: unit, Flags: keyeventfUnicode},
+			keybdInput{Type: inputKeyboard, Scan: unit, Flags: keyeventfUnicode | keyeventfKeyup},
+		)
+	}
+	return sendKeys(inputs)
+}
+
+func mouseFlags(button string) (down, up uint32, err error) {
+	switch strings.ToLower(strings.TrimSpace(button)) {
+	case "", "left":
+		return mouseLeftDown, mouseLeftUp, nil
+	case "right":
+		return mouseRightDown, mouseRightUp, nil
+	default:
+		return 0, 0, ErrBadButton
+	}
+}
+
+func sendMouse(inputs []mouseInput) error {
+	n, _, err := procSendInput.Call(uintptr(len(inputs)), uintptr(unsafe.Pointer(&inputs[0])), uintptr(unsafe.Sizeof(inputs[0])))
+	if n == 0 {
+		return err
+	}
+	return nil
+}
+
+func sendKeys(inputs []keybdInput) error {
+	n, _, err := procSendInput.Call(uintptr(len(inputs)), uintptr(unsafe.Pointer(&inputs[0])), uintptr(unsafe.Sizeof(inputs[0])))
+	if n == 0 {
+		return err
+	}
+	return nil
+}