Desktop Automation

Desktop APIs require a desktop-capable template such as desktop.

Screenshot and display

from nullspace import Machine

with Machine.create(
    template="desktop",
    timeout=300,
    resolution=(1440, 900),
    dpi=144,
) as machine:
    png = machine.desktop.screenshot()
    width, height = machine.desktop.screen_size()
    display = machine.desktop.display_info()
    print(len(png), width, height)
    print(display.virtual_screen.width, display.virtual_screen.height)

    for monitor in display.monitors:
        print(monitor.id, monitor.width, monitor.height, monitor.is_primary)

nullspace machine desktop screenshot mch_123 -o screenshot.png
nullspace machine desktop screen-size mch_123
nullspace machine desktop display mch_123

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/screenshot" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -o screenshot.png

curl "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/screen_size" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}"

curl "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/display" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}"

Regions and compressed screenshots

region_png = machine.desktop.screenshot_region(0, 0, 400, 300)
jpeg = machine.desktop.screenshot_compressed(
    format="jpeg",
    quality=70,
    scale=0.5,
    show_cursor=True,
    region=(0, 0, 800, 600),
)

caps = machine.desktop.capabilities()
print(caps.capture.compressed_formats)

nullspace machine desktop screenshot-region mch_123 0 0 400 300 -o region.png
nullspace machine desktop screenshot-compressed mch_123 \
  --format jpeg --quality 70 --scale 0.5 --show-cursor \
  --x 0 --y 0 --width 800 --height 600 -o region.jpeg
nullspace machine desktop capabilities mch_123

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/screenshot/region" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{"x": 0, "y": 0, "width": 400, "height": 300}' \
  -o region.png

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/screenshot/compressed" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{"format": "jpeg", "quality": 70, "scale": 0.5, "show_cursor": true, "region": {"x": 0, "y": 0, "width": 800, "height": 600}}' \
  -o region.jpeg

curl "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/capabilities" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}"

Mouse and keyboard

machine.desktop.move(100, 100)
machine.desktop.click(100, 100)
machine.desktop.left_click()
machine.desktop.right_click(100, 100)
machine.desktop.mouse_press("left", 100, 100)
machine.desktop.mouse_release("left", 300, 300)
machine.desktop.double_click(200, 200)
machine.desktop.drag(100, 100, 300, 300)
machine.desktop.scroll(400, 400, "down", amount=3)

machine.desktop.type("hello world")
machine.desktop.key("Return")
machine.desktop.hotkey("ctrl+a")
machine.desktop.press("c", modifiers=["ctrl"])

nullspace machine desktop move mch_123 100 100
nullspace machine desktop click mch_123 100 100
nullspace machine desktop left-click mch_123
nullspace machine desktop right-click mch_123 --x 100 --y 100
nullspace machine desktop mouse-down mch_123 --button left --x 100 --y 100
nullspace machine desktop mouse-up mch_123 --button left --x 300 --y 300
nullspace machine desktop double-click mch_123 200 200
nullspace machine desktop drag mch_123 100 100 300 300
nullspace machine desktop scroll mch_123 400 400 down --amount 3

nullspace machine desktop type mch_123 "hello world"
nullspace machine desktop key mch_123 Return
nullspace machine desktop hotkey mch_123 "ctrl+a"
nullspace machine desktop press mch_123 c --modifier ctrl

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/move" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"x": 100, "y": 100}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/click" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"x": 100, "y": 100, "button": "left"}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/mouse/down" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"button": "left", "x": 100, "y": 100}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/mouse/up" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"button": "left", "x": 300, "y": 300}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/click" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"x": 200, "y": 200, "clicks": 2}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/drag" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"x1": 100, "y1": 100, "x2": 300, "y2": 300}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/scroll" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"x": 400, "y": 400, "direction": "down", "amount": 3}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/type" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"text": "hello world"}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/key" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"key": "Return"}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/hotkey" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"keys": "ctrl+a"}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/press" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"key": "c", "modifiers": ["ctrl"]}'

Windows and clipboard

for window in machine.desktop.list_windows():
    print(window.id, window.title)

current = machine.desktop.get_current_window_id()
print(machine.desktop.get_window_title(current))

machine.desktop.set_clipboard_text("copied from sdk")
print(machine.desktop.get_clipboard_text())

nullspace machine desktop windows mch_123
nullspace machine desktop current-window mch_123
nullspace machine desktop window-title mch_123 0x1a00007
nullspace machine desktop clipboard set mch_123 "copied from sdk"
nullspace machine desktop clipboard get mch_123

curl "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/windows" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}"

curl "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/window/current" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}"

curl "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/window/0x1a00007/title" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}"

curl -X PUT "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/clipboard" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"text": "copied from sdk"}'

curl "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/clipboard" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}"

The window title route is read-only (GET /desktop/window/{window_id}/title). There is no endpoint or command to set a window title. The pointer position and per-deployment capability matrix are also read-only surfaces:

x, y = machine.desktop.cursor_position()
caps = machine.desktop.capabilities()
print(x, y, caps.capture.compressed_formats)

nullspace machine desktop cursor mch_123
nullspace machine desktop capabilities mch_123

curl "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/cursor" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}"

curl "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/capabilities" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}"

Launch and open

machine.desktop.launch_executable("xterm")
machine.desktop.launch_desktop_entry("firefox.desktop", uri="https://example.com")
machine.desktop.open("https://example.com")

nullspace machine desktop launch executable mch_123 xterm
nullspace machine desktop launch desktop-entry mch_123 firefox.desktop \
  --uri "https://example.com"
nullspace machine desktop open mch_123 "https://example.com"

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/launch" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{"mode": "executable", "command": "xterm", "args": []}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/launch" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" \
  -d '{"mode": "desktop_entry", "desktop_entry": "firefox.desktop", "uri": "https://example.com"}'

curl -X POST "${NULLSPACE_API_URL}/v1/machines/mch_123/desktop/open" \
  -H "Authorization: Bearer ${NULLSPACE_API_KEY}" \
  -H "Content-Type: application/json" -d '{"target": "https://example.com"}'

​Screenshot and display

​Regions and compressed screenshots

​Mouse and keyboard

​Windows and clipboard

​Launch and open

​Related

Screenshot and display

Regions and compressed screenshots

Mouse and keyboard

Windows and clipboard

Launch and open

Related