impplement pageing between native host and browser extension

browser_cli/native_host.py

@@ -22,7 +22,26 @@ from browser_cli.platform import DEFAULT_ALIAS, endpoint_for_alias, is_windows,
 SOCKET_PATH: str = ""          # set after hello handshake
 PENDING: dict[str, queue.Queue] = {}
 PENDING_LOCK = threading.Lock()
+WRITE_LOCK = threading.Lock()
 REGISTRY_PATH = registry_path()
+PAGE_SIZE = int(os.environ.get("BROWSER_CLI_PAGE_SIZE", "100"))
+PAGEABLE_COMMANDS = {
+    "tabs.list",
+    "tabs.filter",
+    "tabs.query",
+    "group.list",
+    "group.tabs",
+    "group.query",
+    "windows.list",
+    "dom.query",
+    "dom.text",
+    "dom.attr",
+    "extract.links",
+    "extract.images",
+    "extract.json",
+    "cookies.list",
+    "session.list",
+}
 
 # --- Native Messaging protocol (4-byte LE length prefix + UTF-8 JSON) ---
 
@@ -143,21 +162,7 @@ def handle_cli_connection(conn, listener=None) -> None:
         if "id" not in cmd:
             cmd["id"] = str(uuid.uuid4())
 
-        msg_id = cmd["id"]
+        result = _handle_browser_command(cmd)
-        response_queue: queue.Queue = queue.Queue()
-
-        with PENDING_LOCK:
-            PENDING[msg_id] = response_queue
-
-        write_native_message(sys.stdout.buffer, cmd)
-
-        try:
-            result = response_queue.get(timeout=30)
-        except queue.Empty:
-            result = {"id": msg_id, "success": False, "error": "timeout waiting for browser response"}
-
-        with PENDING_LOCK:
-            PENDING.pop(msg_id, None)
 
         response = json.dumps(result).encode("utf-8")
         if is_windows():
@@ -179,6 +184,69 @@ def handle_cli_connection(conn, listener=None) -> None:
             listener.close()
 
 
+def _handle_browser_command(cmd: dict) -> dict:
+    command = cmd.get("command")
+    if command in PAGEABLE_COMMANDS:
+        return _collect_paged_browser_command(cmd)
+    return _send_browser_command(cmd)
+
+
+def _send_browser_command(cmd: dict, timeout: int = 30) -> dict:
+    msg_id = cmd.get("id") or str(uuid.uuid4())
+    cmd["id"] = msg_id
+    response_queue: queue.Queue = queue.Queue()
+
+    with PENDING_LOCK:
+        PENDING[msg_id] = response_queue
+
+    try:
+        with WRITE_LOCK:
+            write_native_message(sys.stdout.buffer, cmd)
+
+        try:
+            return response_queue.get(timeout=timeout)
+        except queue.Empty:
+            return {"id": msg_id, "success": False, "error": "timeout waiting for browser response"}
+    finally:
+        with PENDING_LOCK:
+            PENDING.pop(msg_id, None)
+
+
+def _collect_paged_browser_command(cmd: dict) -> dict:
+    original_id = cmd.get("id") or str(uuid.uuid4())
+    offset = 0
+    items = []
+    total = None
+
+    while True:
+        page_cmd = dict(cmd)
+        page_cmd["id"] = str(uuid.uuid4())
+        page_args = dict(cmd.get("args") or {})
+        page_args["__page"] = {"offset": offset, "limit": PAGE_SIZE}
+        page_cmd["args"] = page_args
+
+        result = _send_browser_command(page_cmd)
+        result["id"] = original_id
+        if not result.get("success", True):
+            return result
+
+        data = result.get("data")
+        if not isinstance(data, dict) or data.get("__browserCliPage") is not True:
+            return result
+
+        page_items = data.get("items") or []
+        if not isinstance(page_items, list):
+            return {"id": original_id, "success": False, "error": "invalid paged response from browser"}
+        items.extend(page_items)
+        total = data.get("total", total)
+        next_offset = data.get("nextOffset")
+        if next_offset is None:
+            break
+        offset = int(next_offset)
+
+    return {"id": original_id, "success": True, "data": items, "pageSize": PAGE_SIZE, "total": total}
+
+
 # --- Socket helpers (length-prefixed framing) ---
 
 def _send_all(conn: socket.socket, data: bytes) -> None:

extension/background.js

@@ -97,7 +97,11 @@ async function onMessage(msg) {
 
   let data, error;
   try {
-    data = await dispatch(command, args || {});
+    const { __page, ...commandArgs } = args || {};
+    data = await dispatch(command, commandArgs);
+    if (__page && Array.isArray(data)) {
+      data = makePagedData(data, __page);
+    }
   } catch (e) {
     error = e.message || String(e);
   }
@@ -117,6 +121,22 @@ async function onMessage(msg) {
   }
 }
 
+function makePagedData(items, page) {
+  const total = items.length;
+  const offset = Math.max(0, Number(page.offset) || 0);
+  const requestedLimit = Math.max(1, Number(page.limit) || 100);
+  const limit = Math.min(requestedLimit, 1000);
+  const end = Math.min(offset + limit, total);
+  return {
+    __browserCliPage: true,
+    items: items.slice(offset, end),
+    offset,
+    limit,
+    total,
+    nextOffset: end < total ? end : null,
+  };
+}
+
 async function dispatch(command, args) {
   switch (command) {
     // ── Navigation ────────────────────────────────────────────────────────

tests/test_native_host.py

@@ -84,3 +84,47 @@ def test_stdin_reader_routes_response_messages(monkeypatch):
 
     assert response_queue.get_nowait() == {"id": "msg-1", "success": True}
     native_host.PENDING.clear()
+
+
+def test_collect_paged_browser_command_accumulates_pages(monkeypatch):
+    calls = []
+    pages = iter([
+        {"success": True, "data": {"__browserCliPage": True, "items": [1, 2], "total": 3, "nextOffset": 2}},
+        {"success": True, "data": {"__browserCliPage": True, "items": [3], "total": 3, "nextOffset": None}},
+    ])
+
+    def fake_send(cmd):
+        calls.append(cmd)
+        return next(pages)
+
+    monkeypatch.setattr(native_host, "PAGE_SIZE", 2)
+    monkeypatch.setattr(native_host, "_send_browser_command", fake_send)
+
+    result = native_host._collect_paged_browser_command({"id": "orig", "command": "tabs.list", "args": {"foo": "bar"}})
+
+    assert result == {"id": "orig", "success": True, "data": [1, 2, 3], "pageSize": 2, "total": 3}
+    assert [call["args"]["__page"] for call in calls] == [
+        {"offset": 0, "limit": 2},
+        {"offset": 2, "limit": 2},
+    ]
+    assert all(call["args"]["foo"] == "bar" for call in calls)
+    assert all(call["id"] != "orig" for call in calls)
+
+
+def test_collect_paged_browser_command_passes_through_non_paged_response(monkeypatch):
+    monkeypatch.setattr(native_host, "_send_browser_command", lambda cmd: {"id": cmd["id"], "success": True, "data": {"value": 1}})
+
+    result = native_host._collect_paged_browser_command({"id": "orig", "command": "tabs.list", "args": {}})
+
+    assert result == {"id": "orig", "success": True, "data": {"value": 1}}
+
+
+def test_handle_browser_command_pages_known_list_commands(monkeypatch):
+    seen = []
+
+    monkeypatch.setattr(native_host, "_collect_paged_browser_command", lambda cmd: seen.append(cmd) or {"success": True, "data": []})
+
+    result = native_host._handle_browser_command({"id": "orig", "command": "tabs.list", "args": {}})
+
+    assert result == {"success": True, "data": []}
+    assert seen[0]["command"] == "tabs.list"