From fb78fd0471eeb866dc782e74dd7bdf24043b815d Mon Sep 17 00:00:00 2001 From: Daniel Dolezal Date: Fri, 1 May 2026 19:07:46 +0200 Subject: [PATCH] impplement pageing between native host and browser extension --- browser_cli/native_host.py | 98 ++++++++++++++++++++++++++++++++------ extension/background.js | 22 ++++++++- tests/test_native_host.py | 44 +++++++++++++++++ 3 files changed, 148 insertions(+), 16 deletions(-) diff --git a/browser_cli/native_host.py b/browser_cli/native_host.py index 3c70386..1fae9c3 100644 --- a/browser_cli/native_host.py +++ b/browser_cli/native_host.py @@ -22,7 +22,26 @@ from browser_cli.platform import DEFAULT_ALIAS, endpoint_for_alias, is_windows, SOCKET_PATH: str = "" # set after hello handshake PENDING: dict[str, queue.Queue] = {} PENDING_LOCK = threading.Lock() +WRITE_LOCK = threading.Lock() REGISTRY_PATH = registry_path() +PAGE_SIZE = int(os.environ.get("BROWSER_CLI_PAGE_SIZE", "100")) +PAGEABLE_COMMANDS = { + "tabs.list", + "tabs.filter", + "tabs.query", + "group.list", + "group.tabs", + "group.query", + "windows.list", + "dom.query", + "dom.text", + "dom.attr", + "extract.links", + "extract.images", + "extract.json", + "cookies.list", + "session.list", +} # --- Native Messaging protocol (4-byte LE length prefix + UTF-8 JSON) --- @@ -143,21 +162,7 @@ def handle_cli_connection(conn, listener=None) -> None: if "id" not in cmd: cmd["id"] = str(uuid.uuid4()) - msg_id = cmd["id"] - response_queue: queue.Queue = queue.Queue() - - with PENDING_LOCK: - PENDING[msg_id] = response_queue - - write_native_message(sys.stdout.buffer, cmd) - - try: - result = response_queue.get(timeout=30) - except queue.Empty: - result = {"id": msg_id, "success": False, "error": "timeout waiting for browser response"} - - with PENDING_LOCK: - PENDING.pop(msg_id, None) + result = _handle_browser_command(cmd) response = json.dumps(result).encode("utf-8") if is_windows(): @@ -179,6 +184,69 @@ def handle_cli_connection(conn, listener=None) -> None: listener.close() +def _handle_browser_command(cmd: dict) -> dict: + command = cmd.get("command") + if command in PAGEABLE_COMMANDS: + return _collect_paged_browser_command(cmd) + return _send_browser_command(cmd) + + +def _send_browser_command(cmd: dict, timeout: int = 30) -> dict: + msg_id = cmd.get("id") or str(uuid.uuid4()) + cmd["id"] = msg_id + response_queue: queue.Queue = queue.Queue() + + with PENDING_LOCK: + PENDING[msg_id] = response_queue + + try: + with WRITE_LOCK: + write_native_message(sys.stdout.buffer, cmd) + + try: + return response_queue.get(timeout=timeout) + except queue.Empty: + return {"id": msg_id, "success": False, "error": "timeout waiting for browser response"} + finally: + with PENDING_LOCK: + PENDING.pop(msg_id, None) + + +def _collect_paged_browser_command(cmd: dict) -> dict: + original_id = cmd.get("id") or str(uuid.uuid4()) + offset = 0 + items = [] + total = None + + while True: + page_cmd = dict(cmd) + page_cmd["id"] = str(uuid.uuid4()) + page_args = dict(cmd.get("args") or {}) + page_args["__page"] = {"offset": offset, "limit": PAGE_SIZE} + page_cmd["args"] = page_args + + result = _send_browser_command(page_cmd) + result["id"] = original_id + if not result.get("success", True): + return result + + data = result.get("data") + if not isinstance(data, dict) or data.get("__browserCliPage") is not True: + return result + + page_items = data.get("items") or [] + if not isinstance(page_items, list): + return {"id": original_id, "success": False, "error": "invalid paged response from browser"} + items.extend(page_items) + total = data.get("total", total) + next_offset = data.get("nextOffset") + if next_offset is None: + break + offset = int(next_offset) + + return {"id": original_id, "success": True, "data": items, "pageSize": PAGE_SIZE, "total": total} + + # --- Socket helpers (length-prefixed framing) --- def _send_all(conn: socket.socket, data: bytes) -> None: diff --git a/extension/background.js b/extension/background.js index 988496a..7a3b1e6 100644 --- a/extension/background.js +++ b/extension/background.js @@ -97,7 +97,11 @@ async function onMessage(msg) { let data, error; try { - data = await dispatch(command, args || {}); + const { __page, ...commandArgs } = args || {}; + data = await dispatch(command, commandArgs); + if (__page && Array.isArray(data)) { + data = makePagedData(data, __page); + } } catch (e) { error = e.message || String(e); } @@ -117,6 +121,22 @@ async function onMessage(msg) { } } +function makePagedData(items, page) { + const total = items.length; + const offset = Math.max(0, Number(page.offset) || 0); + const requestedLimit = Math.max(1, Number(page.limit) || 100); + const limit = Math.min(requestedLimit, 1000); + const end = Math.min(offset + limit, total); + return { + __browserCliPage: true, + items: items.slice(offset, end), + offset, + limit, + total, + nextOffset: end < total ? end : null, + }; +} + async function dispatch(command, args) { switch (command) { // ── Navigation ──────────────────────────────────────────────────────── diff --git a/tests/test_native_host.py b/tests/test_native_host.py index 4469d38..0f85f0d 100644 --- a/tests/test_native_host.py +++ b/tests/test_native_host.py @@ -84,3 +84,47 @@ def test_stdin_reader_routes_response_messages(monkeypatch): assert response_queue.get_nowait() == {"id": "msg-1", "success": True} native_host.PENDING.clear() + + +def test_collect_paged_browser_command_accumulates_pages(monkeypatch): + calls = [] + pages = iter([ + {"success": True, "data": {"__browserCliPage": True, "items": [1, 2], "total": 3, "nextOffset": 2}}, + {"success": True, "data": {"__browserCliPage": True, "items": [3], "total": 3, "nextOffset": None}}, + ]) + + def fake_send(cmd): + calls.append(cmd) + return next(pages) + + monkeypatch.setattr(native_host, "PAGE_SIZE", 2) + monkeypatch.setattr(native_host, "_send_browser_command", fake_send) + + result = native_host._collect_paged_browser_command({"id": "orig", "command": "tabs.list", "args": {"foo": "bar"}}) + + assert result == {"id": "orig", "success": True, "data": [1, 2, 3], "pageSize": 2, "total": 3} + assert [call["args"]["__page"] for call in calls] == [ + {"offset": 0, "limit": 2}, + {"offset": 2, "limit": 2}, + ] + assert all(call["args"]["foo"] == "bar" for call in calls) + assert all(call["id"] != "orig" for call in calls) + + +def test_collect_paged_browser_command_passes_through_non_paged_response(monkeypatch): + monkeypatch.setattr(native_host, "_send_browser_command", lambda cmd: {"id": cmd["id"], "success": True, "data": {"value": 1}}) + + result = native_host._collect_paged_browser_command({"id": "orig", "command": "tabs.list", "args": {}}) + + assert result == {"id": "orig", "success": True, "data": {"value": 1}} + + +def test_handle_browser_command_pages_known_list_commands(monkeypatch): + seen = [] + + monkeypatch.setattr(native_host, "_collect_paged_browser_command", lambda cmd: seen.append(cmd) or {"success": True, "data": []}) + + result = native_host._handle_browser_command({"id": "orig", "command": "tabs.list", "args": {}}) + + assert result == {"success": True, "data": []} + assert seen[0]["command"] == "tabs.list"