From 2982d44e5548c7e35044560ab82449df7b86ad3b Mon Sep 17 00:00:00 2001 From: Daniel Dolezal Date: Sat, 13 Jun 2026 23:59:35 +0200 Subject: [PATCH] feat: add servicelink RPC mesh endpoint - Add the servicelink submodule and register POST /rpc for node-to-node file operations. - Require bearer tokens with the mesh scope and apply rate/body-size limits to RPC calls. - Map database connectivity failures to the existing 504 database error flow, with JSON responses for API routes. - Cover the new RPC handlers and database error handling with focused pytest tests. - Bump the NanoShare package version to 1.21.0. --- .gitmodules | 3 + my_helpers | 2 +- pyproject.toml | 2 +- quart_common | 2 +- routes/api/link.py | 139 +++++++++++++++++++++++++++ routes/handeling/errorsAndBots.py | 10 ++ run.py | 4 + servicelink | 1 + tests/test_database_error_handler.py | 119 +++++++++++++++++++++++ tests/test_servicelink_link.py | 106 ++++++++++++++++++++ uv.lock | 2 +- 11 files changed, 386 insertions(+), 4 deletions(-) create mode 100644 routes/api/link.py create mode 160000 servicelink create mode 100644 tests/test_database_error_handler.py create mode 100644 tests/test_servicelink_link.py diff --git a/.gitmodules b/.gitmodules index 96a2470..98e8c40 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "quart_common"] path = quart_common url = git@git.yiprawr.dev:submodules/python-quart-common.git +[submodule "servicelink"] + path = servicelink + url = git@git.yiprawr.dev:submodules/servicelink.git diff --git a/my_helpers b/my_helpers index 7793ff7..059338c 160000 --- a/my_helpers +++ b/my_helpers @@ -1 +1 @@ -Subproject commit 7793ff79580c6e7699ec7cf31a989acf1ad40932 +Subproject commit 059338cffb4d4daee64daf43d85b3db106a89d46 diff --git a/pyproject.toml b/pyproject.toml index a3feb85..77ac40f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "nanoshare" -version = "1.20.0" +version = "1.21.0" description = "Add your description here" readme = "README.md" requires-python = ">=3.13" diff --git a/quart_common b/quart_common index 5c35d9e..77823f5 160000 --- a/quart_common +++ b/quart_common @@ -1 +1 @@ -Subproject commit 5c35d9ea2e8e725a9f35ef06731946d3733aa798 +Subproject commit 77823f57d1f87ef0dab3c038096de3d7bf692d79 diff --git a/routes/api/link.py b/routes/api/link.py new file mode 100644 index 0000000..4505cb6 --- /dev/null +++ b/routes/api/link.py @@ -0,0 +1,139 @@ +'''ServiceLink mesh endpoint for the picoshare/NanoShare node. + +Lets other nodes (browser-cli, website) push files in and read file metadata +over the shared servicelink envelope at POST /rpc. Sits alongside the existing +web UI and /api routes. + +Security: every call needs a bearer token that decodes to the `mesh` scope +(or `all`); the endpoint is rate limited and body-size capped. Intended to be +reachable only from the internal node network (lock it down at the reverse +proxy), with the token as defence-in-depth. +''' +from __future__ import annotations + +import base64 + +from quart import Blueprint, Response, current_app, request + +from my_modules.app.setup import LIMITER +from my_modules.expiry import ensure_utc, parse_expires +from my_modules.file_meta import format_size, iso_stamp_filename +from servicelink import Forbidden, InvalidParams, NotFound, Router, Unauthorized, bearer_verifier, handle_envelope + +# Cap the /rpc body. base64 inflates ~33%, so this bounds upload size too. +MAX_RPC_BODY = 16 * 1024 * 1024 +MESH_SCOPE = 'mesh' + +router = Router('picoshare') + +def _user_id(ctx): + if ctx.principal is None: + raise Unauthorized('authentication required') + return ctx.principal.subject + +@router.method('files.upload') +async def files_upload(params, ctx): + user_id = _user_id(ctx) + text = params.get('text') + content_b64 = params.get('content_b64') + note = params.get('note', '') + expires_at = ensure_utc(parse_expires(params.get('expires', ''))) + + if content_b64: + data = base64.b64decode(content_b64) + content_type = params.get('content_type') or 'application/octet-stream' + default_ext = 'bin' + elif text is not None: + data = text.encode('utf-8') + content_type = 'text/plain' + default_ext = 'txt' + else: + raise InvalidParams('provide text or content_b64') + + file_name = params.get('file_name') or iso_stamp_filename('mesh', default_ext) + storage_id = await current_app.convex.send_to_storage(data=data, content_type=content_type) + await current_app.convex.add_file( + file_name=file_name, + file_size=format_size(len(data)), + note=note, + content_type=content_type, + expires_at=expires_at, + storage_id=storage_id, + user_id=user_id, + ) + return {'file_name': file_name, 'size': len(data), 'content_type': content_type} + +@router.method('files.list') +async def files_list(params, ctx): + return {'files': await current_app.convex.get_files(user_id=_user_id(ctx))} + +@router.method('files.get') +async def files_get(params, ctx): + file_id = params.get('file_id') + if not file_id: + raise InvalidParams('file_id is required') + meta = await current_app.convex.get_file(file_id) + if not meta: + raise NotFound('no such file', data={'file_id': file_id}) + return meta + +@router.method('files.info') +async def files_info(params, ctx): + file_id = params.get('file_id') + if not file_id: + raise InvalidParams('file_id is required') + return await current_app.convex.get_file_informations(file_id, _user_id(ctx)) + +@router.method('files.update') +async def files_update(params, ctx): + file_id = params.get('file_id') + file_name = params.get('file_name') + if not file_id or not file_name: + raise InvalidParams('file_id and file_name are required') + await current_app.convex.update_file( + file_id=file_id, + file_name=file_name, + note=params.get('note', ''), + expires_at=ensure_utc(parse_expires(params.get('expires', ''))), + user_id=_user_id(ctx), + ) + return {'updated': True} + +@router.method('files.delete') +async def files_delete(params, ctx): + file_id = params.get('file_id') + if not file_id: + raise InvalidParams('file_id is required') + await current_app.convex.delete_file(file_id, _user_id(ctx)) + return {'deleted': True} + +async def _decode_access_token(token): + payload = await current_app.convex.decode_access_token_payload(access_token=token) + if not payload or payload.get('error') or not payload.get('sub'): + raise ValueError((payload or {}).get('error', 'invalid token')) + return payload + +_base_verify = bearer_verifier(_decode_access_token) + +async def _verify(authorization, req): + principal = await _base_verify(authorization, req) + if principal is None or not principal.has_scope(MESH_SCOPE): + raise Forbidden(f'token lacks the {MESH_SCOPE!r} scope') + return principal + +link_bp = Blueprint('servicelink_picoshare', __name__) + +@link_bp.post('/rpc') +@LIMITER.limit('30 per minute') +async def rpc_endpoint(): + if (request.content_length or 0) > MAX_RPC_BODY: + return Response('{"error":"payload too large"}', status=413, content_type='application/json') + raw = await request.get_data() + status, body, content_type = await handle_envelope( + router, + raw, + authorization=request.headers.get('Authorization'), + verify=_verify, + content_type=request.headers.get('Content-Type', 'application/json'), + ) + return Response(body, status=status, content_type=content_type) diff --git a/routes/handeling/errorsAndBots.py b/routes/handeling/errorsAndBots.py index 34d2ea3..701683c 100644 --- a/routes/handeling/errorsAndBots.py +++ b/routes/handeling/errorsAndBots.py @@ -13,6 +13,7 @@ from my_modules.functions import ( ) from quart_common.web.env import is_development_environment from quart_common.web.wide_event import add_wide_event_context +import httpx IGNORED_404_PATHS = [ "/.well-known/", @@ -186,6 +187,11 @@ async def internal_server_error(e): file={'name': '500.webp', 'alt': "Astronaut jumping and clicking on random Buttons as a red alert gone off - They is a Text on the Image saying: Why don't shit Work!?!"}, ), 500 +@app.errorhandler(httpx.HTTPError) +@app.errorhandler(TimeoutError) +async def database_unavailable_error(e): + return await database_server_error(e) + @app.errorhandler(504) async def database_server_error(e): try: @@ -193,8 +199,12 @@ async def database_server_error(e): except LookupError as e: return await to_many_requests(e) + context = get_request_context() add_wide_event_context(error={"type": type(e).__name__, "message": str(e)}) logger.error(e) + if context.path.startswith("/api"): + return jsonify({"error": "Database Error", "message": "The database is currently unavailable. Please try again in a moment."}), 504 + return await render_template('views/basics/error.htm', title='Database Error', header={'title': '504 - Database Error', 'message': "It looks like something is broke on our end... but don't worry, we're fixing it! Either way, thanks for helping us find new ways to crash our system. Stay curious, hacker-friend!"}, diff --git a/run.py b/run.py index d9bab6a..d62a0de 100755 --- a/run.py +++ b/run.py @@ -13,6 +13,7 @@ from routes import ( upload_bp, health_bp ) +from routes.api.link import link_bp as servicelink_bp # Views for Requests adding the uris app.register_blueprint(basic_bp) @@ -21,6 +22,9 @@ app.register_blueprint(auth_login_bp) app.register_blueprint(side_main_bp) app.register_blueprint(upload_bp) +# ServiceLink node-to-node mesh endpoint (POST /rpc) +app.register_blueprint(servicelink_bp) + app.register_blueprint(health_bp, url_prefix='/health') if __name__ == '__main__': diff --git a/servicelink b/servicelink new file mode 160000 index 0000000..946d79c --- /dev/null +++ b/servicelink @@ -0,0 +1 @@ +Subproject commit 946d79c95da8762cfab8f2dbcaf10391e0cfbc05 diff --git a/tests/test_database_error_handler.py b/tests/test_database_error_handler.py new file mode 100644 index 0000000..9750bb2 --- /dev/null +++ b/tests/test_database_error_handler.py @@ -0,0 +1,119 @@ +import asyncio +import importlib.util +import sys +import types +from pathlib import Path + +import httpx +from quart import Blueprint, Quart, request + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +class FakeLimiter: + def exempt(self, func): + return func + + def limit(self, *_args, **_kwargs): + def decorator(func): + return func + return decorator + +class FakeLogger: + def __init__(self): + self.errors = [] + + def error(self, message): + self.errors.append(message) + + def warning(self, message): + pass + +class FailingConvex: + async def get_current_favicon(self): + raise httpx.ConnectError("[Errno -2] Name or service not known") + +def load_module(module_name, module_path): + spec = importlib.util.spec_from_file_location(module_name, module_path) + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + +def install_route_test_modules(monkeypatch, app, logger): + fake_setup = types.ModuleType("my_modules.app.setup") + fake_setup.app = app + fake_setup.LIMITER = FakeLimiter() + monkeypatch.setitem(sys.modules, "my_modules.app.setup", fake_setup) + + fake_constens = types.ModuleType("my_modules.app.constens") + fake_constens.BLOCKED_IPS_ACCESSING_TIMES = 3 + fake_constens.BLOCKED_IPS_STORED_TIMEFRAME = 60 + monkeypatch.setitem(sys.modules, "my_modules.app.constens", fake_constens) + + fake_logger_module = types.ModuleType("my_modules.app.logger") + fake_logger_module.logger = logger + monkeypatch.setitem(sys.modules, "my_modules.app.logger", fake_logger_module) + + fake_functions = types.ModuleType("my_modules.functions") + fake_functions.get_ip = lambda: "203.0.113.10" + fake_functions.enforce_custom_limit = lambda *_args, **_kwargs: None + fake_functions.get_request_context = lambda: types.SimpleNamespace(path=request.path) + fake_functions.is_valid_uuid = lambda value: True + monkeypatch.setitem(sys.modules, "my_modules.functions", fake_functions) + +def register_template_routes(app): + side_main = Blueprint("side_main", __name__) + + async def index(): + return "ok" + + side_main.add_url_rule("/", "index", index) + side_main.add_url_rule("/files", "files_list", index) + side_main.add_url_rule("/access", "access_list", index) + app.register_blueprint(side_main) + + auth_login = Blueprint("auth_login", __name__) + auth_login.add_url_rule("/login", "login", index) + auth_login.add_url_rule("/logout", "logout", index) + app.register_blueprint(auth_login) + + +def load_errors_and_basics(monkeypatch, app): + logger = FakeLogger() + install_route_test_modules(monkeypatch, app, logger) + root = Path(__file__).resolve().parents[1] + errors = load_module("test_routes_handeling_errorsAndBots", root / "routes" / "handeling" / "errorsAndBots.py") + basics = load_module("test_routes_handeling_basics", root / "routes" / "handeling" / "basics.py") + return errors, basics, logger + +def test_convex_connect_error_is_returned_as_global_database_error(monkeypatch): + async def run_test(): + app = Quart(__name__, template_folder=str(Path(__file__).resolve().parents[1] / "templates" / "side")) + register_template_routes(app) + app.convex = FailingConvex() + _errors, basics, logger = load_errors_and_basics(monkeypatch, app) + app.register_blueprint(basics.basic_bp) + + response = await app.test_client().get("/favicon.ico") + + assert response.status_code == 504 + assert any("Name or service not known" in str(error) for error in logger.errors) + + asyncio.run(run_test()) + +def test_api_convex_connect_error_returns_json_database_error(monkeypatch): + async def run_test(): + app = Quart(__name__, template_folder=str(Path(__file__).resolve().parents[1] / "templates" / "side")) + load_errors_and_basics(monkeypatch, app) + + @app.get("/api/failing") + async def failing_api(): + raise httpx.ConnectError("[Errno -2] Name or service not known") + + response = await app.test_client().get("/api/failing") + payload = await response.get_json() + + assert response.status_code == 504 + assert payload["error"] == "Database Error" + + asyncio.run(run_test()) diff --git a/tests/test_servicelink_link.py b/tests/test_servicelink_link.py new file mode 100644 index 0000000..e534c2d --- /dev/null +++ b/tests/test_servicelink_link.py @@ -0,0 +1,106 @@ +import asyncio +import importlib.util +import json +import sys +import types +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from quart import Quart + +from servicelink import Request + +# Load routes/api/link.py in isolation. Importing it as `routes.api.link` would +# run routes/__init__.py -> my_modules.app.setup -> constens (asyncio.run at +# import), which fails under pytest's output capture. We stub my_modules.app.setup +# with a no-op limiter; link.py's other deps (my_modules.expiry/file_meta, +# servicelink, quart) are side-effect free. +class _NoOpLimiter: + def limit(self, *args, **kwargs): + return lambda fn: fn + +_setup_stub = types.ModuleType('my_modules.app.setup') +_setup_stub.LIMITER = _NoOpLimiter() +sys.modules.setdefault('my_modules.app.setup', _setup_stub) + +_LINK_PATH = Path(__file__).resolve().parents[1] / 'routes' / 'api' / 'link.py' +_spec = importlib.util.spec_from_file_location('picoshare_link_under_test', _LINK_PATH) +link = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(link) + +class FakeConvex: + def __init__(self): + self.files = {} + + async def decode_access_token_payload(self, access_token, required_scope=None, required_scopes=None): + if access_token == 'good': + return {'sub': 'user_123', 'scope': 'all'} + if access_token == 'nomesh': + return {'sub': 'user_123', 'scope': 'files'} + return {'error': 'invalid token'} + + async def send_to_storage(self, data, content_type): + return 'storage_1' + + async def add_file(self, file_name, file_size, note, content_type, expires_at, storage_id, user_id): + self.files[file_name] = { + 'file_name': file_name, + 'note': note, + 'content_type': content_type, + 'storage_id': storage_id, + 'user_id': user_id, + } + return self.files[file_name] + + async def get_file(self, file_id): + return self.files.get(file_id) + +def _call(envelope, token=None): + # Call the handler the /rpc route delegates to, bypassing the LIMITER/size-cap + # wrapper; exercises the real verify (mesh scope) + dispatch + handlers. + async def run(): + app = Quart(__name__) + app.convex = FakeConvex() + async with app.test_request_context('/rpc', method='POST'): + status, body, _ = await link.handle_envelope( + link.router, + json.dumps(envelope).encode('utf-8'), + authorization=f'Bearer {token}' if token else None, + verify=link._verify, + content_type='application/json', + ) + return status, json.loads(body) + + return asyncio.run(run()) + +def _request(method, params=None): + return Request.create(method, params or {}, source='browser-cli', target='picoshare').to_dict() + +def test_text_upload_roundtrip(): + status, body = _call(_request('files.upload', {'text': 'hello mesh', 'file_name': 'note.txt', 'note': 'scraped'}), token='good') + assert status == 200 + assert body['ok'] is True + assert body['result']['file_name'] == 'note.txt' + assert body['result']['content_type'] == 'text/plain' + assert body['result']['size'] == len('hello mesh') + +def test_upload_requires_content(): + status, body = _call(_request('files.upload', {'file_name': 'x'}), token='good') + assert status == 422 + assert body['error']['code'] == 'invalid_params' + +def test_get_missing_file_is_not_found(): + status, body = _call(_request('files.get', {'file_id': 'ghost'}), token='good') + assert status == 404 + assert body['error']['code'] == 'not_found' + +def test_missing_token_is_unauthorized(): + status, body = _call(_request('files.list')) + assert status == 401 + assert body['error']['code'] == 'unauthorized' + +def test_token_without_mesh_scope_is_forbidden(): + status, body = _call(_request('files.list'), token='nomesh') + assert status == 403 + assert body['error']['code'] == 'forbidden' diff --git a/uv.lock b/uv.lock index f6616dc..2f5960c 100644 --- a/uv.lock +++ b/uv.lock @@ -695,7 +695,7 @@ wheels = [ [[package]] name = "nanoshare" -version = "1.20.0" +version = "1.21.0" source = { virtual = "." } dependencies = [ { name = "aiohttp" },