Compare commits

..

3 Commits

Author SHA1 Message Date
daniel156161 08d0d154c9 fix(api): keep file event streams alive
Build and Push Docker Container / build-and-push (push) Successful in 2m7s
- Disable the Quart response timeout for authenticated file event streams.
- Send SSE keepalives more often to avoid idle proxy disconnects.
- Treat remote stream interruptions as reconnectable disconnects in watch output.
- Bump NanoShare to 1.26.0 and the standalone CLI to 0.4.1.
2026-07-27 22:51:03 +02:00
daniel156161 d3c914285c feat(cli): watch files with event streams
Build and Push Docker Container / build-and-push (push) Successful in 2m3s
- Add a private SSE endpoint that emits file change events for authenticated clients.
- Replace the watch sleep loop with watchdog local file events and remote SSE triggers.
- Debounce local event bursts and keep watch running after transient sync failures.
- Add watchdog as a standalone CLI dependency and cover watch behavior in tests.
- Bump NanoShare to 1.25.0 and the standalone CLI to 0.4.0.
2026-07-27 22:18:09 +02:00
daniel156161 ffdcc979e6 fix(cli): ignore generated folders by default
- Skip .comments, virtualenvs, env files, VCS folders, and OS metadata in sync.
- Apply defaults before project .nanoshareignore and repeatable --ignore rules.
- Cover default ignores for local uploads and direct pattern matching.
2026-07-27 22:01:52 +02:00
11 changed files with 276 additions and 14 deletions
+2 -7
View File
@@ -3,7 +3,6 @@ from __future__ import annotations
import argparse import argparse
import json import json
import sys import sys
import time
from pathlib import Path from pathlib import Path
from .app import make_client from .app import make_client
@@ -14,6 +13,7 @@ from .config import DEFAULT_CONFIG
from .remote_path import join_remote_path from .remote_path import join_remote_path
from .sync import sync_once from .sync import sync_once
from .table import format_table from .table import format_table
from .watch import watch_loop
DEFAULT_NODE = 'picoshare' DEFAULT_NODE = 'picoshare'
@@ -99,12 +99,7 @@ def _cmd_sync(args) -> int:
client.close() client.close()
def _cmd_watch(args) -> int: def _cmd_watch(args) -> int:
print(f'watching {args.folder} every {args.interval}s') return watch_loop(args, _cmd_sync)
while True:
code = _cmd_sync(args)
if code:
return code
time.sleep(args.interval)
def _cmd_completion(args) -> int: def _cmd_completion(args) -> int:
try: try:
+8 -2
View File
@@ -45,11 +45,17 @@ def list_remote(client, node: str) -> list[dict]:
files = result.get('files', []) if isinstance(result, dict) else [] files = result.get('files', []) if isinstance(result, dict) else []
return files if isinstance(files, list) else [] return files if isinstance(files, list) else []
def download_url(client, node: str, file_id: str) -> str: def node_url(client, node: str, path: str) -> str:
base = client.registry.get(node) base = client.registry.get(node)
if not base: if not base:
raise RuntimeError(f'unknown node: {node}') raise RuntimeError(f'unknown node: {node}')
return f"{base.rstrip('/')}/api/files/{quote(file_id, safe='')}/download" return f"{base.rstrip('/')}{path}"
def download_url(client, node: str, file_id: str) -> str:
return node_url(client, node, f"/api/files/{quote(file_id, safe='')}/download")
def events_url(client, node: str) -> str:
return node_url(client, node, '/api/files/events')
def download(client, node: str, file_id: str, path: Path) -> None: def download(client, node: str, file_id: str, path: Path) -> None:
path.parent.mkdir(parents=True, exist_ok=True) path.parent.mkdir(parents=True, exist_ok=True)
+21 -1
View File
@@ -4,9 +4,29 @@ from fnmatch import fnmatch
from pathlib import Path from pathlib import Path
IGNORE_FILE_NAME = '.nanoshareignore' IGNORE_FILE_NAME = '.nanoshareignore'
DEFAULT_IGNORE_PATTERNS = [
IGNORE_FILE_NAME,
'.comments/',
'.env',
'.env.*',
'.venv/',
'venv/',
'__pycache__/',
'.pytest_cache/',
'.mypy_cache/',
'.ruff_cache/',
'.git/',
'.hg/',
'.svn/',
'.DS_Store',
'Thumbs.db',
'desktop.ini',
'$RECYCLE.BIN/',
'System Volume Information/',
]
def load_ignore_patterns(root: Path, extra_patterns: list[str] | None = None) -> list[str]: def load_ignore_patterns(root: Path, extra_patterns: list[str] | None = None) -> list[str]:
patterns = [IGNORE_FILE_NAME] patterns = list(DEFAULT_IGNORE_PATTERNS)
ignore_file = root / IGNORE_FILE_NAME ignore_file = root / IGNORE_FILE_NAME
if ignore_file.is_file(): if ignore_file.is_file():
for line in ignore_file.read_text().splitlines(): for line in ignore_file.read_text().splitlines():
+127
View File
@@ -0,0 +1,127 @@
from __future__ import annotations
import sys
import threading
import time
from pathlib import Path
from typing import Callable
import httpx
from .client import events_url
from .ignore import is_ignored, load_ignore_patterns
from .sync import relative, state_path, SYNC_DIR_NAME
SyncFunc = Callable[[object], int]
class _ChangeFlag:
def __init__(self) -> None:
self.event = threading.Event()
def notify(self) -> None:
self.event.set()
def wait(self, timeout: float) -> bool:
return self.event.wait(timeout)
def clear(self) -> None:
self.event.clear()
def _is_relevant_path(root: Path, path: str, ignore_patterns: list[str]) -> bool:
try:
candidate = Path(path).expanduser().resolve()
rel = relative(root, candidate)
except ValueError:
return False
if candidate == state_path(root) or root / SYNC_DIR_NAME in candidate.parents:
return False
return not is_ignored(rel, ignore_patterns)
def _remote_events_thread(args, flag: _ChangeFlag, stop: threading.Event) -> threading.Thread:
def run() -> None:
while not stop.is_set():
client = None
try:
from .app import make_client
client = make_client(args)
headers = {'Accept': 'text/event-stream', **client.auth_headers()}
with httpx.Client(timeout=None) as http:
with http.stream('GET', events_url(client, args.node), headers=headers) as response:
response.raise_for_status()
for line in response.iter_lines():
if stop.is_set():
return
if line.startswith('event: files.changed'):
flag.notify()
except Exception as exc:
if not stop.is_set():
print(f'remote event stream disconnected; retrying in 30s: {exc}', file=sys.stderr)
stop.wait(30)
finally:
if client:
client.close()
thread = threading.Thread(target=run, daemon=True)
thread.start()
return thread
def _watchdog_observer(root: Path, ignore_patterns: list[str], flag: _ChangeFlag):
from watchdog.events import FileSystemEventHandler
from watchdog.observers import Observer
class Handler(FileSystemEventHandler):
def on_any_event(self, event):
paths = [event.src_path]
dest_path = getattr(event, 'dest_path', None)
if dest_path:
paths.append(dest_path)
if any(_is_relevant_path(root, path, ignore_patterns) for path in paths):
flag.notify()
observer = Observer()
observer.schedule(Handler(), str(root), recursive=True)
observer.start()
return observer
def watch_loop(args, sync_func: SyncFunc) -> int:
root = Path(args.folder).expanduser().resolve()
root.mkdir(parents=True, exist_ok=True)
ignore_patterns = load_ignore_patterns(root, getattr(args, 'ignore', None))
interval = max(float(getattr(args, 'interval', 10.0)), 0.1)
flag = _ChangeFlag()
observer = None
stop_events = threading.Event()
events_thread = None
try:
try:
observer = _watchdog_observer(root, ignore_patterns, flag)
print(f'watching {args.folder} for local changes; remote events with {interval}s fallback')
except Exception as exc:
print(f'watchdog unavailable, polling every {interval}s: {exc}', file=sys.stderr)
print(f'watching {args.folder} every {interval}s')
events_thread = _remote_events_thread(args, flag, stop_events)
while True:
flag.clear()
code = sync_func(args)
if code:
print(f'watch sync failed; retrying in {interval}s', file=sys.stderr)
flag.wait(interval)
continue
if flag.wait(interval):
flag.clear()
time.sleep(1.0)
while flag.wait(1.0):
flag.clear()
except KeyboardInterrupt:
return 0
finally:
stop_events.set()
if observer:
observer.stop()
observer.join(timeout=5)
if events_thread:
events_thread.join(timeout=5)
+2 -1
View File
@@ -1,11 +1,12 @@
[project] [project]
name = "nanoshare-cli" name = "nanoshare-cli"
version = "0.3.0" version = "0.4.1"
description = "NanoShare desktop CLI and folder sync client" description = "NanoShare desktop CLI and folder sync client"
readme = "README.md" readme = "README.md"
requires-python = ">=3.11" requires-python = ">=3.11"
dependencies = [ dependencies = [
"httpx==0.28.1", "httpx==0.28.1",
"watchdog==6.0.0",
] ]
[project.scripts] [project.scripts]
+1 -1
View File
@@ -1,6 +1,6 @@
[project] [project]
name = "nanoshare" name = "nanoshare"
version = "1.24.0" version = "1.26.0"
description = "Add your description here" description = "Add your description here"
readme = "README.md" readme = "README.md"
requires-python = ">=3.13" requires-python = ">=3.13"
+1
View File
@@ -21,6 +21,7 @@ from .side.upload import upload_bp
from .api.cli_auth import cli_auth_bp from .api.cli_auth import cli_auth_bp
from .api.download import api_download_bp from .api.download import api_download_bp
from .api.events import api_events_bp
# Health # Health
from .api.health import health_bp from .api.health import health_bp
+53
View File
@@ -0,0 +1,53 @@
from __future__ import annotations
import asyncio
import json
from my_modules.app.setup import LIMITER
from my_modules.decoratory.header import token_required
from quart import Blueprint, Response, current_app, stream_with_context
api_events_bp = Blueprint('api_events', __name__)
async def _snapshot(user_id: str) -> str:
files = await current_app.convex.get_files(user_id)
rows = []
for item in files or []:
if not isinstance(item, dict):
continue
rows.append({
'file_id': item.get('file_id'),
'file_name': item.get('file_name'),
'file_path': item.get('file_path') or '',
'file_size': item.get('file_size'),
'expires_at': item.get('expires_at'),
'updated_at': item.get('updated_at') or item.get('uploaded_at'),
})
rows.sort(key=lambda row: str(row.get('file_id') or ''))
return json.dumps(rows, sort_keys=True, separators=(',', ':'))
@api_events_bp.get('/api/files/events')
@LIMITER.limit('6 per minute;60 per hour;')
@token_required(['files', 'mesh'])
async def api_file_events(user: dict):
user_id = user['sub']
@stream_with_context
async def stream():
previous = await _snapshot(user_id)
yield 'event: ready\ndata: {}\n\n'
while True:
await asyncio.sleep(15)
current = await _snapshot(user_id)
if current != previous:
previous = current
yield f'event: files.changed\ndata: {current}\n\n'
else:
yield ': keepalive\n\n'
response = Response(stream(), content_type='text/event-stream', headers={
'Cache-Control': 'no-cache',
'X-Accel-Buffering': 'no',
})
response.timeout = None
return response
+2
View File
@@ -13,6 +13,7 @@ from routes import (
upload_bp, upload_bp,
cli_auth_bp, cli_auth_bp,
api_download_bp, api_download_bp,
api_events_bp,
health_bp health_bp
) )
from routes.api.link import link_bp as servicelink_bp from routes.api.link import link_bp as servicelink_bp
@@ -25,6 +26,7 @@ app.register_blueprint(side_main_bp)
app.register_blueprint(upload_bp) app.register_blueprint(upload_bp)
app.register_blueprint(cli_auth_bp) app.register_blueprint(cli_auth_bp)
app.register_blueprint(api_download_bp) app.register_blueprint(api_download_bp)
app.register_blueprint(api_events_bp)
# ServiceLink node-to-node mesh endpoint (POST /rpc) # ServiceLink node-to-node mesh endpoint (POST /rpc)
app.register_blueprint(servicelink_bp) app.register_blueprint(servicelink_bp)
+58 -1
View File
@@ -9,8 +9,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'cli'))
from nanoshare_client import cli from nanoshare_client import cli
from nanoshare_client.auth import _CallbackServer from nanoshare_client.auth import _CallbackServer
from nanoshare_client.completion import script as completion_script from nanoshare_client.completion import script as completion_script
from nanoshare_client.ignore import is_ignored from nanoshare_client.ignore import load_ignore_patterns, is_ignored
from nanoshare_client.table import format_table from nanoshare_client.table import format_table
from nanoshare_client.watch import _is_relevant_path
class FakeClient: class FakeClient:
def __init__(self): def __init__(self):
@@ -210,6 +211,12 @@ def test_sync_ignores_local_files_from_nanoshareignore(tmp_path, monkeypatch):
(tmp_path / 'cache').mkdir() (tmp_path / 'cache').mkdir()
(tmp_path / 'cache' / 'ignored.txt').write_text('ignored') (tmp_path / 'cache' / 'ignored.txt').write_text('ignored')
(tmp_path / 'scratch.tmp').write_text('ignored') (tmp_path / 'scratch.tmp').write_text('ignored')
(tmp_path / '.comments').mkdir()
(tmp_path / '.comments' / 'ignored.xml').write_text('<comment />')
(tmp_path / '.env').write_text('SECRET=ignored')
(tmp_path / '.venv').mkdir()
(tmp_path / '.venv' / 'ignored.py').write_text('ignored')
(tmp_path / 'Thumbs.db').write_text('ignored')
code = cli.main([ code = cli.main([
'sync', 'sync',
@@ -250,6 +257,56 @@ def test_ignore_patterns_match_files_and_directories():
assert is_ignored('docs/page.draft.md', patterns) assert is_ignored('docs/page.draft.md', patterns)
assert not is_ignored('docs/page.md', patterns) assert not is_ignored('docs/page.md', patterns)
def test_default_ignore_patterns_skip_common_generated_files(tmp_path):
patterns = load_ignore_patterns(tmp_path)
assert is_ignored('.comments/comment.xml', patterns)
assert is_ignored('.env', patterns)
assert is_ignored('.env.local', patterns)
assert is_ignored('.venv/bin/python', patterns)
assert is_ignored('venv/bin/python', patterns)
assert is_ignored('__pycache__/mod.pyc', patterns)
assert is_ignored('nested/Thumbs.db', patterns)
assert not is_ignored('docs/note.txt', patterns)
def test_watch_loop_retries_after_sync_error(monkeypatch, tmp_path):
from nanoshare_client import watch as watch_module
calls = []
class FakeFlag:
def __init__(self):
pass
def clear(self):
pass
def wait(self, timeout):
return False
def fake_sync(args):
calls.append(args.folder)
if len(calls) == 1:
return 1
raise KeyboardInterrupt
monkeypatch.setattr(watch_module, '_ChangeFlag', FakeFlag)
monkeypatch.setattr(watch_module, '_watchdog_observer', lambda *args: None)
args = type('Args', (), {'folder': str(tmp_path), 'ignore': [], 'interval': 0.1})()
code = watch_module.watch_loop(args, fake_sync)
assert code == 0
assert calls == [str(tmp_path), str(tmp_path)]
def test_watch_relevance_ignores_sync_state_and_default_ignores(tmp_path):
patterns = load_ignore_patterns(tmp_path)
assert _is_relevant_path(tmp_path, str(tmp_path / 'note.txt'), patterns)
assert not _is_relevant_path(tmp_path, str(tmp_path / '.nanoshare-sync' / 'state.json'), patterns)
assert not _is_relevant_path(tmp_path, str(tmp_path / '.comments' / 'comment.xml'), patterns)
assert not _is_relevant_path(tmp_path, str(tmp_path / '.env'), patterns)
def test_completion_scripts_include_commands(): def test_completion_scripts_include_commands():
zsh = completion_script('zsh') zsh = completion_script('zsh')
bash = completion_script('bash') bash = completion_script('bash')
Generated
+1 -1
View File
@@ -720,7 +720,7 @@ wheels = [
[[package]] [[package]]
name = "nanoshare" name = "nanoshare"
version = "1.24.0" version = "1.26.0"
source = { virtual = "." } source = { virtual = "." }
dependencies = [ dependencies = [
{ name = "aiohttp" }, { name = "aiohttp" },