Compare commits
3 Commits
472ed11e9f
...
dev
| Author | SHA1 | Date | |
|---|---|---|---|
|
08d0d154c9
|
|||
|
d3c914285c
|
|||
|
ffdcc979e6
|
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import time
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from .app import make_client
|
from .app import make_client
|
||||||
@@ -14,6 +13,7 @@ from .config import DEFAULT_CONFIG
|
|||||||
from .remote_path import join_remote_path
|
from .remote_path import join_remote_path
|
||||||
from .sync import sync_once
|
from .sync import sync_once
|
||||||
from .table import format_table
|
from .table import format_table
|
||||||
|
from .watch import watch_loop
|
||||||
|
|
||||||
DEFAULT_NODE = 'picoshare'
|
DEFAULT_NODE = 'picoshare'
|
||||||
|
|
||||||
@@ -99,12 +99,7 @@ def _cmd_sync(args) -> int:
|
|||||||
client.close()
|
client.close()
|
||||||
|
|
||||||
def _cmd_watch(args) -> int:
|
def _cmd_watch(args) -> int:
|
||||||
print(f'watching {args.folder} every {args.interval}s')
|
return watch_loop(args, _cmd_sync)
|
||||||
while True:
|
|
||||||
code = _cmd_sync(args)
|
|
||||||
if code:
|
|
||||||
return code
|
|
||||||
time.sleep(args.interval)
|
|
||||||
|
|
||||||
def _cmd_completion(args) -> int:
|
def _cmd_completion(args) -> int:
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -45,11 +45,17 @@ def list_remote(client, node: str) -> list[dict]:
|
|||||||
files = result.get('files', []) if isinstance(result, dict) else []
|
files = result.get('files', []) if isinstance(result, dict) else []
|
||||||
return files if isinstance(files, list) else []
|
return files if isinstance(files, list) else []
|
||||||
|
|
||||||
def download_url(client, node: str, file_id: str) -> str:
|
def node_url(client, node: str, path: str) -> str:
|
||||||
base = client.registry.get(node)
|
base = client.registry.get(node)
|
||||||
if not base:
|
if not base:
|
||||||
raise RuntimeError(f'unknown node: {node}')
|
raise RuntimeError(f'unknown node: {node}')
|
||||||
return f"{base.rstrip('/')}/api/files/{quote(file_id, safe='')}/download"
|
return f"{base.rstrip('/')}{path}"
|
||||||
|
|
||||||
|
def download_url(client, node: str, file_id: str) -> str:
|
||||||
|
return node_url(client, node, f"/api/files/{quote(file_id, safe='')}/download")
|
||||||
|
|
||||||
|
def events_url(client, node: str) -> str:
|
||||||
|
return node_url(client, node, '/api/files/events')
|
||||||
|
|
||||||
def download(client, node: str, file_id: str, path: Path) -> None:
|
def download(client, node: str, file_id: str, path: Path) -> None:
|
||||||
path.parent.mkdir(parents=True, exist_ok=True)
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
|||||||
@@ -4,9 +4,29 @@ from fnmatch import fnmatch
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
IGNORE_FILE_NAME = '.nanoshareignore'
|
IGNORE_FILE_NAME = '.nanoshareignore'
|
||||||
|
DEFAULT_IGNORE_PATTERNS = [
|
||||||
|
IGNORE_FILE_NAME,
|
||||||
|
'.comments/',
|
||||||
|
'.env',
|
||||||
|
'.env.*',
|
||||||
|
'.venv/',
|
||||||
|
'venv/',
|
||||||
|
'__pycache__/',
|
||||||
|
'.pytest_cache/',
|
||||||
|
'.mypy_cache/',
|
||||||
|
'.ruff_cache/',
|
||||||
|
'.git/',
|
||||||
|
'.hg/',
|
||||||
|
'.svn/',
|
||||||
|
'.DS_Store',
|
||||||
|
'Thumbs.db',
|
||||||
|
'desktop.ini',
|
||||||
|
'$RECYCLE.BIN/',
|
||||||
|
'System Volume Information/',
|
||||||
|
]
|
||||||
|
|
||||||
def load_ignore_patterns(root: Path, extra_patterns: list[str] | None = None) -> list[str]:
|
def load_ignore_patterns(root: Path, extra_patterns: list[str] | None = None) -> list[str]:
|
||||||
patterns = [IGNORE_FILE_NAME]
|
patterns = list(DEFAULT_IGNORE_PATTERNS)
|
||||||
ignore_file = root / IGNORE_FILE_NAME
|
ignore_file = root / IGNORE_FILE_NAME
|
||||||
if ignore_file.is_file():
|
if ignore_file.is_file():
|
||||||
for line in ignore_file.read_text().splitlines():
|
for line in ignore_file.read_text().splitlines():
|
||||||
|
|||||||
@@ -0,0 +1,127 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Callable
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from .client import events_url
|
||||||
|
from .ignore import is_ignored, load_ignore_patterns
|
||||||
|
from .sync import relative, state_path, SYNC_DIR_NAME
|
||||||
|
|
||||||
|
SyncFunc = Callable[[object], int]
|
||||||
|
|
||||||
|
class _ChangeFlag:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.event = threading.Event()
|
||||||
|
|
||||||
|
def notify(self) -> None:
|
||||||
|
self.event.set()
|
||||||
|
|
||||||
|
def wait(self, timeout: float) -> bool:
|
||||||
|
return self.event.wait(timeout)
|
||||||
|
|
||||||
|
def clear(self) -> None:
|
||||||
|
self.event.clear()
|
||||||
|
|
||||||
|
def _is_relevant_path(root: Path, path: str, ignore_patterns: list[str]) -> bool:
|
||||||
|
try:
|
||||||
|
candidate = Path(path).expanduser().resolve()
|
||||||
|
rel = relative(root, candidate)
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
if candidate == state_path(root) or root / SYNC_DIR_NAME in candidate.parents:
|
||||||
|
return False
|
||||||
|
return not is_ignored(rel, ignore_patterns)
|
||||||
|
|
||||||
|
def _remote_events_thread(args, flag: _ChangeFlag, stop: threading.Event) -> threading.Thread:
|
||||||
|
def run() -> None:
|
||||||
|
while not stop.is_set():
|
||||||
|
client = None
|
||||||
|
try:
|
||||||
|
from .app import make_client
|
||||||
|
client = make_client(args)
|
||||||
|
headers = {'Accept': 'text/event-stream', **client.auth_headers()}
|
||||||
|
with httpx.Client(timeout=None) as http:
|
||||||
|
with http.stream('GET', events_url(client, args.node), headers=headers) as response:
|
||||||
|
response.raise_for_status()
|
||||||
|
for line in response.iter_lines():
|
||||||
|
if stop.is_set():
|
||||||
|
return
|
||||||
|
if line.startswith('event: files.changed'):
|
||||||
|
flag.notify()
|
||||||
|
except Exception as exc:
|
||||||
|
if not stop.is_set():
|
||||||
|
print(f'remote event stream disconnected; retrying in 30s: {exc}', file=sys.stderr)
|
||||||
|
stop.wait(30)
|
||||||
|
finally:
|
||||||
|
if client:
|
||||||
|
client.close()
|
||||||
|
|
||||||
|
thread = threading.Thread(target=run, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
return thread
|
||||||
|
|
||||||
|
def _watchdog_observer(root: Path, ignore_patterns: list[str], flag: _ChangeFlag):
|
||||||
|
from watchdog.events import FileSystemEventHandler
|
||||||
|
from watchdog.observers import Observer
|
||||||
|
|
||||||
|
class Handler(FileSystemEventHandler):
|
||||||
|
def on_any_event(self, event):
|
||||||
|
paths = [event.src_path]
|
||||||
|
dest_path = getattr(event, 'dest_path', None)
|
||||||
|
if dest_path:
|
||||||
|
paths.append(dest_path)
|
||||||
|
if any(_is_relevant_path(root, path, ignore_patterns) for path in paths):
|
||||||
|
flag.notify()
|
||||||
|
|
||||||
|
observer = Observer()
|
||||||
|
observer.schedule(Handler(), str(root), recursive=True)
|
||||||
|
observer.start()
|
||||||
|
return observer
|
||||||
|
|
||||||
|
def watch_loop(args, sync_func: SyncFunc) -> int:
|
||||||
|
root = Path(args.folder).expanduser().resolve()
|
||||||
|
root.mkdir(parents=True, exist_ok=True)
|
||||||
|
ignore_patterns = load_ignore_patterns(root, getattr(args, 'ignore', None))
|
||||||
|
interval = max(float(getattr(args, 'interval', 10.0)), 0.1)
|
||||||
|
flag = _ChangeFlag()
|
||||||
|
observer = None
|
||||||
|
stop_events = threading.Event()
|
||||||
|
events_thread = None
|
||||||
|
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
observer = _watchdog_observer(root, ignore_patterns, flag)
|
||||||
|
print(f'watching {args.folder} for local changes; remote events with {interval}s fallback')
|
||||||
|
except Exception as exc:
|
||||||
|
print(f'watchdog unavailable, polling every {interval}s: {exc}', file=sys.stderr)
|
||||||
|
print(f'watching {args.folder} every {interval}s')
|
||||||
|
|
||||||
|
events_thread = _remote_events_thread(args, flag, stop_events)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
flag.clear()
|
||||||
|
code = sync_func(args)
|
||||||
|
if code:
|
||||||
|
print(f'watch sync failed; retrying in {interval}s', file=sys.stderr)
|
||||||
|
flag.wait(interval)
|
||||||
|
continue
|
||||||
|
|
||||||
|
if flag.wait(interval):
|
||||||
|
flag.clear()
|
||||||
|
time.sleep(1.0)
|
||||||
|
while flag.wait(1.0):
|
||||||
|
flag.clear()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
return 0
|
||||||
|
finally:
|
||||||
|
stop_events.set()
|
||||||
|
if observer:
|
||||||
|
observer.stop()
|
||||||
|
observer.join(timeout=5)
|
||||||
|
if events_thread:
|
||||||
|
events_thread.join(timeout=5)
|
||||||
+2
-1
@@ -1,11 +1,12 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "nanoshare-cli"
|
name = "nanoshare-cli"
|
||||||
version = "0.3.0"
|
version = "0.4.1"
|
||||||
description = "NanoShare desktop CLI and folder sync client"
|
description = "NanoShare desktop CLI and folder sync client"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.11"
|
requires-python = ">=3.11"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"httpx==0.28.1",
|
"httpx==0.28.1",
|
||||||
|
"watchdog==6.0.0",
|
||||||
]
|
]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
|
|||||||
+1
-1
@@ -1,6 +1,6 @@
|
|||||||
[project]
|
[project]
|
||||||
name = "nanoshare"
|
name = "nanoshare"
|
||||||
version = "1.24.0"
|
version = "1.26.0"
|
||||||
description = "Add your description here"
|
description = "Add your description here"
|
||||||
readme = "README.md"
|
readme = "README.md"
|
||||||
requires-python = ">=3.13"
|
requires-python = ">=3.13"
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ from .side.upload import upload_bp
|
|||||||
|
|
||||||
from .api.cli_auth import cli_auth_bp
|
from .api.cli_auth import cli_auth_bp
|
||||||
from .api.download import api_download_bp
|
from .api.download import api_download_bp
|
||||||
|
from .api.events import api_events_bp
|
||||||
|
|
||||||
# Health
|
# Health
|
||||||
from .api.health import health_bp
|
from .api.health import health_bp
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import json
|
||||||
|
|
||||||
|
from my_modules.app.setup import LIMITER
|
||||||
|
from my_modules.decoratory.header import token_required
|
||||||
|
from quart import Blueprint, Response, current_app, stream_with_context
|
||||||
|
|
||||||
|
api_events_bp = Blueprint('api_events', __name__)
|
||||||
|
|
||||||
|
async def _snapshot(user_id: str) -> str:
|
||||||
|
files = await current_app.convex.get_files(user_id)
|
||||||
|
rows = []
|
||||||
|
for item in files or []:
|
||||||
|
if not isinstance(item, dict):
|
||||||
|
continue
|
||||||
|
rows.append({
|
||||||
|
'file_id': item.get('file_id'),
|
||||||
|
'file_name': item.get('file_name'),
|
||||||
|
'file_path': item.get('file_path') or '',
|
||||||
|
'file_size': item.get('file_size'),
|
||||||
|
'expires_at': item.get('expires_at'),
|
||||||
|
'updated_at': item.get('updated_at') or item.get('uploaded_at'),
|
||||||
|
})
|
||||||
|
rows.sort(key=lambda row: str(row.get('file_id') or ''))
|
||||||
|
return json.dumps(rows, sort_keys=True, separators=(',', ':'))
|
||||||
|
|
||||||
|
@api_events_bp.get('/api/files/events')
|
||||||
|
@LIMITER.limit('6 per minute;60 per hour;')
|
||||||
|
@token_required(['files', 'mesh'])
|
||||||
|
async def api_file_events(user: dict):
|
||||||
|
user_id = user['sub']
|
||||||
|
|
||||||
|
@stream_with_context
|
||||||
|
async def stream():
|
||||||
|
previous = await _snapshot(user_id)
|
||||||
|
yield 'event: ready\ndata: {}\n\n'
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(15)
|
||||||
|
current = await _snapshot(user_id)
|
||||||
|
if current != previous:
|
||||||
|
previous = current
|
||||||
|
yield f'event: files.changed\ndata: {current}\n\n'
|
||||||
|
else:
|
||||||
|
yield ': keepalive\n\n'
|
||||||
|
|
||||||
|
response = Response(stream(), content_type='text/event-stream', headers={
|
||||||
|
'Cache-Control': 'no-cache',
|
||||||
|
'X-Accel-Buffering': 'no',
|
||||||
|
})
|
||||||
|
response.timeout = None
|
||||||
|
return response
|
||||||
@@ -13,6 +13,7 @@ from routes import (
|
|||||||
upload_bp,
|
upload_bp,
|
||||||
cli_auth_bp,
|
cli_auth_bp,
|
||||||
api_download_bp,
|
api_download_bp,
|
||||||
|
api_events_bp,
|
||||||
health_bp
|
health_bp
|
||||||
)
|
)
|
||||||
from routes.api.link import link_bp as servicelink_bp
|
from routes.api.link import link_bp as servicelink_bp
|
||||||
@@ -25,6 +26,7 @@ app.register_blueprint(side_main_bp)
|
|||||||
app.register_blueprint(upload_bp)
|
app.register_blueprint(upload_bp)
|
||||||
app.register_blueprint(cli_auth_bp)
|
app.register_blueprint(cli_auth_bp)
|
||||||
app.register_blueprint(api_download_bp)
|
app.register_blueprint(api_download_bp)
|
||||||
|
app.register_blueprint(api_events_bp)
|
||||||
|
|
||||||
# ServiceLink node-to-node mesh endpoint (POST /rpc)
|
# ServiceLink node-to-node mesh endpoint (POST /rpc)
|
||||||
app.register_blueprint(servicelink_bp)
|
app.register_blueprint(servicelink_bp)
|
||||||
|
|||||||
@@ -9,8 +9,9 @@ sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'cli'))
|
|||||||
from nanoshare_client import cli
|
from nanoshare_client import cli
|
||||||
from nanoshare_client.auth import _CallbackServer
|
from nanoshare_client.auth import _CallbackServer
|
||||||
from nanoshare_client.completion import script as completion_script
|
from nanoshare_client.completion import script as completion_script
|
||||||
from nanoshare_client.ignore import is_ignored
|
from nanoshare_client.ignore import load_ignore_patterns, is_ignored
|
||||||
from nanoshare_client.table import format_table
|
from nanoshare_client.table import format_table
|
||||||
|
from nanoshare_client.watch import _is_relevant_path
|
||||||
|
|
||||||
class FakeClient:
|
class FakeClient:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
@@ -210,6 +211,12 @@ def test_sync_ignores_local_files_from_nanoshareignore(tmp_path, monkeypatch):
|
|||||||
(tmp_path / 'cache').mkdir()
|
(tmp_path / 'cache').mkdir()
|
||||||
(tmp_path / 'cache' / 'ignored.txt').write_text('ignored')
|
(tmp_path / 'cache' / 'ignored.txt').write_text('ignored')
|
||||||
(tmp_path / 'scratch.tmp').write_text('ignored')
|
(tmp_path / 'scratch.tmp').write_text('ignored')
|
||||||
|
(tmp_path / '.comments').mkdir()
|
||||||
|
(tmp_path / '.comments' / 'ignored.xml').write_text('<comment />')
|
||||||
|
(tmp_path / '.env').write_text('SECRET=ignored')
|
||||||
|
(tmp_path / '.venv').mkdir()
|
||||||
|
(tmp_path / '.venv' / 'ignored.py').write_text('ignored')
|
||||||
|
(tmp_path / 'Thumbs.db').write_text('ignored')
|
||||||
|
|
||||||
code = cli.main([
|
code = cli.main([
|
||||||
'sync',
|
'sync',
|
||||||
@@ -250,6 +257,56 @@ def test_ignore_patterns_match_files_and_directories():
|
|||||||
assert is_ignored('docs/page.draft.md', patterns)
|
assert is_ignored('docs/page.draft.md', patterns)
|
||||||
assert not is_ignored('docs/page.md', patterns)
|
assert not is_ignored('docs/page.md', patterns)
|
||||||
|
|
||||||
|
def test_default_ignore_patterns_skip_common_generated_files(tmp_path):
|
||||||
|
patterns = load_ignore_patterns(tmp_path)
|
||||||
|
|
||||||
|
assert is_ignored('.comments/comment.xml', patterns)
|
||||||
|
assert is_ignored('.env', patterns)
|
||||||
|
assert is_ignored('.env.local', patterns)
|
||||||
|
assert is_ignored('.venv/bin/python', patterns)
|
||||||
|
assert is_ignored('venv/bin/python', patterns)
|
||||||
|
assert is_ignored('__pycache__/mod.pyc', patterns)
|
||||||
|
assert is_ignored('nested/Thumbs.db', patterns)
|
||||||
|
assert not is_ignored('docs/note.txt', patterns)
|
||||||
|
|
||||||
|
def test_watch_loop_retries_after_sync_error(monkeypatch, tmp_path):
|
||||||
|
from nanoshare_client import watch as watch_module
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
class FakeFlag:
|
||||||
|
def __init__(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def clear(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def wait(self, timeout):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def fake_sync(args):
|
||||||
|
calls.append(args.folder)
|
||||||
|
if len(calls) == 1:
|
||||||
|
return 1
|
||||||
|
raise KeyboardInterrupt
|
||||||
|
|
||||||
|
monkeypatch.setattr(watch_module, '_ChangeFlag', FakeFlag)
|
||||||
|
monkeypatch.setattr(watch_module, '_watchdog_observer', lambda *args: None)
|
||||||
|
args = type('Args', (), {'folder': str(tmp_path), 'ignore': [], 'interval': 0.1})()
|
||||||
|
|
||||||
|
code = watch_module.watch_loop(args, fake_sync)
|
||||||
|
|
||||||
|
assert code == 0
|
||||||
|
assert calls == [str(tmp_path), str(tmp_path)]
|
||||||
|
|
||||||
|
def test_watch_relevance_ignores_sync_state_and_default_ignores(tmp_path):
|
||||||
|
patterns = load_ignore_patterns(tmp_path)
|
||||||
|
|
||||||
|
assert _is_relevant_path(tmp_path, str(tmp_path / 'note.txt'), patterns)
|
||||||
|
assert not _is_relevant_path(tmp_path, str(tmp_path / '.nanoshare-sync' / 'state.json'), patterns)
|
||||||
|
assert not _is_relevant_path(tmp_path, str(tmp_path / '.comments' / 'comment.xml'), patterns)
|
||||||
|
assert not _is_relevant_path(tmp_path, str(tmp_path / '.env'), patterns)
|
||||||
|
|
||||||
def test_completion_scripts_include_commands():
|
def test_completion_scripts_include_commands():
|
||||||
zsh = completion_script('zsh')
|
zsh = completion_script('zsh')
|
||||||
bash = completion_script('bash')
|
bash = completion_script('bash')
|
||||||
|
|||||||
Reference in New Issue
Block a user