feat(cli): sync folders with remote paths
Build and Push Docker Container / build-and-push (push) Successful in 1m36s

- Store synced folder locations in file_path instead of embedding them in file names.
- Add ignore rules from .nanoshareignore and repeatable sync --ignore flags.
- Adopt existing remote files only after SHA-256 verification.
- Move adopted remotes with metadata updates instead of uploading duplicates.
- Bump NanoShare to 1.23.0 and the standalone CLI to 0.2.0.
This commit is contained in:
2026-07-27 20:53:00 +02:00
parent 77c76b16c4
commit 993337be9f
13 changed files with 369 additions and 41 deletions
+96 -10
View File
@@ -5,10 +5,13 @@ import json
import os
import shutil
import sys
import tempfile
import time
from pathlib import Path
from .client import delete_remote, download, list_remote, upload
from .client import delete_remote, download, list_remote, update_remote, upload
from .ignore import is_ignored, load_ignore_patterns
from .remote_path import join_remote_path, split_remote_path
SYNC_DIR_NAME = '.nanoshare-sync'
STATE_FILE_NAME = 'state.json'
@@ -44,21 +47,23 @@ def save_state(root: Path, state: dict) -> None:
tmp.write_text(json.dumps(state, indent=2, sort_keys=True))
os.replace(tmp, path)
def iter_local_files(root: Path):
def iter_local_files(root: Path, ignore_patterns: list[str] | None = None):
sync_dir = root / SYNC_DIR_NAME
patterns = ignore_patterns or []
for path in sorted(root.rglob('*')):
if not path.is_file():
continue
if path == state_path(root) or sync_dir in path.parents:
continue
if is_ignored(relative(root, path), patterns):
continue
yield path
def relative(root: Path, path: Path) -> str:
return path.relative_to(root).as_posix()
def safe_remote_name(name: str, file_id: str) -> str:
candidate = Path(name or file_id).name
return candidate or file_id
return join_remote_path(name, file_id=file_id)
def unique_path(path: Path) -> Path:
if not path.exists():
@@ -80,19 +85,74 @@ def remote_by_name(remote_files: list[dict]) -> dict[str, dict]:
result = {}
for item in remote_files:
if isinstance(item, dict) and item.get('file_name') and item.get('file_id'):
result.setdefault(item['file_name'], item)
result.setdefault(join_remote_path(item['file_name'], item.get('file_path') or ''), item)
return result
def parse_size(value: object) -> int | None:
if isinstance(value, int):
return value
text = str(value or '').strip()
if not text:
return None
parts = text.split()
try:
number = float(parts[0])
except (ValueError, IndexError):
return None
unit = parts[1].lower() if len(parts) > 1 else 'bytes'
factors = {
'byte': 1,
'bytes': 1,
'b': 1,
'kb': 1024,
'mb': 1024 ** 2,
'gb': 1024 ** 3,
'tb': 1024 ** 4,
}
factor = factors.get(unit)
return int(number * factor) if factor else None
def find_adoptable_remote(client, node: str, path: Path, local_hash: str, remote_files: list[dict], known_ids: set[str]) -> dict | None:
size = path.stat().st_size
name = path.name
candidates = []
for remote in remote_files:
if not isinstance(remote, dict):
continue
file_id = remote.get('file_id')
if not file_id or file_id in known_ids:
continue
if remote.get('file_name') != name:
continue
remote_size = parse_size(remote.get('size_bytes') or remote.get('file_size'))
if remote_size is not None and remote_size != size:
continue
candidates.append(remote)
hash_matches = []
with tempfile.TemporaryDirectory(prefix='nanoshare-adopt-') as temp_dir:
for remote in candidates:
temp_path = Path(temp_dir) / str(remote['file_id'])
try:
download(client, node, str(remote['file_id']), temp_path)
except Exception as exc:
print(f"warning: could not verify remote {remote['file_id']} for adoption: {exc}", file=sys.stderr)
continue
if sha256(temp_path) == local_hash:
hash_matches.append(remote)
return hash_matches[0] if len(hash_matches) == 1 else None
def sync_once(args, client) -> int:
root = Path(args.folder).expanduser().resolve()
root.mkdir(parents=True, exist_ok=True)
state = load_state(root)
known = state.setdefault('files', {})
try:
ignore_patterns = load_ignore_patterns(root, getattr(args, 'ignore', None))
remote_files = list_remote(client, args.node)
remote_ids = remote_by_id(remote_files)
remote_names = remote_by_name(remote_files)
local_paths = {relative(root, path): path for path in iter_local_files(root)}
local_paths = {relative(root, path): path for path in iter_local_files(root, ignore_patterns)}
for rel, path in local_paths.items():
current_hash = sha256(path)
@@ -103,13 +163,34 @@ def sync_once(args, client) -> int:
if old_hash == current_hash and remote_id in remote_ids:
continue
remote_file_name, remote_file_path = split_remote_path(rel)
if not entry:
adopted = find_adoptable_remote(client, args.node, path, current_hash, remote_files, {value.get('file_id') for value in known.values() if isinstance(value, dict)})
if adopted:
adopted_id = adopted['file_id']
current_remote_name = join_remote_path(adopted.get('file_name') or '', adopted.get('file_path') or '', adopted_id)
if current_remote_name != rel:
update_remote(client, args.node, adopted_id, remote_file_name, remote_file_path, args.note or '', args.expires or '')
print(f'move remote: {current_remote_name} -> {rel}')
known[rel] = {
'file_id': adopted_id,
'sha256': current_hash,
'local_mtime': path.stat().st_mtime,
'file_name': remote_file_name,
'file_path': remote_file_path,
}
remote_files = list_remote(client, args.node)
remote_ids = remote_by_id(remote_files)
remote_names = remote_by_name(remote_files)
continue
if remote_id and remote_id not in remote_ids and old_hash != current_hash:
conflict_path = unique_path(path)
shutil.copy2(path, conflict_path)
print(f'conflict: kept changed local copy at {conflict_path}')
print(f'upload: {rel}')
result = upload(client, args.node, path, rel, args.note or '', args.expires or '')
result = upload(client, args.node, path, remote_file_name, args.note or '', args.expires or '', remote_file_path)
new_file_id = result.get('file_id') or result.get('id')
if not new_file_id:
matching = remote_names.get(rel)
@@ -134,7 +215,8 @@ def sync_once(args, client) -> int:
'file_id': new_file_id,
'sha256': current_hash,
'local_mtime': path.stat().st_mtime,
'file_name': rel,
'file_name': remote_file_name,
'file_path': remote_file_path,
}
known_ids = {entry.get('file_id') for entry in known.values() if isinstance(entry, dict)}
@@ -142,15 +224,19 @@ def sync_once(args, client) -> int:
file_id = remote.get('file_id')
if not file_id or file_id in known_ids:
continue
name = safe_remote_name(remote.get('file_name') or '', file_id)
name = join_remote_path(remote.get('file_name') or '', remote.get('file_path') or '', file_id)
if is_ignored(name, ignore_patterns):
continue
path = unique_path(root / name)
print(f'download: {file_id} -> {relative(root, path)}')
download(client, args.node, file_id, path)
remote_file_name, remote_file_path = split_remote_path(name)
known[relative(root, path)] = {
'file_id': file_id,
'sha256': sha256(path),
'local_mtime': path.stat().st_mtime,
'file_name': relative(root, path),
'file_name': remote_file_name,
'file_path': remote_file_path,
}
if args.delete: