daniel156161
2b0215cd94
Build and Push Docker Container / build-and-push (push) Successful in 1m52s
- Leave sync and watch notes empty by default instead of writing placeholder text. - Preserve existing remote notes when moving or adopting files without --note. - Detect tracked local moves by SHA-256 and update remote metadata safely. - Skip files that vanish during watch scans instead of aborting the sync cycle. - Bump NanoShare to 1.24.0 and the standalone CLI to 0.3.0.
289 lines
11 KiB
Python
289 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
from .client import delete_remote, download, list_remote, update_remote, upload
|
|
from .ignore import is_ignored, load_ignore_patterns
|
|
from .remote_path import join_remote_path, split_remote_path
|
|
|
|
SYNC_DIR_NAME = '.nanoshare-sync'
|
|
STATE_FILE_NAME = 'state.json'
|
|
|
|
def sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open('rb') as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b''):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
def state_path(root: Path) -> Path:
|
|
return root / SYNC_DIR_NAME / STATE_FILE_NAME
|
|
|
|
def load_state(root: Path) -> dict:
|
|
path = state_path(root)
|
|
if not path.is_file():
|
|
return {'version': 1, 'files': {}}
|
|
try:
|
|
data = json.loads(path.read_text())
|
|
except (OSError, ValueError):
|
|
return {'version': 1, 'files': {}}
|
|
if not isinstance(data, dict):
|
|
return {'version': 1, 'files': {}}
|
|
data.setdefault('version', 1)
|
|
data.setdefault('files', {})
|
|
return data
|
|
|
|
def save_state(root: Path, state: dict) -> None:
|
|
path = state_path(root)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
tmp = path.with_name(f'{path.name}.{os.getpid()}.tmp')
|
|
tmp.write_text(json.dumps(state, indent=2, sort_keys=True))
|
|
os.replace(tmp, path)
|
|
|
|
def iter_local_files(root: Path, ignore_patterns: list[str] | None = None):
|
|
sync_dir = root / SYNC_DIR_NAME
|
|
patterns = ignore_patterns or []
|
|
for path in sorted(root.rglob('*')):
|
|
if not path.is_file():
|
|
continue
|
|
if path == state_path(root) or sync_dir in path.parents:
|
|
continue
|
|
if is_ignored(relative(root, path), patterns):
|
|
continue
|
|
yield path
|
|
|
|
def relative(root: Path, path: Path) -> str:
|
|
return path.relative_to(root).as_posix()
|
|
|
|
def safe_remote_name(name: str, file_id: str) -> str:
|
|
return join_remote_path(name, file_id=file_id)
|
|
|
|
def unique_path(path: Path) -> Path:
|
|
if not path.exists():
|
|
return path
|
|
stamp = time.strftime('%Y%m%d-%H%M%S')
|
|
suffix = ''.join(path.suffixes)
|
|
stem = path.name[:-len(suffix)] if suffix else path.name
|
|
candidate = path.with_name(f'{stem}.conflict-{stamp}{suffix}')
|
|
counter = 2
|
|
while candidate.exists():
|
|
candidate = path.with_name(f'{stem}.conflict-{stamp}-{counter}{suffix}')
|
|
counter += 1
|
|
return candidate
|
|
|
|
def remote_by_id(remote_files: list[dict]) -> dict[str, dict]:
|
|
return {item['file_id']: item for item in remote_files if isinstance(item, dict) and item.get('file_id')}
|
|
|
|
def remote_by_name(remote_files: list[dict]) -> dict[str, dict]:
|
|
result = {}
|
|
for item in remote_files:
|
|
if isinstance(item, dict) and item.get('file_name') and item.get('file_id'):
|
|
result.setdefault(join_remote_path(item['file_name'], item.get('file_path') or ''), item)
|
|
return result
|
|
|
|
def parse_size(value: object) -> int | None:
|
|
if isinstance(value, int):
|
|
return value
|
|
text = str(value or '').strip()
|
|
if not text:
|
|
return None
|
|
parts = text.split()
|
|
try:
|
|
number = float(parts[0])
|
|
except (ValueError, IndexError):
|
|
return None
|
|
unit = parts[1].lower() if len(parts) > 1 else 'bytes'
|
|
factors = {
|
|
'byte': 1,
|
|
'bytes': 1,
|
|
'b': 1,
|
|
'kb': 1024,
|
|
'mb': 1024 ** 2,
|
|
'gb': 1024 ** 3,
|
|
'tb': 1024 ** 4,
|
|
}
|
|
factor = factors.get(unit)
|
|
return int(number * factor) if factor else None
|
|
|
|
def find_moved_entry(root: Path, known: dict, rel: str, local_hash: str, remote_ids: dict[str, dict]) -> tuple[str, dict] | None:
|
|
for old_rel, entry in known.items():
|
|
if old_rel == rel or not isinstance(entry, dict):
|
|
continue
|
|
file_id = entry.get('file_id')
|
|
if entry.get('sha256') == local_hash and file_id in remote_ids and not (root / old_rel).is_file():
|
|
return old_rel, entry
|
|
return None
|
|
|
|
def find_adoptable_remote(client, node: str, path: Path, local_hash: str, remote_files: list[dict], known_ids: set[str]) -> dict | None:
|
|
size = path.stat().st_size
|
|
name = path.name
|
|
candidates = []
|
|
for remote in remote_files:
|
|
if not isinstance(remote, dict):
|
|
continue
|
|
file_id = remote.get('file_id')
|
|
if not file_id or file_id in known_ids:
|
|
continue
|
|
if remote.get('file_name') != name:
|
|
continue
|
|
remote_size = parse_size(remote.get('size_bytes') or remote.get('file_size'))
|
|
if remote_size is not None and remote_size != size:
|
|
continue
|
|
candidates.append(remote)
|
|
|
|
hash_matches = []
|
|
with tempfile.TemporaryDirectory(prefix='nanoshare-adopt-') as temp_dir:
|
|
for remote in candidates:
|
|
temp_path = Path(temp_dir) / str(remote['file_id'])
|
|
try:
|
|
download(client, node, str(remote['file_id']), temp_path)
|
|
except Exception as exc:
|
|
print(f"warning: could not verify remote {remote['file_id']} for adoption: {exc}", file=sys.stderr)
|
|
continue
|
|
if sha256(temp_path) == local_hash:
|
|
hash_matches.append(remote)
|
|
return hash_matches[0] if len(hash_matches) == 1 else None
|
|
|
|
def sync_once(args, client) -> int:
|
|
root = Path(args.folder).expanduser().resolve()
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
state = load_state(root)
|
|
known = state.setdefault('files', {})
|
|
try:
|
|
ignore_patterns = load_ignore_patterns(root, getattr(args, 'ignore', None))
|
|
remote_files = list_remote(client, args.node)
|
|
remote_ids = remote_by_id(remote_files)
|
|
remote_names = remote_by_name(remote_files)
|
|
|
|
for path in iter_local_files(root, ignore_patterns):
|
|
rel = relative(root, path)
|
|
try:
|
|
current_hash = sha256(path)
|
|
except FileNotFoundError:
|
|
print(f'skip vanished local file: {rel}')
|
|
continue
|
|
entry = known.get(rel)
|
|
remote_id = entry.get('file_id') if isinstance(entry, dict) else None
|
|
old_hash = entry.get('sha256') if isinstance(entry, dict) else None
|
|
|
|
if old_hash == current_hash and remote_id in remote_ids:
|
|
continue
|
|
|
|
remote_file_name, remote_file_path = split_remote_path(rel)
|
|
if not entry:
|
|
moved = find_moved_entry(root, known, rel, current_hash, remote_ids)
|
|
if moved:
|
|
old_rel, moved_entry = moved
|
|
moved_id = moved_entry['file_id']
|
|
update_remote(client, args.node, moved_id, remote_file_name, remote_file_path, args.note if args.note else None, args.expires or '')
|
|
print(f'move remote: {old_rel} -> {rel}')
|
|
known.pop(old_rel, None)
|
|
known[rel] = {
|
|
'file_id': moved_id,
|
|
'sha256': current_hash,
|
|
'local_mtime': path.stat().st_mtime,
|
|
'file_name': remote_file_name,
|
|
'file_path': remote_file_path,
|
|
}
|
|
remote_files = list_remote(client, args.node)
|
|
remote_ids = remote_by_id(remote_files)
|
|
remote_names = remote_by_name(remote_files)
|
|
continue
|
|
|
|
adopted = find_adoptable_remote(client, args.node, path, current_hash, remote_files, {value.get('file_id') for value in known.values() if isinstance(value, dict)})
|
|
if adopted:
|
|
adopted_id = adopted['file_id']
|
|
current_remote_name = join_remote_path(adopted.get('file_name') or '', adopted.get('file_path') or '', adopted_id)
|
|
if current_remote_name != rel:
|
|
update_remote(client, args.node, adopted_id, remote_file_name, remote_file_path, args.note if args.note else None, args.expires or '')
|
|
print(f'move remote: {current_remote_name} -> {rel}')
|
|
known[rel] = {
|
|
'file_id': adopted_id,
|
|
'sha256': current_hash,
|
|
'local_mtime': path.stat().st_mtime,
|
|
'file_name': remote_file_name,
|
|
'file_path': remote_file_path,
|
|
}
|
|
remote_files = list_remote(client, args.node)
|
|
remote_ids = remote_by_id(remote_files)
|
|
remote_names = remote_by_name(remote_files)
|
|
continue
|
|
|
|
if remote_id and remote_id not in remote_ids and old_hash != current_hash:
|
|
conflict_path = unique_path(path)
|
|
shutil.copy2(path, conflict_path)
|
|
print(f'conflict: kept changed local copy at {conflict_path}')
|
|
|
|
print(f'upload: {rel}')
|
|
result = upload(client, args.node, path, remote_file_name, args.note or '', args.expires or '', remote_file_path)
|
|
new_file_id = result.get('file_id') or result.get('id')
|
|
if not new_file_id:
|
|
matching = remote_names.get(rel)
|
|
refreshed = list_remote(client, args.node)
|
|
new_file_id = (remote_by_name(refreshed).get(rel) or matching or {}).get('file_id')
|
|
remote_files = refreshed
|
|
remote_ids = remote_by_id(remote_files)
|
|
remote_names = remote_by_name(remote_files)
|
|
if not new_file_id:
|
|
print(f'warning: upload succeeded but file_id is unknown for {rel}', file=sys.stderr)
|
|
continue
|
|
if remote_id and remote_id in remote_ids and remote_id != new_file_id:
|
|
try:
|
|
delete_remote(client, args.node, remote_id)
|
|
print(f'delete old remote: {rel}')
|
|
remote_files = [item for item in remote_files if item.get('file_id') != remote_id]
|
|
remote_ids = remote_by_id(remote_files)
|
|
remote_names = remote_by_name(remote_files)
|
|
except Exception as exc:
|
|
print(f'warning: could not delete old remote {remote_id}: {exc}', file=sys.stderr)
|
|
known[rel] = {
|
|
'file_id': new_file_id,
|
|
'sha256': current_hash,
|
|
'local_mtime': path.stat().st_mtime,
|
|
'file_name': remote_file_name,
|
|
'file_path': remote_file_path,
|
|
}
|
|
|
|
known_ids = {entry.get('file_id') for entry in known.values() if isinstance(entry, dict)}
|
|
for remote in remote_files:
|
|
file_id = remote.get('file_id')
|
|
if not file_id or file_id in known_ids:
|
|
continue
|
|
name = join_remote_path(remote.get('file_name') or '', remote.get('file_path') or '', file_id)
|
|
if is_ignored(name, ignore_patterns):
|
|
continue
|
|
path = unique_path(root / name)
|
|
print(f'download: {file_id} -> {relative(root, path)}')
|
|
download(client, args.node, file_id, path)
|
|
remote_file_name, remote_file_path = split_remote_path(name)
|
|
known[relative(root, path)] = {
|
|
'file_id': file_id,
|
|
'sha256': sha256(path),
|
|
'local_mtime': path.stat().st_mtime,
|
|
'file_name': remote_file_name,
|
|
'file_path': remote_file_path,
|
|
}
|
|
|
|
if args.delete:
|
|
for rel, entry in list(known.items()):
|
|
if (root / rel).is_file():
|
|
continue
|
|
file_id = entry.get('file_id') if isinstance(entry, dict) else None
|
|
if file_id and file_id in remote_ids:
|
|
print(f'delete remote missing locally: {rel}')
|
|
delete_remote(client, args.node, file_id)
|
|
known.pop(rel, None)
|
|
|
|
save_state(root, state)
|
|
except Exception as exc:
|
|
print(f'error: {exc}', file=sys.stderr)
|
|
return 1
|
|
return 0
|