move database and file storage from edgedb and disk to convex

This commit is contained in:
2025-12-22 02:04:10 +01:00
parent 88d72e3ee1
commit d635da039f
8 changed files with 439 additions and 482 deletions
+10 -61
View File
@@ -68,12 +68,12 @@ async def read_all(uploaded) -> bytes:
return await data
return data
def ensure_utc(dt):
def ensure_utc(dt:datetime):
"""Ensure a timezone-aware UTC datetime or None."""
if dt is None:
return None
return None
if dt.tzinfo is None:
return dt.replace(tzinfo=timezone.utc)
return dt.replace(tzinfo=timezone.utc)
return dt.astimezone(timezone.utc)
# --- Routes ------------------------------------------------------
@@ -121,24 +121,20 @@ async def api_upload(user):
fname = iso_stamp_filename("pasted", ext)
fname = safe_name(fname)
path = current_app.upload_folder / fname
data = await read_all(uploaded)
# write to disk
async with aiofiles.open(path, "wb") as f:
await f.write(data)
storage_id = await current_app.convex.send_to_storage(data=data, content_type=content_type)
size_bytes = len(data)
file_size_pretty = format_size(size_bytes)
await current_app.edgedb.add_file(
await current_app.convex.add_file(
file_name=fname,
file_size=file_size_pretty,
note=note,
content_type=content_type,
uploaded_at=datetime.now(timezone.utc),
expires_at=expires_at_dt,
storage_id=storage_id,
user_id=user['sub'],
)
@@ -151,66 +147,19 @@ async def api_upload(user):
async with aiofiles.open(path, "wb") as f:
await f.write(data)
storage_id = await current_app.convex.send_to_storage(data=data, content_type="text/plain")
size_bytes = len(data)
file_size_pretty = format_size(size_bytes)
await current_app.edgedb.add_file(
await current_app.convex.add_file(
file_name=fname,
file_size=file_size_pretty,
note=note,
content_type="text/plain",
uploaded_at=datetime.now(timezone.utc),
expires_at=expires_at_dt,
storage_id=storage_id,
user_id=user['sub'],
)
return jsonify({"ok": True})
# --- Background cleanup ------------------------------------------------------
async def cleanup_task():
"""Hourly cleanup of expired files based on EdgeDB."""
await asyncio.sleep(3) # allow app startup
while True:
try:
now = datetime.now(timezone.utc)
expired = await current_app.edgedb.get_expired_files(now)
if not expired:
await asyncio.sleep(3600)
continue
upload_dir: Path = current_app.upload_folder # ensure Path
removed_ids: list[str] = []
for rec in expired:
try:
# Defensive: only touch files under your upload dir
fpath = (upload_dir / rec['file_name']).resolve()
if upload_dir.resolve() in fpath.parents or fpath == upload_dir.resolve():
fpath.unlink(missing_ok=True)
removed_ids.append(rec['file_id'])
else:
current_app.logger.warning("Refusing to delete outside upload dir: %s", fpath)
except Exception as e:
current_app.logger.exception("Failed to delete file %s (%s)", rec['file_name'], rec['file_id'])
# Remove DB rows for files we actually deleted from disk
if removed_ids:
try:
await current_app.edgedb.delete_files_by_ids(removed_ids)
current_app.logger.info("Deleted %d expired files from disk and database: %s", len(removed_ids), ", ".join(removed_ids))
except Exception:
current_app.logger.exception("Failed to delete DB rows for expired files")
else:
current_app.logger.info("No files where expired or deleted at %s", now.isoformat())
except Exception:
current_app.logger.exception("Cleanup task iteration failed")
await asyncio.sleep(3600) # every hour
@upload_bp.before_app_serving
async def start_cleanup():
asyncio.create_task(cleanup_task())