move database and file storage from edgedb and disk to convex
This commit is contained in:
+10
-61
@@ -68,12 +68,12 @@ async def read_all(uploaded) -> bytes:
|
||||
return await data
|
||||
return data
|
||||
|
||||
def ensure_utc(dt):
|
||||
def ensure_utc(dt:datetime):
|
||||
"""Ensure a timezone-aware UTC datetime or None."""
|
||||
if dt is None:
|
||||
return None
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
# --- Routes ------------------------------------------------------
|
||||
@@ -121,24 +121,20 @@ async def api_upload(user):
|
||||
fname = iso_stamp_filename("pasted", ext)
|
||||
|
||||
fname = safe_name(fname)
|
||||
path = current_app.upload_folder / fname
|
||||
|
||||
data = await read_all(uploaded)
|
||||
|
||||
# write to disk
|
||||
async with aiofiles.open(path, "wb") as f:
|
||||
await f.write(data)
|
||||
storage_id = await current_app.convex.send_to_storage(data=data, content_type=content_type)
|
||||
|
||||
size_bytes = len(data)
|
||||
file_size_pretty = format_size(size_bytes)
|
||||
|
||||
await current_app.edgedb.add_file(
|
||||
await current_app.convex.add_file(
|
||||
file_name=fname,
|
||||
file_size=file_size_pretty,
|
||||
note=note,
|
||||
content_type=content_type,
|
||||
uploaded_at=datetime.now(timezone.utc),
|
||||
expires_at=expires_at_dt,
|
||||
storage_id=storage_id,
|
||||
user_id=user['sub'],
|
||||
)
|
||||
|
||||
@@ -151,66 +147,19 @@ async def api_upload(user):
|
||||
async with aiofiles.open(path, "wb") as f:
|
||||
await f.write(data)
|
||||
|
||||
storage_id = await current_app.convex.send_to_storage(data=data, content_type="text/plain")
|
||||
|
||||
size_bytes = len(data)
|
||||
file_size_pretty = format_size(size_bytes)
|
||||
|
||||
await current_app.edgedb.add_file(
|
||||
await current_app.convex.add_file(
|
||||
file_name=fname,
|
||||
file_size=file_size_pretty,
|
||||
note=note,
|
||||
content_type="text/plain",
|
||||
uploaded_at=datetime.now(timezone.utc),
|
||||
expires_at=expires_at_dt,
|
||||
storage_id=storage_id,
|
||||
user_id=user['sub'],
|
||||
)
|
||||
|
||||
return jsonify({"ok": True})
|
||||
|
||||
# --- Background cleanup ------------------------------------------------------
|
||||
|
||||
async def cleanup_task():
|
||||
"""Hourly cleanup of expired files based on EdgeDB."""
|
||||
await asyncio.sleep(3) # allow app startup
|
||||
while True:
|
||||
try:
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
expired = await current_app.edgedb.get_expired_files(now)
|
||||
if not expired:
|
||||
await asyncio.sleep(3600)
|
||||
continue
|
||||
|
||||
upload_dir: Path = current_app.upload_folder # ensure Path
|
||||
removed_ids: list[str] = []
|
||||
|
||||
for rec in expired:
|
||||
try:
|
||||
# Defensive: only touch files under your upload dir
|
||||
fpath = (upload_dir / rec['file_name']).resolve()
|
||||
if upload_dir.resolve() in fpath.parents or fpath == upload_dir.resolve():
|
||||
fpath.unlink(missing_ok=True)
|
||||
removed_ids.append(rec['file_id'])
|
||||
else:
|
||||
current_app.logger.warning("Refusing to delete outside upload dir: %s", fpath)
|
||||
except Exception as e:
|
||||
current_app.logger.exception("Failed to delete file %s (%s)", rec['file_name'], rec['file_id'])
|
||||
|
||||
# Remove DB rows for files we actually deleted from disk
|
||||
if removed_ids:
|
||||
try:
|
||||
await current_app.edgedb.delete_files_by_ids(removed_ids)
|
||||
current_app.logger.info("Deleted %d expired files from disk and database: %s", len(removed_ids), ", ".join(removed_ids))
|
||||
except Exception:
|
||||
current_app.logger.exception("Failed to delete DB rows for expired files")
|
||||
else:
|
||||
current_app.logger.info("No files where expired or deleted at %s", now.isoformat())
|
||||
|
||||
except Exception:
|
||||
current_app.logger.exception("Cleanup task iteration failed")
|
||||
|
||||
await asyncio.sleep(3600) # every hour
|
||||
|
||||
|
||||
@upload_bp.before_app_serving
|
||||
async def start_cleanup():
|
||||
asyncio.create_task(cleanup_task())
|
||||
|
||||
Reference in New Issue
Block a user