move database and file storage from edgedb and disk to convex
This commit is contained in:
+15
-18
@@ -1,11 +1,9 @@
|
||||
from my_modules.file_helper_functions import is_expired, verify_signed_url
|
||||
from my_modules.decoratory.header import login_required
|
||||
from my_modules.functions import get_ip
|
||||
from my_modules.app.setup import LIMITER
|
||||
from my_modules.app.logger import logger
|
||||
|
||||
from quart import Blueprint, request, session, Response, send_from_directory, render_template, abort, current_app
|
||||
from datetime import datetime, timezone
|
||||
from quart import Blueprint, request, session, Response, send_file, render_template, abort, current_app
|
||||
|
||||
side_main_bp = Blueprint('side_main', __name__)
|
||||
|
||||
@@ -19,31 +17,31 @@ async def index():
|
||||
@side_main_bp.route('/access')
|
||||
@login_required
|
||||
async def access_list(user):
|
||||
access_data = await current_app.edgedb.get_all_access_of_user(user_id=user['sub'])
|
||||
access_data = await current_app.convex.get_all_access(user_id=user['sub'])
|
||||
return await render_template("views/webpage/access/list.htm", access_logs=access_data)
|
||||
|
||||
@side_main_bp.route('/files')
|
||||
@login_required
|
||||
async def files_list(user):
|
||||
files_data = await current_app.edgedb.get_files(current_datetime=datetime.now(timezone.utc), user_id=user['sub'])
|
||||
files_data = await current_app.convex.get_files(user_id=user['sub'])
|
||||
return await render_template("views/webpage/files/list.htm", files=files_data)
|
||||
|
||||
@side_main_bp.route('/files/<path:file_id>/info')
|
||||
@login_required
|
||||
async def file_info(file_id, user):
|
||||
files_data = await current_app.edgedb.get_files(user_id=user['sub'])
|
||||
files_data = await current_app.convex.get_files(user_id=user['sub'])
|
||||
return await render_template("views/webpage/files/info.htm", files=files_data)
|
||||
|
||||
@side_main_bp.route('/files/<path:file_id>/edit')
|
||||
@login_required
|
||||
async def file_edit(file_id, user):
|
||||
files_data = await current_app.edgedb.get_files(user_id=user['sub'])
|
||||
files_data = await current_app.convex.get_files(user_id=user['sub'])
|
||||
return await render_template("views/webpage/files/edit.htm", files=files_data)
|
||||
|
||||
@side_main_bp.route("/-<file_id>")
|
||||
@LIMITER.limit("10 per minute;500 per hour;")
|
||||
async def serve_file(file_id: str):
|
||||
file_data = await current_app.edgedb.get_file(file_id=file_id)
|
||||
file_data = await current_app.convex.get_file(file_id=file_id)
|
||||
disable_logging = False
|
||||
|
||||
if not file_data:
|
||||
@@ -53,9 +51,9 @@ async def serve_file(file_id: str):
|
||||
if user and user['sub'] == file_data['user_id']:
|
||||
disable_logging = True
|
||||
|
||||
if is_expired(file_data.get("expires_at")):
|
||||
if file_data.get("expired", None):
|
||||
if not disable_logging:
|
||||
await current_app.edgedb.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="expired", accessed_at=datetime.now(timezone.utc))
|
||||
await current_app.convex.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="expired")
|
||||
return Response("This file has expired.", status=410, headers={
|
||||
"Cache-Control": "no-store",
|
||||
"X-Content-Type-Options": "nosniff",
|
||||
@@ -66,21 +64,20 @@ async def serve_file(file_id: str):
|
||||
|
||||
force_download = request.args.get("download") in {"1", "true", "yes"}
|
||||
|
||||
path = current_app.upload_folder / file_name
|
||||
if not path.exists() or not path.is_file():
|
||||
if not file_data.get('db_image_url', None):
|
||||
if not disable_logging:
|
||||
await current_app.edgedb.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="error", accessed_at=datetime.now(timezone.utc))
|
||||
await current_app.convex.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="error")
|
||||
abort(404)
|
||||
|
||||
if not disable_logging:
|
||||
await current_app.edgedb.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="ok", accessed_at=datetime.now(timezone.utc))
|
||||
return await send_from_directory(
|
||||
directory=current_app.upload_folder,
|
||||
file_name=file_name,
|
||||
await current_app.convex.add_file_access(file_id=file_id, ip_address=get_ip(), user_agent=request.user_agent, status="ok")
|
||||
|
||||
return await send_file(
|
||||
filename_or_io=await current_app.convex.get_from_storage(file_data.get('db_image_url')),
|
||||
mimetype=content_type,
|
||||
as_attachment=force_download,
|
||||
attachment_filename=file_name,
|
||||
conditional=True,
|
||||
cache_timeout=60,
|
||||
last_modified=path.stat().st_mtime
|
||||
last_modified=int(file_data['uploaded_at']) / 1000
|
||||
)
|
||||
|
||||
+10
-61
@@ -68,12 +68,12 @@ async def read_all(uploaded) -> bytes:
|
||||
return await data
|
||||
return data
|
||||
|
||||
def ensure_utc(dt):
|
||||
def ensure_utc(dt:datetime):
|
||||
"""Ensure a timezone-aware UTC datetime or None."""
|
||||
if dt is None:
|
||||
return None
|
||||
return None
|
||||
if dt.tzinfo is None:
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt.replace(tzinfo=timezone.utc)
|
||||
return dt.astimezone(timezone.utc)
|
||||
|
||||
# --- Routes ------------------------------------------------------
|
||||
@@ -121,24 +121,20 @@ async def api_upload(user):
|
||||
fname = iso_stamp_filename("pasted", ext)
|
||||
|
||||
fname = safe_name(fname)
|
||||
path = current_app.upload_folder / fname
|
||||
|
||||
data = await read_all(uploaded)
|
||||
|
||||
# write to disk
|
||||
async with aiofiles.open(path, "wb") as f:
|
||||
await f.write(data)
|
||||
storage_id = await current_app.convex.send_to_storage(data=data, content_type=content_type)
|
||||
|
||||
size_bytes = len(data)
|
||||
file_size_pretty = format_size(size_bytes)
|
||||
|
||||
await current_app.edgedb.add_file(
|
||||
await current_app.convex.add_file(
|
||||
file_name=fname,
|
||||
file_size=file_size_pretty,
|
||||
note=note,
|
||||
content_type=content_type,
|
||||
uploaded_at=datetime.now(timezone.utc),
|
||||
expires_at=expires_at_dt,
|
||||
storage_id=storage_id,
|
||||
user_id=user['sub'],
|
||||
)
|
||||
|
||||
@@ -151,66 +147,19 @@ async def api_upload(user):
|
||||
async with aiofiles.open(path, "wb") as f:
|
||||
await f.write(data)
|
||||
|
||||
storage_id = await current_app.convex.send_to_storage(data=data, content_type="text/plain")
|
||||
|
||||
size_bytes = len(data)
|
||||
file_size_pretty = format_size(size_bytes)
|
||||
|
||||
await current_app.edgedb.add_file(
|
||||
await current_app.convex.add_file(
|
||||
file_name=fname,
|
||||
file_size=file_size_pretty,
|
||||
note=note,
|
||||
content_type="text/plain",
|
||||
uploaded_at=datetime.now(timezone.utc),
|
||||
expires_at=expires_at_dt,
|
||||
storage_id=storage_id,
|
||||
user_id=user['sub'],
|
||||
)
|
||||
|
||||
return jsonify({"ok": True})
|
||||
|
||||
# --- Background cleanup ------------------------------------------------------
|
||||
|
||||
async def cleanup_task():
|
||||
"""Hourly cleanup of expired files based on EdgeDB."""
|
||||
await asyncio.sleep(3) # allow app startup
|
||||
while True:
|
||||
try:
|
||||
now = datetime.now(timezone.utc)
|
||||
|
||||
expired = await current_app.edgedb.get_expired_files(now)
|
||||
if not expired:
|
||||
await asyncio.sleep(3600)
|
||||
continue
|
||||
|
||||
upload_dir: Path = current_app.upload_folder # ensure Path
|
||||
removed_ids: list[str] = []
|
||||
|
||||
for rec in expired:
|
||||
try:
|
||||
# Defensive: only touch files under your upload dir
|
||||
fpath = (upload_dir / rec['file_name']).resolve()
|
||||
if upload_dir.resolve() in fpath.parents or fpath == upload_dir.resolve():
|
||||
fpath.unlink(missing_ok=True)
|
||||
removed_ids.append(rec['file_id'])
|
||||
else:
|
||||
current_app.logger.warning("Refusing to delete outside upload dir: %s", fpath)
|
||||
except Exception as e:
|
||||
current_app.logger.exception("Failed to delete file %s (%s)", rec['file_name'], rec['file_id'])
|
||||
|
||||
# Remove DB rows for files we actually deleted from disk
|
||||
if removed_ids:
|
||||
try:
|
||||
await current_app.edgedb.delete_files_by_ids(removed_ids)
|
||||
current_app.logger.info("Deleted %d expired files from disk and database: %s", len(removed_ids), ", ".join(removed_ids))
|
||||
except Exception:
|
||||
current_app.logger.exception("Failed to delete DB rows for expired files")
|
||||
else:
|
||||
current_app.logger.info("No files where expired or deleted at %s", now.isoformat())
|
||||
|
||||
except Exception:
|
||||
current_app.logger.exception("Cleanup task iteration failed")
|
||||
|
||||
await asyncio.sleep(3600) # every hour
|
||||
|
||||
|
||||
@upload_bp.before_app_serving
|
||||
async def start_cleanup():
|
||||
asyncio.create_task(cleanup_task())
|
||||
|
||||
Reference in New Issue
Block a user